new speed up

[presentation_reservoir.git] / reservoir.tex

diff --git a/reservoir.tex b/reservoir.tex
index 594ad47468798fea9fe6739db11006fd85e3dec7..950da0f384bdb3061b24d9d5624b9b99cc3a5bb4 100644 (file)
--- a/reservoir.tex
+++ b/reservoir.tex
@@ -16,7 +16,7 @@
 of the neuromorphic simulation code.
 Application on the MNIST problem} 
 
 of the neuromorphic simulation code.
 Application on the MNIST problem} 
 

-\author{Rapha\"el Couturier, {\bf Michel Salomon}}
+\author{Rapha\"el Couturier, Michel Salomon}

 
 \institute[FEMTO-ST Institute]{\textit{FEMTO-ST - DISC Department - AND Team}}
 
 
 \institute[FEMTO-ST Institute]{\textit{FEMTO-ST - DISC Department - AND Team}}
 


@@ -69,7 +69,7 @@ Application on the MNIST problem}
     \begin{itemize}
     \item Study the concept of Reservoir Computing
     \item Design a faster simulation code
     \begin{itemize}
     \item Study the concept of Reservoir Computing
     \item Design a faster simulation code

-    \item Apply it to a new problem
+    \item Apply it to new problems

     \end{itemize}
   \end{femtoBlock}
 \end{frame}
     \end{itemize}
   \end{femtoBlock}
 \end{frame}


@@ -88,7 +88,7 @@ Application on the MNIST problem}
     \centering
     \includegraphics[width=7.5cm]{ntc.png}
     \begin{itemize}
     \centering
     \includegraphics[width=7.5cm]{ntc.png}
     \begin{itemize}

-    \item $\delta_t \rightarrow \mbox{temporal spacing}; \tau_D \rightarrow \mbox{time delay}$
+    \item $\delta \tau \rightarrow \mbox{temporal spacing}; \tau_D \rightarrow \mbox{time delay}$

     \item $f(x) \rightarrow \mbox{nonlinear transformation}; h(t) \rightarrow \mbox{impulse response}$
     \end{itemize}
   \end{femtoBlock}
     \item $f(x) \rightarrow \mbox{nonlinear transformation}; h(t) \rightarrow \mbox{impulse response}$
     \end{itemize}
   \end{femtoBlock}


@@ -173,108 +173,199 @@ Application on the MNIST problem}
   % Grandes lignes a partir du pdf de Laurent
   \begin{femtoBlock}
     {Main lines\\}
   % Grandes lignes a partir du pdf de Laurent
   \begin{femtoBlock}
     {Main lines\\}

-    \begin{itemize}
-    \item Numerical integration to compute the nonlinear transient
-      response (runge kuta in C)
-    \item[] $\Rightarrow$ computation of matrices $A$ and $B$
-    \item Computation of the Readout
-    \item Test of the solution (cross validation)
-    \end{itemize}
+    \begin{enumerate}
+    \item Pre-processing
+      \begin{itemize}
+      \item Input data formatting (1D~vector; sampling period $\rightarrow \delta \tau$)
+      \item $W^I$ initialization (randomly; normalization)
+      \end{itemize}
+    \item Concatenation of 1D~vectors $\rightarrow$ batch processing
+    \item Nonlinear transient computation
+      \begin{itemize}
+      \item Numerical integration using a Runge-Kutta C routine
+      \item Computation of matrices $A$ and $B$
+      \end{itemize}
+    \item Training of the Read-out $\rightarrow$ More-Penrose matrix inversion
+    \item Testing of the solution (cross-validation)
+    \end{enumerate}

   \end{femtoBlock}
   \end{femtoBlock}

+  \smallskip

   % Inconvenient de ce code => temps de calcul
   \begin{femtoBlock}
     {Computation time\\}
   % Inconvenient de ce code => temps de calcul
   \begin{femtoBlock}
     {Computation time\\}

-    12 min for 500 words recognition (2013)
+    \vspace{0.125cm}
+    \centering
+    12 min for 306~``neurons'' on a quad-core i7 1,8~GHz (2013)

   \end{femtoBlock}
 \end{frame}
 
 \section{Parallelization and optimization}
 
 \begin{frame}{Parallelization Scheme} % Slide 8
   \end{femtoBlock}
 \end{frame}
 
 \section{Parallelization and optimization}
 
 \begin{frame}{Parallelization Scheme} % Slide 8

-  \begin{itemize}
-  \item Rewrite the code in C++
-  \item Parallelization with MPI (Message Passing Interface)
-  \item Computation of data response (sound, image) is independent so
-    it can be parallelized (computation of matrices A and B)
-   \item Different regression tests are also independent
-  \item Test of new idea? First test with matlab and then adapt to C++
-    with MPI
-  \end{itemize}
-  % 1 - Comment le paralleliser
-  % 2 - Langage et bibliotheque
+  \begin{femtoBlock}
+    {Guidelines\\}
+    \begin{itemize}
+    \item Reservoir response is independent, whatever the data \\
+      $\rightarrow$ computation of matrices A and B can be parallelized
+    \item Different regression tests are also independent
+    \end{itemize}
+  \end{femtoBlock}

 \end{frame}
 
 \end{frame}
 

-\begin{frame}{Finding the Optimal Parameters} % Slide 9
-  % 1 - Quels parametres et pourquoi ?
+\begin{frame}{Parallelization Scheme} % Slide 9

   \begin{femtoBlock}
   \begin{femtoBlock}

-    {What parameters can be optimized?\\}
-    Currently:
+    {In practice\\}

     \begin{itemize}
     \begin{itemize}

-    \item Pitch 
-    \item Delta 
-    \item Beta 
-    \item Phi
-    \item Lambda
+    \item Simulation code rewritten in C++
+    \item Eigen C++ library for linear algebra operations
+    \item InterProcess~Communication \\
+      $\rightarrow$ {\bf M}essage~{\bf P}assing~{\bf I}nterface

     \end{itemize}
     \end{itemize}

-    Next:
+  \end{femtoBlock}
+  \smallskip
+  \begin{femtoBlock}
+    {Performance on speech recognition problem\\}

     \begin{itemize}
     \begin{itemize}

-    \item Number of nodes that significantly improve the solution (threshold)
-    \item Input filter (convolutional filter for images)
-    \item Potentially any parameters
+    \item Same WER $\rightarrow$ similar classification accuracy
+    \item Reduced computation time

     \end{itemize}
     \end{itemize}

-
+    \centering
+    \vspace{0.125cm}
+    {\bf We can study problems with huge Matlab computation time}
+  \end{femtoBlock}
+  \medskip
+  \begin{femtoBlock}
+    {Testing a new idea?\\}
+    \vspace{0.25cm}
+    \centering
+    First test with Matlab and then adapt to C++ with MPI

   \end{femtoBlock}
 \end{frame}
 
   \end{femtoBlock}
 \end{frame}
 

-\begin{frame}{Finding the Optimal Parameters} % Slide 10
-  % 2 - Optimisation par recuit simule
+\begin{frame}{Finding Optimal Parameters} % Slide 10
+  % 1 - Quels parametres et pourquoi ?

   \begin{femtoBlock}
   \begin{femtoBlock}

-    {Optimization heuristics\\}
+    {What parameters can be optimized?\\}

     \begin{itemize}
     \begin{itemize}

-    \item Now: Simulated annealing
-    \item[] $\Rightarrow$ probabilistic technique for approximating the global optimum of a given function.
-    \item Next: maybe other heuristics
+    \item Currently
+      \begin{itemize}
+      \item Pitch of the Read-Out
+      \item Amplitude parameters $\rightarrow \delta; \beta; \phi_0$
+      \item Regression parameter $\rightarrow \lambda$
+      \end{itemize}
+    \item Next
+      \begin{itemize}
+      \item Number of nodes significantly improving the solution (threshold)
+      \item Input data filter (convolution filter for images)
+      \end{itemize}

     \end{itemize}
     \end{itemize}

+    \centering
+    Potentially any parameter can be optimized

   \end{femtoBlock}
   \end{femtoBlock}

+  \smallskip

   \begin{femtoBlock}
   \begin{femtoBlock}

-    {Similar results with the SDR problem\\}
+    {Optimization heuristics\\}
+    \begin{itemize}
+    \item Currently $\rightarrow$ simulated annealing \\
+      {\small (probabilistic global search controlled by a cooling schedule)}
+    \item Next $\rightarrow$ other metaheuristics like evolutionary algorithms
+    \end{itemize}

   \end{femtoBlock}
 \end{frame}
 
   \end{femtoBlock}
 \end{frame}
 

-\begin{frame}{Performances} % Slide 11
-  % 1 - Taux d'erreur en terme de classification
-  % 2 - Gain en temps d'execution / speedup curve
-\end{frame}
-

 \section{Performances on the MNIST problem}
 
 \section{Performances on the MNIST problem}
 

-\begin{frame}{Application on the MNIST problem} % Slide 12
+\begin{frame}{Application on the MNIST problem} % Slide 11
+  \begin{femtoBlock}
+    {Task of handwritten digits recognition\\}
+    \centering
+    \vspace{0.125cm}
+    National Institute of Standards and Technology database
+    \begin{itemize}
+    \item Training dataset $\rightarrow$ american census bureau employees
+    \item Test dataset $\rightarrow$ american high school students
+    \end{itemize}
+  \end{femtoBlock}
+  \smallskip
+  \begin{femtoBlock}
+    {Mixed-NIST database is widely used in machine learning\\}
+    \centering
+    \vspace{0.125cm}
+    Mixing of both datasets and improved images
+    \begin{columns}
+      \begin{column}{7.5cm}
+        \begin{itemize}
+        \item Datasets
+          \begin{itemize}
+          \item Training $\rightarrow$ 60,000 samples
+          \item Test $\rightarrow$ 10,000 samples
+          \end{itemize}
+        \item Grayscale Images      
+          \begin{itemize}
+          \item Normalized to fit into a $20\times20$ pixel bounding box
+          \item Centered and anti-aliased
+          \end{itemize}
+        \end{itemize}
+      \end{column}
+      \begin{column}{3cm}
+        \centering
+        \includegraphics[width=3cm]{mnist.png}
+      \end{column}
+    \end{columns}
+  \end{femtoBlock}

   % 1 - Decrire ce qu'est le MNIST
   % 2 - Setup de notre reservoir
 \end{frame}
 
   % 1 - Decrire ce qu'est le MNIST
   % 2 - Setup de notre reservoir
 \end{frame}
 

-\begin{frame}{Comparison with other approaches} % Slide 13
+\begin{frame}{Performances of the parallel code} % Slide 12
+  % To be completed
+  10000 images:\\
+  1000 reservoirs of 2 neurons, error : 3.85\%\\
+  1 reservoir of 2000 neurons, error : 7.14\%
+  \includegraphics[width=7.5cm]{speedup.pdf}
+\end{frame}
+
+\begin{frame}{Exploring ways to improve the results} % Slide 13
+  % Tableau recapitulant les performances
+\end{frame}
+
+\begin{frame}{Comparison with other approaches} % Slide 14

   % 1 - Convolutional Neural Networks
   % 2 - Reservoir en pipeline (papier de 2015)
 \end{frame}
 
   % 1 - Convolutional Neural Networks
   % 2 - Reservoir en pipeline (papier de 2015)
 \end{frame}
 

-\begin{frame}{Comparison with other approaches} % Slide 14
+\begin{frame}{Comparison with other approaches} % Slide 15

   % Tableau recapitulant les performances
 \end{frame}
 
 \section{Conclusion and perspectives}
 
   % Tableau recapitulant les performances
 \end{frame}
 
 \section{Conclusion and perspectives}
 

-\begin{frame}{Conclusion and perspectives} % Slide 15
-
-
-  Many perspectives (we are just beginning)
-  Improvement of the code\\
-  Test of many ideas : number of comities\\
-  One reservoir to learn and another one to learn error and correct
-  them\\
-  Test other large problems in simulation before in real\\
-  => Try to test many configuration and to find optimal parameters
-  
+\begin{frame}{Conclusion and perspectives} % Slide 16
+  \begin{femtoBlock}
+    {Results\\}
+    \begin{itemize}
+    \item A parallel code allowing fast simulations
+    \item An evaluation on the MNIST problem
+    \end{itemize}
+  \end{femtoBlock}
+  \begin{femtoBlock}
+    {Future works\\}
+    \begin{itemize}
+    \item Further code improvement
+    \item Use of several reservoirs
+      \begin{itemize}
+      \item Committees
+      \item Correct errors of a reservoir by another one
+      \end{itemize}
+    \end{itemize}
+  \end{femtoBlock}
+  %Many perspectives (we are just beginning)
+  %Improvement of the code\\
+  %Test of many ideas : number of comities\\
+  %One reservoir to learn and another one to learn error and correct
+  %them\\
+  %Test other large problems in simulation before in real\\
+  %=> Try to test many configuration and to find optimal parameters

 \end{frame}
 
 \begin{frame}{Thank you for your attention}
 \end{frame}
 
 \begin{frame}{Thank you for your attention}