X-Git-Url: https://bilbo.iut-bm.univ-fcomte.fr/and/gitweb/presentation_reservoir.git/blobdiff_plain/9f893c74c4b2bb41bdacbbd8f340390d1ab67224..d137c8b8a5eaf8701ecfecbc23b7c7187e77819e:/reservoir.tex?ds=inline diff --git a/reservoir.tex b/reservoir.tex index 594ad47..14c47ef 100644 --- a/reservoir.tex +++ b/reservoir.tex @@ -16,7 +16,7 @@ of the neuromorphic simulation code. Application on the MNIST problem} -\author{Rapha\"el Couturier, {\bf Michel Salomon}} +\author{Rapha\"el Couturier, Michel Salomon} \institute[FEMTO-ST Institute]{\textit{FEMTO-ST - DISC Department - AND Team}} @@ -69,7 +69,7 @@ Application on the MNIST problem} \begin{itemize} \item Study the concept of Reservoir Computing \item Design a faster simulation code - \item Apply it to a new problem + \item Apply it to new problems \end{itemize} \end{femtoBlock} \end{frame} @@ -88,7 +88,7 @@ Application on the MNIST problem} \centering \includegraphics[width=7.5cm]{ntc.png} \begin{itemize} - \item $\delta_t \rightarrow \mbox{temporal spacing}; \tau_D \rightarrow \mbox{time delay}$ + \item $\delta \tau \rightarrow \mbox{temporal spacing}; \tau_D \rightarrow \mbox{time delay}$ \item $f(x) \rightarrow \mbox{nonlinear transformation}; h(t) \rightarrow \mbox{impulse response}$ \end{itemize} \end{femtoBlock} @@ -173,108 +173,230 @@ Application on the MNIST problem} % Grandes lignes a partir du pdf de Laurent \begin{femtoBlock} {Main lines\\} - \begin{itemize} - \item Numerical integration to compute the nonlinear transient - response (runge kuta in C) - \item[] $\Rightarrow$ computation of matrices $A$ and $B$ - \item Computation of the Readout - \item Test of the solution (cross validation) - \end{itemize} + \begin{enumerate} + \item Pre-processing + \begin{itemize} + \item Input data formatting (1D~vector; sampling period $\rightarrow \delta \tau$) + \item $W^I$ initialization (randomly; normalization) + \end{itemize} + \item Concatenation of 1D~vectors $\rightarrow$ batch processing + \item Nonlinear transient computation + \begin{itemize} + \item Numerical integration using a Runge-Kutta C routine + \item Computation of matrices $A$ and $B$ + \end{itemize} + \item Training of the Read-out $\rightarrow$ More-Penrose matrix inversion + \item Testing of the solution (cross-validation) + \end{enumerate} \end{femtoBlock} + \smallskip % Inconvenient de ce code => temps de calcul \begin{femtoBlock} {Computation time\\} - 12 min for 500 words recognition (2013) + \vspace{0.125cm} + \centering + 12 min for 306~``neurons'' on a quad-core i7 1,8~GHz (2013) \end{femtoBlock} \end{frame} \section{Parallelization and optimization} \begin{frame}{Parallelization Scheme} % Slide 8 - \begin{itemize} - \item Rewrite the code in C++ - \item Parallelization with MPI (Message Passing Interface) - \item Computation of data response (sound, image) is independent so - it can be parallelized (computation of matrices A and B) - \item Different regression tests are also independent - \item Test of new idea? First test with matlab and then adapt to C++ - with MPI - \end{itemize} - % 1 - Comment le paralleliser - % 2 - Langage et bibliotheque + \begin{femtoBlock} + {Guidelines\\} + \begin{itemize} + \item Reservoir response is independent, whatever the data \\ + $\rightarrow$ computation of matrices A and B can be parallelized + \item Different regression tests are also independent + \end{itemize} + \end{femtoBlock} \end{frame} -\begin{frame}{Finding the Optimal Parameters} % Slide 9 - % 1 - Quels parametres et pourquoi ? +\begin{frame}{Parallelization Scheme} % Slide 9 \begin{femtoBlock} - {What parameters can be optimized?\\} - Currently: + {In practice\\} \begin{itemize} - \item Pitch - \item Delta - \item Beta - \item Phi - \item Lambda + \item Simulation code rewritten in C++ + \item Eigen C++ library for linear algebra operations + \item InterProcess~Communication \\ + $\rightarrow$ {\bf M}essage~{\bf P}assing~{\bf I}nterface \end{itemize} - Next: + \end{femtoBlock} + \smallskip + \begin{femtoBlock} + {Performance on speech recognition problem\\} \begin{itemize} - \item Number of nodes that significantly improve the solution (threshold) - \item Input filter (convolutional filter for images) - \item Potentially any parameters + \item Same WER $\rightarrow$ similar classification accuracy + \item Reduced computation time \end{itemize} - + \centering + \vspace{0.125cm} + {\bf We can study problems with huge Matlab computation time} + \end{femtoBlock} + \medskip + \begin{femtoBlock} + {Testing a new idea?\\} + \vspace{0.25cm} + \centering + First test with Matlab and then adapt to C++ with MPI \end{femtoBlock} \end{frame} -\begin{frame}{Finding the Optimal Parameters} % Slide 10 - % 2 - Optimisation par recuit simule +\begin{frame}{Finding Optimal Parameters} % Slide 10 + % 1 - Quels parametres et pourquoi ? \begin{femtoBlock} - {Optimization heuristics\\} + {What parameters can be optimized?\\} \begin{itemize} - \item Now: Simulated annealing - \item[] $\Rightarrow$ probabilistic technique for approximating the global optimum of a given function. - \item Next: maybe other heuristics + \item Currently + \begin{itemize} + \item Pitch of the Read-Out + \item Amplitude parameters $\rightarrow \delta; \beta; \phi_0$ + \item Regression parameter $\rightarrow \lambda$ + \end{itemize} + \item Next + \begin{itemize} + \item Number of nodes significantly improving the solution (threshold) + \item Input data filter (convolution filter for images) + \end{itemize} \end{itemize} + \centering + Potentially any parameter can be optimized \end{femtoBlock} + \smallskip \begin{femtoBlock} - {Similar results with the SDR problem\\} + {Optimization heuristics\\} + \begin{itemize} + \item Currently $\rightarrow$ simulated annealing \\ + {\small (probabilistic global search controlled by a cooling schedule)} + \item Next $\rightarrow$ other metaheuristics like evolutionary algorithms + \end{itemize} \end{femtoBlock} \end{frame} -\begin{frame}{Performances} % Slide 11 - % 1 - Taux d'erreur en terme de classification - % 2 - Gain en temps d'execution / speedup curve -\end{frame} - \section{Performances on the MNIST problem} -\begin{frame}{Application on the MNIST problem} % Slide 12 +\begin{frame}{Application on the MNIST problem} % Slide 11 + \begin{femtoBlock} + {Task of handwritten digits recognition\\} + \centering + \vspace{0.125cm} + National Institute of Standards and Technology database + \begin{itemize} + \item Training dataset $\rightarrow$ american census bureau employees + \item Test dataset $\rightarrow$ american high school students + \end{itemize} + \end{femtoBlock} + \smallskip + \begin{femtoBlock} + {Mixed-NIST database is widely used in machine learning\\} + \centering + \vspace{0.125cm} + Mixing of both datasets and improved images + \begin{columns} + \begin{column}{7.5cm} + \begin{itemize} + \item Datasets + \begin{itemize} + \item Training $\rightarrow$ 60,000 samples + \item Test $\rightarrow$ 10,000 samples + \end{itemize} + \item Grayscale Images + \begin{itemize} + \item Normalized to fit into a $20\times20$ pixel bounding box + \item Centered and anti-aliased + \end{itemize} + \end{itemize} + \end{column} + \begin{column}{3cm} + \centering + \includegraphics[width=3cm]{mnist.png} + \end{column} + \end{columns} + \end{femtoBlock} % 1 - Decrire ce qu'est le MNIST % 2 - Setup de notre reservoir \end{frame} -\begin{frame}{Comparison with other approaches} % Slide 13 - % 1 - Convolutional Neural Networks - % 2 - Reservoir en pipeline (papier de 2015) +\begin{frame}{Performances of the parallel code} % Slide 12 + \begin{femtoBlock} + {Classification error for 10000 images\\} + \begin{itemize} + \item 1000 reservoirs of 2 neurons $\rightarrow$ error : 3.85\%\\ + \item 1 reservoir of 2000 neurons $\rightarrow$ error : 7.14\% + \end{itemize} + \end{femtoBlock} + \begin{femtoBlock} + {Speedup\\} + \centering + \includegraphics[width=7.25cm]{speedup.pdf} + \end{femtoBlock} +\end{frame} + +\begin{frame}{Exploring ways to improve the results} % Slide 13 + \begin{femtoBlock} + {Using the parallel NTC code\\} + % 1 - Partitionnement d'un reservoir en plein de petits sous-reservoirs + % 2 - Pre-traitement pour convolution + \end{femtoBlock} + \smallskip + \begin{femtoBlock} + {Using the Ogger library\\} + % Parler des travaux de Niels + % 1 - Augmentation du jeu de donnees par des images deformees + % 2 - Comites de reservoir + \end{femtoBlock} \end{frame} \begin{frame}{Comparison with other approaches} % Slide 14 + \begin{femtoBlock} + {Convolutional Neural Networks\\} + % 1 - Convolutional Neural Networks + \end{femtoBlock} + \begin{femtoBlock} + {Multilayer reservoir computing} + % 2 - Reservoir en pipeline (papier des belges de 2015) + \end{femtoBlock} +\end{frame} + +\begin{frame}{Comparison with other approaches} % Slide 15 % Tableau recapitulant les performances \end{frame} \section{Conclusion and perspectives} -\begin{frame}{Conclusion and perspectives} % Slide 15 - - - Many perspectives (we are just beginning) - Improvement of the code\\ - Test of many ideas : number of comities\\ - One reservoir to learn and another one to learn error and correct - them\\ - Test other large problems in simulation before in real\\ - => Try to test many configuration and to find optimal parameters - +\begin{frame}{Conclusion and perspectives} % Slide 16 + \begin{femtoBlock} + {Results\\} + \begin{itemize} + \item A parallel code allowing fast simulations + \item An evaluation on the MNIST problem + \end{itemize} + \end{femtoBlock} + \smallskip + \begin{femtoBlock} + {Future works\\} + \begin{itemize} + \item Further code improvement + \item Use of several reservoirs + \begin{itemize} + \item Committees + \item Correct errors of a reservoir by another one + \end{itemize} + \item Other applications + \begin{itemize} + \item Simulation of lung motion + \item Airflow prediction + \item etc. + \end{itemize} + \end{itemize} + \end{femtoBlock} + %Many perspectives (we are just beginning) + %Improvement of the code\\ + %Test of many ideas : number of comities\\ + %One reservoir to learn and another one to learn error and correct + %them\\ + %Test other large problems in simulation before in real\\ + %=> Try to test many configuration and to find optimal parameters \end{frame} \begin{frame}{Thank you for your attention}