\documentclass{beamer}

\usepackage{beamerthemefemto}

\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsfonts,amsmath,amssymb,stmaryrd}
\usepackage[frenchb]{babel}


\usepackage{graphicx}
\DeclareGraphicsExtensions{.jpg, .png , .pdf, .bmp, .pdftex}


\title{Parallelization and optimization \\
of the neuromorphic simulation code.
Application on the MNIST problem} 

\author{Rapha\"el Couturier, {\bf Michel Salomon}}

\institute[FEMTO-ST Institute]{\textit{FEMTO-ST - DISC Department - AND Team}}

\date{November 2 \& 3, 2015 / Besançon \\
  Dynamical Systems and Brain-inspired Information Processing Workshop}

%  ____  _____ ____  _   _ _____ 
% |  _ \| ____| __ )| | | |_   _|
% | | | |  _| |  _ \| | | | | |  
% | |_| | |___| |_) | |_| | | |  
% |____/|_____|____/ \___/  |_|  
% 

\begin{document}
\setbeamertemplate{background}{\titrefemto}

\begin{frame}[plain]
\titlepage
\end{frame}
 
\setbeamertemplate{background}{\pagefemto}

\begin{frame}{Introduction} % Slide 1
  % 1 - Photonic implementation of reservoir computing through NLDDE in Laurent's team
  % 2 - Motivation
  % -> Success of reservoir computing
  % -> Hardware implementation allowing fast processing
  % 3 - Simulation by a Matlab code - pre and post-processing by computer
  % -> Drawback = redhibitory computation time
  \begin{femtoBlock}
    {Background}
    \begin{itemize}
    \item Emergence of hardware RC implementation
      \begin{center}
        Analogue electronic; {\bf optoelectronic}; fully optical
      \end{center}
      {\scriptsize Larger {\em et al.} - {\it Photonic information processing beyond Turing: an optoelectronic
      implementation of reservoir computing}, Opt. Express 20, 3241-3249 (2012)}
    \item Simulation code 
      \begin{itemize}
      \item Study processing conditions
      \item Tuning parameters
      \item Pre and post-processing by computer
      \end{itemize}
    \end{itemize}
  \end{femtoBlock}
  \medskip
  \begin{femtoBlock}
    {Motivation}
    \begin{itemize}
    \item Study the concept of Reservoir Computing
    \item Design a faster simulation code
    \item Apply it to a new problem
    \end{itemize}
  \end{femtoBlock}
\end{frame}

\begin{frame}{Outline}
\setbeamertemplate{section in toc}[sections numbered] 
\tableofcontents
\end{frame}

\section{Neuromorphic processing}

\begin{frame}{Delay Dynamics as a Reservoir} % Slide 2
  \begin{femtoBlock}
    % Reprendre le schema de la figure 2 du pdf NTC de Laurent
    {Spatio-temporal viewpoint of a DDE ({\NoAutoSpaceBeforeFDP \tiny Larger {\em et al.} - Opt. Express 20:3 2012})\\}
    \centering
    \includegraphics[width=7.5cm]{ntc.png}
    \begin{itemize}
    \item $\delta_t \rightarrow \mbox{temporal spacing}; \tau_D \rightarrow \mbox{time delay}$
    \item $f(x) \rightarrow \mbox{nonlinear transformation}; h(t) \rightarrow \mbox{impulse response}$
    \end{itemize}
  \end{femtoBlock}
  % Slide 21 + equation a retard du NTC
  \begin{femtoBlock}
    {Computer simulation with an Ikeda type NLDDE}
    $$
    \tau \frac{dx}{dt}(t)=-x(t)+\beta \sin^2 [\alpha\,x(t-\tau_D)+\rho\,u_{\mbox{in}}(t-\tau_D)+\Phi_0]
    $$
    \centering
    $\alpha \rightarrow \mbox{feedback scaling}; \beta \rightarrow \mbox{gain}; \rho \rightarrow \mbox{amplification};
    \Phi_0 \rightarrow \mbox{offset}$
  \end{femtoBlock}
\end{frame}

\begin{frame}{Photonic Reservoir Computing} % Slide 3
  \begin{femtoBlock}
    {Optoelectronic setup ({\NoAutoSpaceBeforeFDP \tiny Larger~{\em et al.}~-~Opt.~Express~20:3~2012})\\}
    \vspace{0.25cm}
    \centering
    \includegraphics[width=7.5cm]{photonic-RC.png}
    \vspace{0.25cm}
  \end{femtoBlock}
  \begin{femtoBlock}
    {Spoken digits recognition task}
    \begin{itemize}
    \item Input data $\rightarrow$ {\small matrix obtained by Lyon ear model transformation}
    \item Output data $\rightarrow$ {\small matrix describing the spoken digit}
    \end{itemize}
  \end{femtoBlock}
\end{frame}

\begin{frame}{Spoken Digits Recognition} % Slide 4
  \begin{femtoBlock}
    {Input (pre-processing) \\}
    \begin{itemize}
    \item Lyon ear model transformation \\
      $\rightarrow$ 60 samples $\times$ 86 frequency channels
    \item Channels connection to reservoir (400 neurons) \\
      $\rightarrow$ sparse and random 
    \end{itemize}
    \centering
    \includegraphics[width=7.5cm]{sdr.png}
  \end{femtoBlock}
  \begin{femtoBlock}
    {Reservoir transient response\\}
    \vspace{0.125cm}
    \centering
    Temporal series recorded for Read-Out processing
  \end{femtoBlock}
\end{frame}

\begin{frame}{Spoken Digits Recognition} % Slide 5
  \begin{femtoBlock}
    {Output (post-processing)\\}
    \begin{itemize}
    \item Training of the Read-Out \\
      $\rightarrow$ optimize $W^R$ matrix for the digits of the training set
    \item Regression problem for $A \times W^R \approx B$
      \vspace{-0.25cm}
      $$
      W^R_{\mbox{opt}}=\left(A^TA - \lambda I\right)^{-1} A^T B
      $$
      \vspace{-0.5cm}
      \begin{itemize}
      \item $A$ = concatenates reservoir transient response for each digit
      \item $B$ = concatenates target matrices
      \end{itemize}
    \end{itemize}
  \end{femtoBlock}
  \begin{femtoBlock}
    {Testing\\}
    \begin{itemize}
    \item Dataset of 500 speech samples $\rightarrow$ 5 female speakers
    \item 20-fold cross-validation $\rightarrow$ 20 $\times$ 25 test samples
    \item Performance evaluation $\rightarrow$ {\bf W}ord {\bf E}rror {\bf R}ate
    \end{itemize}
  \end{femtoBlock}
\end{frame}

\begin{frame}{Matlab Simulation Code} % Slide 7
  % Grandes lignes a partir du pdf de Laurent
  \begin{femtoBlock}
    {Main lines\\}
    \begin{itemize}
    \item Numerical integration to compute the nonlinear transient
      response (runge kuta in C)
    \item[] $\Rightarrow$ computation of matrices $A$ and $B$
    \item Computation of the Readout
    \item Test of the solution (cross validation)
    \end{itemize}
  \end{femtoBlock}
  % Inconvenient de ce code => temps de calcul
  \begin{femtoBlock}
    {Computation time\\}
    12 min for 500 words recognition (2013)
  \end{femtoBlock}
\end{frame}

\section{Parallelization and optimization}

\begin{frame}{Parallelization Scheme} % Slide 8
  \begin{itemize}
  \item Rewrite the code in C++
  \item Parallelization with MPI (Message Passing Interface)
  \item Computation of data response (sound, image) is independent so
    it can be parallelized (computation of matrices A and B)
   \item Different regression tests are also independent
  \item Test of new idea? First test with matlab and then adapt to C++
    with MPI
  \end{itemize}
  % 1 - Comment le paralleliser
  % 2 - Langage et bibliotheque
\end{frame}

\begin{frame}{Finding the Optimal Parameters} % Slide 9
  % 1 - Quels parametres et pourquoi ?
  \begin{femtoBlock}
    {What parameters can be optimized?\\}
    Currently:
    \begin{itemize}
    \item Pitch 
    \item Delta 
    \item Beta 
    \item Phi
    \item Lambda
    \end{itemize}
    Next:
    \begin{itemize}
    \item Number of nodes that significantly improve the solution (threshold)
    \item Input filter (convolutional filter for images)
    \item Potentially any parameters
    \end{itemize}

  \end{femtoBlock}
\end{frame}

\begin{frame}{Finding the Optimal Parameters} % Slide 10
  % 2 - Optimisation par recuit simule
  \begin{femtoBlock}
    {Optimization heuristics\\}
    \begin{itemize}
    \item Now: Simulated annealing
    \item[] $\Rightarrow$ probabilistic technique for approximating the global optimum of a given function.
    \item Next: maybe other heuristics
    \end{itemize}
  \end{femtoBlock}
  \begin{femtoBlock}
    {Similar results with the SDR problem\\}
  \end{femtoBlock}
\end{frame}

\begin{frame}{Performances} % Slide 11
  % 1 - Taux d'erreur en terme de classification
  % 2 - Gain en temps d'execution / speedup curve
\end{frame}

\section{Performances on the MNIST problem}

\begin{frame}{Application on the MNIST problem} % Slide 12
  % 1 - Decrire ce qu'est le MNIST
  % 2 - Setup de notre reservoir
\end{frame}

\begin{frame}{Comparison with other approaches} % Slide 13
  % 1 - Convolutional Neural Networks
  % 2 - Reservoir en pipeline (papier de 2015)
\end{frame}

\begin{frame}{Comparison with other approaches} % Slide 14
  % Tableau recapitulant les performances
\end{frame}

\section{Conclusion and perspectives}

\begin{frame}{Conclusion and perspectives} % Slide 15


  Many perspectives (we are just beginning)
  Improvement of the code\\
  Test of many ideas : number of comities\\
  One reservoir to learn and another one to learn error and correct
  them\\
  Test other large problems in simulation before in real\\
  => Try to test many configuration and to find optimal parameters
  
\end{frame}

\begin{frame}{Thank you for your attention}
  \begin{center}
    \LARGE Questions ?
  \end{center}
\end{frame}
  
%\begin{frame}{Titre du slide}

%	\begin{femtoBlock}
%  		{Ceci est un femtoBlock\\}
%		Et ici le texte dans le femtoBlock
%  		\begin{itemize}
%  			\item premier item
%  			\item deuxième item
%  		\end{itemize}%
%	\end{femtoBlock}
%	\begin{femtoBlock}
%  		{Autre femtoBlock\\}
%  		\begin{itemize}
%  			\item bla bla
%  			\item bla bla
%  		\end{itemize}
%	\end{femtoBlock}
%\end{frame}

%\section{Titre2}

%\begin{frame}{Autre titre}

%	\begin{femtoBlock}
%  		{Utilisation possible de block (standard)\\}
  		
%  		\begin{block}{Titre du block}
%  		texte contenu dans le block
%  		\begin{itemize}
%  			\item bla bla
%  			\item bla bla
%  		\end{itemize}
%  		\end{block}
%	\end{femtoBlock}
	
%\end{frame}
 
\end{document}
%  _____ ___ _   _ 
% |  ___|_ _| \ | |
% | |_   | ||  \| |
% |  _|  | || |\  |
% |_|   |___|_| \_|
%
