src/and/hpcvm/Server.java

   1 package and.hpcvm ;
   2
   3 import java.io.BufferedReader;
   4 import java.io.File;
   5 import java.io.IOException;
   6 import java.io.InputStreamReader;
   7 import java.rmi.Naming;
   8 import java.rmi.RemoteException;
   9 import java.rmi.registry.LocateRegistry;
  10 import java.rmi.registry.Registry;
  11 import java.rmi.server.UnicastRemoteObject;
  12 import java.util.ArrayList;
  13 import java.util.Iterator;
  14 import java.util.concurrent.Semaphore;
  15
  16
  17 public class Server extends UnicastRemoteObject implements ServicesServer
  18 {
  19         private class DiscCount
  20         {
  21                 private int nb ;
  22
  23                 DiscCount() { nb = 0 ; }
  24
  25                 protected void inc() { nb++ ; }
  26
  27                 protected void dec() {
  28                         if( nb > 0 )
  29                         {
  30                                 nb-- ;
  31                         }
  32                 }
  33
  34                 protected int getNb() { return nb ; }
  35         }
  36
  37
  38         private class IPAssociation
  39         {
  40                 private String vmIP ;
  41                 private String hostIP ;
  42
  43                 IPAssociation()
  44                 {
  45                         vmIP = "" ;
  46                         hostIP = "" ;
  47                 }
  48
  49                 protected void setVmIP( String _vmIP )
  50                 {
  51                         vmIP = _vmIP ;
  52                 }
  53
  54                 protected void setHostIP( String _hostIP )
  55                 {
  56                         hostIP = _hostIP ;
  57                 }
  58
  59                 protected String getVmIP()
  60                 {
  61                         return vmIP ;
  62                 }
  63
  64                 protected String getHostIP()
  65                 {
  66                         return hostIP ;
  67                 }
  68         }
  69
  70
  71
  72         private static final long serialVersionUID = 1L ;
  73         private int port ;
  74         private ArrayList<ConnectedClient> clients ;
  75         private ArrayList<ComputingClient> computingClients ;
  76         private ArrayList<RunningApplication> applications ;
  77         private int max_timeout ;
  78         private ConnectedMonitor monitor ;
  79         private DiscCount counter ;
  80         private ArrayList<IPAssociation> vmIPs ;
  81         private String working_directory ;
  82         private long save_interleave ;
  83         private Semaphore semaSave ;
  84         private OperatingClients startingClients ;
  85         private OperatingClients deployingClients ;
  86         private LimitThread limitThread ;
  87         private int maxThread ;
  88         private int ind ;
  89         private boolean running ;
  90
  91
  92         protected Server() throws RemoteException
  93         {
  94                 super() ;
  95         }
  96
  97
  98
  99         @Override
 100         public Integer register( ServicesClient _stub )
 101         {
 102                 if( _stub != null )
 103                 {
 104                         synchronized( clients )
 105                         {
 106                                 String ip = "" ;
 107                                 try {
 108                                         ip = _stub.getIPHost() ;
 109                                 } catch (RemoteException e) {
 110                                         e.printStackTrace() ;
 111                                         return 1 ;
 112                                 }
 113
 114                                 boolean exists = false ;
 115                                 int i ;
 116
 117                                 for( i = 0 ; i < clients.size() ; i++ )
 118                                 {
 119                                         if( ip.equals( clients.get( i ).getIP() ) )
 120                                         {
 121                                                 exists = true ;
 122                                                 System.out.println( "Client already connected!" ) ;
 123                                                 break ;
 124                                         }
 125                                 }
 126
 127                                 if( exists )
 128                                 {
 129                                         System.out.println( "The client stub will be replaced." ) ;
 130                                         clients.get( i ).setStub( _stub ) ;
 131                                         System.out.println( "(reconnection of " + clients.get( i ).getName() + ")" ) ;
 132                                         return 2 ;
 133                                 } else {
 134                                         System.out.println( "New connection!" ) ;
 135                                         clients.add( new ConnectedClient( _stub ) ) ;
 136                                         System.out.println( "(connection of " + clients.get( clients.size() - 1 ).getName() + ")" ) ;
 137                                         generateVmIP( ip ) ;
 138
 139                                         if( clients.size() == 0 )
 140                                         {
 141                                                 System.out.println( "There is no client connected." ) ;
 142                                         } else if( clients.size() == 1 ) {
 143                                                 System.out.println( "There is one client connected." ) ;
 144                                         } else {
 145                                                 System.out.println( "There are " + clients.size() + " clients connected." ) ;
 146                                         }
 147
 148                                         return 0 ;
 149                                 }
 150                         }
 151                 }
 152
 153                 return 1 ;
 154         }
 155
 156
 157         private void generateVmIP( String _ip )
 158         {
 159                 if( _ip != null && ! _ip.equals( "" ) )
 160                 {
 161                         for( int i = 0 ; i < vmIPs.size() ; i++ )
 162                         {
 163                                 if( vmIPs.get( i ).getHostIP().equalsIgnoreCase( "" ) )
 164                                 {
 165                                         vmIPs.get( i ).setHostIP( _ip ) ;
 166
 167                                         break ;
 168                                 }
 169                         }
 170                 }
 171         }
 172
 173
 174         @Override
 175         public void ping( String _ip )
 176         {
 177                 if( _ip != null )
 178                 {
 179                         for( int i = 0 ; i < clients.size() ; i++ )
 180                         {
 181                                 if( _ip.equals( clients.get( i ).getIP() ) )
 182                                 {
 183                                         clients.get( i ).resetTimeout() ;
 184                                         break ;
 185                                 }
 186                         }
 187                 }
 188         }
 189
 190
 191         @Override
 192         public void changeStatus( String _ip, String _status )
 193         {
 194                 if( _ip != null && _status != null )
 195                 {
 196                         for( int i = 0 ; i < clients.size() ; i++ )
 197                         {
 198                                 if( _ip.equals( clients.get( i ).getIP() ) )
 199                                 {
 200                                         clients.get( i ).setStatus( _status ) ;
 201                                         System.out.println( "Client " + clients.get( i ).getName() + " changed its status to: " + _status ) ;
 202
 203                                         break ;
 204                                 }
 205                         }
 206                 }
 207         }
 208
 209
 210         public void init( int _port )
 211         {
 212                 port = _port ;
 213                 max_timeout = 4 ;
 214
 215                 clients = new ArrayList<ConnectedClient>() ;
 216                 computingClients = new ArrayList<ComputingClient>() ;
 217                 applications = new ArrayList<RunningApplication>() ;
 218                 monitor = null ;
 219
 220                 startingClients = new OperatingClients() ;
 221                 deployingClients = new OperatingClients() ;
 222                 limitThread = new LimitThread() ;
 223                 maxThread = 20 ;
 224
 225                 ind = -1 ;
 226                 running = false ;
 227
 228                 working_directory = "/localhome/vmware" ;
 229
 230                 save_interleave  = 30 * 60 * 1000 ;
 231
 232                 semaSave = new Semaphore( 1 ) ;
 233
 234                 exportObject() ;
 235
 236                 vmIPs = new ArrayList<IPAssociation>() ;
 237                 // TODO initialisation of VM IPs
 238                 for( int i = 2 ; i < 101 ; i++ )
 239                 {
 240                         vmIPs.add( new IPAssociation() ) ;
 241                         vmIPs.get( vmIPs.size() - 1 ).setVmIP( "10.11.10." + i ) ;
 242                 }
 243
 244                 clients = new ArrayList<ConnectedClient>() ;
 245
 246                 counter = new DiscCount() ;
 247
 248                 monitor = new ConnectedMonitor() ;
 249                 monitor.start() ;
 250
 251                 // TODO
 252                 // Check if there are running applications ... and restart them :)
 253         }
 254
 255
 256         public void stop()
 257         {
 258                 if( monitor != null ) { monitor.stopMonitor() ; }
 259
 260                 for( int i = 0 ; i < clients.size() ; i++ )
 261                 {
 262                         try {
 263                                 clients.get( i ).getStub().emergencyStop() ;
 264                                 clients.get( i ).getStub().stop() ;
 265                         } catch (RemoteException e) {
 266                                 System.err.println( "Unable to send stop signal to " + clients.get( i ).getName() ) ;
 267                                 e.printStackTrace() ;
 268                         }
 269                 }
 270
 271                 // unexportObject ?
 272
 273                 System.exit( 0 ) ;
 274         }
 275
 276
 277         private void exportObject()
 278         {
 279                 ServicesServer ref = null ;
 280                 Registry reg = null ;
 281
 282                 try
 283                 {
 284                         while( true )
 285                         {
 286                                 reg = LocateRegistry.getRegistry( port ) ;
 287
 288                                 String tab[] = reg.list() ;
 289
 290                                 System.out.println( "There is an existing RMI Registry on port " +
 291                                                 port + " with " + tab.length + " entries!" ) ;
 292                                 for( int i = 0 ; i < tab.length ; i++ )
 293                                 {
 294                                         try {
 295                                                 if( UnicastRemoteObject.unexportObject( Naming.lookup(tab[i]), true ) )
 296                                                 {
 297                                                         System.out.println( "Register successfuly deleted!" ) ;
 298                                                 } else {
 299                                                         System.err.println( "Register undeleted !!!" ) ;
 300                                                 }
 301                                         } catch( Exception e ) {
 302                                                 e.printStackTrace() ;
 303                                         }
 304                                 }
 305                         }
 306                 } catch( RemoteException e ) {
 307                 }
 308
 309                 try {
 310                         if ( System.getSecurityManager() == null )
 311                         {
 312                     System.setSecurityManager( new SecurityManager() ) ;
 313                 }
 314
 315                         LocateRegistry.createRegistry( port ) ;
 316                         LocateRegistry.getRegistry( port ).rebind( "Server", this ) ;
 317                         ref = (ServicesServer) Naming.lookup( "rmi://"
 318                                         + LocalHost.Instance().getIP() + ":" + port
 319                                         + "/Server" ) ;
 320                 } catch ( Exception e ) {
 321                         System.err.println( "Error in Server.exportObject() when creating local services:" + e ) ;
 322                         System.err.println( "Exit from Server.exportObject" ) ;
 323                         System.exit( 1 ) ;
 324                 }
 325
 326                 LocalHost.Instance().setServerStub( ref ) ;
 327
 328                 System.out.println( "Server launched on IP " + LocalHost.Instance().getIP() +
 329                                 " on port " + port + "." ) ;
 330         }
 331
 332         /** Fault manager thread **/
 333         private class FaultManager extends Thread
 334         {
 335                 ConnectedClient cl ;
 336
 337                 FaultManager( ConnectedClient _cl )
 338                 {
 339                         cl = _cl ;
 340                 }
 341
 342                 @Override
 343                 public void run()
 344                 {
 345                         if( cl != null && cl.getStatus().equalsIgnoreCase( "running" ) ||
 346                                         cl.getStatus().equalsIgnoreCase( "saving" ) )
 347                         {
 348                                 ComputingClient cc = cl.getComputingClient() ;
 349 //                              ServicesClient dead = cc.getClient().getStub() ;
 350                                 String ipDead = cc.getClient().getIP() ;
 351                                 SaveNeighbor snDead = null ;
 352                                 for( int i = 0 ; i < computingClients.size() ; i++ )
 353                                 {
 354                                         if( computingClients.get( i ).getSaveNeighbor().getIPHost().equalsIgnoreCase( ipDead ) )
 355                                         {
 356                                                 snDead = computingClients.get( i ).getSaveNeighbor() ;
 357                                                 break ;
 358                                         }
 359                                 }
 360
 361                                 boolean ok = false ;
 362
 363                                 for( int i = 0 ; i < clients.size() ; i++ )
 364                                 {
 365                                         if( clients.get( i ).getStatus().equalsIgnoreCase( "connected" ) )
 366                                         {
 367 //                                              int res = 1 ;
 368 //                                              try {
 369 //                                                      res = clients.get( i ).getStub().startVM() ;
 370 //                                              } catch( RemoteException e ) {
 371 //                                                      e.printStackTrace();
 372 //                                              }
 373 //
 374 //                                              if( res == 0 )
 375 //                                              {
 376                                                         //clients.get(i).setStatus( "running" ) ;
 377
 378                                                 int pos = computingClients.indexOf( cc )  ;
 379                                                 if( pos == -1 )
 380                                                 {
 381                                                         System.err.println( "Dead client not found in the computing clients list!" ) ;
 382                                                 } else {
 383                                                         System.out.println( "Trying to replace " + cc.getClient().getName() + " with " +
 384                                                                         clients.get(i).getName() + " ... " ) ;
 385
 386                                                         String save_name = computingClients.get( pos ).getLastSave() ;
 387
 388                                                         ComputingClient ccl = new ComputingClient( clients.get(i) ) ;
 389                                                         clients.get( i ).setComputingClient( ccl ) ;
 390                                                         SaveNeighbor sn = computingClients.get( pos ).getSaveNeighbor() ;
 391                                                         ccl.setSaveNeighbor( sn ) ;
 392                                                         computingClients.set( pos, ccl ) ;
 393
 394
 395                                                         int res = 1 ;
 396                                                         try {
 397                                                                 res = computingClients.get( pos ).getClient().getStub().
 398                                                                       retrieveSave( save_name ) ;
 399                                                         } catch( RemoteException e ) {
 400                                                                 System.err.println( "Unable to indicate to client to retrieve last save!" ) ;
 401                                                                 e.printStackTrace() ;
 402                                                                 yield() ;
 403                                                         }
 404
 405                                                         if( res == 0 )
 406                                                         {
 407                                                                 ok = true ;
 408
 409                                                                 boolean ok_new = false, ok_old = false ;
 410
 411                                                                 // replace dead client in vmIPs
 412                                                                 for( int j = 0 ; j < vmIPs.size() ; j++ )
 413                                                                 {
 414                                                                         if( vmIPs.get( j ).getHostIP().equalsIgnoreCase( computingClients.get( pos ).getClient().getIP() ) )
 415                                                                         {
 416                                                                                 vmIPs.get( j ).setHostIP( "" ) ;
 417                                                                                 ok_new = true ;
 418                                                                         }
 419                                                                         if( vmIPs.get( j ).getHostIP().equalsIgnoreCase( ipDead ) )
 420                                                                         {
 421                                                                                 String vmIP = vmIPs.get( j ).getVmIP() ;
 422                                                                                 vmIPs.get( j ).setHostIP( computingClients.get( pos ).getClient().getIP() ) ;
 423                                                                                 ok_old = true ;
 424
 425                                                                                 try {
 426                                                                                         computingClients.get( pos ).getClient().getStub().setIPVM( vmIP ) ;
 427                                                                                 } catch( RemoteException e ) {
 428                                                                                         System.err.println( "Unable to set the new VM IP on the replacing client!" ) ;
 429                                                                                         e.printStackTrace() ;
 430                                                                                         yield() ;
 431                                                                                 }
 432
 433                                                                                 if( ok_new && ok_old )
 434                                                                                 {
 435                                                                                         break ;
 436                                                                                 }
 437                                                                         }
 438                                                                 }
 439
 440                                                                 // Replacing in RunningApplication
 441                                                                 applications.get( ind ).replaceComputingClient( cc, ccl ) ;
 442
 443                                                                 for( int l = 0 ; l < applications.get( ind ).getComputingClients().size() ; l++ )
 444                                                                 {
 445                                                                         applications.get( ind ).getComputingClients().get( l ).setSaveRequest( false ) ;
 446                                                                 }
 447
 448
 449                                                                 System.out.println( "Successful redeployment of the VM." ) ;
 450                                                         } else {
 451                                                                 System.err.println( "Unable to deploy the save on the new computing client!" ) ;
 452                                                         }
 453                                                 }
 454
 455                                                 if( ok )
 456                                                 {
 457                                                         for( int k = 0 ; k < computingClients.size() ; k++ )
 458                                                         {
 459                                                                 try {
 460                                                                         computingClients.get( k ).getClient().getStub().
 461                                                                                 replaceSaveNeighbor( snDead, new SaveNeighbor( clients.get( i ).getStub() ) ) ;
 462                                                                 } catch( RemoteException e ) {
 463                                                                         System.err.println( "Unable to inform " + computingClients.get( k ).getClient().getName() +
 464                                                                                         " of the replacement of a save neighbor!" ) ;
 465                                                                         e.printStackTrace() ;
 466                                                                 }
 467                                                         }
 468
 469                                                         System.out.println( "Dead client successfully replaced." ) ;
 470
 471                                                         break ;
 472                                                 } else {
 473                                                         System.err.println( "Dead client not replaced!!" ) ;
 474                                                 }
 475                                         }
 476                                 }
 477                         }
 478
 479                         try {
 480                                 synchronized( counter ) {
 481                                         counter.dec() ;
 482                                         counter.notifyAll() ;}
 483                         } catch( Exception e ) {}
 484                 }
 485         }
 486
 487
 488         /** Monitoring thread **/
 489         private class ConnectedMonitor extends Thread
 490         {
 491                 boolean run ;
 492
 493                 ConnectedMonitor()
 494                 {
 495                         run = true ;
 496                 }
 497
 498                 protected void stopMonitor() { run = false ; }
 499
 500                 @Override
 501                 public void run()
 502                 {
 503                         boolean change, dead ;
 504
 505                         while( run )
 506                         {
 507                                 change = false ;
 508                                 Iterator<ConnectedClient> it = clients.iterator() ;
 509                                 int nb_disconnections = 0 ;
 510                                 int nb_disconnections_computing = 0 ;
 511
 512                                 while( it.hasNext() )
 513                                 {
 514                                         ConnectedClient cl = it.next() ;
 515                                         cl.incTimeout() ;
 516                                         dead = false ;
 517
 518                                         if( cl.getTimeout() > max_timeout || cl.getFail() )
 519                                         {
 520                                                 dead = true ;
 521                                                 if( ! cl.getFail() )
 522                                                 {
 523                                                         try {
 524                                                                 cl.getStub().echo() ;
 525                                                                 cl.resetTimeout() ;
 526                                                                 dead = false ;
 527                                                         } catch( RemoteException e ) {
 528                                                                 dead = true ;
 529                                                         }
 530                                                 }
 531
 532                                                 if( dead )
 533                                                 {
 534                                                         System.out.println( "Disconnection of " + cl.getName() ) ;
 535                                                         if( cl.getStatus().equalsIgnoreCase( "running" ) || cl.getStatus().equalsIgnoreCase( "saving" ) )
 536                                                         {
 537                                                                 System.out.println( "A VM was running on it!!" ) ;
 538                                                                 System.out.println( "I will redeploy a save and restart all VM ..." ) ;
 539
 540 //                                                              for( int i = 0 ; i < computingClients.size() ; i++ )
 541 //                                                              {
 542 //                                                                      if( computingClients.get( i ).getClient().getIP().equals( cl.getIP() ) )
 543 //                                                                      {
 544 //                                                                              computingClients.remove( i ) ;
 545 //                                                                              break ;
 546 //                                                                      }
 547 //                                                              }
 548                                                                 synchronized( counter )
 549                                                                 {
 550                                                                         counter.inc() ;
 551                                                                 }
 552
 553                                                                 new Server.FaultManager( cl ).start() ;
 554                                                                 nb_disconnections_computing++ ;
 555                                                         } else {
 556                                                                 System.out.println( "There was no VM running on it." ) ;
 557                                                                 System.out.println( "Maybe it will come back later :)" ) ;
 558                                                         }
 559
 560                                                         synchronized( clients )
 561                                                         {
 562                                                                 it.remove() ;
 563                                                         }
 564                                                         nb_disconnections++ ;
 565                                                         change = true ;
 566                                                 }
 567                                         }
 568                                 }
 569
 570                                 if( change )
 571                                 {
 572                                         synchronized( clients )
 573                                         {
 574                                                 if( clients.size() == 0 )
 575                                                 {
 576                                                         System.out.println( "There is no client connected." ) ;
 577                                                 } else if( clients.size() == 1 ) {
 578                                                         System.out.println( "There is one client connected." ) ;
 579                                                 } else {
 580                                                         System.out.println( "There are " + clients.size() + " clients connected." ) ;
 581                                                 }
 582                                         }
 583                                 }
 584
 585
 586                                 if( nb_disconnections_computing > 0 )
 587                                 {
 588                                         System.out.println( "Sending emergency stop signal to all computing nodes ... " ) ;
 589
 590                                         for( int i = 0 ; i < clients.size() ; i++ )
 591                                         {
 592                                                 if( clients.get( i ).getStatus().equalsIgnoreCase( "running" )
 593                                                         || clients.get( i ).getStatus().equalsIgnoreCase( "saving" ) )
 594                                                 {
 595                                                         try {
 596                                                                 clients.get( i ).getStub().emergencyStop() ;
 597                                                         } catch( RemoteException e ) {
 598                                                                 System.err.println( "Unable to invoke emergency stop signal on " + clients.get( i ).getName() ) ;
 599                                                                 e.printStackTrace() ;
 600                                                                 yield() ;
 601                                                         }
 602                                                 }
 603                                         }
 604
 605                                         System.out.println( "I will redeploy save and restart VMs ... " ) ;
 606
 607                                         synchronized( counter )
 608                                         {
 609                                                 if( counter.getNb() > 0 )
 610                                                 {
 611                                                         System.out.println( "... waiting all redeployments done ..." ) ;
 612                                                 }
 613
 614                                                 while( counter.getNb() != 0 )
 615                                                 {
 616                                                         try {
 617                                                                 counter.wait() ;
 618                                                         } catch( InterruptedException e ) {
 619                                                                 e.printStackTrace() ;
 620                                                                 yield() ;
 621                                                         }
 622                                                 }
 623                                         }
 624
 625                                         for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
 626                                         {
 627                                                 applications.get( ind ).getComputingClients().get( i ).setRestartOk( false ) ;
 628
 629                                                 new RestartVM( applications.get( ind ).getComputingClients().get( i ).getClient() ).start() ;
 630
 631 //                                              final ServicesClient sc = applications.get( ind ).getComputingClients().get( i ).getClient().getStub() ;
 632
 633 //                                              new Thread( new Runnable() {
 634 //
 635 //                                                      @Override
 636 //                                                      public void run()
 637 //                                                      {
 638 //                                                              try {
 639 //                                                                      if( sc.restartVMAfterCrash() != 0 )
 640 //                                                                      {
 641 //                                                                              System.err.println( "Problem while restarting VM on " +sc.getName() + "!" ) ;
 642 //                                                                      }
 643 //                                                              } catch( RemoteException e ) {
 644 //                                                                      try {
 645 //                                                                              System.err.println( "Problem while restarting VM on " + sc.getName() + "!" ) ;
 646 //                                                                      } catch( RemoteException e1 ) {
 647 //                                                                              System.err.println( "Problem while restarting a VM!" ) ;
 648 //                                                                              e1.printStackTrace() ;
 649 //                                                                      }
 650 //                                                                      e.printStackTrace() ;
 651 //                                                                      yield() ;
 652 //                                                              }
 653 //                                                      }
 654 //                                              } ).start() ;
 655                                         }
 656                                 }
 657
 658                                 try
 659                                 {
 660                                         Thread.sleep( 2000 ) ;
 661                                 } catch( InterruptedException e ) {
 662                                         e.printStackTrace() ;
 663                                         yield() ;
 664                                 }
 665                         }
 666                 }
 667         }
 668
 669         @Override
 670         public Integer saveOk( String _ip, String _saveName )
 671         {
 672                 int i ;
 673                 for( i = 0 ; i < computingClients.size() ; i ++ )
 674                 {
 675                         if( computingClients.get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 676                         {
 677                                 computingClients.get( i ).setLastSave( _saveName ) ;
 678                                 computingClients.get( i ).setSaveStatus( true ) ;
 679                                 break ;
 680                         }
 681                 }
 682
 683
 684                 boolean all_ok = true ;
 685                 i = 0 ;
 686
 687                 while( all_ok && i < computingClients.size() )
 688                 {
 689                         all_ok = all_ok & computingClients.get( i ).getSaveStatus() ;
 690                         i++ ;
 691                 }
 692
 693                 if( all_ok )
 694                 {
 695                         for( i = 0 ; i < computingClients.size() ; i++ )
 696                         {
 697                                 try {
 698                                         computingClients.get( i ).getClient().getStub().saveOk() ;
 699                                 } catch (RemoteException e) {
 700                                         System.err.println( "Unable to invoke the saveOk method on " + computingClients.get( i ).getClient().getName() ) ;
 701                                         e.printStackTrace() ;
 702                                 }
 703                                 computingClients.get( i ).setSaveStatus( false ) ;
 704                         }
 705
 706                         applications.get( ind ).setLastSaveDate( System.currentTimeMillis() ) ;
 707                 }
 708
 709                 return 0 ;
 710         }
 711
 712
 713         public Integer changeSaveName( String _ip, String _saveName )
 714         {
 715                 if( _ip != null && _ip.length() > 0 && _saveName != null && _saveName.length() > 0 )
 716                 {
 717                         for( int i = 0 ; i < computingClients.size() ; i ++ )
 718                         {
 719                                 if( computingClients.get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 720                                 {
 721                                         computingClients.get( i ).setLastSave( _saveName ) ;
 722                                         System.out.println( "Save name successfully changed on " + computingClients.get( i ).getClient().getName() ) ;
 723                                         return 0 ;
 724                                 }
 725                         }
 726
 727                         System.err.println( "Unable to found computing client with IP " + _ip + "!" ) ;
 728                         return 1 ;
 729                 }
 730
 731                 System.err.println( "Unable to change save name. IP or save name empty ! (IP: " + _ip + " ; save name: " + _saveName +")" ) ;
 732
 733                 return 1 ;
 734         }
 735
 736
 737         @Override
 738         public ArrayList<ServicesClient> startApplication( int _nb )
 739         {
 740                 int nb = clients.size() - computingClients.size() ;
 741
 742                 if( nb > _nb && ! running )
 743                 {
 744                         running = true ;
 745
 746                         final ArrayList<ServicesClient> ac = new ArrayList<ServicesClient>() ;
 747                         final ArrayList<ComputingClient> tmp = new ArrayList<ComputingClient>() ;
 748
 749                         RunningApplication app = new RunningApplication( "Test" ) ;
 750
 751                         int i = 0 ;
 752                         boolean ok ;
 753
 754                         while( i < clients.size() && ac.size() < _nb )
 755                         {
 756                                 ok = false ;
 757                                 if( clients.get( i ).getStatus().equalsIgnoreCase( "connected" ) )
 758                                 {
 759                                         synchronized( startingClients )
 760                                         {
 761                                                 while( ac.size() + startingClients.getNb() >= _nb )
 762                                                 {
 763                                                         if( ac.size() == _nb ) break ;
 764
 765                                                         try {
 766                                                                 startingClients.wait() ;
 767                                                         } catch( InterruptedException e ) {
 768                                                                 e.printStackTrace() ;
 769                                                         }
 770                                                 }
 771
 772                                                 if( ac.size() < _nb )
 773                                                 {
 774                                                         startingClients.inc() ;
 775                                                         ok = true ;
 776                                                 }
 777                                         }
 778
 779                                         if( ok )
 780                                         {
 781                                                 synchronized( limitThread )
 782                                                 {
 783                                                         while( limitThread.getNb() >= maxThread )
 784                                                         {
 785                                                                 try {
 786                                                                         limitThread.wait() ;
 787                                                                 } catch (InterruptedException e) {
 788                                                                         e.printStackTrace();
 789                                                                 }
 790                                                         }
 791
 792                                                         limitThread.inc() ;
 793                                                 }
 794
 795                                                 final int indice = i ;
 796                                                 new Thread( new Runnable()
 797                                                 {
 798                                                         @Override
 799                                                         public void run()
 800                                                         {
 801                                                                 int res = 1 ;
 802                                                                 try {
 803                                                                         res = clients.get( indice ).getStub().startVM( 0 ) ;
 804                                                                 } catch( RemoteException e ) {
 805                                                                         e.printStackTrace() ;
 806                                                                 }
 807
 808                                                                 if( res == 0 )
 809                                                                 {
 810                                                                         ac.add( clients.get( indice ).getStub() ) ;
 811                                                                         clients.get( indice ).setStatus( "running" ) ;
 812                                                                         ComputingClient cl = new ComputingClient( clients.get( indice ) ) ;
 813                                                                         clients.get( indice ).setComputingClient( cl ) ;
 814                                                                         computingClients.add( cl ) ;
 815                                                                         tmp.add( cl ) ;
 816                                                                 } else {
 817                                                                         System.err.println( "Problem while launching the VM on "
 818                                                                                         + clients.get( indice ).getName() + "!" ) ;
 819                                                                 }
 820
 821                                                                 synchronized( limitThread )
 822                                                                 {
 823                                                                         limitThread.dec() ;
 824                                                                         limitThread.notifyAll() ;
 825                                                                 }
 826
 827                                                                 synchronized( startingClients )
 828                                                                 {
 829                                                                         startingClients.dec() ;
 830                                                                         startingClients.notifyAll() ;
 831                                                                 }
 832                                                         }
 833                                                 }).start() ;
 834                                         }
 835                                 }
 836
 837                                 i++ ;
 838                         }
 839
 840                         if( ac.size() == _nb )
 841                         {
 842                                 app.setComputingClients( tmp ) ;
 843                                 app.setRunning( true ) ;
 844 //                              app.setStartTime( System.currentTimeMillis() ) ;
 845
 846                                 int index, index2 ;
 847                                 /* Choosing save neighbors */
 848                                 for( i = 0 ; i < tmp.size() ; i++ )
 849                                 {
 850                                         if( i == tmp.size() - 1 )
 851                                         {
 852                                                 index = computingClients.indexOf( tmp.get( i ) ) ;
 853                                                 index2 = computingClients.indexOf( tmp.get( 0 ) ) ;
 854
 855                                                 if( index == -1 || index2 == -1 )
 856                                                 {
 857                                                         System.err.println( "Problem in ComputingClients list!" ) ;
 858                                                 } else {
 859                                                         try {
 860                                                                 computingClients.get( index ).setSaveNeighbor( new SaveNeighbor( computingClients.get( index2 ).getClient().getStub() )) ;
 861                                                         } catch( RemoteException e ) {
 862                                                                 System.err.println( "Unable to create the save neighbor!" ) ;
 863                                                                 e.printStackTrace() ;
 864                                                         }
 865                                                 }
 866                                         } else {
 867                                                 index = computingClients.indexOf( tmp.get( i ) ) ;
 868                                                 index2 = computingClients.indexOf( tmp.get( i + 1 ) ) ;
 869
 870                                                 if( index == -1 || index2 == -1 )
 871                                                 {
 872                                                         System.err.println( "Problem in ComputingClients list!" ) ;
 873                                                 } else {
 874                                                         try {
 875                                                                 computingClients.get( index ).setSaveNeighbor( new SaveNeighbor( computingClients.get( index2 ).getClient().getStub() ) ) ;
 876                                                         } catch( RemoteException e ) {
 877                                                                 System.err.println( "Unable to create the save neighbor!" ) ;
 878                                                                 e.printStackTrace() ;
 879                                                         }
 880                                                 }
 881                                         }
 882                                 }
 883                         }
 884
 885                         applications.add( app ) ;
 886
 887                         ind = applications.indexOf( app ) ;
 888
 889                         return ac ;
 890                 }
 891
 892                 return null ;
 893         }
 894
 895
 896         @Override
 897         public void requestSave( String _ip )
 898         {
 899                 try {
 900                         semaSave.acquire() ;
 901                 } catch( InterruptedException e ) {
 902                         System.err.println( "Unable to obtain the semaphore for semaSave!" ) ;
 903                         e.printStackTrace() ;
 904                 }
 905
 906                 final String ip = _ip ;
 907
 908                 new Thread( new Runnable() {
 909
 910                         @Override
 911                         public void run()
 912                         {
 913                                 treatRequestSave( ip ) ;
 914                         }
 915                 } ).start() ;
 916         }
 917
 918
 919         public void treatRequestSave( String _ip )
 920         {
 921                 if( applications.size() > 0 && _ip != null && _ip.length() > 0 )
 922                 {
 923                         if( (System.currentTimeMillis() - applications.get( ind ).getLastSaveDate()) > save_interleave )
 924                         {
 925                                 // Mark it as a requester
 926                                 for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
 927                                 {
 928                                         if( applications.get( ind ).getComputingClients().get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 929                                         {
 930                                                 applications.get( ind ).getComputingClients().get( i ).setSaveRequest( true ) ;
 931
 932                                                 break ;
 933                                         }
 934                                 }
 935
 936                                 semaSave.release() ;
 937
 938                                 // Is every body ok?
 939                                 boolean ok = false ;
 940                                 for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
 941                                 {
 942                                         if( i == 0 )
 943                                         {
 944                                                 ok = applications.get( ind ).getComputingClients().get( i ).getSaveRequest() ;
 945                                         } else {
 946                                                 ok = ok & applications.get( ind ).getComputingClients().get( i ).getSaveRequest() ;
 947                                         }
 948
 949                                         if( ! ok )
 950                                         {
 951                                                 break ;
 952                                         }
 953                                 }
 954
 955                                 if( ok )
 956                                 {
 957 //                                      try {
 958 //                                              Thread.sleep( 5000 ) ;
 959 //                                      } catch( InterruptedException e1 ) {
 960 //                                              e1.printStackTrace() ;
 961 //                                      }
 962
 963                                         for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
 964                                         {
 965                                                 try {
 966                                                         applications.get( ind ).getComputingClients().get( i ).getClient().getStub().responseSave( true ) ;
 967                                                         applications.get( ind ).getComputingClients().get( i ).setSaveRequest( false ) ;
 968                                                 } catch( RemoteException e ) {
 969                                                         System.err.println( "Unable to send the save request response to " +
 970                                                                         applications.get( ind ).getComputingClients().get( i ).getClient().getName() + "!" ) ;
 971                                                         e.printStackTrace() ;
 972                                                 }
 973                                         }
 974
 975                                         applications.get( ind ).setLastSaveDate( System.currentTimeMillis() ) ;
 976                                 }
 977
 978                         } else {
 979                                 semaSave.release() ;
 980
 981                                 for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
 982                                 {
 983                                         if( applications.get( ind ).getComputingClients().get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 984                                         {
 985                                                 try {
 986                                                         applications.get( ind ).getComputingClients().get( i ).getClient().getStub().responseSave( false ) ;
 987                                                 } catch( RemoteException e ) {
 988                                                         System.err.println( "Unable to send the save request response to " +
 989                                                                         applications.get( ind ).getComputingClients().get( i ).getClient().getName() + "!" ) ;
 990                                                         e.printStackTrace() ;
 991                                                 }
 992                                                 break ;
 993                                         }
 994                                 }
 995                         }
 996                 } else {
 997                         semaSave.release() ;
 998                         System.err.println( "!! Serious problem in treatRequestSave method!!" ) ;
 999                 }
1000         }
1001
1002
1003         @Override
1004         public void restartOk( String _ip )
1005         {
1006                 if( applications.size() > 0 && _ip != null && _ip.length() > 0 )
1007                 {
1008                         System.out.println( "Client " + _ip + " has finished to restart ("+applications.get( ind ).getComputingClients().size()+") ... " ) ;
1009 //                      if( (System.currentTimeMillis() - applications.get( ind ).getLastSaveDate()) > save_interleave )
1010                         {
1011                                 // Has it already finished?
1012                                 for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
1013                                 {
1014                                         if( applications.get( ind ).getComputingClients().get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
1015                                         {
1016                                                 applications.get( ind ).getComputingClients().get( i ).setRestartOk( true ) ;
1017
1018                                                 break ;
1019                                         }
1020                                 }
1021
1022                                 // Is everybody ok?
1023                                 boolean ok = false ;
1024                                 for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
1025                                 {
1026                                         if( i == 0 )
1027                                         {
1028                                                 ok = applications.get( ind ).getComputingClients().get( i ).getRestartOk() ;
1029                                         } else {
1030                                                 ok = ok & applications.get( ind ).getComputingClients().get( i ).getRestartOk() ;
1031                                         }
1032
1033                                         if( ! ok )
1034                                         {
1035                                                 break ;
1036                                         }
1037                                 }
1038
1039                                 if( ok )
1040                                 {
1041                                         applications.get( ind ).setLastSaveDate( System.currentTimeMillis() ) ;
1042
1043                                         for( int i = 0 ; i < applications.get( ind ).getComputingClients().size() ; i++ )
1044                                         {
1045                                                 applications.get( ind ).getComputingClients().get( i ).setRestartOk( false ) ;
1046                                         }
1047                                 }
1048
1049                         }
1050                 }
1051         }
1052
1053
1054         @Override
1055         public void goApplication()
1056         {
1057                 synchronized( applications ) {
1058                 if( running && applications.get( ind ).getStartTime() != 0 )
1059                 {
1060                         applications.get( ind ).setStartTime( System.currentTimeMillis() ) ;
1061                         applications.get( ind ).setLastSaveDate( System.currentTimeMillis() ) ;
1062                 }}
1063         }
1064
1065
1066         @Override
1067         public void endApplication()
1068         {
1069                 synchronized( applications )
1070                 {
1071                         if( running )
1072                         {
1073                                 applications.get( ind ).setEndTime( System.currentTimeMillis() ) ;
1074                                 applications.get( ind ).setRunning( false ) ;
1075                                 applications.get( ind ).clear() ;
1076
1077                                 Iterator<ComputingClient> it = computingClients.iterator() ;
1078
1079                                 while( it.hasNext() )
1080                                 {
1081                                         ComputingClient cl = it.next() ;
1082
1083                                         try {
1084                                                 cl.getClient().getStub().emergencyStop() ;
1085                                         } catch (RemoteException e) {
1086                                                 e.printStackTrace();
1087                                         }
1088
1089                                         cl.getClient().setStatus( "connected" ) ;
1090                                         cl.getClient().setComputingClient( null ) ;
1091                                         it.remove() ;
1092                                         cl = null ;
1093                                 }
1094
1095
1096                                 applications.get( ind ).clear() ;
1097
1098                                 running = false ;
1099
1100                                 System.out.println( "Application " + applications.get( ind ).getName() + " ends in " +
1101                                         applications.get( ind ).getExecutionTime() + " seconds." ) ;
1102                         }
1103                 }
1104         }
1105
1106
1107
1108         @Override
1109         public String getAssociatedIP( String _ip )
1110         {
1111                 String ret = null ;
1112
1113                 for( int i = 0 ; i < vmIPs.size() ; i++ )
1114                 {
1115                         if( vmIPs.get( i ).getHostIP().equalsIgnoreCase( _ip ) )
1116                         {
1117                                 ret = vmIPs.get( i ).getVmIP() ;
1118                                 break ;
1119                         }
1120                 }
1121
1122                 return ret ;
1123         }
1124
1125
1126         public Integer deployVM( final String _name, final String _archive, final String _directory )
1127         {
1128                 int nb = 0, pb = 0 ;
1129
1130                 if( _name != null && _name.length() > 0 && _archive != null && _name.length() > 0
1131                                 && _directory != null && _directory.length() > 0 )
1132                 {
1133                         System.out.println( "Deploying the VM " + _name + " (" + _archive + ") ... " ) ;
1134
1135                         File file = new File( working_directory + "/" + _archive ) ;
1136                         if( ! file.exists() )
1137                         {
1138                                 System.err.println( "There is no archive named " + _archive + " in my working directory!" ) ;
1139                                 file = null ;
1140                                 return 1 ;
1141                         } else if( file.isDirectory() ) {
1142                                 System.err.println( _archive + " is a directory!" ) ;
1143                                 file = null ;
1144                                 return 1 ;
1145                         }
1146
1147                         file = null ;
1148
1149                         // TODO do a better deployment !!
1150 //                      int ret ;
1151 //                      boolean error, ok, server ;
1152 //                      ArrayList<ConnectedClient> deployed = new ArrayList<ConnectedClient>() ;
1153
1154 //                      boolean server = true ;
1155 //                      boolean ok ;
1156
1157                         for( int i = 0 ; i < clients.size() ; i++ )
1158                         {
1159 //                              ret = 1 ;
1160                                 nb++ ;
1161 //                              error = false ;
1162 //                              ok = false ;
1163                                 if( clients.get( i ).getStatus().equalsIgnoreCase( "connected" ) )
1164                                 {
1165                                         synchronized( limitThread )
1166                                         {
1167                                                 while( limitThread.getNb() >= maxThread )
1168                                                 {
1169                                                         try {
1170                                                                 limitThread.wait() ;
1171                                                         } catch (InterruptedException e) {
1172                                                                 e.printStackTrace();
1173                                                         }
1174                                                 }
1175
1176                                                 limitThread.inc() ;
1177                                         }
1178
1179                                         final int indice = i ;
1180                                         new Thread( new Runnable() {
1181
1182                                                 @Override
1183                                                 public void run() {
1184                                                         int ret = -1 ;
1185                                                         boolean error = true  ;
1186
1187                                                         try {
1188                                                                 ret = clients.get( indice ).getStub().deployVM( _name, _archive, _directory ) ;
1189                                                         } catch( RemoteException e ) {
1190                                                                 System.err.println( "Unable to deploy the VM on " + clients.get( indice ).getName() + "!" ) ;
1191                                                                 e.printStackTrace() ;
1192                                                         }
1193
1194                                                         // The client does not have the archive, we have to send it.
1195                                                         if( ret == 2 )
1196                                                         {
1197                                                                 // Attention au multi-envois !!!
1198                                                                 System.out.print( "Sending VM archive to " + clients.get( indice ).getName() + " ... " ) ;
1199
1200                                                                 String wd = "" ;
1201                                                                 String snIP = "" ;
1202                                                                 error = false ;
1203
1204                                                                 try {
1205                                                                         wd = clients.get( indice ).getStub().getWorkingDirectory() ;
1206                                                                         snIP = clients.get( indice ).getStub().getIPHost() ;
1207                                                                 } catch (RemoteException e2) {
1208                                                                         System.err.println( "Unable to retrieve information on " + clients.get( indice ).getName() + "!" ) ;
1209                                                                         e2.printStackTrace() ;
1210                                                                         error = true ;
1211                                                                 }
1212
1213                                                                 String[] command = new String[]{ "/usr/bin/scp", working_directory + "/" + _archive,
1214                                                                                 snIP + ":" + wd } ;
1215
1216                                                                 if( ! error )
1217                                                                 {
1218                                                                         try {
1219                                                                                 Process proc = Runtime.getRuntime().exec( command ) ;
1220                                                                                 proc.waitFor() ;
1221
1222                                                                                 if( proc.exitValue() == 0 )
1223                                                                                 {
1224                                                                                         System.out.println( "Initial VM archive successfully sent." ) ;
1225                                                                                 } else {
1226                                                                                         System.err.println( "Initial VM archive not sent!" ) ;
1227                                                                                         System.err.println( "Error: " + proc.exitValue() ) ;
1228                                                                                         BufferedReader b = new BufferedReader( new InputStreamReader( proc.getErrorStream() ) ) ;
1229
1230                                                                                         String l ;
1231                                                                                         try {
1232                                                                                                 while( (l = b.readLine()) != null )
1233                                                                                                 {
1234                                                                                                         System.err.println( l ) ;
1235                                                                                                 }
1236                                                                                         } catch( IOException e ) {
1237                                                                                                 e.printStackTrace() ;
1238                                                                                         }
1239
1240                                                                                         error = true ;
1241                                                                                 }
1242                                                                         } catch( IOException e ) {
1243                                                                                 System.err.println( "Error during initial VM archive send command: " ) ;
1244                                                                                 e.printStackTrace() ;
1245                                                                                 error = true ;
1246                                                                         } catch( InterruptedException e ) {
1247                                                                                 e.printStackTrace() ;
1248                                                                                 error = true ;
1249                                                                         }
1250                                                                 }
1251
1252
1253                                                                 if( ! error )
1254                                                                 {
1255                                                                         // Second try ...
1256                                                                         ret = 1 ;
1257                                                                         try {
1258                                                                                 ret = clients.get( indice ).getStub().deployVM( _name, _archive, _directory ) ;
1259                                                                         } catch( RemoteException e ) {
1260                                                                                 System.err.println( "Unable to deploy the VM on " + clients.get( indice ).getName() + "!" ) ;
1261                                                                                 e.printStackTrace() ;
1262                                                                         }
1263                                                                 }
1264                                                         }
1265
1266                                                         if( ret == 0 )
1267                                                         {
1268                                                                 System.out.println( "Initial VM archive successfully deployed on " + clients.get( indice ).getName() + "." ) ;
1269
1270                                                                 synchronized( deployingClients )
1271                                                                 {
1272                                                                         deployingClients.inc() ;
1273                                                                 }
1274                                                         }
1275
1276                                                         synchronized( limitThread )
1277                                                         {
1278                                                                 limitThread.dec() ;
1279                                                                 limitThread.notifyAll() ;
1280                                                         }
1281                                                 }
1282                                         }).start() ;
1283                                 }
1284                         }
1285                 }
1286
1287                 synchronized( limitThread )
1288                 {
1289                         while( limitThread.getNb() > 0  )
1290                         {
1291                                 try {
1292                                         limitThread.wait() ;
1293                                 } catch( InterruptedException e ) {
1294                                         e.printStackTrace() ;
1295                                 }
1296                         }
1297                 }
1298
1299                 if( nb - deployingClients.getNb() > 0 )
1300                 {
1301                         if( pb == 1 )
1302                                 System.err.println( "** " + pb + " machine is not deployed!" ) ;
1303                         if( pb > 1 )
1304                                 System.err.println( "** " + pb + " machines are not deployed!" ) ;
1305                 }
1306
1307                 return nb ;
1308         }
1309
1310
1311         public String getWorkingDirectory()
1312         {
1313                 return working_directory ;
1314         }
1315
1316
1317         private class OperatingClients
1318         {
1319                 private int nb ;
1320
1321                 OperatingClients() { nb = 0 ; }
1322
1323                 protected void inc() { nb++ ; }
1324
1325                 protected void dec() { nb-- ; }
1326
1327                 protected int getNb() { return nb ; }
1328         }
1329
1330
1331         private class LimitThread
1332         {
1333                 private int nb ;
1334
1335                 LimitThread() { nb = 0 ; }
1336
1337                 protected void inc() { nb++ ; }
1338
1339                 protected void dec() { nb-- ; }
1340
1341                 protected int getNb() { return nb ; }
1342         }
1343
1344
1345         private class RestartVM extends Thread
1346         {
1347                 private ConnectedClient cc ;
1348
1349                 protected RestartVM( ConnectedClient _cc )
1350                 {
1351                         cc = _cc ;
1352                 }
1353
1354                 public void run()
1355                 {
1356                         boolean error = false ;
1357                         if( cc != null )
1358                         {
1359                                 try {
1360                                         if( cc.getStub().restartVMAfterCrash() != 0 )
1361                                         {
1362                                                 System.err.println( "Problem while restarting VM on " + cc.getName() + "!" ) ;
1363                                                 error = true ;
1364                                         }
1365                                 } catch( RemoteException e ) {
1366                                         e.printStackTrace() ;
1367                                         error = true ;
1368                                         yield() ;
1369                                 }
1370                         } else {
1371                                 System.err.println( "The client to restart is null!" ) ;
1372                         }
1373
1374                         if( error )
1375                         {
1376                                 cc.setFail( true ) ;
1377
1378                                 try {
1379                                         System.out.print( "Trying to stop the client ... " ) ;
1380                                         cc.getStub().stop() ;
1381                                         System.out.println( "successful client stop." );
1382                                 } catch( RemoteException e ) {
1383                                         System.out.println( "unsuccessful client stop!" ) ;
1384                                         e.printStackTrace() ;
1385                                 }
1386                         }
1387                 }
1388         }
1389
1390 }
1391
1392 /** La programmation est un art, respectons ceux qui la pratiquent !! **/
1393