src/and/hpcvm/Server.java

   1 package and.hpcvm ;
   2
   3 import java.io.BufferedReader;
   4 import java.io.File;
   5 import java.io.IOException;
   6 import java.io.InputStreamReader;
   7 import java.rmi.Naming;
   8 import java.rmi.RemoteException;
   9 import java.rmi.registry.LocateRegistry;
  10 import java.rmi.registry.Registry;
  11 import java.rmi.server.UnicastRemoteObject;
  12 import java.util.ArrayList;
  13 import java.util.Iterator;
  14 import java.util.concurrent.Semaphore;
  15
  16
  17 public class Server extends UnicastRemoteObject implements ServicesServer
  18 {
  19         private class DiscCount
  20         {
  21                 private int nb ;
  22
  23                 DiscCount() { nb = 0 ; }
  24
  25                 protected void inc() { nb++ ; }
  26
  27                 protected void dec() {
  28                         if( nb > 0 )
  29                         {
  30                                 nb-- ;
  31                         }
  32                 }
  33
  34                 protected int getNb() { return nb ; }
  35         }
  36
  37
  38         private class IPAssociation
  39         {
  40                 private String vmIP ;
  41                 private String hostIP ;
  42
  43                 IPAssociation()
  44                 {
  45                         vmIP = "" ;
  46                         hostIP = "" ;
  47                 }
  48
  49                 protected void setVmIP( String _vmIP )
  50                 {
  51                         vmIP = _vmIP ;
  52                 }
  53
  54                 protected void setHostIP( String _hostIP )
  55                 {
  56                         hostIP = _hostIP ;
  57                 }
  58
  59                 protected String getVmIP()
  60                 {
  61                         return vmIP ;
  62                 }
  63
  64                 protected String getHostIP()
  65                 {
  66                         return hostIP ;
  67                 }
  68         }
  69
  70
  71
  72         private static final long serialVersionUID = 1L ;
  73         private int port ;
  74         private ArrayList<ConnectedClient> clients ;
  75         private ArrayList<ComputingClient> computingClients ;
  76         private ArrayList<RunningApplication> applications ;
  77         private int max_timeout ;
  78         private ConnectedMonitor monitor ;
  79         private DiscCount counter ;
  80         private ArrayList<IPAssociation> vmIPs ;
  81         private String working_directory ;
  82         private long save_interleave ;
  83         private Semaphore semaSave ;
  84         private OperatingClients startingClients ;
  85         private OperatingClients deployingClients ;
  86         private LimitThread limitThread ;
  87         private int maxThread;
  88
  89
  90         protected Server() throws RemoteException
  91         {
  92                 super() ;
  93         }
  94
  95
  96
  97         @Override
  98         public Integer register( ServicesClient _stub )
  99         {
 100                 if( _stub != null )
 101                 {
 102                         synchronized( clients )
 103                         {
 104                                 String ip = "" ;
 105                                 try {
 106                                         ip = _stub.getIPHost() ;
 107                                 } catch (RemoteException e) {
 108                                         e.printStackTrace() ;
 109                                         return 1 ;
 110                                 }
 111
 112                                 boolean exists = false ;
 113                                 int i ;
 114
 115                                 for( i = 0 ; i < clients.size() ; i++ )
 116                                 {
 117                                         if( ip.equals( clients.get( i ).getIP() ) )
 118                                         {
 119                                                 exists = true ;
 120                                                 System.out.println( "Client already connected!" ) ;
 121                                                 break ;
 122                                         }
 123                                 }
 124
 125                                 if( exists )
 126                                 {
 127                                         System.out.println( "The client stub will be replaced." ) ;
 128                                         clients.get( i ).setStub( _stub ) ;
 129                                         System.out.println( "(reconnection of " + clients.get( i ).getName() + ")" ) ;
 130                                         return 2 ;
 131                                 } else {
 132                                         System.out.println( "New connection!" ) ;
 133                                         clients.add( new ConnectedClient( _stub ) ) ;
 134                                         System.out.println( "(connection of " + clients.get( clients.size() - 1 ).getName() + ")" ) ;
 135                                         generateVmIP( ip ) ;
 136
 137                                         if( clients.size() == 0 )
 138                                         {
 139                                                 System.out.println( "There is no client connected." ) ;
 140                                         } else if( clients.size() == 1 ) {
 141                                                 System.out.println( "There is one client connected." ) ;
 142                                         } else {
 143                                                 System.out.println( "There are " + clients.size() + " clients connected." ) ;
 144                                         }
 145
 146                                         return 0 ;
 147                                 }
 148                         }
 149                 }
 150
 151                 return 1 ;
 152         }
 153
 154
 155         private void generateVmIP( String _ip )
 156         {
 157                 if( _ip != null && ! _ip.equals( "" ) )
 158                 {
 159                         for( int i = 0 ; i < vmIPs.size() ; i++ )
 160                         {
 161                                 if( vmIPs.get( i ).getHostIP().equalsIgnoreCase( "" ) )
 162                                 {
 163                                         vmIPs.get( i ).setHostIP( _ip ) ;
 164
 165                                         break ;
 166                                 }
 167                         }
 168                 }
 169         }
 170
 171
 172         @Override
 173         public void ping( String _ip )
 174         {
 175                 if( _ip != null )
 176                 {
 177                         for( int i = 0 ; i < clients.size() ; i++ )
 178                         {
 179                                 if( _ip.equals( clients.get( i ).getIP() ) )
 180                                 {
 181                                         clients.get( i ).resetTimeout() ;
 182                                         break ;
 183                                 }
 184                         }
 185                 }
 186         }
 187
 188
 189         @Override
 190         public void changeStatus( String _ip, String _status )
 191         {
 192                 if( _ip != null && _status != null )
 193                 {
 194                         for( int i = 0 ; i < clients.size() ; i++ )
 195                         {
 196                                 if( _ip.equals( clients.get( i ).getIP() ) )
 197                                 {
 198                                         if( _status.equalsIgnoreCase( "stopped" ) )
 199                                         {
 200                                                 System.out.println( "Client " + clients.get( i ).getName() + " is stopped." ) ;
 201                                                 clients.get( i ).setStatus( "connected" ) ;
 202                                                 ComputingClient cc = clients.get( i ).getComputingClient() ;
 203                                                 computingClients.remove( cc ) ;
 204                                                 clients.get( i ).setComputingClient( null ) ;
 205                                         } else {
 206                                                 clients.get( i ).setStatus( _status ) ;
 207                                                 System.out.println( "Client " + clients.get( i ).getName() + " changed its status to: " + _status ) ;
 208                                         }
 209
 210                                         break ;
 211                                 }
 212                         }
 213                 }
 214         }
 215
 216
 217         public void init( int _port )
 218         {
 219                 port = _port ;
 220                 max_timeout = 4 ;
 221
 222                 clients = new ArrayList<ConnectedClient>() ;
 223                 computingClients = new ArrayList<ComputingClient>() ;
 224                 applications = new ArrayList<RunningApplication>() ;
 225                 monitor = null ;
 226
 227                 startingClients = new OperatingClients() ;
 228                 deployingClients = new OperatingClients() ;
 229                 limitThread = new LimitThread() ;
 230                 maxThread = 20 ;
 231
 232                 working_directory = "/localhome/vmware" ;
 233
 234                 save_interleave  = 30 * 60 * 1000 ;
 235
 236                 semaSave = new Semaphore( 1 ) ;
 237
 238                 exportObject() ;
 239
 240                 vmIPs = new ArrayList<IPAssociation>() ;
 241                 // TODO initialisation of VM IPs
 242                 for( int i = 2 ; i < 101 ; i++ )
 243                 {
 244                         vmIPs.add( new IPAssociation() ) ;
 245                         vmIPs.get( vmIPs.size() - 1 ).setVmIP( "10.11.10." + i ) ;
 246                 }
 247
 248                 clients = new ArrayList<ConnectedClient>() ;
 249
 250                 counter = new DiscCount() ;
 251
 252                 monitor = new ConnectedMonitor() ;
 253                 monitor.start() ;
 254
 255                 // TODO
 256                 // Check if there are running applications ... and restart them :)
 257         }
 258
 259
 260         public void stop()
 261         {
 262                 if( monitor != null ) { monitor.stopMonitor() ; }
 263
 264                 for( int i = 0 ; i < clients.size() ; i++ )
 265                 {
 266                         try {
 267                                 clients.get( i ).getStub().emergencyStop() ;
 268                                 clients.get( i ).getStub().stop() ;
 269                         } catch (RemoteException e) {
 270                                 e.printStackTrace();
 271                         }
 272                 }
 273
 274                 // unexportObject ?
 275
 276                 System.exit( 0 ) ;
 277         }
 278
 279
 280         private void exportObject()
 281         {
 282                 ServicesServer ref = null ;
 283                 Registry reg = null ;
 284
 285                 try
 286                 {
 287                         while( true )
 288                         {
 289                                 reg = LocateRegistry.getRegistry( port ) ;
 290
 291                                 String tab[] = reg.list() ;
 292
 293                                 System.out.println( "There is an existing RMI Registry on port " +
 294                                                 port + " with " + tab.length + " entries!" ) ;
 295                                 for( int i = 0 ; i < tab.length ; i++ )
 296                                 {
 297                                         try {
 298                                                 if( UnicastRemoteObject.unexportObject( Naming.lookup(tab[i]), true ) )
 299                                                 {
 300                                                         System.out.println( "Register successfuly deleted!" ) ;
 301                                                 } else {
 302                                                         System.err.println( "Register undeleted !!!" ) ;
 303                                                 }
 304                                         } catch( Exception e ) {
 305                                                 e.printStackTrace() ;
 306                                         }
 307                                 }
 308                         }
 309                 } catch( RemoteException e ) {
 310                 }
 311
 312                 try {
 313                         if ( System.getSecurityManager() == null )
 314                         {
 315                     System.setSecurityManager( new SecurityManager() ) ;
 316                 }
 317
 318                         LocateRegistry.createRegistry( port ) ;
 319                         LocateRegistry.getRegistry( port ).rebind( "Server", this ) ;
 320                         ref = (ServicesServer) Naming.lookup( "rmi://"
 321                                         + LocalHost.Instance().getIP() + ":" + port
 322                                         + "/Server" ) ;
 323                 } catch ( Exception e ) {
 324                         System.err.println( "Error in Server.exportObject() when creating local services:" + e ) ;
 325                         System.err.println( "Exit from Server.exportObject" ) ;
 326                         System.exit( 1 ) ;
 327                 }
 328
 329                 LocalHost.Instance().setServerStub( ref ) ;
 330
 331                 System.out.println( "Server launched on IP " + LocalHost.Instance().getIP() +
 332                                 " on port " + port + "." ) ;
 333         }
 334
 335         /** Fault manager thread **/
 336         private class FaultManager extends Thread
 337         {
 338                 ConnectedClient cl ;
 339
 340                 FaultManager( ConnectedClient _cl )
 341                 {
 342                         cl = _cl ;
 343                 }
 344
 345                 @Override
 346                 public void run()
 347                 {
 348                         if( cl != null && cl.getStatus().equalsIgnoreCase( "running" ) ||
 349                                         cl.getStatus().equalsIgnoreCase( "saving" ) )
 350                         {
 351                                 ComputingClient cc = cl.getComputingClient() ;
 352 //                              ServicesClient dead = cc.getClient().getStub() ;
 353                                 String ipDead = cc.getClient().getIP() ;
 354                                 SaveNeighbor snDead = null ;
 355                                 for( int i = 0 ; i < computingClients.size() ; i++ )
 356                                 {
 357                                         if( computingClients.get( i ).getSaveNeighbor().getIPHost().equalsIgnoreCase( ipDead ) )
 358                                         {
 359                                                 snDead = computingClients.get( i ).getSaveNeighbor() ;
 360                                                 break ;
 361                                         }
 362                                 }
 363
 364                                 boolean ok = false ;
 365
 366                                 for( int i = 0 ; i < clients.size() ; i++ )
 367                                 {
 368                                         if( clients.get( i ).getStatus().equalsIgnoreCase( "connected" ) )
 369                                         {
 370 //                                              int res = 1 ;
 371 //                                              try {
 372 //                                                      res = clients.get( i ).getStub().startVM() ;
 373 //                                              } catch( RemoteException e ) {
 374 //                                                      e.printStackTrace();
 375 //                                              }
 376 //
 377 //                                              if( res == 0 )
 378 //                                              {
 379                                                         //clients.get(i).setStatus( "running" ) ;
 380
 381                                                 int pos = computingClients.indexOf( cc )  ;
 382                                                 if( pos == -1 )
 383                                                 {
 384                                                         System.err.println( "Dead client not found in the computing clients list!" ) ;
 385                                                 } else {
 386                                                         System.out.println( "Trying to replace " + cc.getClient().getName() + " with " +
 387                                                                         clients.get(i).getName() + " ... " ) ;
 388
 389                                                         String save_name = computingClients.get( pos ).getLastSave() ;
 390
 391                                                         ComputingClient ccl = new ComputingClient( clients.get(i) ) ;
 392                                                         clients.get( i ).setComputingClient( ccl ) ;
 393                                                         SaveNeighbor sn = computingClients.get( pos ).getSaveNeighbor() ;
 394                                                         ccl.setSaveNeighbor( sn ) ;
 395                                                         computingClients.set( pos, ccl ) ;
 396
 397
 398                                                         int res = 1 ;
 399                                                         try {
 400                                                                 res = computingClients.get( pos ).getClient().getStub().
 401                                                                       retrieveSave( save_name ) ;
 402                                                         } catch( RemoteException e ) {
 403                                                                 System.err.println( "Unable to indicate to client to retrieve last save!" ) ;
 404                                                                 e.printStackTrace() ;
 405                                                         }
 406
 407                                                         if( res == 0 )
 408                                                         {
 409                                                                 ok = true ;
 410
 411                                                                 boolean ok_new = false, ok_old = false ;
 412
 413                                                                 // replace dead client in vmIPs
 414                                                                 for( int j = 0 ; j < vmIPs.size() ; j++ )
 415                                                                 {
 416                                                                         if( vmIPs.get( j ).getHostIP().equalsIgnoreCase( computingClients.get( pos ).getClient().getIP() ) )
 417                                                                         {
 418                                                                                 vmIPs.get( j ).setHostIP( "" ) ;
 419                                                                                 ok_new = true ;
 420                                                                         }
 421                                                                         if( vmIPs.get( j ).getHostIP().equalsIgnoreCase( ipDead ) )
 422                                                                         {
 423                                                                                 String vmIP = vmIPs.get( j ).getVmIP() ;
 424                                                                                 vmIPs.get( j ).setHostIP( computingClients.get( pos ).getClient().getIP() ) ;
 425                                                                                 ok_old = true ;
 426
 427                                                                                 try {
 428                                                                                         computingClients.get( pos ).getClient().getStub().setIPVM( vmIP ) ;
 429                                                                                 } catch( RemoteException e ) {
 430                                                                                         System.err.println( "Unable to set the new VM IP on the replacing client!" ) ;
 431                                                                                         e.printStackTrace() ;
 432                                                                                 }
 433
 434                                                                                 if( ok_new && ok_old )
 435                                                                                 {
 436                                                                                         break ;
 437                                                                                 }
 438                                                                         }
 439                                                                 }
 440
 441                                                                 // Replacing in RunningApplication
 442                                                                 applications.get( 0 ).replaceComputingClient( cc, ccl ) ;
 443
 444                                                                 for( int l = 0 ; l < applications.get(0).getComputingClients().size() ; l++ )
 445                                                                 {
 446                                                                         applications.get(0).getComputingClients().get( l ).setSaveRequest( false ) ;
 447                                                                 }
 448
 449
 450                                                                 System.out.println( "Successful redeployment of the VM." ) ;
 451                                                         } else {
 452                                                                 System.err.println( "Unable to deploy the save on the new computing client!" ) ;
 453                                                         }
 454                                                 }
 455 //                                              } else {
 456 //                                                      System.err.println( "Problem while launching the VM on "
 457 //                                                                      + clients.get(i).getName() + "!" ) ;
 458 //                                              }
 459
 460                                                 if( ok )
 461                                                 {
 462                                                         for( int k = 0 ; k < computingClients.size() ; k++ )
 463                                                         {
 464                                                                 try {
 465                                                                         computingClients.get( k ).getClient().getStub().
 466                                                                                 replaceSaveNeighbor( snDead, new SaveNeighbor( clients.get( i ).getStub() ) ) ;
 467                                                                 } catch( RemoteException e ) {
 468                                                                         System.err.println( "Unable to inform " + computingClients.get( k ).getClient().getName() +
 469                                                                                         " of the replacement of a save neighbor!" ) ;
 470                                                                         e.printStackTrace() ;
 471                                                                 }
 472                                                         }
 473
 474                                                         System.out.println( "Dead client successfully replaced." ) ;
 475
 476                                                         break ;
 477                                                 } else {
 478                                                         System.err.println( "Dead client not replaced!!" ) ;
 479                                                 }
 480                                         }
 481                                 }
 482                         }
 483
 484                         try {
 485                                 synchronized( counter ) {
 486                                         counter.dec() ;
 487                                         counter.notifyAll() ;}
 488                         } catch( Exception e ) {}
 489                 }
 490         }
 491
 492
 493         /** Monitoring thread **/
 494         private class ConnectedMonitor extends Thread
 495         {
 496                 boolean run ;
 497
 498                 ConnectedMonitor()
 499                 {
 500                         run = true ;
 501                 }
 502
 503                 protected void stopMonitor() { run = false ; }
 504
 505                 @Override
 506                 public void run()
 507                 {
 508                         boolean change ;
 509
 510                         while( run )
 511                         {
 512                                 change = false ;
 513                                 Iterator<ConnectedClient> it = clients.iterator() ;
 514                                 int nb_disconnections = 0 ;
 515                                 int nb_disconnections_computing = 0 ;
 516
 517                                 while( it.hasNext() )
 518                                 {
 519                                         ConnectedClient cl = it.next() ;
 520                                         cl.incTimeout() ;
 521
 522                                         if( cl.getTimeout() > max_timeout )
 523                                         {
 524                                                 System.out.println( "Disconnection of " + cl.getName() ) ;
 525                                                 if( cl.getStatus().equalsIgnoreCase( "running" ) || cl.getStatus().equalsIgnoreCase( "saving" ) )
 526                                                 {
 527                                                         System.out.println( "A VM was running on it!!" ) ;
 528                                                         System.out.println( "I will redeploy a save and restart all VM ..." ) ;
 529
 530 //                                                      for( int i = 0 ; i < computingClients.size() ; i++ )
 531 //                                                      {
 532 //                                                              if( computingClients.get( i ).getClient().getIP().equals( cl.getIP() ) )
 533 //                                                              {
 534 //                                                                      computingClients.remove( i ) ;
 535 //                                                                      break ;
 536 //                                                              }
 537 //                                                      }
 538                                                         synchronized( counter ){
 539                                                                 counter.inc() ;}
 540
 541
 542                                                         new Server.FaultManager( cl ).start() ;
 543                                                         nb_disconnections_computing++ ;
 544                                                 } else {
 545                                                         System.out.println( "There was no VM running on it." ) ;
 546                                                         System.out.println( "Maybe it will come back later :)" ) ;
 547                                                 }
 548
 549                                                 synchronized( clients )
 550                                                 {
 551                                                         it.remove() ;
 552                                                 }
 553                                                 nb_disconnections++ ;
 554                                                 change = true ;
 555                                         }
 556                                 }
 557
 558                                 if( change )
 559                                 {
 560                                         synchronized( clients )
 561                                         {
 562                                                 if( clients.size() == 0 )
 563                                                 {
 564                                                         System.out.println( "There is no client connected." ) ;
 565                                                 } else if( clients.size() == 1 ) {
 566                                                         System.out.println( "There is one client connected." ) ;
 567                                                 } else {
 568                                                         System.out.println( "There are " + clients.size() + " clients connected." ) ;
 569                                                 }
 570                                         }
 571                                 }
 572
 573
 574                                 if( nb_disconnections_computing > 0 )
 575                                 {
 576                                         System.out.println( "Sending emergency stop signal to all computing nodes ... " ) ;
 577
 578                                         for( int i = 0 ; i < clients.size() ; i++ )
 579                                         {
 580                                                 if( clients.get( i ).getStatus().equalsIgnoreCase( "running" )
 581                                                         || clients.get( i ).getStatus().equalsIgnoreCase( "saving" ) )
 582                                                 {
 583                                                         try {
 584                                                                 clients.get( i ).getStub().emergencyStop() ;
 585                                                         } catch( RemoteException e ) {
 586                                                                 System.err.println( "Unable to invoke emergency stop signal on " + clients.get( i ).getName() ) ;
 587                                                                 e.printStackTrace() ;
 588                                                         }
 589                                                 }
 590                                         }
 591
 592                                         System.out.println( "I will redeploy save and restart VMs ... " ) ;
 593
 594                                         synchronized( counter )
 595                                         {
 596                                                 if( counter.getNb() > 0 )
 597                                                 {
 598                                                         System.out.println( "... waiting all redeployments done ..." ) ;
 599                                                 }
 600
 601                                                 while( counter.getNb() != 0 )
 602                                                 {
 603                                                         try {
 604                                                                 counter.wait() ;
 605                                                         } catch( InterruptedException e ) {
 606                                                                 e.printStackTrace() ;
 607                                                         }
 608                                                 }
 609                                         }
 610
 611                                         for( int  i = 0 ; i < applications.get(0).getComputingClients().size() ; i++ )
 612                                         {
 613                                                 final ServicesClient sc = applications.get(0).getComputingClients().get( i ).getClient().getStub() ;
 614
 615                                                 applications.get( 0 ).getComputingClients().get( i ).setRestartOk( false ) ;
 616
 617                                                 new Thread( new Runnable() {
 618
 619                                                         @Override
 620                                                         public void run()
 621                                                         {
 622                                                                 try {
 623                                                                         sc.restartVMAfterCrash() ;
 624                                                                 } catch( RemoteException e ) {
 625                                                                         e.printStackTrace() ;
 626                                                                 }
 627                                                         }
 628                                                 } ).start() ;
 629                                         }
 630                                 }
 631
 632                                 try
 633                                 {
 634                                         Thread.sleep( 2000 ) ;
 635                                 } catch( InterruptedException e ) {
 636                                         e.printStackTrace() ;
 637                                 }
 638                         }
 639                 }
 640         }
 641
 642         @Override
 643         public Integer saveOk( String _ip, String _saveName )
 644         {
 645                 int i ;
 646                 for( i = 0 ; i < computingClients.size() ; i ++ )
 647                 {
 648                         if( computingClients.get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 649                         {
 650                                 computingClients.get( i ).setLastSave( _saveName ) ;
 651                                 computingClients.get( i ).setSaveStatus( true ) ;
 652                                 break ;
 653                         }
 654                 }
 655
 656
 657                 boolean all_ok = true ;
 658                 i = 0 ;
 659
 660                 while( all_ok && i < computingClients.size() )
 661                 {
 662                         all_ok = all_ok & computingClients.get( i ).getSaveStatus() ;
 663                         i++ ;
 664                 }
 665
 666                 if( all_ok )
 667                 {
 668                         for( i = 0 ; i < computingClients.size() ; i++ )
 669                         {
 670                                 try {
 671                                         computingClients.get( i ).getClient().getStub().saveOk() ;
 672                                 } catch (RemoteException e) {
 673                                         e.printStackTrace();
 674                                 }
 675                                 computingClients.get( i ).setSaveStatus( false ) ;
 676                         }
 677
 678                         applications.get( 0 ).setLastSaveDate( System.currentTimeMillis() ) ;
 679                 }
 680
 681                 return 0 ;
 682         }
 683
 684
 685         public Integer changeSaveName( String _ip, String _saveName )
 686         {
 687                 if( _ip != null && _ip.length() > 0 && _saveName != null && _saveName.length() > 0 )
 688                 {
 689                         for( int i = 0 ; i < computingClients.size() ; i ++ )
 690                         {
 691                                 if( computingClients.get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 692                                 {
 693                                         computingClients.get( i ).setLastSave( _saveName ) ;
 694                                         System.out.println( "Save name successfully changed for " + _ip ) ;
 695                                         return 0 ;
 696                                 }
 697                         }
 698                 }
 699
 700                 return 1 ;
 701         }
 702
 703         @Override
 704         public ArrayList<ServicesClient> startApplication( int _nb )
 705         {
 706                 int nb = clients.size() - computingClients.size() ;
 707
 708                 if( nb > _nb )
 709                 {
 710                         final ArrayList<ServicesClient> ac = new ArrayList<ServicesClient>() ;
 711                         final ArrayList<ComputingClient> tmp = new ArrayList<ComputingClient>() ;
 712
 713                         RunningApplication app = new RunningApplication( "Test" ) ;
 714
 715                         int i = 0 ;
 716                         boolean ok ;
 717
 718                         while( i < clients.size() && ac.size() < _nb )
 719                         {
 720                                 ok = false ;
 721                                 if( clients.get( i ).getStatus().equalsIgnoreCase( "connected" ) )
 722                                 {
 723                                         synchronized( startingClients )
 724                                         {
 725                                                 while( ac.size() + startingClients.getNb() >= _nb )
 726                                                 {
 727                                                         if( ac.size() == _nb ) break ;
 728
 729                                                         try {
 730                                                                 startingClients.wait() ;
 731                                                         } catch (InterruptedException e) {
 732                                                                 e.printStackTrace();
 733                                                         }
 734                                                 }
 735
 736                                                 if( ac.size() < _nb )
 737                                                 {
 738                                                         startingClients.inc() ;
 739                                                         ok = true ;
 740                                                 }
 741                                         }
 742
 743                                         if( ok )
 744                                         {
 745                                                 synchronized( limitThread )
 746                                                 {
 747                                                         while( limitThread.getNb() >= maxThread )
 748                                                         {
 749                                                                 try {
 750                                                                         limitThread.wait() ;
 751                                                                 } catch (InterruptedException e) {
 752                                                                         e.printStackTrace();
 753                                                                 }
 754                                                         }
 755
 756                                                         limitThread.inc() ;
 757                                                 }
 758
 759                                                 final int ind = i ;
 760                                                 new Thread( new Runnable(){
 761
 762                                                         @Override
 763                                                         public void run()
 764                                                         {
 765                                                                 int res = 1 ;
 766                                                                 try {
 767                                                                         res = clients.get( ind ).getStub().startVM( 0 ) ;
 768                                                                 } catch( RemoteException e ) {
 769                                                                         e.printStackTrace();
 770                                                                 }
 771
 772                                                                 if( res == 0 )
 773                                                                 {
 774                                                                         ac.add( clients.get( ind ).getStub() ) ;
 775                                                                         clients.get( ind ).setStatus( "running" ) ;
 776                                                                         ComputingClient cl = new ComputingClient( clients.get( ind ) ) ;
 777                                                                         clients.get( ind ).setComputingClient( cl ) ;
 778                                                                         computingClients.add( cl ) ;
 779                                                                         tmp.add( cl ) ;
 780                                                                 } else {
 781                                                                         System.err.println( "Problem while launching the VM on "
 782                                                                                         + clients.get(ind).getName() + "!" ) ;
 783                                                                 }
 784
 785                                                                 synchronized( limitThread )
 786                                                                 {
 787                                                                         limitThread.dec() ;
 788                                                                         limitThread.notifyAll() ;
 789                                                                 }
 790
 791                                                                 synchronized( startingClients )
 792                                                                 {
 793                                                                         startingClients.dec() ;
 794                                                                         startingClients.notifyAll() ;
 795                                                                 }
 796                                                         }
 797                                                 }).start() ;
 798                                         }
 799                                 }
 800
 801                                 i++ ;
 802                         }
 803
 804                         if( ac.size() == _nb )
 805                         {
 806                                 app.setComputingClients( tmp ) ;
 807                                 app.setRunning( true ) ;
 808                                 app.setStartTime( System.currentTimeMillis() ) ;
 809
 810                                 int index, index2 ;
 811                                 /* Choosing save neighbors */
 812                                 for( i = 0 ; i < tmp.size() ; i++ )
 813                                 {
 814                                         if( i == tmp.size() - 1 )
 815                                         {
 816                                                 index = computingClients.indexOf( tmp.get( i ) ) ;
 817                                                 index2 = computingClients.indexOf( tmp.get( 0 ) ) ;
 818
 819                                                 if( index == -1 || index2 == -1 )
 820                                                 {
 821                                                         System.err.println( "Problem in ComputingClients list!" ) ;
 822                                                 } else {
 823                                                         try {
 824                                                                 computingClients.get( index ).setSaveNeighbor( new SaveNeighbor( computingClients.get( index2 ).getClient().getStub() )) ;
 825                                                         } catch( RemoteException e ) {
 826                                                                 System.err.println( "Unable to create the save neighbor!" ) ;
 827                                                                 e.printStackTrace() ;
 828                                                         }
 829                                                 }
 830                                         } else {
 831                                                 index = computingClients.indexOf( tmp.get( i ) ) ;
 832                                                 index2 = computingClients.indexOf( tmp.get( i + 1 ) ) ;
 833
 834                                                 if( index == -1 || index2 == -1 )
 835                                                 {
 836                                                         System.err.println( "Problem in ComputingClients list!" ) ;
 837                                                 } else {
 838                                                         try {
 839                                                                 computingClients.get( index ).setSaveNeighbor( new SaveNeighbor( computingClients.get( index2 ).getClient().getStub() ) ) ;
 840                                                         } catch( RemoteException e ) {
 841                                                                 System.err.println( "Unable to create the save neighbor!" ) ;
 842                                                                 e.printStackTrace() ;
 843                                                         }
 844                                                 }
 845                                         }
 846                                 }
 847                         }
 848
 849                         applications.add( app ) ;
 850
 851                         return ac ;
 852                 }
 853
 854                 return null ;
 855         }
 856
 857
 858         @Override
 859         public void requestSave( String _ip )
 860         {
 861                 try {
 862                         semaSave.acquire() ;
 863                 } catch( InterruptedException e ) {
 864                         System.err.println( "Unable to obtain the semaphore for semaSave!" ) ;
 865                         e.printStackTrace() ;
 866                 }
 867
 868                 final String ip = _ip ;
 869
 870                 new Thread( new Runnable() {
 871
 872                         @Override
 873                         public void run()
 874                         {
 875                                 treatRequestSave( ip ) ;
 876                         }
 877                 } ).start() ;
 878         }
 879
 880
 881         public void treatRequestSave( String _ip )
 882         {
 883                 if( applications.size() > 0 && _ip != null && _ip.length() > 0 )
 884                 {
 885                         if( (System.currentTimeMillis() - applications.get( 0 ).getLastSaveDate()) > save_interleave )
 886                         {
 887                                 // Mark it as a requester
 888                                 for( int i = 0 ; i < applications.get( 0 ).getComputingClients().size() ; i++ )
 889                                 {
 890                                         if( applications.get( 0 ).getComputingClients().get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 891                                         {
 892                                                 applications.get( 0 ).getComputingClients().get( i ).setSaveRequest( true ) ;
 893
 894                                                 break ;
 895                                         }
 896                                 }
 897
 898                                 semaSave.release() ;
 899
 900                                 // Is every body ok?
 901                                 boolean ok = false ;
 902                                 for( int i = 0 ; i < applications.get( 0 ).getComputingClients().size() ; i++ )
 903                                 {
 904                                         if( i == 0 )
 905                                         {
 906                                                 ok = applications.get( 0 ).getComputingClients().get( i ).getSaveRequest() ;
 907                                         } else {
 908                                                 ok = ok & applications.get( 0 ).getComputingClients().get( i ).getSaveRequest() ;
 909                                         }
 910
 911                                         if( ! ok )
 912                                         {
 913                                                 break ;
 914                                         }
 915                                 }
 916
 917                                 if( ok )
 918                                 {
 919 //                                      try {
 920 //                                              Thread.sleep( 5000 ) ;
 921 //                                      } catch( InterruptedException e1 ) {
 922 //                                              e1.printStackTrace() ;
 923 //                                      }
 924
 925                                         for( int i = 0 ; i < applications.get( 0 ).getComputingClients().size() ; i++ )
 926                                         {
 927                                                 try {
 928                                                         applications.get( 0 ).getComputingClients().get( i ).getClient().getStub().responseSave( true ) ;
 929                                                         applications.get( 0 ).getComputingClients().get( i ).setSaveRequest( false ) ;
 930                                                 } catch( RemoteException e ) {
 931                                                         System.err.println( "Unable to send the save request response to " +
 932                                                                         applications.get( 0 ).getComputingClients().get( i ).getClient().getName() + "!" ) ;
 933                                                         e.printStackTrace() ;
 934                                                 }
 935                                         }
 936
 937                                         applications.get( 0 ).setLastSaveDate( System.currentTimeMillis() ) ;
 938                                 }
 939
 940                         } else {
 941                                 semaSave.release() ;
 942
 943                                 for( int i = 0 ; i < applications.get( 0 ).getComputingClients().size() ; i++ )
 944                                 {
 945                                         if( applications.get( 0 ).getComputingClients().get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 946                                         {
 947                                                 try {
 948                                                         applications.get( 0 ).getComputingClients().get( i ).getClient().getStub().responseSave( false ) ;
 949                                                 } catch( RemoteException e ) {
 950                                                         System.err.println( "Unable to send the save request response to " +
 951                                                                         applications.get( 0 ).getComputingClients().get( i ).getClient().getName() + "!" ) ;
 952                                                         e.printStackTrace() ;
 953                                                 }
 954                                                 break ;
 955                                         }
 956                                 }
 957                         }
 958                 } else {
 959                         semaSave.release() ;
 960                         System.err.println( "!! Serious problem in treatRequestSave method!!" ) ;
 961                 }
 962         }
 963
 964
 965         @Override
 966         public void restartOk( String _ip )
 967         {
 968                 if( applications.size() > 0 && _ip != null && _ip.length() > 0 )
 969                 {
 970                         System.out.println( "Client " + _ip + " has finished to restart ("+applications.get(0).getComputingClients().size()+") ... " ) ;
 971                         if( (System.currentTimeMillis() - applications.get( 0 ).getLastSaveDate()) > save_interleave )
 972                         {
 973                                 // Has it already finished?
 974                                 for( int i = 0 ; i < applications.get( 0 ).getComputingClients().size() ; i++ )
 975                                 {
 976                                         if( applications.get( 0 ).getComputingClients().get( i ).getClient().getIP().equalsIgnoreCase( _ip ) )
 977                                         {
 978                                                 applications.get( 0 ).getComputingClients().get( i ).setRestartOk( true ) ;
 979
 980                                                 break ;
 981                                         }
 982                                 }
 983
 984                                 // Is every body ok?
 985                                 boolean ok = false ;
 986                                 for( int i = 0 ; i < applications.get( 0 ).getComputingClients().size() ; i++ )
 987                                 {
 988                                         if( i == 0 )
 989                                         {
 990                                                 ok = applications.get( 0 ).getComputingClients().get( i ).getRestartOk() ;
 991                                         } else {
 992                                                 ok = ok & applications.get( 0 ).getComputingClients().get( i ).getRestartOk() ;
 993                                         }
 994
 995                                         if( ! ok )
 996                                         {
 997                                                 break ;
 998                                         }
 999                                 }
1000
1001                                 if( ok )
1002                                 {
1003                                         applications.get( 0 ).setLastSaveDate( System.currentTimeMillis() ) ;
1004
1005                                         for( int i = 0 ; i < applications.get( 0 ).getComputingClients().size() ; i++ )
1006                                         {
1007                                                 applications.get( 0 ).getComputingClients().get( i ).setRestartOk( false ) ;
1008                                         }
1009                                 }
1010
1011                         }
1012                 }
1013         }
1014
1015
1016         @Override
1017         public void endApplication()
1018         {
1019                 Iterator<ComputingClient> it = computingClients.iterator() ;
1020
1021                 while( it.hasNext() )
1022                 {
1023                         ComputingClient cl = it.next() ;
1024
1025                         try {
1026                                 cl.getClient().getStub().emergencyStop() ;
1027                         } catch (RemoteException e) {
1028                                 e.printStackTrace();
1029                         }
1030
1031                         cl.getClient().setStatus( "connected" ) ;
1032                         cl.getClient().setComputingClient( null ) ;
1033                         it.remove() ;
1034                         cl = null ;
1035                 }
1036
1037                 applications.get( 0 ).setEndTime( System.currentTimeMillis() ) ;
1038                 applications.get( 0 ).setRunning( false ) ;
1039                 applications.get( 0 ).clear() ;
1040
1041         }
1042
1043
1044
1045         @Override
1046         public String getAssociatedIP( String _ip )
1047         {
1048                 String ret = null ;
1049
1050                 for( int i = 0 ; i < vmIPs.size() ; i++ )
1051                 {
1052                         if( vmIPs.get( i ).getHostIP().equalsIgnoreCase( _ip ) )
1053                         {
1054                                 ret = vmIPs.get( i ).getVmIP() ;
1055                                 break ;
1056                         }
1057                 }
1058
1059                 return ret ;
1060         }
1061
1062
1063         public Integer deployVM( final String _name, final String _archive, final String _directory )
1064         {
1065                 int nb = 0, pb = 0 ;
1066
1067                 if( _name != null && _name.length() > 0 && _archive != null && _name.length() > 0
1068                                 && _directory != null && _directory.length() > 0 )
1069                 {
1070                         System.out.println( "Deploying the VM " + _name + " (" + _archive + ") ... " ) ;
1071
1072                         File file = new File( working_directory + "/" + _archive ) ;
1073                         if( ! file.exists() )
1074                         {
1075                                 System.err.println( "There is no archive named " + _archive + " in my working directory!" ) ;
1076                                 file = null ;
1077                                 return 1 ;
1078                         } else if( file.isDirectory() ) {
1079                                 System.err.println( _archive + " is a directory!" ) ;
1080                                 file = null ;
1081                                 return 1 ;
1082                         }
1083
1084                         file = null ;
1085
1086                         // TODO do a better deployment !!
1087 //                      int ret ;
1088 //                      boolean error, ok, server ;
1089 //                      ArrayList<ConnectedClient> deployed = new ArrayList<ConnectedClient>() ;
1090
1091                         boolean server = true ;
1092                         boolean ok ;
1093
1094                         for( int i = 0 ; i < clients.size() ; i++ )
1095                         {
1096 //                              ret = 1 ;
1097                                 nb++ ;
1098 //                              error = false ;
1099 //                              ok = false ;
1100                                 if( clients.get( i ).getStatus().equalsIgnoreCase( "connected" ) )
1101                                 {
1102                                         synchronized( limitThread )
1103                                         {
1104                                                 while( limitThread.getNb() >= maxThread )
1105                                                 {
1106                                                         try {
1107                                                                 limitThread.wait() ;
1108                                                         } catch (InterruptedException e) {
1109                                                                 e.printStackTrace();
1110                                                         }
1111                                                 }
1112
1113                                                 limitThread.inc() ;
1114                                         }
1115
1116                                         final int ind = i ;
1117                                         new Thread( new Runnable() {
1118
1119                                                 @Override
1120                                                 public void run() {
1121                                                         int ret = -1 ;
1122                                                         boolean error = true  ;
1123
1124                                                         try {
1125                                                                 ret = clients.get( ind ).getStub().deployVM( _name, _archive, _directory ) ;
1126                                                         } catch( RemoteException e ) {
1127                                                                 System.err.println( "Unable to deploy the VM on " + clients.get( ind ).getName() + "!" ) ;
1128                                                                 e.printStackTrace() ;
1129                                                         }
1130
1131                                                         // The client does not have the archive, we have to send it.
1132                                                         if( ret == 2 )
1133                                                         {
1134                                                                 // Attention au multi-envois !!!
1135                                                                 System.out.print( "Sending VM archive to " + clients.get( ind ).getName() + " ... " ) ;
1136
1137                                                                 String wd = "" ;
1138                                                                 String snIP = "" ;
1139                                                                 error = false ;
1140
1141                                                                 try {
1142                                                                         wd = clients.get( ind ).getStub().getWorkingDirectory() ;
1143                                                                         snIP = clients.get( ind ).getStub().getIPHost() ;
1144                                                                 } catch (RemoteException e2) {
1145                                                                         System.err.println( "Unable to retrieve information on " + clients.get( ind ).getName() + "!" ) ;
1146                                                                         e2.printStackTrace() ;
1147                                                                         error = true ;
1148                                                                 }
1149
1150                                                                 String[] command = new String[]{ "/usr/bin/scp", working_directory + "/" + _archive,
1151                                                                                 snIP + ":" + wd } ;
1152
1153                                                                 if( ! error )
1154                                                                 {
1155                                                                         try {
1156                                                                                 Process proc = Runtime.getRuntime().exec( command ) ;
1157                                                                                 proc.waitFor() ;
1158
1159                                                                                 if( proc.exitValue() == 0 )
1160                                                                                 {
1161                                                                                         System.out.println( "Initial VM archive successfully sent." ) ;
1162                                                                                 } else {
1163                                                                                         System.err.println( "Initial VM archive not sent!" ) ;
1164                                                                                         System.err.println( "Error: " + proc.exitValue() ) ;
1165                                                                                         BufferedReader b = new BufferedReader( new InputStreamReader( proc.getErrorStream() ) ) ;
1166
1167                                                                                         String l ;
1168                                                                                         try {
1169                                                                                                 while( (l = b.readLine()) != null )
1170                                                                                                 {
1171                                                                                                         System.err.println( l ) ;
1172                                                                                                 }
1173                                                                                         } catch( IOException e ) {
1174                                                                                                 e.printStackTrace() ;
1175                                                                                         }
1176
1177                                                                                         error = true ;
1178                                                                                 }
1179                                                                         } catch( IOException e ) {
1180                                                                                 System.err.println( "Error during initial VM archive send command: " ) ;
1181                                                                                 e.printStackTrace() ;
1182                                                                                 error = true ;
1183                                                                         } catch( InterruptedException e ) {
1184                                                                                 e.printStackTrace() ;
1185                                                                                 error = true ;
1186                                                                         }
1187                                                                 }
1188
1189
1190                                                                 if( ! error )
1191                                                                 {
1192                                                                         // Second try ...
1193                                                                         ret = 1 ;
1194                                                                         try {
1195                                                                                 ret = clients.get( ind ).getStub().deployVM( _name, _archive, _directory ) ;
1196                                                                         } catch( RemoteException e ) {
1197                                                                                 System.err.println( "Unable to deploy the VM on " + clients.get( ind ).getName() + "!" ) ;
1198                                                                                 e.printStackTrace() ;
1199                                                                         }
1200                                                                 }
1201                                                         }
1202
1203                                                         if( ret == 0 )
1204                                                         {
1205                                                                 System.out.println( "Initial VM archive successfully deployed on " + clients.get( ind ).getName() + "." ) ;
1206
1207                                                                 synchronized( deployingClients )
1208                                                                 {
1209                                                                         deployingClients.inc() ;
1210                                                                 }
1211                                                         }
1212
1213                                                         synchronized( limitThread )
1214                                                         {
1215                                                                 limitThread.dec() ;
1216                                                                 limitThread.notifyAll() ;
1217                                                         }
1218                                                 }
1219                                         }).start() ;
1220                                 }
1221                         }
1222                 }
1223
1224                 synchronized( limitThread )
1225                 {
1226                         while( limitThread.getNb() > 0  )
1227                         {
1228                                 try {
1229                                         limitThread.wait() ;
1230                                 } catch( InterruptedException e ) {
1231                                         e.printStackTrace() ;
1232                                 }
1233                         }
1234                 }
1235
1236                 if( nb - deployingClients.getNb() > 0 )
1237                 {
1238                         if( pb == 1 )
1239                                 System.err.println( "** " + pb + " machine is not deployed!" ) ;
1240                         if( pb > 1 )
1241                                 System.err.println( "** " + pb + " machines are not deployed!" ) ;
1242                 }
1243
1244                 return nb ;
1245         }
1246
1247
1248         public String getWorkingDirectory()
1249         {
1250                 return working_directory ;
1251         }
1252
1253
1254         private class OperatingClients
1255         {
1256                 private int nb ;
1257
1258                 OperatingClients() { nb = 0 ; }
1259
1260                 protected void inc() { nb++ ; }
1261
1262                 protected void dec() { nb-- ; }
1263
1264                 protected int getNb() { return nb ; }
1265         }
1266
1267
1268         private class LimitThread
1269         {
1270                 private int nb ;
1271
1272                 LimitThread() { nb = 0 ; }
1273
1274                 protected void inc() { nb++ ; }
1275
1276                 protected void dec() { nb-- ; }
1277
1278                 protected int getNb() { return nb ; }
1279         }
1280
1281 }
1282
1283 /** La programmation est un art, respectons ceux qui la pratiquent !! **/
1284