Merge branch 'charm' of charmgit:charm into harshitha/adaptive_lb
authorHarshitha <gplkrsh2@illinois.edu>
Fri, 12 Oct 2012 05:16:19 +0000 (00:16 -0500)
committerHarshitha <gplkrsh2@illinois.edu>
Fri, 12 Oct 2012 05:16:19 +0000 (00:16 -0500)
Conflicts:
examples/charm++/jacobi1d/jacobi1d.C
src/ck-core/cklocation.C
src/ck-ldb/CommAwareRefineLB.C
src/scripts/Makefile

13 files changed:
1  2 
src/ck-core/ckarray.C
src/ck-core/cklocation.C
src/ck-core/cklocation.h
src/ck-core/init.C
src/ck-ldb/BaseLB.h
src/ck-ldb/CentralLB.C
src/ck-ldb/CentralLB.h
src/ck-ldb/CommAwareRefineLB.C
src/ck-ldb/LBDatabase.C
src/ck-ldb/LBDatabase.h
src/ck-ldb/lbdb.C
src/ck-ldb/lbdb.h
src/scripts/Makefile

Simple merge
index 1d3eadb93ebf29e84a8db19f3093cca663913385,c33807bab89b9e469e81b8b42ac708b42b2d4927..10373ccaf6c37d9151aece1cff08fec2da9fd52d
  
  #if CMK_LBDB_ON
  #include "LBDatabase.h"
 +#include "MetaBalancer.h"
+ #if CMK_GLOBAL_LOCATION_UPDATE
+ #include "BaseLB.h"
+ #include "init.h"
+ #endif
  #endif // CMK_LBDB_ON
  
  #if CMK_GRID_QUEUE_AVAILABLE
@@@ -55,9 -57,11 +59,12 @@@ static const char *idx2str(const CkArra
  #   define DEBB(x) /*CkPrintf x*/
  #   define str(x) /**/
  #   define DEBUG(x)   /**/
 +#   define DEBAD(x) /*CkPrintf x*/
  #endif
  
+ //whether to use block mapping in the SMP node level
+ bool useNodeBlkMapping;
  #if CMK_LBDB_ON
  /*LBDB object handles are fixed-sized, and not necc.
  the same size as ArrayIndices.
@@@ -1322,9 -1308,12 +1399,13 @@@ CkLocRec_local::CkLocRec_local(CkLocMg
          enable_measure = CmiTrue;
        bounced  = CmiFalse;
        the_lbdb=mgr->getLBDB();
 +  the_metalb=mgr->getMetaBalancer();
+         LDObjid ldid = idx2LDObjid(idx);
+ #if CMK_GLOBAL_LOCATION_UPDATE
+         ldid.locMgrGid = mgr->getGroupID().idx;
+ #endif        
        ldHandle=the_lbdb->RegisterObj(mgr->getOMHandle(),
-               idx2LDObjid(idx),(void *)this,1);
+               ldid,(void *)this,1);
        if (fromMigration) {
                DEBL((AA"Element %s migrated in\n"AB,idx2str(idx)));
                if (!ignoreArrival)  {
index d884f7cd6e5191554df276225564a73189b21856,00076bfde4cbf66b0844f27429a803ae6a4b62be..7dd90c6350202c3080f92dc5fffd80ddde6210de
@@@ -696,11 -670,7 +697,10 @@@ private
  
        /// Call this member function on each element of this location:
        typedef void (CkMigratable::* CkMigratable_voidfn_t)(void);
-       void callMethod(CkLocRec_local *rec,CkMigratable_voidfn_t fn);
  
 +      typedef void (CkMigratable::* CkMigratable_voidfn_arg_t)(void*);
 +      void callMethod(CkLocRec_local *rec,CkMigratable_voidfn_arg_t fn, void*);
 +
        CmiBool deliverUnknown(CkArrayMessage *msg,CkDeliver_t type,int opts);
  
        /// Create a new local record at this array index.
Simple merge
Simple merge
Simple merge
Simple merge
index 338a21e120280392264b4582ce2491099013dc4d,33d88b1ad21d225b03df9e678fde5f3483408174..67d8eeec524e0322d77ac6f6f4c7ad022e86ccee
@@@ -269,100 -219,84 +219,103 @@@ void CommAwareRefineLB::work(LDStats* s
  
    // Keep on loadbalancing until the number of above avg processors is 0
    while (parr_above_avg.size() != 0 && total_swaps > 0 && parr_below_avg.size() != 0) {
-       // CkPrintf("Above avg : %d Below avg : %d Total swaps: %d\n", parr_above_avg.size(),
-       //    parr_below_avg.size(), total_swaps);
-       obj_allocated = false;
-       num_tries = 0;
-       // Pop the heaviest processor
-       int p_index = popFromProcHeap(parr_above_avg, parr);
-       ProcInfo& p = parr->procs[p_index];
+     // CkPrintf("Above avg : %d Below avg : %d Total swaps: %d\n", parr_above_avg.size(),
+     //    parr_below_avg.size(), total_swaps);
+     obj_allocated = false;
+     num_tries = 0;
+     // Pop the heaviest processor
+     int p_index = popFromProcHeap(parr_above_avg, parr);
+     ProcInfo& p = parr->procs[p_index];
+     while (!obj_allocated && num_tries < parr_objs[p.getProcId()].size()) {
+       // It might so happen that due to overhead load, it might not have any
+       // more objects in its list
+       if (parr_objs[p.getProcId()].size() == 0) {
+         // CkPrintf("No obj left to be allocated\n");
+         obj_allocated = true;
+         break;
+       }
  
-       while (!obj_allocated && num_tries < 5) {
+       int randd = rand();
+       random = randd % parr_objs[p.getProcId()].size();
+       randomly_obj_id = parr_objs[p.getProcId()][random];
+       obj_load = ogr->vertices[randomly_obj_id].getVertexLoad();
  
-         // It might so happen that due to overhead load, it might not have any
-         // more objects in its list
-         if (parr_objs[p.getProcId()].size() == 0) {
-           // CkPrintf("No obj left to be allocated\n");
-           obj_allocated = true;
-           break;
-         }
+       // CkPrintf("Heavy %d: Parr obj size : %d random : %d random obj id : %d\n", p_index,
+       //     parr_objs[p.getProcId()].size(), randd, randomly_obj_id);
+       std::vector<int> possible_pes;
+       getPossiblePes(possible_pes, randomly_obj_id, ogr, parr);
+       for (i = 0; i < possible_pes.size(); i++) {
  
-         int randd = rand();
-         random = randd % parr_objs[p.getProcId()].size();
-         randomly_obj_id = parr_objs[p.getProcId()][random];
-         obj_load = ogr->vertices[randomly_obj_id].getVertexLoad();
+         // If the heaviest communicating processor is there in the list, then
+         // assign it to that.
+         possible_pe = possible_pes[i];
  
-         // CkPrintf("Heavy %d: Parr obj size : %d random : %d random obj id : %d\n", p_index,
-         //     parr_objs[p.getProcId()].size(), randd, randomly_obj_id);
-         for (i = 0; i < objpcomm[randomly_obj_id].pcomm.size(); i++) {
+         if ((parr->procs[possible_pe].getTotalLoad() + obj_load) < upper_threshold) {
+          // CkPrintf("**  Transfered %d(Load %lf) from %d:%d(Load %lf) to %d:%d(Load %lf)\n",
+          //     randomly_obj_id, obj_load, CkNodeOf(p.getProcId()), p.getProcId(), p.getTotalLoad(),
+          //     CkNodeOf(possible_pe), possible_pe,
+          //     parr->procs[possible_pe].getTotalLoad());
  
-           // If the heaviest communicating processor is there in the list, then
-           // assign it to that.
-           possible_pe = objpcomm[randomly_obj_id].pcomm[i].pe_id;
+           handleTransfer(randomly_obj_id, p, possible_pe, parr_objs, ogr, parr);
+           obj_allocated = true;
+           total_swaps--;
+           updateLoadInfo(p_index, possible_pe, upper_threshold, lower_threshold,
+               parr_above_avg, parr_below_avg, proc_load_info, parr);
  
-           if ((parr->procs[possible_pe].getTotalLoad() + obj_load) < upper_threshold) {
-             //CkPrintf("** Iter[%d] : Transfered %d from %d : Load %E to %d : Load %E Comm : %d\n",
-             //    i, randomly_obj_id, p.getProcId(), p.getTotalLoad(),
-             //    possible_pe, parr->procs[possible_pe].getTotalLoad(),
-             //    objpcomm[randomly_obj_id].pcomm[i].num_msg);
+           break;
+         }
+       }
  
-             handleTransfer(randomly_obj_id, p, possible_pe, parr_objs, ogr, parr);
+       // Since there is no processor in the least loaded list with which this
+       // chare communicates, pick a random least loaded processor.
+       if (!obj_allocated) {
+         //CkPrintf(":( Could not transfer to the nearest communicating ones\n");
+         for (int x = 0; x < parr_below_avg.size(); x++) {
+           int random_pe = parr_below_avg[x];
+           if ((parr->procs[random_pe].getTotalLoad() + obj_load) < upper_threshold) {
              obj_allocated = true;
              total_swaps--;
-             updateLoadInfo(p_index, possible_pe, upper_threshold, lower_threshold,
+             handleTransfer(randomly_obj_id, p, random_pe, parr_objs, ogr, parr);
+             updateLoadInfo(p_index, random_pe, upper_threshold, lower_threshold,
                  parr_above_avg, parr_below_avg, proc_load_info, parr);
              break;
            }
-         }
-         // Since there is no processor in the least loaded list with which this
-         // chare communicates, pick a random least loaded processor.
-         if (!obj_allocated) {
-           // CkPrintf(":( Could not transfer to the nearest communicating ones\n");
-           for (int x = 0; x < parr_below_avg.size(); x++) {
-             int random_pe = parr_below_avg[x];
-             if ((parr->procs[random_pe].getTotalLoad() + obj_load) < upper_threshold) {
-               obj_allocated = true;
-               total_swaps--;
-               handleTransfer(randomly_obj_id, p, random_pe, parr_objs, ogr, parr);
-               updateLoadInfo(p_index, random_pe, upper_threshold, lower_threshold,
-                   parr_above_avg, parr_below_avg, proc_load_info, parr);
-               break;
-             }
-             num_tries++;
-           }
+           num_tries++;
          }
        }
+     }
  
-       if (!obj_allocated) {
-         CkPrintf("!!!! Could not handle the heavy proc %d so giving up\n", p_index);
-        // parr_above_avg.push_back(p_index);
-        // std::push_heap(parr_above_avg.begin(), parr_above_avg.end(),
-        //     ProcLoadGreater(parr));
-       }
+     if (!obj_allocated) {
+       //CkPrintf("!!!! Could not handle the heavy proc %d so giving up\n", p_index);
+       // parr_above_avg.push_back(p_index);
+       // std::push_heap(parr_above_avg.begin(), parr_above_avg.end(),
+       //     ProcLoadGreater(parr));
      }
+   }
  
 +  int local_comm = 0;
 +  int remote_comm = 0;
 +  for (int i=0; i< ogr->vertices.size(); i++) {
 +    for(int j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
 +      vert = ogr->vertices[i].sendToList[j].getNeighborId();
 +      if(ogr->vertices[i].getNewPe() == ogr->vertices[vert].getNewPe()) {
 +        local_comm++;
 +      } else {
 +        remote_comm++;
 +      }
 +    }
 +  }
 +  CkPrintf("Local communication: %d Remote communication: %d\n", local_comm, remote_comm);
 +  stats->local_comm = local_comm;
 +  stats->remote_comm = remote_comm;
 +
 +  stats->is_prev_lb_refine = 3;
 +  stats->after_lb_max = 0;
 +  stats->after_lb_avg = 0;
+   //CkPrintf("CommAwareRefine> After lb max load: %lf avg load: %lf\n", max_load, avg_load/parr->procs.size());
  
    /** ============================== CLEANUP ================================ */
    ogr->convertDecisions(stats);         // Send decisions back to LDStats
index c5e1630cf40d392e7fc635eea6d10dd36a6b1105,ca2cc0667a991a39a1acf487b9b3743d33f9a59d..186b0cb044de1d8ea96ecb78a6b6693e7ab9e8a5
@@@ -576,9 -513,22 +576,22 @@@ extern "C" void LBTurnInstrumentOff() 
      _lb_args.statsOn() = 0;
  #endif
  }
+ extern "C" void LBTurnCommOn() {
+ #if CMK_LBDB_ON
+   _lb_args.traceComm() = 1;
+ #endif
+ }
+ extern "C" void LBTurnCommOff() {
+ #if CMK_LBDB_ON
+   _lb_args.traceComm() = 0;
+ #endif
+ }
  void LBClearLoads() {
  #if CMK_LBDB_ON
 -  LBDatabase::Object()->ClearLoads(); 
 +  LBDatabase::Object()->ClearLoads();
  #endif
  }
  
Simple merge
Simple merge
Simple merge
index 3564c5d49cb1454442f6155981d57e8d1762b3a7,df0aac5039ddbfebd2a3a2ec12e4bc4b32750d58..c96c58fbb705459f094b11258fb217aa85f28103
@@@ -709,8 -712,8 +712,8 @@@ CK_LIBS_CORE=$(L)/libck.
  LIBCK_CORE=trace-common.o tracec.o tracef.o init.o register.o qd.o ck.o main.o \
           msgalloc.o ckfutures.o ckIgetControl.o debug-message.o debug-charm.o ckcallback.o \
           cklocation.o ckarray.o ckreduction.o ckarrayreductionmgr.o \
 -           tempo.o waitqd.o LBDatabase.o lbdb.o lbdbf.o charisma.o ckobjQ.o  \
 +           tempo.o waitqd.o LBDatabase.o MetaBalancer.o lbdb.o lbdbf.o charisma.o ckobjQ.o  \
-          LBAgent.o LBProfit.o ckcheckpoint.o ckmemcheckpoint.o ckevacuation.o ckmessagelogging.o ckcausalmlog.o\
+          LBAgent.o LBProfit.o ckcheckpoint.o ckmemcheckpoint.o ckevacuation.o ckmessagelogging.o ckcausalmlog.o ckobjid.o\
             LBDBManager.o LBComm.o LBObj.o LBMachineUtil.o CentralPredictor.o \
           BaseLB.o CentralLB.o HybridBaseLB.o NborBaseLB.o \
             ckgraph.o LButil.o Refiner.o RefinerApprox.o  \