Refactor metabalancer portion and move it out of lbdatabase
authorHarshitha <gplkrsh2@illinois.edu>
Tue, 9 Oct 2012 16:17:05 +0000 (11:17 -0500)
committerHarshitha <gplkrsh2@illinois.edu>
Tue, 9 Oct 2012 16:17:05 +0000 (11:17 -0500)
17 files changed:
src/ck-core/ckarray.C
src/ck-core/cklocation.C
src/ck-core/cklocation.ci
src/ck-core/cklocation.h
src/ck-core/init.C
src/ck-ldb/CentralLB.C
src/ck-ldb/CentralLB.h
src/ck-ldb/EveryLB.ci
src/ck-ldb/LBDatabase.C
src/ck-ldb/LBDatabase.ci
src/ck-ldb/LBDatabase.h
src/ck-ldb/Make.lb
src/ck-ldb/Makefile_lb.sh
src/ck-ldb/MetaBalancer.C [new file with mode: 0644]
src/ck-ldb/MetaBalancer.ci [new file with mode: 0644]
src/ck-ldb/MetaBalancer.h [new file with mode: 0644]
src/scripts/Makefile

index 65dd8a1eb64c85b9cdeb1d21e712db1882088d04..9a24377d13ca26118ac0f6320f8b0f60a8cbf8bd 100644 (file)
@@ -617,10 +617,11 @@ CkArrayID CProxy_ArrayBase::ckCreateArray(CkArrayMessage *m,int ctor,
   { //Create a new location manager
 #if !CMK_LBDB_ON
     CkGroupID _lbdb;
+    CkGroupID _metalb;
 #endif
     CkEntryOptions  e_opts;
     e_opts.setGroupDepID(opts.getMap());       // group creation dependence
-    locMgr = CProxy_CkLocMgr::ckNew(opts.getMap(),_lbdb,opts.getNumInitial(),&e_opts);
+    locMgr = CProxy_CkLocMgr::ckNew(opts.getMap(),_lbdb,_metalb,opts.getNumInitial(),&e_opts);
     opts.setLocationManager(locMgr);
   }
   //Create the array manager
index 3b5aac4451000a580377a0163e085288141a7ebe..b08bd799a1652ddb38f424fd46dd6a612e2f1831 100644 (file)
@@ -18,6 +18,7 @@
 
 #if CMK_LBDB_ON
 #include "LBDatabase.h"
+#include "MetaBalancer.h"
 #endif // CMK_LBDB_ON
 
 #if CMK_GRID_QUEUE_AVAILABLE
@@ -977,7 +978,7 @@ void CkMigratable::commonInit(void) {
   local_state = OFF;
   prev_load = 0.0;
   can_reset = false;
-  atsync_iteration = myRec->getLBDB()->get_iteration();
+  atsync_iteration = myRec->getMetaBalancer()->get_iteration();
 
        /*
        FAULT_EVAC
@@ -1098,7 +1099,7 @@ void CkMigratable::recvLBPeriod(void *data) {
 
   bool is_tentative;
   if (local_state == LOAD_BALANCE) {
-    CkAssert(lb_period == myRec->getLBDB()->getPredictedLBPeriod(is_tentative));
+    CkAssert(lb_period == myRec->getMetaBalancer()->getPredictedLBPeriod(is_tentative));
     return;
   }
 
@@ -1146,7 +1147,8 @@ void CkMigratable::AtSync(int waitForMigration)
   // model-based load balancing, ask user to provide cpu load
   if (usesAutoMeasure == CmiFalse) UserSetLBLoad();
   
-  //   myRec->getLBDB()->AtLocalBarrier(ldBarrierHandle);
+//  myRec->getLBDB()->AtLocalBarrier(ldBarrierHandle);
+
   if (atsync_iteration == -1) {
     can_reset = false;
     local_state = OFF;
@@ -1156,23 +1158,23 @@ void CkMigratable::AtSync(int waitForMigration)
   atsync_iteration++;
   // CkPrintf("[pe %s] atsync_iter %d && predicted period %d state: %d\n",
   //     idx2str(thisIndexMax), atsync_iteration,
-  //     myRec->getLBDB()->getPredictedLBPeriod(), local_state);
+  //     myRec->getMetaBalancer()->getPredictedLBPeriod(), local_state);
   double tmp = prev_load;
   prev_load = myRec->getObjTime();
   double current_load = prev_load - tmp;
-  if (atsync_iteration < myRec->getLBDB()->get_iteration()) {
+  if (atsync_iteration < myRec->getMetaBalancer()->get_iteration()) {
     CkPrintf("[%d:%s] Error!! Contributing to iter %d < current iter %d\n",
       CkMyPe(), idx2str(thisIndexMax), atsync_iteration,
-      myRec->getLBDB()->get_iteration());
+      myRec->getMetaBalancer()->get_iteration());
     CkAbort("Not contributing to the right iteration\n");
   }
 
   if (atsync_iteration != 0) {
-    myRec->getLBDB()->AddLoad(atsync_iteration, current_load);
+    myRec->getMetaBalancer()->AddLoad(atsync_iteration, current_load);
   }
 
   bool is_tentative;
-  if (atsync_iteration < myRec->getLBDB()->getPredictedLBPeriod(is_tentative)) {
+  if (atsync_iteration < myRec->getMetaBalancer()->getPredictedLBPeriod(is_tentative)) {
     ResumeFromSync();
   } else if (is_tentative) {
     local_state = PAUSE;
@@ -1320,6 +1322,7 @@ CkLocRec_local::CkLocRec_local(CkLocMgr *mgr,CmiBool fromMigration,
         enable_measure = CmiTrue;
        bounced  = CmiFalse;
        the_lbdb=mgr->getLBDB();
+  the_metalb=mgr->getMetaBalancer();
        ldHandle=the_lbdb->RegisterObj(mgr->getOMHandle(),
                idx2LDObjid(idx),(void *)this,1);
        if (fromMigration) {
@@ -1925,7 +1928,7 @@ void CkLocMgr::callForAllRecords(CkLocFn fnPointer,CkArray *arr,void *data){
 #endif
 
 /*************************** LocMgr: CREATION *****************************/
-CkLocMgr::CkLocMgr(CkGroupID mapID_,CkGroupID lbdbID_,CkArrayIndex& numInitial)
+CkLocMgr::CkLocMgr(CkGroupID mapID_,CkGroupID lbdbID_,CkGroupID metalbID_,CkArrayIndex& numInitial)
        :thisProxy(thisgroup),thislocalproxy(thisgroup,CkMyPe()),
         hash(17,0.3)
 {
@@ -1949,7 +1952,8 @@ CkLocMgr::CkLocMgr(CkGroupID mapID_,CkGroupID lbdbID_,CkArrayIndex& numInitial)
 
 //Find and register with the load balancer
        lbdbID = lbdbID_;
-       initLB(lbdbID_);
+  metalbID = metalbID_;
+       initLB(lbdbID_, metalbID_);
        hashImmLock = CmiCreateImmediateLock();
 }
 
@@ -1971,6 +1975,7 @@ void CkLocMgr::pup(PUP::er &p){
        p|mapID;
        p|mapHandle;
        p|lbdbID;
+  p|metalbID;
        mapID = _defaultArrayMapID;
        if(p.isUnpacking()){
                thisProxy=thisgroup;
@@ -1982,7 +1987,7 @@ void CkLocMgr::pup(PUP::er &p){
                 CkArrayIndex emptyIndex;
                map->registerArray(emptyIndex,thisgroup);
                // _lbdb is the fixed global groupID
-               initLB(lbdbID);
+               initLB(lbdbID, metalbID);
 
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))     
         int count;
@@ -3020,7 +3025,7 @@ unsigned int CkLocMgr::numLocalElements()
 
 #if !CMK_LBDB_ON
 //Empty versions of all load balancer calls
-void CkLocMgr::initLB(CkGroupID lbdbID_) {}
+void CkLocMgr::initLB(CkGroupID lbdbID_, CkGroupID metalbID_) {}
 void CkLocMgr::startInserting(void) {}
 void CkLocMgr::doneInserting(void) {}
 void CkLocMgr::dummyAtSync(void) {}
@@ -3028,12 +3033,15 @@ void CkLocMgr::dummyAtSync(void) {}
 
 
 #if CMK_LBDB_ON
-void CkLocMgr::initLB(CkGroupID lbdbID_)
+void CkLocMgr::initLB(CkGroupID lbdbID_, CkGroupID metalbID_)
 { //Find and register with the load balancer
        the_lbdb = (LBDatabase *)CkLocalBranch(lbdbID_);
        if (the_lbdb == 0)
                CkAbort("LBDatabase not yet created?\n");
        DEBL((AA"Connected to load balancer %p\n"AB,the_lbdb));
+  the_metalb = (MetaBalancer *)CkLocalBranch(metalbID_);
+       if (the_metalb == 0)
+               CkAbort("MetaBalancer not yet created?\n");
 
        // Register myself as an object manager
        LDOMid myId;
index 3a2271ee2d6e393c10f081eb9231e1250462b220..6235a0699a79aecb4576060873ebb0da30b55638 100644 (file)
@@ -6,7 +6,7 @@ module CkLocation {
   };
 
   group [migratable] CkLocMgr {
-       entry CkLocMgr(CkGroupID map,CkGroupID _lbdb,CkArrayIndex numInitial);
+       entry CkLocMgr(CkGroupID map,CkGroupID _lbdb,CkGroupID _metalb,CkArrayIndex numInitial);
        entry void deliverInline(CkMessage *m);
        entry [expedited] void immigrate(CkArrayElementMigrateMessage *msg);
        entry [expedited] void updateLocation(CkArrayIndex idx,int nowOnPe);
index 733e9a435eb9457a6bdb8c80106a7e4dc05eabcd..d884f7cd6e5191554df276225564a73189b21856 100644 (file)
@@ -28,7 +28,9 @@ public:
 /* Utility */
 //#if CMK_LBDB_ON
 #include "LBDatabase.h"
+#include "MetaBalancer.h"
 class LBDatabase;
+class MetaBalancer;
 //#endif
 
 //Forward declarations
@@ -216,6 +218,7 @@ public:
 #if CMK_LBDB_ON
 public:
   inline LBDatabase *getLBDB(void) const {return the_lbdb;}
+  inline MetaBalancer *getMetaBalancer(void) const {return the_metalb;}
   inline LDObjHandle getLdHandle() const{return ldHandle;}
   static void staticMigrate(LDObjHandle h, int dest);
   static void staticAdaptResumeSync(LDObjHandle h, int lb_ideal_period);
@@ -234,6 +237,7 @@ public:
   inline void setMeasure(CmiBool status) { enable_measure = status; }
 private:
   LBDatabase *the_lbdb;
+  MetaBalancer *the_metalb;
   LDObjHandle ldHandle;
   CmiBool  asyncMigrate;  /// if readyMove is inited
   CmiBool  readyMigrate;    /// status whether it is ready to migrate
@@ -306,6 +310,7 @@ public:
   //Begin load balancer measurements again (e.g., after CthSuspend)
   inline void ckStartTiming(void) {myRec->startTiming();}
   inline LBDatabase *getLBDB(void) const {return myRec->getLBDB();}
+  inline MetaBalancer *getMetaBalancer(void) const {return myRec->getMetaBalancer();}
 #else
   inline void ckStopTiming(void) { }
   inline void ckStartTiming(void) { }
@@ -539,7 +544,7 @@ typedef void (*CkLocFn)(CkArray *,void *,CkLocRec *,CkArrayIndex *);
 class CkLocMgr : public IrrGroup {
        CkMagicNumber<CkMigratable> magic; //To detect heap corruption
 public:
-       CkLocMgr(CkGroupID map,CkGroupID _lbdb,CkArrayIndex& numInitial);
+       CkLocMgr(CkGroupID map,CkGroupID _lbdb,CkGroupID _metalb,CkArrayIndex& numInitial);
        CkLocMgr(CkMigrateMessage *m);
        inline CmiBool isLocMgr(void) { return CmiTrue; }
        CkGroupID &getGroupID(void) {return thisgroup;}
@@ -604,6 +609,7 @@ public:
 
 #if CMK_LBDB_ON
        LBDatabase *getLBDB(void) const { return the_lbdb; }
+  MetaBalancer *getMetaBalancer(void) const { return the_metalb;}
        const LDOMHandle &getOMHandle(void) const { return myLBHandle; }
 #endif
 
@@ -757,8 +763,10 @@ CkLocRec_local *createLocal(const CkArrayIndex &idx,
        CkArrayMap *map;
 
        CkGroupID lbdbID;
+       CkGroupID metalbID;
 #if CMK_LBDB_ON
        LBDatabase *the_lbdb;
+  MetaBalancer *the_metalb;
        LDBarrierClient dummyBarrierHandle;
        static void staticDummyResumeFromSync(void* data);
        void dummyResumeFromSync(void);
@@ -766,7 +774,7 @@ CkLocRec_local *createLocal(const CkArrayIndex &idx,
        void recvAtSync(void);
        LDOMHandle myLBHandle;
 #endif
-       void initLB(CkGroupID lbdbID);
+       void initLB(CkGroupID lbdbID, CkGroupID metalbID);
 
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 public:
index 776e8525d7fa81e00742f5fb4fbaa6a8e6db6461..31d5a59dc5a5ec0d446e39733b977ac08ea15eca 100644 (file)
@@ -868,6 +868,7 @@ static void _nullFn(void *, void *)
 }
 
 extern void _registerLBDatabase(void);
+extern void _registerMetaBalancer(void);
 extern void _registerPathHistory(void);
 #if CMK_WITH_CONTROLPOINT
 extern void _registerControlPoints(void);
@@ -876,6 +877,7 @@ extern void _registerTraceControlPoints();
 extern void _registerExternalModules(char **argv);
 extern void _ckModuleInit(void);
 extern void _loadbalancerInit();
+extern void _metabalancerInit();
 extern void _initChareTables();
 #if CMK_MEM_CHECKPOINT
 extern void init_memcheckpt(char **argv);
@@ -1061,6 +1063,7 @@ void _initCharm(int unused_argc, char **argv)
 
        _futuresModuleInit(); // part of futures implementation is a converse module
        _loadbalancerInit();
+  _metabalancerInit();
        
 #if CMK_MEM_CHECKPOINT
         init_memcheckpt(argv);
@@ -1116,6 +1119,7 @@ void _initCharm(int unused_argc, char **argv)
                _registerCkFutures();
                _registerCkArray();
                _registerLBDatabase();
+    _registerMetaBalancer();
                _registerCkCallback();
                _registertempo();
                _registerwaitqd();
index fd4b3b6494c0016513d598485ca9acdc1f5961f6..453ebc90ac148607fd59e7325219fedac080ff45 100644 (file)
@@ -984,7 +984,7 @@ void CentralLB::LoadBalance()
   for (proc = 0; proc < clients; proc++) statsMsgsList[proc] = NULL;
 #endif
 
-  theLbdb->ResetAdaptive();
+//NOTE  theLbdb->ResetAdaptive();
   if (!_lb_args.samePeSpeed()) statsData->normalize_speed();
 
   if (_lb_args.debug()) 
@@ -1452,7 +1452,7 @@ void CentralLB::CheckMigrationComplete()
     }
 
     //FIX ME!!! adaptive_struct.lb_migration_cost = (CkWallTimer() - start_lb_time);
-    theLbdb->SetMigrationCost(CkWallTimer() - start_lb_time);
+//NOTE    theLbdb->SetMigrationCost(CkWallTimer() - start_lb_time);
 
     lbdone = 0;
     future_migrates_expected = -1;
@@ -1506,7 +1506,7 @@ LBMigrateMsg* CentralLB::Strategy(LDStats* stats)
   info.getSummary(mLoad, mCpuLoad, totalLoad);
   //CkPrintf("CharmLB> Max load w/o comm %lf Max cpu load %lf Avg load %lf\n", mLoad, mCpuLoad, totalLoad/clients);
   //info.print();
-  theLbdb->UpdateAfterLBData(mLoad, mCpuLoad, totalLoad/clients);
+//NOTE!!  theLbdb->UpdateAfterLBData(mLoad, mCpuLoad, totalLoad/clients);
 
   //getPredictedLoadWithMsg(stats, clients, msg, info,1);
   //info.getSummary(mLoad, mCpuLoad, totalLoadWComm);
@@ -1536,7 +1536,7 @@ LBMigrateMsg* CentralLB::Strategy(LDStats* stats)
     CkPrintf("CharmLB> %s: PE [%d] strategy finished at %f duration %f s\n",
              lbname, cur_ld_balancer, strat_end_time, strat_end_time-strat_start_time);
     //CkPrintf("Strategy cost %f %f %f\n", strat_end_time, strat_start_time, adaptive_struct.lb_strategy_cost);
-    theLbdb->SetStrategyCost(strat_end_time - strat_start_time);
+//NOTE    theLbdb->SetStrategyCost(strat_end_time - strat_start_time);
   }
   return msg;
 #else
index 31e0d5a3f6fc4b2555823aec353600c6f8d85086..b7242f63086ed6fa983a3a88f369b1006b1ae8f8 100644 (file)
@@ -241,19 +241,19 @@ protected:
 
   virtual void UpdateLBDBWithData(int is_prev_lb_refine, double lb_max,
       double lb_avg, double local_comm, double remote_comm) {
-    theLbdb->UpdateAfterLBData(is_prev_lb_refine, lb_max, lb_avg, local_comm,
-        remote_comm);
+//NOTE    theLbdb->UpdateAfterLBData(is_prev_lb_refine, lb_max, lb_avg, local_comm,
+//        remote_comm);
   }
 
   virtual void GetPrevLBData(int& is_prev_lb_refine, double& lb_max_avg_ratio,
       double& local_remote_comm_ratio) {
-    theLbdb->GetPrevLBData(is_prev_lb_refine, lb_max_avg_ratio,
-        local_remote_comm_ratio);
+// NOTE    theLbdb->GetPrevLBData(is_prev_lb_refine, lb_max_avg_ratio,
+//        local_remote_comm_ratio);
   }
 
   virtual void GetLBDataForLB(int prev_lb, double& lb_max_avg_ratio, double&
       local_remote_comm_ratio) {
-    theLbdb->GetLBDataForLB(prev_lb, lb_max_avg_ratio, local_remote_comm_ratio);
+// NOTE    theLbdb->GetLBDataForLB(prev_lb, lb_max_avg_ratio, local_remote_comm_ratio);
   }
 
 
index 11eda6e8bb34d3175c7855c8620054837f0bf66f..2b98083882c6ebc6a4436117485034c61734c302 100644 (file)
@@ -33,6 +33,8 @@ module EveryLB {
   extern module RefineTopoLB;
   extern module TopoCentLB;
   extern module TopoLB;
+  extern module AdaptiveLB;
+  extern module MetisLB;
 
   initnode void initEveryLB(void);
 };
index 0450d1d822e7304174aea0e8c6048deb51a84b58..74d90f0aceb5eb2dbc57fbe6aee3ffc7337b503c 100644 (file)
 #   define DEBAD(x) /*CkPrintf x*/
 #   define EXTRA_FEATURE 0
 
-struct AdaptiveData {
-  double iteration;
-  double max_load;
-  double avg_load;
-  double utilization;
-  double idle_time;
-};
-
-struct AdaptiveLBDatabase {
-  std::vector<AdaptiveData> history_data;
-  int lb_iter_no;
-};
-
-struct AdaptiveLBInfo {
-  AdaptiveLBInfo() {
-    max_avg_ratio = 1;
-    remote_local_ratio = 1;
-  }
-  double max_avg_ratio;
-  double remote_local_ratio;
-};
-
-// TODO: Separate out the datastructure required by just the central and on all
-// processors
-struct AdaptiveLBStructure {
-  int tentative_period;
-  int final_lb_period;
-  // This is based on the linear extrapolation
-  int lb_calculated_period;
-  int lb_iteration_no;
-  // This is set when all the processor sends the maximum iteration no
-  int global_max_iter_no;
-  // This keeps track of what was the max iteration no we had previously
-  // received. TODO: Mostly global_max_iter_no should be sufficied.
-  int tentative_max_iter_no;
-  // TODO: Use reduction to collect max iteration. Then we don't need this
-  // counter.
-  int global_recv_iter_counter;
-  // true indicates it is in Inform->ReceiveMaxIter->FinalLBPeriod stage.
-  bool in_progress;
-  double lb_strategy_cost;
-  double lb_migration_cost;
-  bool doCommStrategy;
-  int lb_msg_send_no;
-  int lb_msg_recv_no;
-  // Total AtSync calls from all the chares residing on the processor
-  int total_syncs_called;
-  int last_lb_type;
-  AdaptiveLBInfo greedy_info;
-  AdaptiveLBInfo refine_info;
-  AdaptiveLBInfo comm_info;
-  AdaptiveLBInfo comm_refine_info;
-};
-
-
-CkReductionMsg* lbDataCollection(int nMsg, CkReductionMsg** msgs) {
-  double lb_data[8];
-  lb_data[1] = 0.0; // total number of processors contributing
-  lb_data[2] = 0.0; // total load
-  lb_data[3] = 0.0; // max load
-  lb_data[4] = 0.0; // idle time
-  lb_data[5] = 1.0; // utilization
-  lb_data[6] = 0.0; // total load with bg
-  lb_data[7] = 0.0; // max load with bg
-  for (int i = 0; i < nMsg; i++) {
-    CkAssert(msgs[i]->getSize() == 8*sizeof(double));
-    if (msgs[i]->getSize() != 8*sizeof(double)) {
-      CkPrintf("Error!!! Reduction not correct. Msg size is %d\n", msgs[i]->getSize());
-    }
-    double* m = (double *)msgs[i]->getData();
-    // Total count
-    lb_data[1] += m[1];
-    // Avg load
-    lb_data[2] += m[2];
-    // Max load
-    lb_data[3] = ((m[3] > lb_data[3])? m[3] : lb_data[3]);
-    // Avg idle
-    lb_data[4] += m[4];
-    // Get least utilization
-    lb_data[5] = ((m[5] < lb_data[5]) ? m[5] : lb_data[5]);
-    // Get Avg load with bg
-    lb_data[6] += m[6];
-    // Get Max load with bg
-    lb_data[7] = ((m[7] > lb_data[7])? m[7] : lb_data[7]);
-    if (i == 0) {
-      // Iteration no
-      lb_data[0] = m[0];
-    }
-    if (m[0] != lb_data[0]) {
-      CkPrintf("Error!!! Reduction is intermingled between iteration %lf and\
-      %lf\n", lb_data[0], m[0]);
-      CkAbort("Intermingling iterations\n");
-    }
-  }
-  return CkReductionMsg::buildNew(8*sizeof(double), lb_data);
-}
-
-/*global*/ CkReduction::reducerType lbDataCollectionType;
-/*initcall*/ void registerLBDataCollection(void) {
-  lbDataCollectionType = CkReduction::addReducer(lbDataCollection);
-}
-
 CkGroupID _lbdb;
 
 CkpvDeclare(int, numLoadBalancers);  /**< num of lb created */
 CkpvDeclare(int, hasNullLB);         /**< true if NullLB is created */
 CkpvDeclare(int, lbdatabaseInited);  /**< true if lbdatabase is inited */
 
-CkpvDeclare(AdaptiveLBDatabase, adaptive_lbdb);
-CkpvDeclare(AdaptiveLBStructure, adaptive_struct);
-
 // command line options
 CkLBArgs _lb_args;
 int _lb_predict=0;
@@ -273,6 +168,7 @@ LBDBInit::LBDBInit(CkArgMsg *m)
 // called from init.C
 void _loadbalancerInit()
 {
+  CkPrintf("loadbalancerInit\n");
   CkpvInitialize(int, lbdatabaseInited);
   CkpvAccess(lbdatabaseInited) = 0;
   CkpvInitialize(int, numLoadBalancers);
@@ -463,42 +359,6 @@ void LBDatabase::init(void)
   if (manualOn) TurnManualLBOn();
 #endif
 
-  total_load_vec.resize(VEC_SIZE, 0.0);
-  total_count_vec.resize(VEC_SIZE, 0);
-  max_iteration = -1;
-  prev_idle = 0.0;
-  alpha_beta_cost_to_load = 1.0; // Some random value. TODO: Find the actual
-
-  CkpvInitialize(AdaptiveLBDatabase, adaptive_lbdb);
-  CkpvInitialize(AdaptiveLBStructure, adaptive_struct);
-
-  CkpvAccess(adaptive_lbdb).lb_iter_no = -1;
-
-  // If metabalancer enabled, initialize the variables
-  CkpvAccess(adaptive_struct).tentative_period =  INT_MAX;
-  CkpvAccess(adaptive_struct).final_lb_period =  INT_MAX;
-  CkpvAccess(adaptive_struct).lb_calculated_period = INT_MAX;
-  CkpvAccess(adaptive_struct).lb_iteration_no = -1;
-  CkpvAccess(adaptive_struct).global_max_iter_no = 0;
-  CkpvAccess(adaptive_struct).tentative_max_iter_no = -1;
-  CkpvAccess(adaptive_struct).global_recv_iter_counter = 0;
-  CkpvAccess(adaptive_struct).in_progress = false;
-  CkpvAccess(adaptive_struct).lb_strategy_cost = 0.0;
-  CkpvAccess(adaptive_struct).lb_migration_cost = 0.0;
-  CkpvAccess(adaptive_struct).lb_msg_send_no = 0;
-  CkpvAccess(adaptive_struct).lb_msg_recv_no = 0;
-  CkpvAccess(adaptive_struct).total_syncs_called = 0;
-  CkpvAccess(adaptive_struct).last_lb_type = -1;
-
-
-  // This is indicating if the load balancing strategy and migration started.
-  // This is mainly used to register callbacks for noobj pes. They would
-  // register as soon as resumefromsync is called. On receiving the handles at
-  // the central pe, it clears the previous handlers and sets lb_in_progress
-  // to false so that it doesn't clear the handles.
-  lb_in_progress = false;
-
-  is_prev_lb_refine = -1;
 }
 
 LBDatabase::LastLBInfo::LastLBInfo()
@@ -596,6 +456,7 @@ const char *LBDatabase::loadbalancer(int seq) {
 
 void LBDatabase::pup(PUP::er& p)
 {
+  // NOTE if unpacking set metabalancer using the id
        IrrGroup::pup(p);
        // the memory should be already allocated
        int np;
@@ -612,7 +473,10 @@ void LBDatabase::pup(PUP::er& p)
        }
        p(avail_vector, np);
        p|mystep;
-       if(p.isUnpacking()) nloadbalancers = 0;
+       if(p.isUnpacking()) {
+    nloadbalancers = 0;
+    metabalancer = (MetaBalancer*)CkLocalBranch(_metalb);
+  }
 }
 
 
@@ -627,640 +491,17 @@ void LBDatabase::EstObjLoad(const LDObjHandle &_h, double cputime)
 #endif
 }
 
-void LBDatabase::ResumeClients() {
-  // If metabalancer enabled, initialize the variables
-  CkpvAccess(adaptive_lbdb).history_data.clear();
-
-  CkpvAccess(adaptive_struct).tentative_period =  INT_MAX;
-  CkpvAccess(adaptive_struct).final_lb_period =  INT_MAX;
-  CkpvAccess(adaptive_struct).lb_calculated_period = INT_MAX;
-  CkpvAccess(adaptive_struct).lb_iteration_no = -1;
-  CkpvAccess(adaptive_struct).global_max_iter_no = 0;
-  CkpvAccess(adaptive_struct).tentative_max_iter_no = -1;
-  CkpvAccess(adaptive_struct).global_recv_iter_counter = 0;
-  CkpvAccess(adaptive_struct).in_progress = false;
-  CkpvAccess(adaptive_struct).lb_strategy_cost = 0.0;
-  CkpvAccess(adaptive_struct).lb_migration_cost = 0.0;
-  CkpvAccess(adaptive_struct).lb_msg_send_no = 0;
-  CkpvAccess(adaptive_struct).lb_msg_recv_no = 0;
-  CkpvAccess(adaptive_struct).total_syncs_called = 0;
-
-  prev_idle = 0.0;
-  if (lb_in_progress) {
-    lbdb_no_obj_callback.clear();
-    lb_in_progress = false;
-  }
-
-  // While resuming client, if we find that there are no objects, then handle
-  // the case accordingly.
-  if (getLBDB()->ObjDataCount() == 0) {
-    CkPrintf("%d processor has 0 objs\n", CkMyPe());
-    HandleAdaptiveNoObj();
-  }
-  LDResumeClients(myLDHandle);
-}
-
-int LBDatabase::get_iteration() {
-  return CkpvAccess(adaptive_struct).lb_iteration_no;
+void LBDatabase::RegisterMetaBalancer() {
+  metabalancer = (MetaBalancer *)CkLocalBranch(_metalb);
 }
 
-bool LBDatabase::AddLoad(int it_n, double load) {
-  int index = it_n % VEC_SIZE;
-  total_count_vec[index]++;
-  CkpvAccess(adaptive_struct).total_syncs_called++;
-  DEBAD(("At PE %d Total contribution for iteration %d is %d total objs %d\n",
-  CkMyPe(), it_n,
-    total_count_vec[index], getLBDB()->ObjDataCount()));
-
-  if (it_n < CkpvAccess(adaptive_struct).lb_iteration_no) {
-    CkAbort("Error!! Received load for previous iteration\n");
-  }
-  if (it_n > CkpvAccess(adaptive_struct).lb_iteration_no) {
-    CkpvAccess(adaptive_struct).lb_iteration_no = it_n;
-  }
-  total_load_vec[index] += load;
-  if (total_count_vec[index] > getLBDB()->ObjDataCount()) {
-    CkPrintf("iteration %d received %d contributions and expected %d\n", it_n,
-        total_count_vec[index], getLBDB()->ObjDataCount());
-    CkAbort("Abort!!! Received more contribution");
-  }
-
-  if (total_count_vec[index] == getLBDB()->ObjDataCount()) {
-    double idle_time, bg_walltime, cpu_bgtime;
-    IdleTime(&idle_time);
-    BackgroundLoad(&bg_walltime, &cpu_bgtime);
 
-    int sync_for_bg = CkpvAccess(adaptive_struct).total_syncs_called +
-        getLBDB()->ObjDataCount();
-    bg_walltime = bg_walltime * getLBDB()->ObjDataCount() / sync_for_bg;
-
-    if (it_n < NEGLECT_IDLE) {
-      prev_idle = idle_time;
-    }
-    idle_time -= prev_idle;
-
-    // The chares do not contribute their 0th iteration load. So the total syncs
-    // in reality is total_syncs_called + obj_counts
-    int total_countable_syncs = CkpvAccess(adaptive_struct).total_syncs_called +
-        (1 - NEGLECT_IDLE) * getLBDB()->ObjDataCount(); // TODO: Fix me! weird!
-    if (total_countable_syncs != 0) {
-      idle_time = idle_time * getLBDB()->ObjDataCount() / total_countable_syncs;
-    }
-    //CkPrintf("[%d] Idle time %lf and countable %d for iteration %d\n", CkMyPe(), idle_time, total_countable_syncs, iteration);
-
-    double lb_data[8];
-    lb_data[0] = it_n;
-    lb_data[1] = 1;
-    lb_data[2] = total_load_vec[index]; // For average load
-    lb_data[3] = total_load_vec[index]; // For max load
-    lb_data[4] = idle_time;
-    // Set utilization
-    if (total_load_vec[index] == 0.0) {
-      lb_data[5] = 0.0;
-    } else {
-      lb_data[5] = total_load_vec[index]/(idle_time + total_load_vec[index]);
-    }
-    lb_data[6] = lb_data[2] + bg_walltime; // For Avg load with bg
-    lb_data[7] = lb_data[6]; // For Max load with bg
-    total_load_vec[index] = 0.0;
-    total_count_vec[index] = 0;
-
-    //CkPrintf("   [%d] sends total load %lf idle time %lf ratio of idle/load %lf at iter %d\n", CkMyPe(),
-    //    total_load_vec[iteration], idle_time,
-    //    idle_time/total_load_vec[iteration], adaptive_struct.lb_iteration_no);
-
-    CkCallback cb(CkIndex_LBDatabase::ReceiveMinStats((CkReductionMsg*)NULL), thisProxy[0]);
-    contribute(8*sizeof(double), lb_data, lbDataCollectionType, cb);
-  }
-  return true;
-}
-
-void LBDatabase::ReceiveMinStats(CkReductionMsg *msg) {
-  double* load = (double *) msg->getData();
-  double avg = load[2]/load[1];
-  double max = load[3];
-  double avg_idle = load[4]/load[1];
-  double utilization = load[5];
-  int iteration_n = load[0];
-  double avg_load_bg = load[6]/load[1];
-  double max_load_bg = load[7];
-  DEBAD(("** [%d] Iteration Avg load: %lf Max load: %lf Avg Idle : %lf Max Idle : %lf for %lf procs\n",iteration_n, avg, max, avg_idle, utilization, load[1]));
-  CkPrintf("** [%d] Iteration Avg load: %lf Max load: %lf With bg Avg load: %lf Max load: %lf Avg Idle : %lf Max Idle : %lf for %lf procs\n",iteration_n, avg, max, avg_load_bg, max_load_bg, avg_idle, utilization, load[1]);
-  delete msg;
-
-#if EXTRA_FEATURE
-  if (CkpvAccess(adaptive_struct).final_lb_period != iteration_n) {
-    for (int i = 0; i < lbdb_no_obj_callback.size(); i++) {
-      thisProxy[lbdb_no_obj_callback[i]].TriggerAdaptiveReduction();
-    }
-  }
-#endif
-
-  // Store the data for this iteration
-  CkpvAccess(adaptive_lbdb).lb_iter_no = iteration_n;
-  AdaptiveData data;
-  data.iteration = CkpvAccess(adaptive_lbdb).lb_iter_no;
-  data.max_load = max;
-  data.avg_load = avg;
-  data.utilization = utilization;
-  data.idle_time = avg_idle;
-  CkpvAccess(adaptive_lbdb).history_data.push_back(data);
-
-  // If lb period inform is in progress, dont inform again.
-  // If this is the lb data corresponding to the final lb period informed, then
-  // don't recalculate as some of the processors might have already gone into
-  // LB_STAGE.
-  if (CkpvAccess(adaptive_struct).in_progress || (CkpvAccess(adaptive_struct).final_lb_period == iteration_n)) {
-    return;
-  }
-
-  double utilization_threshold = UTILIZATION_THRESHOLD;
-
-#if EXTRA_FEATURE
-  CkPrintf("alpha_beta_to_load %lf\n", alpha_beta_cost_to_load);
-  if (alpha_beta_cost_to_load < 0.1) {
-    // Ignore the effect of idle time and there by lesser utilization. So we
-    // assign utilization threshold to be 0.0
-    CkPrintf("Changing the idle load tolerance coz this isn't communication intensive benchmark\n");
-    utilization_threshold = 0.0;
-  }
-#endif
-
-  // First generate the lb period based on the cost of lb. Find out what is the
-  // expected imbalance at the calculated lb period.
-  int period;
-  // This is based on the new max load after load balancing. So technically, it
-  // is calculated based on the shifter up avg curve.
-  double ratio_at_t = 1.0;
-  int tmp_lb_type;
-  double tmp_max_avg_ratio, tmp_comm_ratio;
-  GetPrevLBData(tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
-  double tolerate_imb = IMB_TOLERANCE * tmp_max_avg_ratio;
-
-  if (generatePlan(period, ratio_at_t)) {
-    DEBAD(("Generated period and calculated %d and period %d max iter %d\n",
-      CkpvAccess(adaptive_struct).lb_calculated_period, period,
-      CkpvAccess(adaptive_struct).tentative_max_iter_no));
-    // set the imbalance tolerance to be ratio_at_calculated_lb_period
-    if (ratio_at_t != 1.0) {
-      CkPrintf("Changed tolerance to %lf after line eq whereas max/avg is %lf\n", ratio_at_t, max/avg);
-      // Since ratio_at_t is shifter up, max/(tmp_max_avg_ratio * avg) should be
-      // compared with the tolerance
-      tolerate_imb = ratio_at_t * tmp_max_avg_ratio * OUTOFWAY_TOLERANCE;
-    }
-
-    CkPrintf("Prev LB Data Type %d, max/avg %lf, local/remote %lf\n", tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
-
-    if ((utilization < utilization_threshold || max/avg >= tolerate_imb) &&
-          CkpvAccess(adaptive_lbdb).history_data.size() > MIN_STATS) {
-      CkPrintf("Trigger soon even though we calculated lbperiod max/avg(%lf) and utilization ratio (%lf)\n", max/avg, utilization);
-      TriggerSoon(iteration_n, max/avg, tolerate_imb);
-      return;
-    }
-
-    // If the new lb period from linear extrapolation is greater than maximum
-    // iteration known from previously collected data, then inform all the
-    // processors about the new calculated period.
-    if (period > CkpvAccess(adaptive_struct).tentative_max_iter_no && period !=
-          CkpvAccess(adaptive_struct).final_lb_period) {
-      CkpvAccess(adaptive_struct).doCommStrategy = false;
-      CkpvAccess(adaptive_struct).lb_calculated_period = period;
-      CkpvAccess(adaptive_struct).in_progress = true;
-      CkPrintf("Sticking to the calculated period %d\n",
-        CkpvAccess(adaptive_struct).lb_calculated_period);
-      thisProxy.LoadBalanceDecision(CkpvAccess(adaptive_struct).lb_msg_send_no++,
-        CkpvAccess(adaptive_struct).lb_calculated_period);
-      return;
-    }
-    // TODO: Shouldn't we return from here??
-  }
-
-  CkPrintf("Prev LB Data Type %d, max/avg %lf, local/remote %lf\n", tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
-
-  // This would be called when the datasize is not enough to calculate lb period
-  if ((utilization < utilization_threshold || max/avg >= tolerate_imb) && CkpvAccess(adaptive_lbdb).history_data.size() > 4) {
-    CkPrintf("Carry out load balancing step at iter max/avg(%lf) and utilization ratio (%lf)\n", max/avg, utilization);
-    TriggerSoon(iteration_n, max/avg, tolerate_imb);
-    return;
-  }
-
-}
-
-void LBDatabase::TriggerSoon(int iteration_n, double imbalance_ratio,
-    double tolerate_imb) {
-
-  // If the previously calculated_period (not the final decision) is greater
-  // than the iter +1 and if it is greater than the maximum iteration we have
-  // seen so far, then we can inform this
-  if ((iteration_n + 1 > CkpvAccess(adaptive_struct).tentative_max_iter_no) &&
-      (iteration_n+1 < CkpvAccess(adaptive_struct).lb_calculated_period) &&
-      (iteration_n + 1 != CkpvAccess(adaptive_struct).final_lb_period)) {
-    if (imbalance_ratio < tolerate_imb) {
-      CkpvAccess(adaptive_struct).doCommStrategy = true;
-      CkPrintf("No load imbalance but idle time\n");
-    } else {
-      CkpvAccess(adaptive_struct).doCommStrategy = false;
-      CkPrintf("load imbalance \n");
-    }
-    CkpvAccess(adaptive_struct).lb_calculated_period = iteration_n + 1;
-    CkpvAccess(adaptive_struct).in_progress = true;
-    CkPrintf("Informing everyone the lb period is %d\n",
-        CkpvAccess(adaptive_struct).lb_calculated_period);
-    thisProxy.LoadBalanceDecision(CkpvAccess(adaptive_struct).lb_msg_send_no++,
-        CkpvAccess(adaptive_struct).lb_calculated_period);
-  }
-}
-
-bool LBDatabase::generatePlan(int& period, double& ratio_at_t) {
-  if (CkpvAccess(adaptive_lbdb).history_data.size() <= 4) {
-    return false;
-  }
-
-  // Some heuristics for lbperiod
-  // If constant load or almost constant,
-  // then max * new_lb_period > avg * new_lb_period + lb_cost
-  double max = 0.0;
-  double avg = 0.0;
-  AdaptiveData data;
-  for (int i = 0; i < CkpvAccess(adaptive_lbdb).history_data.size(); i++) {
-    data = CkpvAccess(adaptive_lbdb).history_data[i];
-    max += data.max_load;
-    avg += data.avg_load;
-    //DEBAD(("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load));
-    //CkPrintf("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load);
-  }
-//  max /= (adaptive_struct.lb_iteration_no - CkpvAccess(adaptive_lbdb).history_data[0].iteration);
-//  avg /= (adaptive_struct.lb_iteration_no - CkpvAccess(adaptive_lbdb).history_data[0].iteration);
-
-  // If linearly varying load, then find lb_period
-  // area between the max and avg curve
-  // If we can attain perfect balance, then the new load is close to the
-  // average. Hence we pass 1, else pass in some other value which would be the
-  // new max_load after load balancing.
-  int tmp_lb_type;
-  double tmp_max_avg_ratio, tmp_comm_ratio;
-  double tolerate_imb;
-
-#if EXTRA_FEATURE
-  // First get the data for refine.
-  GetLBDataForLB(1, tmp_max_avg_ratio, tmp_comm_ratio);
-  tolerate_imb = tmp_max_avg_ratio;
-
-  // If RefineLB does a good job, then find the period considering RefineLB
-  if (tmp_max_avg_ratio <= 1.01) {
-    if (max/avg < tolerate_imb) {
-      CkPrintf("Resorting to imb = 1.0 coz max/avg (%lf) < imb(%lf)\n", max/avg, tolerate_imb);
-      tolerate_imb = 1.0;
-    }
-    CkPrintf("Will generate plan for refine %lf imb and %lf overhead\n", tolerate_imb, 0.2);
-    return getPeriodForStrategy(tolerate_imb, 0.2, period, ratio_at_t);
-  }
-
-  GetLBDataForLB(0, tmp_max_avg_ratio, tmp_comm_ratio);
-#endif
-
-  GetPrevLBData(tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
-  tolerate_imb = tmp_max_avg_ratio;
-//  if (max/avg < tolerate_imb) {
-//    CkPrintf("Resorting to imb = 1.0 coz max/avg (%lf) < imb(%lf)\n", max/avg, tolerate_imb);
-//    tolerate_imb = 1.0;
-//  }
-  if (max/avg > tolerate_imb) {
-    if (getPeriodForStrategy(tolerate_imb, 1, period, ratio_at_t)) {
-      return true;
-    }
-  }
-
-  max = 0.0;
-  avg = 0.0;
-  for (int i = 0; i < CkpvAccess(adaptive_lbdb).history_data.size(); i++) {
-    data = CkpvAccess(adaptive_lbdb).history_data[i];
-    max += data.max_load;
-    avg += data.avg_load*tolerate_imb;
-    //DEBAD(("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load));
-    //CkPrintf("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load);
-  }
-  max /= CkpvAccess(adaptive_lbdb).history_data.size();
-  avg /= CkpvAccess(adaptive_lbdb).history_data.size();
-  double cost = CkpvAccess(adaptive_struct).lb_strategy_cost + CkpvAccess(adaptive_struct).lb_migration_cost;
-  period = cost/(max - avg); 
-  CkPrintf("Obtained period %d from constant prediction\n", period);
-  if (period < 0) { 
-    period = CkpvAccess(adaptive_struct).final_lb_period;
-    CkPrintf("Obtained -ve period from constant prediction so changing to prev %d\n", period);
-  } 
-  ratio_at_t = max / avg;
-  return true;
-}
-
-bool LBDatabase::getPeriodForStrategy(double new_load_percent,
-    double overhead_percent, int& period, double& ratio_at_t) {
-  double mslope, aslope, mc, ac;
-  getLineEq(new_load_percent, aslope, ac, mslope, mc);
-  CkPrintf("new load percent %lf\n", new_load_percent);
-  CkPrintf("\n max: %fx + %f; avg: %fx + %f\n", mslope, mc, aslope, ac);
-  double a = (mslope - aslope)/2;
-  double b = (mc - ac);
-  double c = -(CkpvAccess(adaptive_struct).lb_strategy_cost +
-      CkpvAccess(adaptive_struct).lb_migration_cost) * overhead_percent;
-  bool got_period = getPeriodForLinear(a, b, c, period);
-  if (!got_period) {
-    return false;
-  }
-
-  if (mslope < 0) {
-    if (period > (-mc/mslope)) {
-      CkPrintf("Max < 0 Period set when max load is -ve\n");
-      return false;
-    }
-  }
-
-  if (aslope < 0) {
-    if (period > (-ac/aslope)) {
-      CkPrintf("Avg < 0 Period set when avg load is -ve\n");
-      return false;
-    }
-  }
-
-  int intersection_t = (mc-ac) / (aslope - mslope);
-  if (intersection_t > 0 && period > intersection_t) {
-    CkPrintf("Avg | Max Period set when curves intersect\n");
-    return false;
-  }
-  ratio_at_t = ((mslope*period + mc)/(aslope*period + ac));
-  CkPrintf("Ratio at t (%lf*%d + %lf) / (%lf*%d+%lf) = %lf\n", mslope, period, mc, aslope, period, ac, ratio_at_t);
-  return true;
-}
-
-bool LBDatabase::getPeriodForLinear(double a, double b, double c, int& period) {
-  CkPrintf("Quadratic Equation %lf X^2 + %lf X + %lf\n", a, b, c);
-  if (a == 0.0) {
-    period = (-c / b);
-    if (period < 0) {
-      CkPrintf("-ve period for -c/b (%d)\n", period);
-      return false;
-    }
-    CkPrintf("Ideal period for linear load %d\n", period);
-    return true;
-  }
-  int x;
-  double t = (b * b) - (4*a*c);
-  if (t < 0) {
-    CkPrintf("(b * b) - (4*a*c) is -ve sqrt : %lf\n", sqrt(t));
-    return false;
-  }
-  t = (-b + sqrt(t)) / (2*a);
-  x = t;
-  if (x < 0) {
-    CkPrintf("boo!!! x (%d) < 0\n", x);
-    x = 0;
-    return false;
-  }
-  period = x;
-  CkPrintf("Ideal period for linear load %d\n", period);
-  return true;
-}
-
-bool LBDatabase::getLineEq(double new_load_percent, double& aslope, double& ac, double& mslope, double& mc) {
-  int total = CkpvAccess(adaptive_lbdb).history_data.size();
-  int iterations = 1 + CkpvAccess(adaptive_lbdb).history_data[total - 1].iteration -
-      CkpvAccess(adaptive_lbdb).history_data[0].iteration;
-  double a1 = 0;
-  double m1 = 0;
-  double a2 = 0;
-  double m2 = 0;
-  AdaptiveData data;
-  int i = 0;
-  for (i = 0; i < total/2; i++) {
-    data = CkpvAccess(adaptive_lbdb).history_data[i];
-    m1 += data.max_load;
-    a1 += data.avg_load;
-    CkPrintf("max (%d, %lf) avg (%d, %lf) adjusted_avg (%d, %lf)\n", i, data.max_load, i, data.avg_load, i, new_load_percent*data.avg_load);
-  }
-  m1 /= i;
-  a1 = (a1 * new_load_percent) / i;
-
-  for (i = total/2; i < total; i++) {
-    data = CkpvAccess(adaptive_lbdb).history_data[i];
-    m2 += data.max_load;
-    a2 += data.avg_load;
-    CkPrintf("max (%d, %lf) avg (%d, %lf) adjusted_avg (%d, %lf)\n", i, data.max_load, i, data.avg_load, i, new_load_percent*data.avg_load);
-  }
-  m2 /= (i - total/2);
-  a2 = (a2 * new_load_percent) / (i - total/2);
-
-  aslope = 2 * (a2 - a1) / iterations;
-  mslope = 2 * (m2 - m1) / iterations;
-  ac = CkpvAccess(adaptive_lbdb).history_data[0].avg_load * new_load_percent;
-  mc = CkpvAccess(adaptive_lbdb).history_data[0].max_load;
-
-  ac = a1 - ((aslope * total)/4);
-  mc = m1 - ((mslope * total)/4);
-
-  //ac = (CkpvAccess(adaptive_lbdb).history_data[1].avg_load * new_load_percent - aslope);
-  //mc = (CkpvAccess(adaptive_lbdb).history_data[1].max_load - mslope);
-
-  return true;
-}
-
-void LBDatabase::LoadBalanceDecision(int req_no, int period) {
-  if (req_no < CkpvAccess(adaptive_struct).lb_msg_recv_no) {
-    CkPrintf("Error!!! Received a request which was already sent or old\n");
-    return;
-  }
-  CkPrintf("[%d] Load balance decision made cur iteration: %d period:%d\n",CkMyPe(), CkpvAccess(adaptive_struct).lb_iteration_no, period);
-  CkpvAccess(adaptive_struct).tentative_period = period;
-  CkpvAccess(adaptive_struct).lb_msg_recv_no = req_no;
-  thisProxy[0].ReceiveIterationNo(req_no, CkpvAccess(adaptive_struct).lb_iteration_no);
-}
-
-void LBDatabase::LoadBalanceDecisionFinal(int req_no, int period) {
-  if (req_no < CkpvAccess(adaptive_struct).lb_msg_recv_no) {
-    return;
-  }
-  DEBAD(("[%d] Final Load balance decision made cur iteration: %d period:%d \n",CkMyPe(), CkpvAccess(adaptive_struct).lb_iteration_no, period));
-  CkpvAccess(adaptive_struct).tentative_period = period;
-  CkpvAccess(adaptive_struct).final_lb_period = period;
-  LDOMAdaptResumeSync(myLDHandle, period);
-}
-
-
-void LBDatabase::ReceiveIterationNo(int req_no, int local_iter_no) {
-  CmiAssert(CkMyPe() == 0);
-
-  CkpvAccess(adaptive_struct).global_recv_iter_counter++;
-  if (local_iter_no > CkpvAccess(adaptive_struct).global_max_iter_no) {
-    CkpvAccess(adaptive_struct).global_max_iter_no = local_iter_no;
-  }
-
-  int period;
-  if (CkNumPes() == CkpvAccess(adaptive_struct).global_recv_iter_counter) {
-
-    if (CkpvAccess(adaptive_struct).global_max_iter_no > CkpvAccess(adaptive_struct).tentative_max_iter_no) {
-      CkpvAccess(adaptive_struct).tentative_max_iter_no = CkpvAccess(adaptive_struct).global_max_iter_no;
-    }
-    period = (CkpvAccess(adaptive_struct).tentative_period > CkpvAccess(adaptive_struct).global_max_iter_no) ? CkpvAccess(adaptive_struct).tentative_period : CkpvAccess(adaptive_struct).global_max_iter_no + 1;
-    // If no one has gone into load balancing stage, then we can safely change
-    // the period otherwise keep the old period.
-    if (CkpvAccess(adaptive_struct).global_max_iter_no < CkpvAccess(adaptive_struct).final_lb_period) {
-      CkpvAccess(adaptive_struct).tentative_period = period;
-      CkPrintf("Final lb_period CHANGED!%d\n", CkpvAccess(adaptive_struct).tentative_period);
-    } else {
-      CkpvAccess(adaptive_struct).tentative_period = CkpvAccess(adaptive_struct).final_lb_period;
-      CkPrintf("Final lb_period NOT CHANGED!%d\n", CkpvAccess(adaptive_struct).tentative_period);
-    }
-    thisProxy.LoadBalanceDecisionFinal(req_no, CkpvAccess(adaptive_struct).tentative_period);
-    CkpvAccess(adaptive_struct).in_progress = false;
-    CkpvAccess(adaptive_struct).global_recv_iter_counter = 0;
-  }
-}
-
-int LBDatabase::getPredictedLBPeriod(bool& is_tentative) {
-  // If tentative and final_lb_period are the same, then the decision has been
-  // made but if not, they are in the middle of consensus, hence return the
-  // lease of the two
-  if (CkpvAccess(adaptive_struct).tentative_period != CkpvAccess(adaptive_struct).final_lb_period) {
-    is_tentative = true;
-  } else {
-    is_tentative = false;
-  }
-  if (CkpvAccess(adaptive_struct).tentative_period < CkpvAccess(adaptive_struct).final_lb_period) {
-    return CkpvAccess(adaptive_struct).tentative_period;
-   } else {
-     return CkpvAccess(adaptive_struct).final_lb_period;
-   }
-}
-
-// Called by CentralLB to indicate that the LB strategy and migration is in
-// progress.
-void LBDatabase::ResetAdaptive() {
-  CkpvAccess(adaptive_lbdb).lb_iter_no = -1;
-  lb_in_progress = true;
-}
-
-void LBDatabase::HandleAdaptiveNoObj() {
-#if EXTRA_FEATURE
-  CkpvAccess(adaptive_struct).lb_iteration_no++;
-  //CkPrintf("HandleAdaptiveNoObj %d\n", adaptive_struct.lb_iteration_no);
-  thisProxy[0].RegisterNoObjCallback(CkMyPe());
-  TriggerAdaptiveReduction();
-#endif
-}
-
-void LBDatabase::RegisterNoObjCallback(int index) {
-#if EXTRA_FEATURE
-  if (lb_in_progress) {
-    lbdb_no_obj_callback.clear();
-    //CkPrintf("Clearing and registering\n");
-    lb_in_progress = false;
-  }
-  lbdb_no_obj_callback.push_back(index);
-  CkPrintf("Registered %d to have no objs.\n", index);
-
-  // If collection has already happened and this is second iteration, then
-  // trigger reduction.
-  if (CkpvAccess(adaptive_lbdb).lb_iter_no != -1) {
-    //CkPrintf("Collection already started now %d so kick in\n", adaptive_struct.lb_iteration_no);
-    thisProxy[index].TriggerAdaptiveReduction();
-  }
-#endif
-}
-
-void LBDatabase::TriggerAdaptiveReduction() {
-#if EXTRA_FEATURE
-  CkpvAccess(adaptive_struct).lb_iteration_no++;
-  //CkPrintf("Trigger adaptive for %d\n", CkpvAccess(adaptive_struct).lb_iteration_no);
-  double lb_data[8];
-  lb_data[0] = CkpvAccess(adaptive_struct).lb_iteration_no;
-  lb_data[1] = 1;
-  lb_data[2] = 0.0;
-  lb_data[3] = 0.0;
-  lb_data[4] = 0.0;
-  lb_data[5] = 0.0;
-  lb_data[6] = 0.0;
-  lb_data[7] = 0.0;
-
-  // CkPrintf("   [%d] sends total load %lf idle time %lf ratio of idle/load %lf at iter %d\n", CkMyPe(),
-  //     total_load_vec[iteration], idle_time,
-  //     idle_time/total_load_vec[iteration], adaptive_struct.lb_iteration_no);
-
-  CkCallback cb(CkIndex_LBDatabase::ReceiveMinStats((CkReductionMsg*)NULL), thisProxy[0]);
-  contribute(8*sizeof(double), lb_data, lbDataCollectionType, cb);
-#endif
-}
-
-
-bool LBDatabase::isStrategyComm() {
-  return CkpvAccess(adaptive_struct).doCommStrategy;
-}
-
-void LBDatabase::SetMigrationCost(double lb_migration_cost) {
-  CkpvAccess(adaptive_struct).lb_migration_cost = lb_migration_cost;
-}
-
-void LBDatabase::SetStrategyCost(double lb_strategy_cost) {
-  CkpvAccess(adaptive_struct).lb_strategy_cost = lb_strategy_cost;
-}
-
-void LBDatabase::UpdateAfterLBData(int lb, double lb_max, double lb_avg, double
-    local_comm, double remote_comm) {
-  CkpvAccess(adaptive_struct).last_lb_type = lb;
-  if (lb == 0) {
-    CkpvAccess(adaptive_struct).greedy_info.max_avg_ratio = lb_max/lb_avg;
-  } else if (lb == 1) {
-    CkpvAccess(adaptive_struct).refine_info.max_avg_ratio = lb_max/lb_avg;
-  } else if (lb == 2) {
-    CkpvAccess(adaptive_struct).comm_info.remote_local_ratio = remote_comm/local_comm;
-  } else if (lb == 3) {
-    CkpvAccess(adaptive_struct).comm_refine_info.remote_local_ratio =
-    remote_comm/local_comm;
-  }
-}
-
-void LBDatabase::UpdateAfterLBData(double max_load, double max_cpu, double
-avg_load) {
-  if (CkpvAccess(adaptive_struct).last_lb_type == -1) {
-    CkpvAccess(adaptive_struct).last_lb_type = 0;
-  }
-  int lb = CkpvAccess(adaptive_struct).last_lb_type;
-  //CkPrintf("Storing data after lb ratio %lf for lb %d\n", max_load/avg_load, lb);
-  if (lb == 0) {
-    CkpvAccess(adaptive_struct).greedy_info.max_avg_ratio = max_load/avg_load;
-  } else if (lb == 1) {
-    CkpvAccess(adaptive_struct).refine_info.max_avg_ratio = max_load/avg_load;
-  } else if (lb == 2) {
-    CkpvAccess(adaptive_struct).comm_info.max_avg_ratio = max_load/avg_load;
-  } else if (lb == 3) {
-    CkpvAccess(adaptive_struct).comm_refine_info.max_avg_ratio = max_load/avg_load;
-  }
-}
-
-void LBDatabase::UpdateAfterLBComm(double alpha_beta_to_load) {
-  CkPrintf("Setting alpha beta %lf\n", alpha_beta_to_load);
-  alpha_beta_cost_to_load = alpha_beta_to_load;
-}
-
-
-void LBDatabase::GetPrevLBData(int& lb_type, double& lb_max_avg_ratio, double&
-    remote_local_comm_ratio) {
-  lb_type = CkpvAccess(adaptive_struct).last_lb_type;
-  lb_max_avg_ratio = 1;
-  remote_local_comm_ratio = 1;
-  GetLBDataForLB(lb_type, lb_max_avg_ratio, remote_local_comm_ratio);
-}
-
-void LBDatabase::GetLBDataForLB(int lb_type, double& lb_max_avg_ratio, double&
-    remote_local_comm_ratio) {
-  if (lb_type == 0) {
-    lb_max_avg_ratio = CkpvAccess(adaptive_struct).greedy_info.max_avg_ratio;
-  } else if (lb_type == 1) {
-    lb_max_avg_ratio = CkpvAccess(adaptive_struct).refine_info.max_avg_ratio;
-  } else if (lb_type == 2) {
-    remote_local_comm_ratio = CkpvAccess(adaptive_struct).comm_info.remote_local_ratio;
-  } else if (lb_type == 3) {
-    remote_local_comm_ratio =
-       CkpvAccess(adaptive_struct).comm_refine_info.remote_local_ratio;
+void LBDatabase::ResumeClients() {
+  if (metabalancer == NULL) {
+    RegisterMetaBalancer();
   }
+  metabalancer->ResumeClients();
+  LDResumeClients(myLDHandle);
 }
 
 /*
index d19c170e910e6f37b00273266be0f008841b7896..32b0b6ba3a087c71cab94f8ca563a15f127e2340 100644 (file)
@@ -8,17 +8,9 @@ module LBDatabase {
     entry LBDBInit(CkArgMsg *m);
   };
 
-  initcall void registerLBDataCollection(void);
   group [migratable] LBDatabase {
     entry void LBDatabase(void);  
     initnode void initnodeFn();
-
-    entry [immediate] void ReceiveMinStats(CkReductionMsg *msg);
-    entry [immediate] void LoadBalanceDecision(int req_no, int period);
-    entry [immediate] void LoadBalanceDecisionFinal(int req_no, int period);
-    entry [immediate] void ReceiveIterationNo(int, int);
-    entry [immediate] void RegisterNoObjCallback(int);
-    entry [immediate] void TriggerAdaptiveReduction();
   }
 
 };
index 98c3d921a4d76ff84808506d1b9d306ecbe7f202..8a5c819c6260e9f33eed0aeb4f208301d433f029 100644 (file)
@@ -12,6 +12,7 @@
 #include "lbdb.h"
 #include "LBDBManager.h"
 #include "lbdb++.h"
+#include "MetaBalancer.h"
 
 #include <vector>
 
@@ -358,62 +359,22 @@ public:
   inline void SetLBPeriod(double s) { LDSetLBPeriod(myLDHandle, s);}
   inline double GetLBPeriod() { return LDGetLBPeriod(myLDHandle);}
 
-  void ResetAdaptive();
-  int get_iteration();
-  bool AddLoad(int iteration, double load);
-  void ReceiveMinStats(CkReductionMsg *);
-  void TriggerSoon(int iteration_no, double imbalance_ratio, double tolerate_imb);
-  void LoadBalanceDecision(int, int);
-  void LoadBalanceDecisionFinal(int, int);
-  void ReceiveIterationNo(int, int); // Receives the current iter no
-  void HandleAdaptiveNoObj();
-  void RegisterNoObjCallback(int index);
-  void TriggerAdaptiveReduction();
-
-  bool generatePlan(int& period, double& ratio_at_t);
-  bool getLineEq(double new_load_percent, double& aslope, double& ac, double& mslope, double& mc);
-  bool getPeriodForLinear(double a, double b, double c, int& period);
-  bool getPeriodForStrategy(double new_load, double overhead_percent, int& period, double& ratio_at_t);
-  int getPredictedLBPeriod(bool& is_tentative);
-
-  bool isStrategyComm();
-
-  void UpdateAfterLBData(int is_lb_refine, double lb_max, double lb_avg, double
-      local_comm, double remote_comm);
-
-  void UpdateAfterLBData(double max_load, double max_cpu, double avg_load);
-  void UpdateAfterLBComm(double alpha_beta_cost);
-  void GetPrevLBData(int& lb_type, double& lb_max_avg_ratio, double&
-      local_remote_comm_ratio);
-  void GetLBDataForLB(int lb_type, double& lb_max_avg_ratio, double&
-      local_remote_comm_ratio);
-
-  void SetMigrationCost(double lb_migration_cost);
-  void SetStrategyCost(double lb_strategy_cost);
+  inline void AdaptResumeSync(int lb_period) {
+    LDOMAdaptResumeSync(myLDHandle, lb_period);
+  }
+
+  void RegisterMetaBalancer();
 
 private:
-  //CProxy_LBDatabase thisProxy;
   int mystep;
   LDHandle myLDHandle;
   static char *avail_vector;   // processor bit vector
   int new_ld_balancer;         // for Node 0
   CkVec<BaseLB *>   loadbalancers;
   int nloadbalancers;
-  // Keeps track of per iteration load.
-  std::vector<double> total_load_vec;
-  // Keeps track of how many local chares contributed
-  std::vector<int> total_count_vec;
-  std::vector<int> lbdb_no_obj_callback;
-  int max_iteration;
-
-  double after_lb_max;
-  double after_lb_avg;
-  double prev_idle;
-  double alpha_beta_cost_to_load;
-  int is_prev_lb_refine;
+  MetaBalancer* metabalancer;
 
 public:
-  bool lb_in_progress;
   BaseLB** getLoadBalancers() {return loadbalancers.getVec();}
   int getNLoadBalancers() {return nloadbalancers;}
 
index 74131d577282d9332990994f2ea628e5cd9560a2..22fd6b8c8718d332a9ffe3a0ff72dd7b1d2dc3b4 100644 (file)
@@ -32,6 +32,8 @@ ALL_LDBS=\
    $(L)/libmoduleRefineTopoLB.a \
    $(L)/libmoduleTopoCentLB.a \
    $(L)/libmoduleTopoLB.a \
+   $(L)/libmoduleAdaptiveLB.a \
+   $(L)/libmoduleMetisLB.a \
    manager.o
 
 
@@ -227,6 +229,18 @@ $(L)/libmoduleTopoLB.a: TopoLB.o
 LBHEADERS += TopoLB.h TopoLB.decl.h
 
 
+$(L)/libmoduleAdaptiveLB.a: AdaptiveLB.o 
+       $(CHARMC) -o $(L)/libmoduleAdaptiveLB.a AdaptiveLB.o 
+       
+LBHEADERS += AdaptiveLB.h AdaptiveLB.decl.h
+
+
+$(L)/libmoduleMetisLB.a: MetisLB.o 
+       $(CHARMC) -o $(L)/libmoduleMetisLB.a MetisLB.o 
+       cp -f libmoduleMetisLB.dep $(L)/
+LBHEADERS += MetisLB.h MetisLB.decl.h
+
+
 $(L)/libmoduleTempAwareGreedyLB.a: TempAwareGreedyLB.o 
        $(CHARMC) -o $(L)/libmoduleTempAwareGreedyLB.a TempAwareGreedyLB.o 
        
@@ -291,6 +305,8 @@ ALL_LB_OBJS=EveryLB.o \
     RefineTopoLB.o \
     TopoCentLB.o \
     TopoLB.o \
+    AdaptiveLB.o \
+    MetisLB.o \
     TempAwareGreedyLB.o \
     MetisLB.o \
     ScotchLB.o \
@@ -335,6 +351,8 @@ EVERYLB_DEPS=EveryLB.o \
     RefineTopoLB.o \
     TopoCentLB.o \
     TopoLB.o \
+    AdaptiveLB.o \
+    MetisLB.o \
     manager.o \
     tm_tree.o  \
     tm_timings.o  \
index 0fdfd88139f14508c6274e3e04c112b2158cfe16..14cf646788af8e7d413539813fdd455360d8f398 100755 (executable)
@@ -2,7 +2,7 @@
 UNCOMMON_LDBS="TempAwareGreedyLB MetisLB ScotchLB TeamLB WSLB"
 COMMON_LDBS="BlockLB CommLB DummyLB GreedyAgentLB GreedyCommLB GreedyLB NeighborCommLB NeighborLB OrbLB PhasebyArrayLB RandCentLB RecBipartLB RefineLB RefineCommLB RotateLB TreeMatchLB RefineSwapLB CommAwareRefineLB"
 OTHER_LDBS="ComboCentLB GraphPartLB GraphBFTLB GridCommLB GridCommRefineLB GridHybridLB GridHybridSeedLB GridMetisLB HbmLB HybridLB RefineKLB RefineTopoLB TopoCentLB TopoLB"
-ALL_LDBS="$COMMON_LDBS $OTHER_LDBS"
+ALL_LDBS="$COMMON_LDBS $OTHER_LDBS AdaptiveLB MetisLB"
 
 out="Make.lb"
 
diff --git a/src/ck-ldb/MetaBalancer.C b/src/ck-ldb/MetaBalancer.C
new file mode 100644 (file)
index 0000000..5e317bd
--- /dev/null
@@ -0,0 +1,851 @@
+/**
+ * \addtogroup CkLdb
+*/
+/*@{*/
+
+#include "converse.h"
+
+/*
+ * This C++ file contains the Charm stub functions
+ */
+
+#include "MetaBalancer.h"
+#include "LBSimulation.h"
+#include "topology.h"
+
+#include "limits.h"
+
+#include "NullLB.h"
+
+#define VEC_SIZE 50
+#define IMB_TOLERANCE 1.1
+#define OUTOFWAY_TOLERANCE 2
+#define UTILIZATION_THRESHOLD 0.7
+#define NEGLECT_IDLE 2 // Should never be == 1
+#define MIN_STATS 6
+
+#   define DEBAD(x) /*CkPrintf x*/
+#   define EXTRA_FEATURE 0
+
+struct AdaptiveData {
+  double iteration;
+  double max_load;
+  double avg_load;
+  double utilization;
+  double idle_time;
+};
+
+struct AdaptiveMetaBalancer {
+  std::vector<AdaptiveData> history_data;
+  int lb_iter_no;
+};
+
+struct AdaptiveLBInfo {
+  AdaptiveLBInfo() {
+    max_avg_ratio = 1;
+    remote_local_ratio = 1;
+  }
+  double max_avg_ratio;
+  double remote_local_ratio;
+};
+
+// TODO: Separate out the datastructure required by just the central and on all
+// processors
+struct AdaptiveLBStructure {
+  int tentative_period;
+  int final_lb_period;
+  // This is based on the linear extrapolation
+  int lb_calculated_period;
+  int lb_iteration_no;
+  // This is set when all the processor sends the maximum iteration no
+  int global_max_iter_no;
+  // This keeps track of what was the max iteration no we had previously
+  // received. TODO: Mostly global_max_iter_no should be sufficied.
+  int tentative_max_iter_no;
+  // TODO: Use reduction to collect max iteration. Then we don't need this
+  // counter.
+  int global_recv_iter_counter;
+  // true indicates it is in Inform->ReceiveMaxIter->FinalLBPeriod stage.
+  bool in_progress;
+  double lb_strategy_cost;
+  double lb_migration_cost;
+  bool doCommStrategy;
+  int lb_msg_send_no;
+  int lb_msg_recv_no;
+  // Total AtSync calls from all the chares residing on the processor
+  int total_syncs_called;
+  int last_lb_type;
+  AdaptiveLBInfo greedy_info;
+  AdaptiveLBInfo refine_info;
+  AdaptiveLBInfo comm_info;
+  AdaptiveLBInfo comm_refine_info;
+};
+
+
+CkReductionMsg* lbDataCollection(int nMsg, CkReductionMsg** msgs) {
+  double lb_data[8];
+  lb_data[1] = 0.0; // total number of processors contributing
+  lb_data[2] = 0.0; // total load
+  lb_data[3] = 0.0; // max load
+  lb_data[4] = 0.0; // idle time
+  lb_data[5] = 1.0; // utilization
+  lb_data[6] = 0.0; // total load with bg
+  lb_data[7] = 0.0; // max load with bg
+  for (int i = 0; i < nMsg; i++) {
+    CkAssert(msgs[i]->getSize() == 8*sizeof(double));
+    if (msgs[i]->getSize() != 8*sizeof(double)) {
+      CkPrintf("Error!!! Reduction not correct. Msg size is %d\n", msgs[i]->getSize());
+    }
+    double* m = (double *)msgs[i]->getData();
+    // Total count
+    lb_data[1] += m[1];
+    // Avg load
+    lb_data[2] += m[2];
+    // Max load
+    lb_data[3] = ((m[3] > lb_data[3])? m[3] : lb_data[3]);
+    // Avg idle
+    lb_data[4] += m[4];
+    // Get least utilization
+    lb_data[5] = ((m[5] < lb_data[5]) ? m[5] : lb_data[5]);
+    // Get Avg load with bg
+    lb_data[6] += m[6];
+    // Get Max load with bg
+    lb_data[7] = ((m[7] > lb_data[7])? m[7] : lb_data[7]);
+    if (i == 0) {
+      // Iteration no
+      lb_data[0] = m[0];
+    }
+    if (m[0] != lb_data[0]) {
+      CkPrintf("Error!!! Reduction is intermingled between iteration %lf and\
+      %lf\n", lb_data[0], m[0]);
+      CkAbort("Intermingling iterations\n");
+    }
+  }
+  return CkReductionMsg::buildNew(8*sizeof(double), lb_data);
+}
+
+/*global*/ CkReduction::reducerType lbDataCollectionType;
+/*initcall*/ void registerLBDataCollection(void) {
+  lbDataCollectionType = CkReduction::addReducer(lbDataCollection);
+}
+
+CkGroupID _metalb;
+
+CkpvDeclare(int, metalbInited);  /**< true if metabalancer is inited */
+
+CkpvDeclare(AdaptiveMetaBalancer, adaptive_lbdb);
+CkpvDeclare(AdaptiveLBStructure, adaptive_struct);
+
+// mainchare
+MetaLBInit::MetaLBInit(CkArgMsg *m)
+{
+#if CMK_LBDB_ON
+  _metalb = CProxy_MetaBalancer::ckNew();
+#endif
+  CkPrintf("META LB Init Called\n");
+  delete m;
+}
+
+// called from init.C
+void _metabalancerInit()
+{
+  CkpvInitialize(int, metalbInited);
+  CkpvAccess(metalbInited) = 0;
+}
+
+void MetaBalancer::initnodeFn()
+{
+}
+
+// called my constructor
+void MetaBalancer::init(void)
+{
+  CkPrintf("Metabalancer init %d lbdb\n", _lbdb);
+  lbdatabase = (LBDatabase *)CkLocalBranch(_lbdb);
+  CkpvAccess(metalbInited) = 1;
+  total_load_vec.resize(VEC_SIZE, 0.0);
+  total_count_vec.resize(VEC_SIZE, 0);
+  max_iteration = -1;
+  prev_idle = 0.0;
+  alpha_beta_cost_to_load = 1.0; // Some random value. TODO: Find the actual
+
+  CkpvInitialize(AdaptiveMetaBalancer, adaptive_lbdb);
+  CkpvInitialize(AdaptiveLBStructure, adaptive_struct);
+
+  CkpvAccess(adaptive_lbdb).lb_iter_no = -1;
+
+  // If metabalancer enabled, initialize the variables
+  CkpvAccess(adaptive_struct).tentative_period =  INT_MAX;
+  CkpvAccess(adaptive_struct).final_lb_period =  INT_MAX;
+  CkpvAccess(adaptive_struct).lb_calculated_period = INT_MAX;
+  CkpvAccess(adaptive_struct).lb_iteration_no = -1;
+  CkpvAccess(adaptive_struct).global_max_iter_no = 0;
+  CkpvAccess(adaptive_struct).tentative_max_iter_no = -1;
+  CkpvAccess(adaptive_struct).global_recv_iter_counter = 0;
+  CkpvAccess(adaptive_struct).in_progress = false;
+  CkpvAccess(adaptive_struct).lb_strategy_cost = 0.0;
+  CkpvAccess(adaptive_struct).lb_migration_cost = 0.0;
+  CkpvAccess(adaptive_struct).lb_msg_send_no = 0;
+  CkpvAccess(adaptive_struct).lb_msg_recv_no = 0;
+  CkpvAccess(adaptive_struct).total_syncs_called = 0;
+  CkpvAccess(adaptive_struct).last_lb_type = -1;
+
+
+  // This is indicating if the load balancing strategy and migration started.
+  // This is mainly used to register callbacks for noobj pes. They would
+  // register as soon as resumefromsync is called. On receiving the handles at
+  // the central pe, it clears the previous handlers and sets lb_in_progress
+  // to false so that it doesn't clear the handles.
+  lb_in_progress = false;
+
+  is_prev_lb_refine = -1;
+}
+
+void MetaBalancer::pup(PUP::er& p)
+{
+       IrrGroup::pup(p);
+  if (p.isUnpacking()) {
+    lbdatabase = (LBDatabase *)CkLocalBranch(_lbdb);
+  }
+  // NOTE set lbdatabase using the id
+}
+
+
+void MetaBalancer::ResumeClients() {
+  // If metabalancer enabled, initialize the variables
+  CkpvAccess(adaptive_lbdb).history_data.clear();
+
+  CkpvAccess(adaptive_struct).tentative_period =  INT_MAX;
+  CkpvAccess(adaptive_struct).final_lb_period =  INT_MAX;
+  CkpvAccess(adaptive_struct).lb_calculated_period = INT_MAX;
+  CkpvAccess(adaptive_struct).lb_iteration_no = -1;
+  CkpvAccess(adaptive_struct).global_max_iter_no = 0;
+  CkpvAccess(adaptive_struct).tentative_max_iter_no = -1;
+  CkpvAccess(adaptive_struct).global_recv_iter_counter = 0;
+  CkpvAccess(adaptive_struct).in_progress = false;
+  CkpvAccess(adaptive_struct).lb_strategy_cost = 0.0;
+  CkpvAccess(adaptive_struct).lb_migration_cost = 0.0;
+  CkpvAccess(adaptive_struct).lb_msg_send_no = 0;
+  CkpvAccess(adaptive_struct).lb_msg_recv_no = 0;
+  CkpvAccess(adaptive_struct).total_syncs_called = 0;
+
+  prev_idle = 0.0;
+  if (lb_in_progress) {
+    lbdb_no_obj_callback.clear();
+    lb_in_progress = false;
+  }
+  // While resuming client, if we find that there are no objects, then handle
+  // the case accordingly.
+  if (lbdatabase->getLBDB()->ObjDataCount() == 0) {
+    CkPrintf("%d processor has 0 objs\n", CkMyPe());
+    HandleAdaptiveNoObj();
+  }
+}
+
+int MetaBalancer::get_iteration() {
+  return CkpvAccess(adaptive_struct).lb_iteration_no;
+}
+
+bool MetaBalancer::AddLoad(int it_n, double load) {
+  int index = it_n % VEC_SIZE;
+  total_count_vec[index]++;
+  CkpvAccess(adaptive_struct).total_syncs_called++;
+  DEBAD(("At PE %d Total contribution for iteration %d is %d total objs %d\n",
+      CkMyPe(), it_n, total_count_vec[index],
+      lbdatabase->getLBDB()->ObjDataCount()));
+
+  if (it_n < CkpvAccess(adaptive_struct).lb_iteration_no) {
+    CkAbort("Error!! Received load for previous iteration\n");
+  }
+  if (it_n > CkpvAccess(adaptive_struct).lb_iteration_no) {
+    CkpvAccess(adaptive_struct).lb_iteration_no = it_n;
+  }
+  total_load_vec[index] += load;
+  if (total_count_vec[index] > lbdatabase->getLBDB()->ObjDataCount()) {
+    CkPrintf("iteration %d received %d contributions and expected %d\n", it_n,
+        total_count_vec[index], lbdatabase->getLBDB()->ObjDataCount());
+    CkAbort("Abort!!! Received more contribution");
+  }
+
+  if (total_count_vec[index] == lbdatabase->getLBDB()->ObjDataCount()) {
+    double idle_time, bg_walltime, cpu_bgtime;
+    lbdatabase->IdleTime(&idle_time);
+    lbdatabase->BackgroundLoad(&bg_walltime, &cpu_bgtime);
+
+    int sync_for_bg = CkpvAccess(adaptive_struct).total_syncs_called +
+        lbdatabase->getLBDB()->ObjDataCount();
+    bg_walltime = bg_walltime * lbdatabase->getLBDB()->ObjDataCount() / sync_for_bg;
+
+    if (it_n < NEGLECT_IDLE) {
+      prev_idle = idle_time;
+    }
+    idle_time -= prev_idle;
+
+    // The chares do not contribute their 0th iteration load. So the total syncs
+    // in reality is total_syncs_called + obj_counts
+    int total_countable_syncs = CkpvAccess(adaptive_struct).total_syncs_called +
+        (1 - NEGLECT_IDLE) * lbdatabase->getLBDB()->ObjDataCount(); // TODO: Fix me! weird!
+    if (total_countable_syncs != 0) {
+      idle_time = idle_time * lbdatabase->getLBDB()->ObjDataCount() / total_countable_syncs;
+    }
+    //CkPrintf("[%d] Idle time %lf and countable %d for iteration %d\n", CkMyPe(), idle_time, total_countable_syncs, iteration);
+
+    double lb_data[8];
+    lb_data[0] = it_n;
+    lb_data[1] = 1;
+    lb_data[2] = total_load_vec[index]; // For average load
+    lb_data[3] = total_load_vec[index]; // For max load
+    lb_data[4] = idle_time;
+    // Set utilization
+    if (total_load_vec[index] == 0.0) {
+      lb_data[5] = 0.0;
+    } else {
+      lb_data[5] = total_load_vec[index]/(idle_time + total_load_vec[index]);
+    }
+    lb_data[6] = lb_data[2] + bg_walltime; // For Avg load with bg
+    lb_data[7] = lb_data[6]; // For Max load with bg
+    total_load_vec[index] = 0.0;
+    total_count_vec[index] = 0;
+
+    //CkPrintf("   [%d] sends total load %lf idle time %lf ratio of idle/load %lf at iter %d\n", CkMyPe(),
+    //    total_load_vec[iteration], idle_time,
+    //    idle_time/total_load_vec[iteration], adaptive_struct.lb_iteration_no);
+
+    CkCallback cb(CkIndex_MetaBalancer::ReceiveMinStats((CkReductionMsg*)NULL), thisProxy[0]);
+    contribute(8*sizeof(double), lb_data, lbDataCollectionType, cb);
+  }
+  return true;
+}
+
+void MetaBalancer::ReceiveMinStats(CkReductionMsg *msg) {
+  double* load = (double *) msg->getData();
+  double avg = load[2]/load[1];
+  double max = load[3];
+  double avg_idle = load[4]/load[1];
+  double utilization = load[5];
+  int iteration_n = load[0];
+  double avg_load_bg = load[6]/load[1];
+  double max_load_bg = load[7];
+  DEBAD(("** [%d] Iteration Avg load: %lf Max load: %lf Avg Idle : %lf Max Idle : %lf for %lf procs\n",iteration_n, avg, max, avg_idle, utilization, load[1]));
+  CkPrintf("** [%d] Iteration Avg load: %lf Max load: %lf With bg Avg load: %lf Max load: %lf Avg Idle : %lf Max Idle : %lf for %lf procs\n",iteration_n, avg, max, avg_load_bg, max_load_bg, avg_idle, utilization, load[1]);
+  delete msg;
+
+#if EXTRA_FEATURE
+  if (CkpvAccess(adaptive_struct).final_lb_period != iteration_n) {
+    for (int i = 0; i < lbdb_no_obj_callback.size(); i++) {
+      thisProxy[lbdb_no_obj_callback[i]].TriggerAdaptiveReduction();
+    }
+  }
+#endif
+
+  // Store the data for this iteration
+  CkpvAccess(adaptive_lbdb).lb_iter_no = iteration_n;
+  AdaptiveData data;
+  data.iteration = CkpvAccess(adaptive_lbdb).lb_iter_no;
+  data.max_load = max;
+  data.avg_load = avg;
+  data.utilization = utilization;
+  data.idle_time = avg_idle;
+  CkpvAccess(adaptive_lbdb).history_data.push_back(data);
+
+  // If lb period inform is in progress, dont inform again.
+  // If this is the lb data corresponding to the final lb period informed, then
+  // don't recalculate as some of the processors might have already gone into
+  // LB_STAGE.
+  if (CkpvAccess(adaptive_struct).in_progress || (CkpvAccess(adaptive_struct).final_lb_period == iteration_n)) {
+    return;
+  }
+
+  double utilization_threshold = UTILIZATION_THRESHOLD;
+
+#if EXTRA_FEATURE
+  CkPrintf("alpha_beta_to_load %lf\n", alpha_beta_cost_to_load);
+  if (alpha_beta_cost_to_load < 0.1) {
+    // Ignore the effect of idle time and there by lesser utilization. So we
+    // assign utilization threshold to be 0.0
+    CkPrintf("Changing the idle load tolerance coz this isn't communication intensive benchmark\n");
+    utilization_threshold = 0.0;
+  }
+#endif
+
+  // First generate the lb period based on the cost of lb. Find out what is the
+  // expected imbalance at the calculated lb period.
+  int period;
+  // This is based on the new max load after load balancing. So technically, it
+  // is calculated based on the shifter up avg curve.
+  double ratio_at_t = 1.0;
+  int tmp_lb_type;
+  double tmp_max_avg_ratio, tmp_comm_ratio;
+  GetPrevLBData(tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
+  double tolerate_imb = IMB_TOLERANCE * tmp_max_avg_ratio;
+
+  if (generatePlan(period, ratio_at_t)) {
+    DEBAD(("Generated period and calculated %d and period %d max iter %d\n",
+      CkpvAccess(adaptive_struct).lb_calculated_period, period,
+      CkpvAccess(adaptive_struct).tentative_max_iter_no));
+    // set the imbalance tolerance to be ratio_at_calculated_lb_period
+    if (ratio_at_t != 1.0) {
+      CkPrintf("Changed tolerance to %lf after line eq whereas max/avg is %lf\n", ratio_at_t, max/avg);
+      // Since ratio_at_t is shifter up, max/(tmp_max_avg_ratio * avg) should be
+      // compared with the tolerance
+      tolerate_imb = ratio_at_t * tmp_max_avg_ratio * OUTOFWAY_TOLERANCE;
+    }
+
+    CkPrintf("Prev LB Data Type %d, max/avg %lf, local/remote %lf\n", tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
+
+    if ((utilization < utilization_threshold || max/avg >= tolerate_imb) &&
+          CkpvAccess(adaptive_lbdb).history_data.size() > MIN_STATS) {
+      CkPrintf("Trigger soon even though we calculated lbperiod max/avg(%lf) and utilization ratio (%lf)\n", max/avg, utilization);
+      TriggerSoon(iteration_n, max/avg, tolerate_imb);
+      return;
+    }
+
+    // If the new lb period from linear extrapolation is greater than maximum
+    // iteration known from previously collected data, then inform all the
+    // processors about the new calculated period.
+    if (period > CkpvAccess(adaptive_struct).tentative_max_iter_no && period !=
+          CkpvAccess(adaptive_struct).final_lb_period) {
+      CkpvAccess(adaptive_struct).doCommStrategy = false;
+      CkpvAccess(adaptive_struct).lb_calculated_period = period;
+      CkpvAccess(adaptive_struct).in_progress = true;
+      CkPrintf("Sticking to the calculated period %d\n",
+        CkpvAccess(adaptive_struct).lb_calculated_period);
+      thisProxy.LoadBalanceDecision(CkpvAccess(adaptive_struct).lb_msg_send_no++,
+        CkpvAccess(adaptive_struct).lb_calculated_period);
+      return;
+    }
+    // TODO: Shouldn't we return from here??
+  }
+
+  CkPrintf("Prev LB Data Type %d, max/avg %lf, local/remote %lf\n", tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
+
+  // This would be called when the datasize is not enough to calculate lb period
+  if ((utilization < utilization_threshold || max/avg >= tolerate_imb) && CkpvAccess(adaptive_lbdb).history_data.size() > 4) {
+    CkPrintf("Carry out load balancing step at iter max/avg(%lf) and utilization ratio (%lf)\n", max/avg, utilization);
+    TriggerSoon(iteration_n, max/avg, tolerate_imb);
+    return;
+  }
+
+}
+
+void MetaBalancer::TriggerSoon(int iteration_n, double imbalance_ratio,
+    double tolerate_imb) {
+
+  // If the previously calculated_period (not the final decision) is greater
+  // than the iter +1 and if it is greater than the maximum iteration we have
+  // seen so far, then we can inform this
+  if ((iteration_n + 1 > CkpvAccess(adaptive_struct).tentative_max_iter_no) &&
+      (iteration_n+1 < CkpvAccess(adaptive_struct).lb_calculated_period) &&
+      (iteration_n + 1 != CkpvAccess(adaptive_struct).final_lb_period)) {
+    if (imbalance_ratio < tolerate_imb) {
+      CkpvAccess(adaptive_struct).doCommStrategy = true;
+      CkPrintf("No load imbalance but idle time\n");
+    } else {
+      CkpvAccess(adaptive_struct).doCommStrategy = false;
+      CkPrintf("load imbalance \n");
+    }
+    CkpvAccess(adaptive_struct).lb_calculated_period = iteration_n + 1;
+    CkpvAccess(adaptive_struct).in_progress = true;
+    CkPrintf("Informing everyone the lb period is %d\n",
+        CkpvAccess(adaptive_struct).lb_calculated_period);
+    thisProxy.LoadBalanceDecision(CkpvAccess(adaptive_struct).lb_msg_send_no++,
+        CkpvAccess(adaptive_struct).lb_calculated_period);
+  }
+}
+
+bool MetaBalancer::generatePlan(int& period, double& ratio_at_t) {
+  if (CkpvAccess(adaptive_lbdb).history_data.size() <= 4) {
+    return false;
+  }
+
+  // Some heuristics for lbperiod
+  // If constant load or almost constant,
+  // then max * new_lb_period > avg * new_lb_period + lb_cost
+  double max = 0.0;
+  double avg = 0.0;
+  AdaptiveData data;
+  for (int i = 0; i < CkpvAccess(adaptive_lbdb).history_data.size(); i++) {
+    data = CkpvAccess(adaptive_lbdb).history_data[i];
+    max += data.max_load;
+    avg += data.avg_load;
+    //DEBAD(("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load));
+    //CkPrintf("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load);
+  }
+//  max /= (adaptive_struct.lb_iteration_no - CkpvAccess(adaptive_lbdb).history_data[0].iteration);
+//  avg /= (adaptive_struct.lb_iteration_no - CkpvAccess(adaptive_lbdb).history_data[0].iteration);
+
+  // If linearly varying load, then find lb_period
+  // area between the max and avg curve
+  // If we can attain perfect balance, then the new load is close to the
+  // average. Hence we pass 1, else pass in some other value which would be the
+  // new max_load after load balancing.
+  int tmp_lb_type;
+  double tmp_max_avg_ratio, tmp_comm_ratio;
+  double tolerate_imb;
+
+#if EXTRA_FEATURE
+  // First get the data for refine.
+  GetLBDataForLB(1, tmp_max_avg_ratio, tmp_comm_ratio);
+  tolerate_imb = tmp_max_avg_ratio;
+
+  // If RefineLB does a good job, then find the period considering RefineLB
+  if (tmp_max_avg_ratio <= 1.01) {
+    if (max/avg < tolerate_imb) {
+      CkPrintf("Resorting to imb = 1.0 coz max/avg (%lf) < imb(%lf)\n", max/avg, tolerate_imb);
+      tolerate_imb = 1.0;
+    }
+    CkPrintf("Will generate plan for refine %lf imb and %lf overhead\n", tolerate_imb, 0.2);
+    return getPeriodForStrategy(tolerate_imb, 0.2, period, ratio_at_t);
+  }
+
+  GetLBDataForLB(0, tmp_max_avg_ratio, tmp_comm_ratio);
+#endif
+
+  GetPrevLBData(tmp_lb_type, tmp_max_avg_ratio, tmp_comm_ratio);
+  tolerate_imb = tmp_max_avg_ratio;
+//  if (max/avg < tolerate_imb) {
+//    CkPrintf("Resorting to imb = 1.0 coz max/avg (%lf) < imb(%lf)\n", max/avg, tolerate_imb);
+//    tolerate_imb = 1.0;
+//  }
+  if (max/avg > tolerate_imb) {
+    if (getPeriodForStrategy(tolerate_imb, 1, period, ratio_at_t)) {
+      return true;
+    }
+  }
+
+  max = 0.0;
+  avg = 0.0;
+  for (int i = 0; i < CkpvAccess(adaptive_lbdb).history_data.size(); i++) {
+    data = CkpvAccess(adaptive_lbdb).history_data[i];
+    max += data.max_load;
+    avg += data.avg_load*tolerate_imb;
+    //DEBAD(("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load));
+    //CkPrintf("max (%d, %lf) avg (%d, %lf)\n", i, data.max_load, i, data.avg_load);
+  }
+  max /= CkpvAccess(adaptive_lbdb).history_data.size();
+  avg /= CkpvAccess(adaptive_lbdb).history_data.size();
+  double cost = CkpvAccess(adaptive_struct).lb_strategy_cost + CkpvAccess(adaptive_struct).lb_migration_cost;
+  period = cost/(max - avg); 
+  CkPrintf("Obtained period %d from constant prediction\n", period);
+  if (period < 0) { 
+    period = CkpvAccess(adaptive_struct).final_lb_period;
+    CkPrintf("Obtained -ve period from constant prediction so changing to prev %d\n", period);
+  } 
+  ratio_at_t = max / avg;
+  return true;
+}
+
+bool MetaBalancer::getPeriodForStrategy(double new_load_percent,
+    double overhead_percent, int& period, double& ratio_at_t) {
+  double mslope, aslope, mc, ac;
+  getLineEq(new_load_percent, aslope, ac, mslope, mc);
+  CkPrintf("new load percent %lf\n", new_load_percent);
+  CkPrintf("\n max: %fx + %f; avg: %fx + %f\n", mslope, mc, aslope, ac);
+  double a = (mslope - aslope)/2;
+  double b = (mc - ac);
+  double c = -(CkpvAccess(adaptive_struct).lb_strategy_cost +
+      CkpvAccess(adaptive_struct).lb_migration_cost) * overhead_percent;
+  bool got_period = getPeriodForLinear(a, b, c, period);
+  if (!got_period) {
+    return false;
+  }
+
+  if (mslope < 0) {
+    if (period > (-mc/mslope)) {
+      CkPrintf("Max < 0 Period set when max load is -ve\n");
+      return false;
+    }
+  }
+
+  if (aslope < 0) {
+    if (period > (-ac/aslope)) {
+      CkPrintf("Avg < 0 Period set when avg load is -ve\n");
+      return false;
+    }
+  }
+
+  int intersection_t = (mc-ac) / (aslope - mslope);
+  if (intersection_t > 0 && period > intersection_t) {
+    CkPrintf("Avg | Max Period set when curves intersect\n");
+    return false;
+  }
+  ratio_at_t = ((mslope*period + mc)/(aslope*period + ac));
+  CkPrintf("Ratio at t (%lf*%d + %lf) / (%lf*%d+%lf) = %lf\n", mslope, period, mc, aslope, period, ac, ratio_at_t);
+  return true;
+}
+
+bool MetaBalancer::getPeriodForLinear(double a, double b, double c, int& period) {
+  CkPrintf("Quadratic Equation %lf X^2 + %lf X + %lf\n", a, b, c);
+  if (a == 0.0) {
+    period = (-c / b);
+    if (period < 0) {
+      CkPrintf("-ve period for -c/b (%d)\n", period);
+      return false;
+    }
+    CkPrintf("Ideal period for linear load %d\n", period);
+    return true;
+  }
+  int x;
+  double t = (b * b) - (4*a*c);
+  if (t < 0) {
+    CkPrintf("(b * b) - (4*a*c) is -ve sqrt : %lf\n", sqrt(t));
+    return false;
+  }
+  t = (-b + sqrt(t)) / (2*a);
+  x = t;
+  if (x < 0) {
+    CkPrintf("boo!!! x (%d) < 0\n", x);
+    x = 0;
+    return false;
+  }
+  period = x;
+  CkPrintf("Ideal period for linear load %d\n", period);
+  return true;
+}
+
+bool MetaBalancer::getLineEq(double new_load_percent, double& aslope, double& ac, double& mslope, double& mc) {
+  int total = CkpvAccess(adaptive_lbdb).history_data.size();
+  int iterations = 1 + CkpvAccess(adaptive_lbdb).history_data[total - 1].iteration -
+      CkpvAccess(adaptive_lbdb).history_data[0].iteration;
+  double a1 = 0;
+  double m1 = 0;
+  double a2 = 0;
+  double m2 = 0;
+  AdaptiveData data;
+  int i = 0;
+  for (i = 0; i < total/2; i++) {
+    data = CkpvAccess(adaptive_lbdb).history_data[i];
+    m1 += data.max_load;
+    a1 += data.avg_load;
+    CkPrintf("max (%d, %lf) avg (%d, %lf) adjusted_avg (%d, %lf)\n", i, data.max_load, i, data.avg_load, i, new_load_percent*data.avg_load);
+  }
+  m1 /= i;
+  a1 = (a1 * new_load_percent) / i;
+
+  for (i = total/2; i < total; i++) {
+    data = CkpvAccess(adaptive_lbdb).history_data[i];
+    m2 += data.max_load;
+    a2 += data.avg_load;
+    CkPrintf("max (%d, %lf) avg (%d, %lf) adjusted_avg (%d, %lf)\n", i, data.max_load, i, data.avg_load, i, new_load_percent*data.avg_load);
+  }
+  m2 /= (i - total/2);
+  a2 = (a2 * new_load_percent) / (i - total/2);
+
+  aslope = 2 * (a2 - a1) / iterations;
+  mslope = 2 * (m2 - m1) / iterations;
+  ac = CkpvAccess(adaptive_lbdb).history_data[0].avg_load * new_load_percent;
+  mc = CkpvAccess(adaptive_lbdb).history_data[0].max_load;
+
+  ac = a1 - ((aslope * total)/4);
+  mc = m1 - ((mslope * total)/4);
+
+  //ac = (CkpvAccess(adaptive_lbdb).history_data[1].avg_load * new_load_percent - aslope);
+  //mc = (CkpvAccess(adaptive_lbdb).history_data[1].max_load - mslope);
+
+  return true;
+}
+
+void MetaBalancer::LoadBalanceDecision(int req_no, int period) {
+  if (req_no < CkpvAccess(adaptive_struct).lb_msg_recv_no) {
+    CkPrintf("Error!!! Received a request which was already sent or old\n");
+    return;
+  }
+  CkPrintf("[%d] Load balance decision made cur iteration: %d period:%d\n",CkMyPe(), CkpvAccess(adaptive_struct).lb_iteration_no, period);
+  CkpvAccess(adaptive_struct).tentative_period = period;
+  CkpvAccess(adaptive_struct).lb_msg_recv_no = req_no;
+  thisProxy[0].ReceiveIterationNo(req_no, CkpvAccess(adaptive_struct).lb_iteration_no);
+}
+
+void MetaBalancer::LoadBalanceDecisionFinal(int req_no, int period) {
+  if (req_no < CkpvAccess(adaptive_struct).lb_msg_recv_no) {
+    return;
+  }
+  DEBAD(("[%d] Final Load balance decision made cur iteration: %d period:%d \n",CkMyPe(), CkpvAccess(adaptive_struct).lb_iteration_no, period));
+  CkpvAccess(adaptive_struct).tentative_period = period;
+  CkpvAccess(adaptive_struct).final_lb_period = period;
+  // NOTE LDOMAdaptResumeSync(myLDHandle, period);
+  lbdatabase->AdaptResumeSync(period);
+}
+
+
+void MetaBalancer::ReceiveIterationNo(int req_no, int local_iter_no) {
+  CmiAssert(CkMyPe() == 0);
+
+  CkpvAccess(adaptive_struct).global_recv_iter_counter++;
+  if (local_iter_no > CkpvAccess(adaptive_struct).global_max_iter_no) {
+    CkpvAccess(adaptive_struct).global_max_iter_no = local_iter_no;
+  }
+
+  int period;
+  if (CkNumPes() == CkpvAccess(adaptive_struct).global_recv_iter_counter) {
+
+    if (CkpvAccess(adaptive_struct).global_max_iter_no > CkpvAccess(adaptive_struct).tentative_max_iter_no) {
+      CkpvAccess(adaptive_struct).tentative_max_iter_no = CkpvAccess(adaptive_struct).global_max_iter_no;
+    }
+    period = (CkpvAccess(adaptive_struct).tentative_period > CkpvAccess(adaptive_struct).global_max_iter_no) ? CkpvAccess(adaptive_struct).tentative_period : CkpvAccess(adaptive_struct).global_max_iter_no + 1;
+    // If no one has gone into load balancing stage, then we can safely change
+    // the period otherwise keep the old period.
+    if (CkpvAccess(adaptive_struct).global_max_iter_no < CkpvAccess(adaptive_struct).final_lb_period) {
+      CkpvAccess(adaptive_struct).tentative_period = period;
+      CkPrintf("Final lb_period CHANGED!%d\n", CkpvAccess(adaptive_struct).tentative_period);
+    } else {
+      CkpvAccess(adaptive_struct).tentative_period = CkpvAccess(adaptive_struct).final_lb_period;
+      CkPrintf("Final lb_period NOT CHANGED!%d\n", CkpvAccess(adaptive_struct).tentative_period);
+    }
+    thisProxy.LoadBalanceDecisionFinal(req_no, CkpvAccess(adaptive_struct).tentative_period);
+    CkpvAccess(adaptive_struct).in_progress = false;
+    CkpvAccess(adaptive_struct).global_recv_iter_counter = 0;
+  }
+}
+
+int MetaBalancer::getPredictedLBPeriod(bool& is_tentative) {
+  // If tentative and final_lb_period are the same, then the decision has been
+  // made but if not, they are in the middle of consensus, hence return the
+  // lease of the two
+  if (CkpvAccess(adaptive_struct).tentative_period != CkpvAccess(adaptive_struct).final_lb_period) {
+    is_tentative = true;
+  } else {
+    is_tentative = false;
+  }
+  if (CkpvAccess(adaptive_struct).tentative_period < CkpvAccess(adaptive_struct).final_lb_period) {
+    return CkpvAccess(adaptive_struct).tentative_period;
+   } else {
+     return CkpvAccess(adaptive_struct).final_lb_period;
+   }
+}
+
+// Called by CentralLB to indicate that the LB strategy and migration is in
+// progress.
+void MetaBalancer::ResetAdaptive() {
+  CkpvAccess(adaptive_lbdb).lb_iter_no = -1;
+  lb_in_progress = true;
+}
+
+void MetaBalancer::HandleAdaptiveNoObj() {
+#if EXTRA_FEATURE
+  CkpvAccess(adaptive_struct).lb_iteration_no++;
+  //CkPrintf("HandleAdaptiveNoObj %d\n", adaptive_struct.lb_iteration_no);
+  thisProxy[0].RegisterNoObjCallback(CkMyPe());
+  TriggerAdaptiveReduction();
+#endif
+}
+
+void MetaBalancer::RegisterNoObjCallback(int index) {
+#if EXTRA_FEATURE
+  if (lb_in_progress) {
+    lbdb_no_obj_callback.clear();
+    //CkPrintf("Clearing and registering\n");
+    lb_in_progress = false;
+  }
+  lbdb_no_obj_callback.push_back(index);
+  CkPrintf("Registered %d to have no objs.\n", index);
+
+  // If collection has already happened and this is second iteration, then
+  // trigger reduction.
+  if (CkpvAccess(adaptive_lbdb).lb_iter_no != -1) {
+    //CkPrintf("Collection already started now %d so kick in\n", adaptive_struct.lb_iteration_no);
+    thisProxy[index].TriggerAdaptiveReduction();
+  }
+#endif
+}
+
+void MetaBalancer::TriggerAdaptiveReduction() {
+#if EXTRA_FEATURE
+  CkpvAccess(adaptive_struct).lb_iteration_no++;
+  //CkPrintf("Trigger adaptive for %d\n", CkpvAccess(adaptive_struct).lb_iteration_no);
+  double lb_data[8];
+  lb_data[0] = CkpvAccess(adaptive_struct).lb_iteration_no;
+  lb_data[1] = 1;
+  lb_data[2] = 0.0;
+  lb_data[3] = 0.0;
+  lb_data[4] = 0.0;
+  lb_data[5] = 0.0;
+  lb_data[6] = 0.0;
+  lb_data[7] = 0.0;
+
+  // CkPrintf("   [%d] sends total load %lf idle time %lf ratio of idle/load %lf at iter %d\n", CkMyPe(),
+  //     total_load_vec[iteration], idle_time,
+  //     idle_time/total_load_vec[iteration], adaptive_struct.lb_iteration_no);
+
+  CkCallback cb(CkIndex_MetaBalancer::ReceiveMinStats((CkReductionMsg*)NULL), thisProxy[0]);
+  contribute(8*sizeof(double), lb_data, lbDataCollectionType, cb);
+#endif
+}
+
+
+bool MetaBalancer::isStrategyComm() {
+  return CkpvAccess(adaptive_struct).doCommStrategy;
+}
+
+void MetaBalancer::SetMigrationCost(double lb_migration_cost) {
+  CkpvAccess(adaptive_struct).lb_migration_cost = lb_migration_cost;
+}
+
+void MetaBalancer::SetStrategyCost(double lb_strategy_cost) {
+  CkpvAccess(adaptive_struct).lb_strategy_cost = lb_strategy_cost;
+}
+
+void MetaBalancer::UpdateAfterLBData(int lb, double lb_max, double lb_avg, double
+    local_comm, double remote_comm) {
+  CkpvAccess(adaptive_struct).last_lb_type = lb;
+  if (lb == 0) {
+    CkpvAccess(adaptive_struct).greedy_info.max_avg_ratio = lb_max/lb_avg;
+  } else if (lb == 1) {
+    CkpvAccess(adaptive_struct).refine_info.max_avg_ratio = lb_max/lb_avg;
+  } else if (lb == 2) {
+    CkpvAccess(adaptive_struct).comm_info.remote_local_ratio = remote_comm/local_comm;
+  } else if (lb == 3) {
+    CkpvAccess(adaptive_struct).comm_refine_info.remote_local_ratio =
+    remote_comm/local_comm;
+  }
+}
+
+void MetaBalancer::UpdateAfterLBData(double max_load, double max_cpu, double
+avg_load) {
+  if (CkpvAccess(adaptive_struct).last_lb_type == -1) {
+    CkpvAccess(adaptive_struct).last_lb_type = 0;
+  }
+  int lb = CkpvAccess(adaptive_struct).last_lb_type;
+  //CkPrintf("Storing data after lb ratio %lf for lb %d\n", max_load/avg_load, lb);
+  if (lb == 0) {
+    CkpvAccess(adaptive_struct).greedy_info.max_avg_ratio = max_load/avg_load;
+  } else if (lb == 1) {
+    CkpvAccess(adaptive_struct).refine_info.max_avg_ratio = max_load/avg_load;
+  } else if (lb == 2) {
+    CkpvAccess(adaptive_struct).comm_info.max_avg_ratio = max_load/avg_load;
+  } else if (lb == 3) {
+    CkpvAccess(adaptive_struct).comm_refine_info.max_avg_ratio = max_load/avg_load;
+  }
+}
+
+void MetaBalancer::UpdateAfterLBComm(double alpha_beta_to_load) {
+  CkPrintf("Setting alpha beta %lf\n", alpha_beta_to_load);
+  alpha_beta_cost_to_load = alpha_beta_to_load;
+}
+
+
+void MetaBalancer::GetPrevLBData(int& lb_type, double& lb_max_avg_ratio, double&
+    remote_local_comm_ratio) {
+  lb_type = CkpvAccess(adaptive_struct).last_lb_type;
+  lb_max_avg_ratio = 1;
+  remote_local_comm_ratio = 1;
+  GetLBDataForLB(lb_type, lb_max_avg_ratio, remote_local_comm_ratio);
+}
+
+void MetaBalancer::GetLBDataForLB(int lb_type, double& lb_max_avg_ratio, double&
+    remote_local_comm_ratio) {
+  if (lb_type == 0) {
+    lb_max_avg_ratio = CkpvAccess(adaptive_struct).greedy_info.max_avg_ratio;
+  } else if (lb_type == 1) {
+    lb_max_avg_ratio = CkpvAccess(adaptive_struct).refine_info.max_avg_ratio;
+  } else if (lb_type == 2) {
+    remote_local_comm_ratio = CkpvAccess(adaptive_struct).comm_info.remote_local_ratio;
+  } else if (lb_type == 3) {
+    remote_local_comm_ratio =
+       CkpvAccess(adaptive_struct).comm_refine_info.remote_local_ratio;
+  }
+}
+
+#include "MetaBalancer.def.h"
+
+/*@}*/
diff --git a/src/ck-ldb/MetaBalancer.ci b/src/ck-ldb/MetaBalancer.ci
new file mode 100644 (file)
index 0000000..87694b3
--- /dev/null
@@ -0,0 +1,24 @@
+module MetaBalancer {
+  //extern module CkMarshall;
+  //extern module NullLB;
+
+  readonly CkGroupID _metalb;
+
+  mainchare MetaLBInit {
+    entry MetaLBInit(CkArgMsg *m);
+  };
+
+  initcall void registerLBDataCollection(void);
+  group [migratable] MetaBalancer {
+    entry void MetaBalancer(void);  
+    initnode void initnodeFn();
+
+    entry [immediate] void ReceiveMinStats(CkReductionMsg *msg);
+    entry [immediate] void LoadBalanceDecision(int req_no, int period);
+    entry [immediate] void LoadBalanceDecisionFinal(int req_no, int period);
+    entry [immediate] void ReceiveIterationNo(int, int);
+    entry [immediate] void RegisterNoObjCallback(int);
+    entry [immediate] void TriggerAdaptiveReduction();
+  }
+
+};
diff --git a/src/ck-ldb/MetaBalancer.h b/src/ck-ldb/MetaBalancer.h
new file mode 100644 (file)
index 0000000..8fbc65d
--- /dev/null
@@ -0,0 +1,114 @@
+/**
+ * \addtogroup CkLdb
+*/
+/*@{*/
+
+#ifndef METABALANCER_H
+#define METABALANCER_H
+
+
+//#include <charm++.h>
+//#include "ckreduction.h" 
+
+//#include "lbdb.h"
+//#include "LBDBManager.h"
+//#include "lbdb++.h"
+#include "LBDatabase.h"
+
+#include <vector>
+
+#include "MetaBalancer.decl.h"
+
+extern CkGroupID _metalb;
+
+CkpvExtern(int, metalbInited);
+
+void _MetaLBInit();
+
+// main chare
+class MetaLBInit : public Chare {
+  public:
+    MetaLBInit(CkArgMsg*);
+    MetaLBInit(CkMigrateMessage *m):Chare(m) {}
+};
+
+class MetaBalancer : public CBase_MetaBalancer {
+public:
+  MetaBalancer(void)  { init(); }
+  MetaBalancer(CkMigrateMessage *m)  { init(); }
+  ~MetaBalancer()  {}
+  
+private:
+  void init();
+public:
+  inline static MetaBalancer * Object() { return CkpvAccess(metalbInited)?(MetaBalancer *)CkLocalBranch(_metalb):NULL; }
+
+  static void initnodeFn(void);
+
+  void pup(PUP::er& p);
+
+  void ResumeClients();
+
+/*  inline void SetLBPeriod(double s) { LDSetLBPeriod(myLDHandle, s);}
+  inline double GetLBPeriod() { return LDGetLBPeriod(myLDHandle);}*/ //NOTE
+
+  void ResetAdaptive();
+  int get_iteration();
+  bool AddLoad(int iteration, double load);
+  void ReceiveMinStats(CkReductionMsg *);
+  void TriggerSoon(int iteration_no, double imbalance_ratio, double tolerate_imb);
+  void LoadBalanceDecision(int, int);
+  void LoadBalanceDecisionFinal(int, int);
+  void ReceiveIterationNo(int, int); // Receives the current iter no
+  void HandleAdaptiveNoObj();
+  void RegisterNoObjCallback(int index);
+  void TriggerAdaptiveReduction();
+
+  bool generatePlan(int& period, double& ratio_at_t);
+  bool getLineEq(double new_load_percent, double& aslope, double& ac, double& mslope, double& mc);
+  bool getPeriodForLinear(double a, double b, double c, int& period);
+  bool getPeriodForStrategy(double new_load, double overhead_percent, int& period, double& ratio_at_t);
+  int getPredictedLBPeriod(bool& is_tentative);
+
+  bool isStrategyComm();
+
+  void UpdateAfterLBData(int is_lb_refine, double lb_max, double lb_avg, double
+      local_comm, double remote_comm);
+
+  void UpdateAfterLBData(double max_load, double max_cpu, double avg_load);
+  void UpdateAfterLBComm(double alpha_beta_cost);
+  void GetPrevLBData(int& lb_type, double& lb_max_avg_ratio, double&
+      local_remote_comm_ratio);
+  void GetLBDataForLB(int lb_type, double& lb_max_avg_ratio, double&
+      local_remote_comm_ratio);
+
+  void SetMigrationCost(double lb_migration_cost);
+  void SetStrategyCost(double lb_strategy_cost);
+
+private:
+  //CProxy_MetaBalancer thisProxy;
+  LBDatabase* lbdatabase;
+  int mystep;
+  std::vector<double> total_load_vec;
+  // Keeps track of how many local chares contributed
+  std::vector<int> total_count_vec;
+  std::vector<int> lbdb_no_obj_callback;
+  int max_iteration;
+
+  double after_lb_max;
+  double after_lb_avg;
+  double prev_idle;
+  double alpha_beta_cost_to_load;
+  int is_prev_lb_refine;
+
+public:
+  bool lb_in_progress;
+
+public:
+};
+
+inline MetaBalancer* MetaBalancerObj() { return MetaBalancer::Object(); }
+
+#endif /* LDATABASE_H */
+
+/*@}*/
index ad34d34bf4cce6e96a73740cdbc3ca16a00322c4..3564c5d49cb1454442f6155981d57e8d1762b3a7 100644 (file)
@@ -222,7 +222,7 @@ CKHEADERS=ck.h ckstream.h envelope.h init.h qd.h charm.h charm++.h \
          ckobjQ.h readonly.h charisma.h \
           comlib.h ComlibArrayListener.h ComlibStrategy.h \
          ComlibLearner.h $(UTILHEADERS) \
-         tempo.h waitqd.h LBDatabase.h lbdb.h lbdb++.h LBProfit.h $(LBHEADERS) \
+         tempo.h waitqd.h LBDatabase.h MetaBalancer.h lbdb.h lbdb++.h LBProfit.h $(LBHEADERS) \
           LBDBManager.h        LBComm.h LBOM.h LBObj.h LBMachineUtil.h LBAgent.h \
          Refiner.h RefinerApprox.h RefinerComm.h ckgraph.h ckheap.h \
           elements.h CommLBHeap.h topology.h manager.h \
@@ -234,7 +234,7 @@ CKHEADERS=ck.h ckstream.h envelope.h init.h qd.h charm.h charm++.h \
           CkReduction.decl.h CkArrayReductionMgr.decl.h\
           tempo.decl.h waitqd.decl.h \
           CkFutures.decl.h CkCheckpoint.decl.h CkMemCheckpoint.decl.h \
-          LBDatabase.decl.h CentralLB.decl.h NullLB.decl.h \
+          LBDatabase.decl.h MetaBalancer.decl.h CentralLB.decl.h NullLB.decl.h \
           BaseLB.decl.h \
          NborBaseLB.decl.h \
           HybridBaseLB.decl.h EveryLB.decl.h CommonLBs.decl.h \
@@ -709,7 +709,7 @@ CK_LIBS_CORE=$(L)/libck.a
 LIBCK_CORE=trace-common.o tracec.o tracef.o init.o register.o qd.o ck.o main.o \
           msgalloc.o ckfutures.o ckIgetControl.o debug-message.o debug-charm.o ckcallback.o \
           cklocation.o ckarray.o ckreduction.o ckarrayreductionmgr.o \
-           tempo.o waitqd.o LBDatabase.o lbdb.o lbdbf.o charisma.o ckobjQ.o  \
+           tempo.o waitqd.o LBDatabase.o MetaBalancer.o lbdb.o lbdbf.o charisma.o ckobjQ.o  \
           LBAgent.o LBProfit.o ckcheckpoint.o ckmemcheckpoint.o ckevacuation.o ckmessagelogging.o ckcausalmlog.o\
            LBDBManager.o LBComm.o LBObj.o LBMachineUtil.o CentralPredictor.o \
           BaseLB.o CentralLB.o HybridBaseLB.o NborBaseLB.o \
@@ -770,8 +770,8 @@ libtrace-Tau.o: trace-Tau.C charm++.h charm.h converse.h conv-config.h \
   conv-ccs.h sockRoutines.h ccs-server.h ckobjQ.h ckreduction.h \
   CkReduction.decl.h CkArrayReductionMgr.decl.h \
   ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
-  cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
-  LBMachineUtil.h lbdb++.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
+  cklocation.h LBDatabase.h MetaBalancer.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
+  LBMachineUtil.h lbdb++.h LBDatabase.decl.h MetaBalancer.decl.h NullLB.decl.h BaseLB.decl.h \
   CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
   convcomlibstrategy.h ComlibLearner.h envelope.h envelope-path.h CkFutures.decl.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \