kNeighbor changes which can be used to test metabalancer
authorHarshitha <gplkrsh2@illinois.edu>
Fri, 9 Mar 2012 02:01:49 +0000 (20:01 -0600)
committerHarshitha <gplkrsh2@illinois.edu>
Fri, 9 Mar 2012 02:01:49 +0000 (20:01 -0600)
examples/charm++/load_balancing/kNeighbor/Makefile
examples/charm++/load_balancing/kNeighbor/kNeighbor.C
examples/charm++/load_balancing/kNeighbor/kNeighbor.ci

index 592dd96a728ebc556cc7eb74536e37abf7776c61..f957eb9bb38698f9f9732344418a953b27e8ca42 100644 (file)
@@ -1,5 +1,6 @@
-OPTS   = -O3
+OPTS   = -g
 CHARMC = ../../../../bin/charmc $(OPTS)
+#CHARMC        = $(HOME)/charm/bin/charmc $(OPTS)
 
 OBJS   = kNeighbor.o
 
index a941c7d39b68f9a60b224fb459bf10b4a1cb2427..63c72526f01282f8dbaf7a71ad05ec0418cb8400 100644 (file)
@@ -18,6 +18,7 @@
 /* readonly */ int num_chares;
 /* readonly */ int gMsgSize;
 /* readonly */ int gLBFreq;
+/* readonly */ int numSteps;
 
 int cmpFunc(const void *a, const void *b) {
   if(*(double *)a < *(double *)b) return -1;
@@ -48,7 +49,6 @@ class Main: public CBase_Main {
   public:
     CProxy_Block array;
 
-    int numSteps;
     int currentStep;
     int currentMsgSize;
 
@@ -57,6 +57,7 @@ class Main: public CBase_Main {
     double maxTime;
     double minTime;
     double *timeRec;
+    double startTime;
 
     double gStarttime;
 
@@ -66,15 +67,15 @@ class Main: public CBase_Main {
       CkPrintf("\nStarting kNeighbor ...\n");
 
       if (m->argc!=4 && m->argc!=5) {
-       CkPrintf("Usage: %s <#elements> <#iterations> <msg size> [ldb freq]\n", m->argv[0]);
-       delete m;
-       CkExit();
+        CkPrintf("Usage: %s <#elements> <#iterations> <msg size> [ldb freq]\n", m->argv[0]);
+        delete m;
+        CkExit();
       }
 
       num_chares = atoi(m->argv[1]);
       if(num_chares < CkNumPes()) {
-       printf("Warning: #elements is forced to be equal to #pes\n");
-       num_chares = CkNumPes();
+        printf("Warning: #elements is forced to be equal to #pes\n");
+        num_chares = CkNumPes();
       }
 
       numSteps = atoi(m->argv[2]);
@@ -82,7 +83,7 @@ class Main: public CBase_Main {
 
       gLBFreq = 100000;
       if(m->argc==5) {
-       gLBFreq = atoi(m->argv[4]);
+        gLBFreq = atoi(m->argv[4]);
       }
 
 #if TURN_ON_LDB
@@ -96,18 +97,23 @@ class Main: public CBase_Main {
       array = CProxy_Block::ckNew(num_chares);
       CkCallback *cb = new CkCallback(CkIndex_Main::nextStep(NULL), thisProxy);
       array.ckSetReductionClient(cb);
+      startTime = CmiWallTimer();
 
       beginIteration();
+      for (int i=0; i<num_chares; i++)
+        array[i].commWithNeighbors();
+
     }
 
     void beginIteration() {
       currentStep++;
       if (currentStep == numSteps) {
-       CkPrintf("kNeighbor program finished!\n\n");
-       //CkCallback *cb = new CkCallback(CkIndex_Main::terminate(NULL), thisProxy);
-       //array.ckSetReductionClient(cb);
-       terminate(NULL);
-       return;
+        CkPrintf("kNeighbor program finished!\n\n");
+        //CkCallback *cb = new CkCallback(CkIndex_Main::terminate(NULL), thisProxy);
+        //array.ckSetReductionClient(cb);
+        CkPrintf("Total time %lf\n", CmiWallTimer() - startTime);
+        terminate(NULL);
+        return;
       }
 
       numElemsRcvd = 0;
@@ -117,13 +123,13 @@ class Main: public CBase_Main {
 
       //currentMsgSize = msgSize;
       if(currentStep!=0 && (currentStep % gLBFreq == 0)) {
-       array.pauseForLB();
-       return;
+     //   array.pauseForLB();
+        return;
       }
 
       gStarttime = CmiWallTimer();
-      for (int i=0; i<num_chares; i++)
-       array[i].commWithNeighbors();
+     // for (int i=0; i<num_chares; i++)
+     //   array[i].commWithNeighbors();
     }
 
     void resumeIter() {
@@ -132,7 +138,7 @@ class Main: public CBase_Main {
 #endif
       gStarttime = CmiWallTimer();
       for (int i=0; i<num_chares; i++)
-       array[i].commWithNeighbors();
+        array[i].commWithNeighbors();
     }
 
     void terminate(CkReductionMsg *msg) {
@@ -140,7 +146,7 @@ class Main: public CBase_Main {
       double total = 0.0;
 
       for (int i=0; i<numSteps; i++)
-       timeRec[i] = timeRec[i]*1e6;
+        timeRec[i] = timeRec[i]*1e6;
 
       qsort(timeRec, numSteps, sizeof(double), cmpFunc);
       printf("Time stats: lowest: %f, median: %f, highest: %f\n", timeRec[0], timeRec[numSteps/2], timeRec[numSteps-1]);
@@ -148,7 +154,7 @@ class Main: public CBase_Main {
       int samples = 100;
       if(numSteps<=samples) samples = numSteps-1;
       for (int i=0; i<samples; i++)
-       total += timeRec[i];
+        total += timeRec[i];
       total /= samples;
 
       CkPrintf("Average time for each %d-Neighbor iteration with msg size %d is %f (us)\n", STRIDEK, currentMsgSize, total);
@@ -161,7 +167,7 @@ class Main: public CBase_Main {
       double wholeStepTime = CmiWallTimer() - gStarttime;
       timeRec[currentStep] = wholeStepTime/CALCPERSTEP;
       if(currentStep % 10 == 0)
-       CkPrintf("Step %d with msg size %d finished: max=%f, total=%f\n", currentStep, currentMsgSize, maxTime/CALCPERSTEP, wholeStepTime/CALCPERSTEP);
+        CkPrintf("Step %d with msg size %d finished: max=%f, total=%f\n", currentStep, currentMsgSize, maxTime/CALCPERSTEP, wholeStepTime/CALCPERSTEP);
       beginIteration();
     }
 
@@ -181,6 +187,7 @@ class Block: public CBase_Block {
     int curIterMsgSize;
     int internalStepCnt;
     int sum;
+    int currentStep;
 
     toNeighborMsg **iterMsg;
 
@@ -195,28 +202,28 @@ class Block: public CBase_Block {
       int nidx=0;
       //setting left neighbors
       for (int i=thisIndex-STRIDEK; i<thisIndex; i++, nidx++) {
-       int tmpnei = i;
-       while (tmpnei<0) tmpnei += num_chares;
-       neighbors[nidx] = tmpnei;
+        int tmpnei = i;
+        while (tmpnei<0) tmpnei += num_chares;
+        neighbors[nidx] = tmpnei;
       }
       //setting right neighbors
       for (int i=thisIndex+1; i<=thisIndex+STRIDEK; i++, nidx++) {
-       int tmpnei = i;
-       while (tmpnei>=num_chares) tmpnei -= num_chares;
-       neighbors[nidx] = tmpnei;
+        int tmpnei = i;
+        while (tmpnei>=num_chares) tmpnei -= num_chares;
+        neighbors[nidx] = tmpnei;
       }
 
       for (int i=0; i<numNeighbors; i++)
-       recvTimes[i] = 0.0;
+        recvTimes[i] = 0.0;
 
       iterMsg = new toNeighborMsg *[numNeighbors];
       for (int i=0; i<numNeighbors; i++)
-       iterMsg[i] = NULL;
+        iterMsg[i] = NULL;
 
 #if DEBUG
       CkPrintf("Neighbors of %d: ", thisIndex);
       for (int i=0; i<numNeighbors; i++)
-       CkPrintf("%d ", neighbors[i]);
+        CkPrintf("%d ", neighbors[i]);
       CkPrintf("\n");
 #endif
 
@@ -235,8 +242,8 @@ class Block: public CBase_Block {
       p(numNborsRcvd);
 
       if(p.isUnpacking()) {
-       neighbors = new int[numNeighbors];
-       recvTimes = new double[numNeighbors];
+        neighbors = new int[numNeighbors];
+        recvTimes = new double[numNeighbors];
       }
       PUParray(p, neighbors, numNeighbors);
       PUParray(p, recvTimes, numNeighbors);
@@ -245,9 +252,10 @@ class Block: public CBase_Block {
       p(curIterMsgSize);
       p(internalStepCnt);
       p(sum);
+      p(currentStep);
       if(p.isUnpacking()) iterMsg = new toNeighborMsg *[numNeighbors];
       for(int i=0; i<numNeighbors; i++){
-       CkPupMessage(p, (void **)&iterMsg[i]);
+        CkPupMessage(p, (void **)&iterMsg[i]);
       }
     }
 
@@ -261,8 +269,9 @@ class Block: public CBase_Block {
     }
 
     void ResumeFromSync(){ //Called by load-balancing framework
-      CkCallback cb(CkIndex_Main::resumeIter(), mainProxy);
-      contribute(0, NULL, CkReduction::sum_int, cb);
+     // CkCallback cb(CkIndex_Main::resumeIter(), mainProxy);
+     // contribute(0, NULL, CkReduction::sum_int, cb);
+     commWithNeighbors();
     }
 
     void startInternalIteration() {
@@ -272,35 +281,55 @@ class Block: public CBase_Block {
 
       numNborsRcvd = 0;
       /* 1: pick a work size and do some computation */
-      int N = (thisIndex * thisIndex / num_chares) * 100;
+      //int N = (thisIndex * thisIndex / num_chares) * 100;
+      int N = 2;
+      if (currentStep < numSteps/4) {
+        if (thisIndex >= num_chares/8 && thisIndex < num_chares/4) {
+          N = 500;
+        }
+      } else if(currentStep < numSteps/2) {
+        if (thisIndex >= num_chares/num_chares && thisIndex < num_chares/8) {
+          N = 500;
+        }
+      } else if (currentStep < 3*numSteps/2) {
+        if (thisIndex >= num_chares/4 && thisIndex < 24) {
+          N = 500;
+        }
+      } else {
+        if (thisIndex >= 3*(num_chares/8) && thisIndex < num_chares/2) {
+          N = 500;
+        }
+      }
+
       for (int i=0; i<N; i++)
-       for (int j=0; j<N; j++) {
-         sum += (thisIndex * i + j);
-       }
+        for (int j=0; j<N; j++) {
+          sum += (thisIndex * i + j);
+        }
 
       /* 2. send msg to K neighbors */
       int msgSize = curIterMsgSize;
 
       // Send msgs to neighbors
       for (int i=0; i<numNeighbors; i++) {
-       //double memtimer = CmiWallTimer();
+        //double memtimer = CmiWallTimer();
 
-       toNeighborMsg *msg = iterMsg[i];
+        toNeighborMsg *msg = iterMsg[i];
 
 #if DEBUG
-       CkPrintf("[%d]: send msg to neighbor[%d]=%d\n", thisIndex, i, neighbors[i]);
+        CkPrintf("[%d]: send msg to neighbor[%d]=%d\n", thisIndex, i, neighbors[i]);
 #endif
-       msg->setMsgSrc(thisIndex, i);
-       //double entrytimer = CmiWallTimer();
-       thisProxy(neighbors[i]).recvMsgs(msg);
-       //double entrylasttimer = CmiWallTimer();
-       //if(thisIndex==0){
-       //      CkPrintf("At current step %d to neighbor %d, msg creation time: %f, entrymethod fire time: %f\n", internalStepCnt, neighbors[i], entrytimer-memtimer, entrylasttimer-entrytimer);
-       //}
+        msg->setMsgSrc(thisIndex, i);
+        //double entrytimer = CmiWallTimer();
+        thisProxy(neighbors[i]).recvMsgs(msg);
+        //double entrylasttimer = CmiWallTimer();
+        //if(thisIndex==0){
+        //     CkPrintf("At current step %d to neighbor %d, msg creation time: %f, entrymethod fire time: %f\n", internalStepCnt, neighbors[i], entrytimer-memtimer, entrylasttimer-entrytimer);
+        //}
       }
     }
 
     void commWithNeighbors() {
+      currentStep++;
       internalStepCnt = 0;
       curIterMsgSize = gMsgSize;
       //currently the work size is only changed every big steps (which
@@ -308,8 +337,8 @@ class Block: public CBase_Block {
       random++;
 
       if(iterMsg[0]==NULL) { //indicating the messages have not been created
-       for(int i=0; i<numNeighbors; i++)
-         iterMsg[i] = new(curIterMsgSize/4, 0) toNeighborMsg(curIterMsgSize/4);
+        for(int i=0; i<numNeighbors; i++)
+          iterMsg[i] = new(curIterMsgSize/4, 0) toNeighborMsg(curIterMsgSize/4);
       }
 
       startTime = CmiWallTimer();
@@ -329,18 +358,26 @@ class Block: public CBase_Block {
       //get one step time and send it back to mainProxy
       numNborsRcvd++;
       if (numNborsRcvd == numNeighbors) {
-       internalStepCnt++;
-       if (internalStepCnt==CALCPERSTEP) {
-         double iterCommTime = CmiWallTimer() - startTime;
-         contribute(sizeof(double), &iterCommTime, CkReduction::max_double);
-         /*if(thisIndex==0){
-           for(int i=0; i<numNeighbors; i++){
-           CkPrintf("RTT time from neighbor %d (actual elem id %d): %lf\n", i, neighbors[i], recvTimes[i]);
-           }
-           }*/
-       } else {
-         startInternalIteration();
-       }
+        internalStepCnt++;
+        if (internalStepCnt==CALCPERSTEP) {
+          double iterCommTime = CmiWallTimer() - startTime;
+          contribute(sizeof(double), &iterCommTime, CkReduction::max_double);
+
+          if(currentStep!=0 && (currentStep % gLBFreq == 0)) {
+            pauseForLB();
+            return; 
+          }
+          if (currentStep != numSteps ) {
+            commWithNeighbors();
+          }
+          /*if(thisIndex==0){
+            for(int i=0; i<numNeighbors; i++){
+            CkPrintf("RTT time from neighbor %d (actual elem id %d): %lf\n", i, neighbors[i], recvTimes[i]);
+            }
+            }*/
+        } else {
+          startInternalIteration();
+        }
       }
     }
 
index 58b6f1c18876d792137a578f57a615a41d36997e..3144754c0a0e854c0a0db73cbdad1115e86883c9 100644 (file)
@@ -3,6 +3,7 @@ mainmodule kNeighbor {
   readonly int num_chares;
   readonly int gMsgSize;
   readonly int gLBFreq;
+  readonly int numSteps;
 
   message toNeighborMsg {
     int data[];