use CldSimpleMultipleSend (one by one sending) since CldMultipleSend in ibverbs are...
authorGengbin Zheng <gzheng@illinois.edu>
Mon, 14 Mar 2011 05:46:46 +0000 (00:46 -0500)
committerGengbin Zheng <gzheng@illinois.edu>
Mon, 14 Mar 2011 05:46:46 +0000 (00:46 -0500)
also modified CldSimpleMultipleSend to work within immediate handlers.

src/conv-ldb/cldb.c
src/conv-ldb/cldb.h
src/conv-ldb/cldb.neighbor.c
src/conv-ldb/cldb.workstealing.c

index f3d59888a251df4d3e37d0faec8a99de419951ee..07d7eaf5809ec037e44b8b5e845b63272ea31caf 100644 (file)
@@ -365,7 +365,7 @@ void CldMultipleSend(int pe, int numToSend, int rank, int immed)
 }
 
 /* simple scheme - just send one by one. useful for multicore */
-void CldSimpleMultipleSend(int pe, int numToSend)
+void CldSimpleMultipleSend(int pe, int numToSend, int rank)
 {
   char *msg;
   int len, queueing, priobits, *msgSizes, i, numSent, done=0;
@@ -379,13 +379,13 @@ void CldSimpleMultipleSend(int pe, int numToSend)
   numSent = 0;
   while (!done) {
     for (i=0; i<numToSend; i++) {
-      CldGetToken(&msg);
+      CldGetTokenFromRank(&msg, rank);
       if (msg != 0) {
        done = 1;
        numToSend--;
        ifn = (CldInfoFn)CmiHandlerToFunction(CmiGetInfo(msg));
        ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
-       CldSwitchHandler(msg, CpvAccessOther(CldBalanceHandlerIndex, pe));
+       CldSwitchHandler(msg, CpvAccessOther(CldBalanceHandlerIndex, rank));
         CmiSyncSendAndFree(pe, len, msg);
         if (numToSend == 0) done = 1;
       }
index 19e0eb2a70f06289a0a98b97f9391840e07b91fd..fe32a496e41f5351af682d34016ef58483272e24 100644 (file)
@@ -14,7 +14,7 @@ CpvExtern(int, CldLoadNotify);
 CpvExtern(CmiNodeLock, cldLock);
 
 void CldMultipleSend(int pe, int numToSend, int rank, int immed);
-void CldSimpleMultipleSend(int pe, int numToSend);
+void CldSimpleMultipleSend(int pe, int numToSend, int rank);
 void CldSetPEBitVector(const char *);
 
 int  CldLoad(void);
index 9446c7053efb9f1477e5cef7e87ea1e95c2e9e00..3eaf653bd32494168329384a87fe8f2d00465e44 100644 (file)
@@ -284,20 +284,21 @@ void CldBalance(void *dummy, double curT)
           maxUnderAvg = avgLoad - CpvAccess(neighbors)[i].load;
         numUnderAvg++;
       }
-    if (numUnderAvg > 0)
+    if (numUnderAvg > 0)  {
+      int myrank = CmiMyRank();
       for (i=0; ((i<nNeighbors) && (overload>0)); i++) {
-          j = (i+CpvAccess(Mindex))%CpvAccess(numNeighbors);
-          if (CpvAccess(neighbors)[j].load < avgLoad) {
-              numToMove = (avgLoad - CpvAccess(neighbors)[j].load);
+        j = (i+CpvAccess(Mindex))%CpvAccess(numNeighbors);
+        if (CpvAccess(neighbors)[j].load < avgLoad) {
+          numToMove = (avgLoad - CpvAccess(neighbors)[j].load);
           if (numToMove > overload)
-            numToMove = overload;
+              numToMove = overload;
           overload -= numToMove;
          CpvAccess(neighbors)[j].load += numToMove;
 #if CMK_MULTICORE
-          CldSimpleMultipleSend(CpvAccess(neighbors)[j].pe, numToMove);
+          CldSimpleMultipleSend(CpvAccess(neighbors)[j].pe, numToMove, myrank);
 #else
           CldMultipleSend(CpvAccess(neighbors)[j].pe, 
-                         numToMove, CmiMyRank()
+                         numToMove, myrank
 #if CMK_SMP
                          0
 #else
@@ -307,6 +308,7 @@ void CldBalance(void *dummy, double curT)
 #endif
         }
       }
+    }             /* end of numUnderAvg > 0 */
   }
   CldSendLoad();
 #if CMK_TRACE_ENABLED && TRACE_USEREVENTS
index fe1335a06581f092241d2ce7d40f0d354ff00460..b611ccd2ff72fef270d2d025f1c4471fa0159e2c 100644 (file)
@@ -108,7 +108,11 @@ static void CldAskLoadHandler(requestmsg *msg)
       if(_stealonly1) sendLoad = 1;
       else sendLoad = myload/2; 
       if(sendLoad > 0)
+#if ! CMK_USE_IBVERBS
           CldMultipleSend(receiver, sendLoad, rank, 0);
+#else
+          CldSimpleMultipleSend(receiver, sendLoad, rank);
+#endif
       CmiFree(msg);
   }else
   {     /* send ack indicating there is no task */