Avoid large messages in memopt atom distribution 61/4461/1
authorJim Phillips <jim@ks.uiuc.edu>
Wed, 8 Aug 2018 19:15:47 +0000 (14:15 -0500)
committerJim Phillips <jim@ks.uiuc.edu>
Wed, 8 Aug 2018 19:19:53 +0000 (14:19 -0500)
When sending atoms from input pes to patches during startup of
memory optimized builds, sending one message per pe can result
in very large messages that break gni persistent messages in
Converse, so send one message per patch instead.

Change-Id: If134460983309da2e6add79d3117ee07d3fae59e

src/ParallelIOMgr.C

index b4a733c..48470d3 100644 (file)
@@ -1298,32 +1298,24 @@ void ParallelIOMgr::sendAtomsToHomePatchProcs()
         int len = procList[i].size();
         if(len==0) continue;
 
-        if ( numAcksOutstanding >= 10 ) {
-          //CkPrintf("Pe %d ParallelIOMgr::sendAtomsToHomePatchProcs suspending at %d of %d pes\n",CkMyPe(),k,pesToSend.size());
-          //fflush(stdout);
-          sendAtomsThread = CthSelf();
-          CthSuspend();
-        }
-        ++numAcksOutstanding;
-
-        //prepare a message to send
-        int patchCnt = len;
-        int totalAtomCnt = 0;
+        // Sending one message per pe can result in very large messages
+        // that break Converse so send one message per patch instead.
         for(int j=0; j<len; j++) {
             int pid = procList[i][j];
             int atomCnt = eachPatchAtomList[pid].size();
-            totalAtomCnt += atomCnt;
-        }
 
-        MovePatchAtomsMsg *msg = new (patchCnt, patchCnt, totalAtomCnt, 0)MovePatchAtomsMsg;
-        msg->from = CkMyPe();
-        msg->patchCnt = patchCnt;
-        int atomIdx = 0;
-        for(int j=0; j<len; j++) {
-            int pid = procList[i][j];
-            int atomCnt = eachPatchAtomList[pid].size();
-            msg->pidList[j] = pid;
-            msg->sizeList[j] = atomCnt;
+            if ( numAcksOutstanding >= 10 ) {
+              sendAtomsThread = CthSelf();
+              CthSuspend();
+            }
+            ++numAcksOutstanding;
+
+            MovePatchAtomsMsg *msg = new (1, 1, atomCnt, 0)MovePatchAtomsMsg;
+            msg->from = CkMyPe();
+            msg->patchCnt = 1;
+            int atomIdx = 0;
+            msg->pidList[0] = pid;
+            msg->sizeList[0] = atomCnt;
             for(int k=0; k<atomCnt; k++, atomIdx++) {
                 int aid = eachPatchAtomList[pid][k];
                 FullAtom one = initAtoms[aid];
@@ -1332,8 +1324,8 @@ void ParallelIOMgr::sendAtomsToHomePatchProcs()
                 one.hydVal = initAtoms[aid].hydList;
                 msg->allAtoms[atomIdx] = one;
             }
+            pIO[i].recvAtomsToHomePatchProcs(msg);
         }
-        pIO[i].recvAtomsToHomePatchProcs(msg);
 
         procList[i].clear();
     }