use pxshm before using persistent
authorGengbin Zheng <gzheng@illinois.edu>
Wed, 21 Mar 2012 02:40:53 +0000 (21:40 -0500)
committerGengbin Zheng <gzheng@illinois.edu>
Wed, 21 Mar 2012 02:40:53 +0000 (21:40 -0500)
src/arch/util/machine-broadcast.c
src/arch/util/machine-common-core.c
src/arch/util/machine-lrts.h

index 0f965a7fd0724b833257ff678ede1591323cf6a2..b6f5d52461d8b553c6256bf6e45ca5c48a87813a 100644 (file)
@@ -1,3 +1,9 @@
+/**
+ *        functions for broadcast
+**/
+
+CmiCommHandle CmiSendNetworkFunc(int destNode, int size, char *msg, int mode);
+
 static void handleOneBcastMsg(int size, char *msg) {
     CmiAssert(CMI_BROADCAST_ROOT(msg)!=0);
 #if CMK_OFFLOAD_BCAST_PROCESS
@@ -106,10 +112,10 @@ static void SendSpanningChildren(int size, char *msg, int rankToAssign, int star
         CmiAssert(nd>=0 && nd!=CmiMyNode());
 #if CMK_BROADCAST_USE_CMIREFERENCE
         CmiReference(msg);
-        LrtsSendNetworkFunc(nd, size, msg, BCAST_SYNC);
+        CmiSendNetworkFunc(nd, size, msg, BCAST_SYNC);
 #else
         newmsg = CopyMsg(msg, size);
-        LrtsSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
+        CmiSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
 #endif
     }
     CMI_DEST_RANK(msg) = oldRank;
@@ -154,10 +160,10 @@ static void SendHyperCube(int size,  char *msg, int rankToAssign, int startNode)
         CmiAssert(nd>=0 && nd!=CmiMyNode());
 #if CMK_BROADCAST_USE_CMIREFERENCE
         CmiReference(msg);
-        LrtsSendNetworkFunc(nd, size, msg, BCAST_SYNC);
+        CmiSendNetworkFunc(nd, size, msg, BCAST_SYNC);
 #else
         char *newmsg = CopyMsg(msg, size);
-        LrtsSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
+        CmiSendNetworkFunc(nd, size, newmsg, BCAST_SYNC);
 #endif
     }
     CMI_DEST_RANK(msg) = oldRank;
@@ -354,3 +360,74 @@ void CmiFreeNodeBroadcastAllFn(int size, char *msg) {
 }
 #endif
 /* ##### End of Functions Related with Message Sending OPs ##### */
+
+#if ! CMK_MULTICAST_LIST_USE_COMMON_CODE
+
+void CmiSyncListSendFn(int npes, int *pes, int len, char *msg)
+{
+  int i;
+#if CMK_BROADCAST_USE_CMIREFERENCE
+  for(i=0;i<npes;i++) {
+    if (pes[i] == CmiMyPe())
+      CmiSyncSend(pes[i], len, msg);
+    else {
+      CmiReference(msg);
+      CmiSyncSendAndFree(pes[i], len, msg);
+    }
+  }
+#else
+  for(i=0;i<npes;i++) {
+    CmiSyncSend(pes[i], len, msg);
+  }
+#endif
+}
+
+CmiCommHandle CmiAsyncListSendFn(int npes, int *pes, int len, char *msg)
+{
+  /* A better asynchronous implementation may be wanted, but at least it works */
+  CmiSyncListSendFn(npes, pes, len, msg);
+  return (CmiCommHandle) 0;
+}
+
+void CmiFreeListSendFn(int npes, int *pes, int len, char *msg)
+{
+  if (npes == 1) {
+      CmiSyncSendAndFree(pes[0], len, msg);
+      return;
+  }
+#if CMK_PERSISTENT_COMM
+  if (CpvAccess(phs) && len > 1024) {
+      int i;
+      for(i=0;i<npes;i++) {
+        if (pes[i] == CmiMyPe())
+          CmiSyncSend(pes[i], len, msg);
+        else {
+          CmiReference(msg);
+          CmiSyncSendAndFree(pes[i], len, msg);
+        }
+      }
+      CmiFree(msg);
+      return;
+  }
+#endif
+  
+#if CMK_BROADCAST_USE_CMIREFERENCE
+  if (npes == 1) {
+    CmiSyncSendAndFree(pes[0], len, msg);
+    return;
+  }
+  CmiSyncListSendFn(npes, pes, len, msg);
+  CmiFree(msg);
+#else
+  int i;
+  for(i=0;i<npes-1;i++) {
+    CmiSyncSend(pes[i], len, msg);
+  }
+  if (npes>0)
+    CmiSyncSendAndFree(pes[npes-1], len, msg);
+  else 
+    CmiFree(msg);
+#endif
+}
+
+#endif
index 245b591280cb52da9c456e0d19d60e21fea13fbd..c207e78105c26482441fa0448a2c9e1168f2b2a5 100644 (file)
@@ -477,7 +477,7 @@ void CmiSyncSendFn(int destPE, int size, char *msg) {
 #include "machine-xpmem.c"
 #endif
 
-int refcount = 0;
+static int refcount = 0;
 
 #if CMK_USE_OOB
 CpvExtern(int, _urgentSend);
@@ -487,7 +487,7 @@ CpvExtern(int, _urgentSend);
 #if CMK_C_INLINE
 inline 
 #endif
-CmiCommHandle LrtsSendNetworkFunc(int destNode, int size, char *msg, int mode)
+CmiCommHandle CmiSendNetworkFunc(int destNode, int size, char *msg, int mode)
 {
         int rank;
 #if CMK_USE_PXSHM
@@ -504,6 +504,15 @@ CmiCommHandle LrtsSendNetworkFunc(int destNode, int size, char *msg, int mode)
           return 0;
         }
 #endif
+#if CMK_PERSISTENT_COMM
+        if (CpvAccess(phs)) {
+          if (size > 1024) {
+            CmiAssert(CpvAccess(curphs) < CpvAccess(phsSize));
+            LrtsSendPersistentMsg(CpvAccess(phs)[CpvAccess(curphs)], destNode, size, msg);
+            return 0;
+          }
+        }
+#endif
 
 #if CMK_WITH_STATS
 if (MSG_STATISTIC)
@@ -527,29 +536,24 @@ void CmiFreeSendFn(int destPE, int size, char *msg) {
 #if CMK_PERSISTENT_COMM
         if (CpvAccess(phs)) CpvAccess(curphs)++;
 #endif
-    } else {
-#if CMK_PERSISTENT_COMM
-        if (CpvAccess(phs)) {
-          if (size > 1024) {
-            CmiAssert(CpvAccess(curphs) < CpvAccess(phsSize));
-            int destNode = CmiNodeOf(destPE);
-            CMI_DEST_RANK(msg) = CmiRankOf(destPE);
-            LrtsSendPersistentMsg(CpvAccess(phs)[CpvAccess(curphs)++], destNode, size, msg);
-            return;
-          }
-          else
-            CpvAccess(curphs)++;
-        }
-#endif
+    } 
+    else {
         int destNode = CmiNodeOf(destPE);
+        int destRank = CmiRankOf(destPE);
 #if CMK_SMP
         if (CmiMyNode()==destNode) {
-            CmiPushPE(CmiRankOf(destPE), msg);
+            CmiPushPE(destRank, msg);
+#if CMK_PERSISTENT_COMM
+        if (CpvAccess(phs)) CpvAccess(curphs)++;
+#endif
             return;
         }
 #endif
-        CMI_DEST_RANK(msg) = CmiRankOf(destPE);
-        LrtsSendNetworkFunc(destNode, size, msg, P2P_SYNC);
+        CMI_DEST_RANK(msg) = destRank;
+        CmiSendNetworkFunc(destNode, size, msg, P2P_SYNC);
+#if CMK_PERSISTENT_COMM
+        if (CpvAccess(phs)) CpvAccess(curphs)++;
+#endif
     }
 }
 #endif
index bf628b59bb2db68642ead02ce94f6a4831e99401..0f793e1d87ab0918bc438d19c218ece12a4d261c 100644 (file)
@@ -5,7 +5,6 @@ void LrtsPrepareEnvelope(char *msg, int size);
 
 /* The machine-specific send function */
 CmiCommHandle LrtsSendFunc(int destNode, int size, char *msg, int mode);
-CmiCommHandle LrtsSendNetworkFunc(int destNode, int size, char *msg, int mode);
 
 #if CMK_PERSISTENT_COMM
 void LrtsSendPersistentMsg(PersistentHandle h, int destPE, int size, void *m);