fixed bugs in persistent comm. Now it works on Gemini.
authorGengbin Zheng <gzheng@illinois.edu>
Tue, 20 Sep 2011 21:37:13 +0000 (14:37 -0700)
committerGengbin Zheng <gzheng@illinois.edu>
Tue, 20 Sep 2011 21:37:13 +0000 (14:37 -0700)
added pingpong test for persistent
compile with persistent using "persistent" build option.

examples/converse/pingpong/pingpong.C
src/arch/common/conv-mach-persistent.h [new file with mode: 0644]
src/arch/common/conv-mach-persistent.sh [new file with mode: 0644]
src/arch/gemini_gni/conv-common.h
src/arch/gemini_gni/machine-persistent.c
src/arch/gemini_gni/machine-persistent.h
src/arch/gemini_gni/machine.c
src/arch/util/machine-common-core.c
src/arch/util/machine-pxshm.c
src/arch/util/persist-comm.c

index 66e0eb5d9d38c6a2a281b9d12977a7a33bfdcda4..a218543df5161e984e2f482a7b97b039ecd5b8d6 100644 (file)
@@ -10,7 +10,7 @@
 #include <converse.h>
 
 enum {nCycles =4096};
-enum { maxMsgSize = 1 << 16 };
+enum { maxMsgSize = 1 << 17 };
 
 CpvDeclare(int,msgSize);
 CpvDeclare(int,cycleNum);
@@ -20,13 +20,19 @@ CpvDeclare(int,node1Handler);
 CpvStaticDeclare(double,startTime);
 CpvStaticDeclare(double,endTime);
 
+#define USE_PERSISTENT     0
+
+#if USE_PERSISTENT
+PersistentHandle h;
+#endif
+
 // Start the pingpong for each message size
 void startRing()
 {
   CpvAccess(cycleNum) = 0;
 
   //Increase message in powers of 4. Also add a converse header to that
-  CpvAccess(msgSize) = (CpvAccess(msgSize)-CmiMsgHeaderSizeBytes)*4 + 
+  CpvAccess(msgSize) = (CpvAccess(msgSize)-CmiMsgHeaderSizeBytes)*2 + 
       CmiMsgHeaderSizeBytes;
 
   char *msg = (char *)CmiAlloc(CpvAccess(msgSize));
@@ -81,7 +87,13 @@ CmiHandler node0HandlerFunc(char *msg)
         CmiSetHandler(msg,CpvAccess(node1Handler));
         *((int *)(msg+CmiMsgHeaderSizeBytes)) = CpvAccess(msgSize);
         
+#if USE_PERSISTENT
+        CmiUsePersistentHandle(&h, 1);
+#endif
         CmiSyncSendAndFree(1,CpvAccess(msgSize),msg);
+#if USE_PERSISTENT
+        CmiUsePersistentHandle(NULL, 0);
+#endif
     }
     return 0;
 }
@@ -91,7 +103,13 @@ CmiHandler node1HandlerFunc(char *msg)
     CpvAccess(msgSize) = *((int *)(msg+CmiMsgHeaderSizeBytes));
     CmiSetHandler(msg,CpvAccess(node0Handler));
     
+#if USE_PERSISTENT
+    CmiUsePersistentHandle(&h, 1);
+#endif
     CmiSyncSendAndFree(0,CpvAccess(msgSize),msg);
+#if USE_PERSISTENT
+    CmiUsePersistentHandle(NULL, 0);
+#endif
     return 0;
 }
 
@@ -116,6 +134,9 @@ CmiStartFn mymain()
     
     int otherPe = CmiMyPe() ^ 1;
     
+#if USE_PERSISTENT
+    h = CmiCreatePersistent(otherPe, maxMsgSize+1024);
+#endif
     
     if (CmiMyPe() == 0)
         startRing();
diff --git a/src/arch/common/conv-mach-persistent.h b/src/arch/common/conv-mach-persistent.h
new file mode 100644 (file)
index 0000000..70b1632
--- /dev/null
@@ -0,0 +1,3 @@
+
+#undef  CMK_PERSISTENT_COMM
+#define   CMK_PERSISTENT_COMM                       1
diff --git a/src/arch/common/conv-mach-persistent.sh b/src/arch/common/conv-mach-persistent.sh
new file mode 100644 (file)
index 0000000..e69de29
index cc46b9626035b5ab8c151fe4334b580eca614cc1..b13332e8edcb42dc5daeedd079a973205b81564b 100644 (file)
@@ -36,5 +36,3 @@
 
 #define CMK_LB_CPUTIMER                                           0
 
-#define CMK_PERSISTENT_COMM                                1
-
index bfb2c907cbdc4d1dc83ef7b6b4355c09cc6f1bc7..3ae2370e4a806f9b5e4f1ade6598eaad8133df18 100644 (file)
@@ -33,18 +33,17 @@ void LrtsSendPersistentMsg(PersistentHandle h, int destPE, int size, void *m)
         CmiAbort("Abort: Invalid size\n");
     }
 
-/*CmiPrintf("[%d] LrtsSendPersistentMsg h=%p hdl=%d destPE=%d destAddress=%p size=%d\n", CmiMyPe(), *phs, CmiGetHandler(m), destPE, slot->destAddress[0], size);*/
+    /* CmiPrintf("[%d] LrtsSendPersistentMsg h=%p hdl=%d destPE=%d destAddress=%p size=%d\n", CmiMyPe(), h, CmiGetHandler(m), destPE, slot->destBuf[0].destAddress, size); */
 
     if (slot->destBuf[0].destAddress) {
         // uGNI part
-     
         MallocPostDesc(pd);
-        if(size < LRTS_GNI_RDMA_THRESHOLD) 
+        if(size < LRTS_GNI_RDMA_PUT_THRESHOLD) 
             pd->type            = GNI_POST_FMA_PUT;
         else
             pd->type            = GNI_POST_RDMA_PUT;
 
-        pd->cq_mode         = GNI_CQMODE_GLOBAL_EVENT |  GNI_CQMODE_REMOTE_EVENT;
+        pd->cq_mode         = GNI_CQMODE_GLOBAL_EVENT;
         pd->dlvr_mode       = GNI_DLVMODE_PERFORMANCE;
         pd->length          = size;
         pd->local_addr      = (uint64_t) m;
index bdc07cddfddaaf63c2f4544ce5cf15f92efb2712..7100af204ddd45b3083e4e4cfe05fbfae7ea5d38 100644 (file)
@@ -41,6 +41,7 @@ extern PersistentReceivesTable *persistentReceivesTableTail;
 
 extern PersistentHandle  *phs;
 extern int phsSize;
+extern int curphs;
 
 void *PerAlloc(int size);
 void PerFree(char *msg);
index fc1f4aac6197f524623fee5cac13d8ca1f33fcda..0017b981bf7177e1d65440263ff2cfab76025037 100644 (file)
@@ -99,6 +99,7 @@ static int  SMSG_MAX_MSG;
 #define MSGQ_MAXSIZE       2048
 /* large message transfer with FMA or BTE */
 #define LRTS_GNI_RDMA_THRESHOLD  2048
+#define LRTS_GNI_RDMA_PUT_THRESHOLD  2048
 
 #define REMOTE_QUEUE_ENTRIES  2048 
 #define LOCAL_QUEUE_ENTRIES   64 
@@ -980,11 +981,25 @@ static void PumpNetworkSmsg()
                 SendRdmaMsg();
                 break;
             }
+#if CMK_PERSISTENT_COMM
             case PUT_DONE_TAG: //persistent message
             {
-                handleOneRecvedMsg(((CONTROL_MSG *) header)->length,((void*) (CONTROL_MSG *) header)); 
+                void *msg = (void *)((CONTROL_MSG *) header)->source_addr;
+                int size = ((CONTROL_MSG *) header)->length;
+#if 0
+                void *dupmsg;
+                dupmsg = CmiAlloc(size);
+                _MEMCHECK(dupmsg);
+                memcpy(dupmsg, msg, size);
+                msg = dupmsg;
+
+#else
+                CmiReference(msg);
+#endif
+                handleOneRecvedMsg(size, msg); 
                 break;
             }
+#endif
             default: {
                 CmiPrintf("weird tag problem\n");
                 CmiAbort("Unknown tag\n");
@@ -1180,23 +1195,27 @@ static void PumpLocalRdmaTransactions()
             //status = GNI_GetCompleted(post_tx_cqh, ev, &tmp_pd);
             status = GNI_GetCompleted(smsg_tx_cqh, ev, &tmp_pd);
             ////Message is sent, free message , put is not used now
-            if(tmp_pd->type == GNI_POST_RDMA_PUT || tmp_pd->type == GNI_POST_FMA_PUT)
-            {
+            switch (tmp_pd->type) {
+#if CMK_PERSISTENT_COMM
+            case GNI_POST_RDMA_PUT:
+            case GNI_POST_FMA_PUT:
 #if useDynamicSMSG
                 SendSmsgConnectMsg();
                 if(tmp_pd->length == sizeof(gni_smsg_attr_t))
                     continue;
 #endif
-                //else  
-                {
-                    //persistent message 
-                    CmiFree((void *)tmp_pd->local_addr);
-                    msg_tag = PUT_DONE_TAG; 
-                }
-            }else if(tmp_pd->type == GNI_POST_RDMA_GET || tmp_pd->type == GNI_POST_FMA_GET)
-            {
+                CmiFree((void *)tmp_pd->local_addr);
+                msg_tag = PUT_DONE_TAG;
+                break;
+#endif
+            case GNI_POST_RDMA_GET:
+            case GNI_POST_FMA_GET:
                 msg_tag = ACK_TAG;  
+                break;
+            default:
+                CmiAbort("PumpLocalRdmaTransactions: unknown type!");
             }
+
             MallocControlMsg(ack_msg_tmp);
             ack_msg_tmp->source             = myrank;
             ack_msg_tmp->source_addr        = tmp_pd->remote_addr;
@@ -1214,13 +1233,19 @@ static void PumpLocalRdmaTransactions()
 #if     !USE_LRTS_MEMPOOL
             MEMORY_DEREGISTER(onesided_hnd, nic_hndl, &tmp_pd->local_mem_hndl, &omdh);
 #endif
-            CmiAssert(SIZEFIELD((void*)(tmp_pd->local_addr)) <= tmp_pd->length);
-            handleOneRecvedMsg(tmp_pd->length, (void*)tmp_pd->local_addr); 
-            SendRdmaMsg(); 
+#if CMK_PERSISTENT_COMM
+            if (tmp_pd->type == GNI_POST_RDMA_GET || tmp_pd->type == GNI_POST_FMA_GET)
+#endif
+            {
+              CmiAssert(SIZEFIELD((void*)(tmp_pd->local_addr)) <= tmp_pd->length);
+              handleOneRecvedMsg(tmp_pd->length, (void*)tmp_pd->local_addr); 
+              SendRdmaMsg(); 
+            }
             FreePostDesc(tmp_pd);
         }
     } //end while
 }
+
 static void SendSmsgConnectMsg()
 {
     gni_return_t            status = GNI_RC_SUCCESS;
@@ -1249,8 +1274,8 @@ static void SendSmsgConnectMsg()
         prev = ptr;
         ptr = ptr->next;
     } //end while
 }
+
 static void  SendRdmaMsg()
 {
     gni_return_t            status = GNI_RC_SUCCESS;
index 76096057799b4c1a45e1e2e50612d9f7fecb0de4..b2ec35c8988486a5bd73b70e40a4d2669c83f8c2 100644 (file)
@@ -455,6 +455,9 @@ void CmiFreeSendFn(int destPE, int size, char *msg) {
     CQdCreate(CpvAccess(cQdState), 1);
     if (CmiMyPe()==destPE) {
         CmiSendSelf(msg);
+#if CMK_PERSISTENT_COMM
+        if (phs) curphs++;
+#endif
     } else {
 #if CMK_USE_PXSHM
         int ret=CmiValidPxshm(destPE, size);
@@ -462,13 +465,16 @@ void CmiFreeSendFn(int destPE, int size, char *msg) {
           CMI_DEST_RANK(msg) = CmiRankOf(destPE);
           CmiSendMessagePxshm(msg, size, destPE, &refcount, CmiRankOf(destPE), 0);
           //for (int i=0; i<refcount; i++) CmiReference(msg);
+#if CMK_PERSISTENT_COMM
+        if (phs) curphs++;
+#endif
           return;
         }
 #endif
 #if CMK_PERSISTENT_COMM
-    if (phs) {
-        CmiAssert(phsSize == 1);
-        LrtsSendPersistentMsg(*phs, destPE, size, msg);
+    if (phs && size > 8192) {
+        CmiAssert(curphs < phsSize);
+        LrtsSendPersistentMsg(phs[curphs++], destPE, size, msg);
         return;
     }
 #endif
index a18af3a735ddf5e6f529674b4999b5e58f6229ca..88d19abcfebfbdb2f9de44c2f6a68e6c659ec5e7 100644 (file)
@@ -100,9 +100,10 @@ enum entities {SENDER,RECEIVER};
 #define NAMESTRLEN 60
 #define PREFIXSTRLEN 50 
 
-#define SHMBUFLEN (1024*1024*1)
+#define SHMBUFLEN (1024*1024*4)
+#define SHMMAXSIZE     (1024*1024)
 
-#define SENDQSTARTSIZE    128
+#define SENDQSTARTSIZE    256
 
 
 /// This struct is used as the first portion of a shared memory region, followed by data
@@ -287,7 +288,7 @@ inline int CmiValidPxshm(int dst, int size){
        }*/
        //replace by bitmap later
        //if(ogm->dst >= pxshmContext->nodestart && ogm->dst <= pxshmContext->nodeend && ogm->size < SHMBUFLEN ){
-       if(dst >= pxshmContext->nodestart && dst <= pxshmContext->nodeend && size < SHMBUFLEN ){
+       if(dst >= pxshmContext->nodestart && dst <= pxshmContext->nodeend && size < SHMMAXSIZE ){
                return 1;
        }else{
                return 0;
@@ -802,8 +803,8 @@ void emptyRecvBuf(sharedBufData *recvBuf){
 }
 
 
-void static inline handoverPxshmMessage(char *newmsg,int total_size,int rank,int broot){
 #if CMK_NET_VERSION
+void static inline handoverPxshmMessage(char *newmsg,int total_size,int rank,int broot){
        CmiAssert(rank == 0);
 #if CMK_BROADCAST_SPANNING_TREE
         if (rank == DGRAM_BROADCAST
@@ -834,8 +835,8 @@ void static inline handoverPxshmMessage(char *newmsg,int total_size,int rank,int
           CmiPushPE(rank, newmsg);
                                }
        }    /* end of switch */
-#endif
 }
+#endif
 
 
 /**************************
index e493887b35014337f0f2b5499d3a1830d8f61fff..f23c7ae1a2d70ae48a414ed2a60c5a33c4362083 100644 (file)
@@ -53,6 +53,7 @@ int persistentDestoryHandlerIdx;
 
 PersistentHandle  *phs = NULL;
 int phsSize;
+int curphs = 0;
 
 /******************************************************************************
      Utilities
@@ -230,11 +231,15 @@ static void persistentRequestHandler(void *env)
 static void persistentReqGrantedHandler(void *env)
 {
   int i;
-  /*CmiPrintf("Persistent handler granted\n");*/
+
+
   PersistentReqGrantedMsg *msg = (PersistentReqGrantedMsg *)env;
   PersistentHandle h = msg->sourceHandlerIndex;
   PersistentSendsTable *slot = (PersistentSendsTable *)h;
   CmiAssert(slot->used == 1);
+
+  /* CmiPrintf("[%d] Persistent handler granted  h:%p\n", CmiMyPe(), h); */
+
   for (i=0; i<PERSIST_BUFFERS_NUM; i++) {
 #if 0
     slot->destAddress[i] = msg->msgAddr[i];
@@ -419,6 +424,7 @@ void CmiUsePersistentHandle(PersistentHandle *p, int n)
 #endif
   phs = p;
   phsSize = n;
+  curphs = 0;
 }
 
 #endif