added CQWrite support for persistent message
authorYanhua Sun <yanhuas@jyc1.(none)>
Mon, 19 Mar 2012 07:10:59 +0000 (02:10 -0500)
committerYanhua Sun <yanhuas@jyc1.(none)>
Mon, 19 Mar 2012 07:10:59 +0000 (02:10 -0500)
examples/charm++/hello/1darray/hello.C
examples/converse/pingpong/pingpong.C
src/arch/gemini_gni/machine-persistent.c
src/arch/gemini_gni/machine.c
tests/charm++/pingpong/Makefile

index a983c5f3eaaa2d82da1a044704150236074dcda4..340b8f6ce3c4e50b6c106928a58aac5a4684aa4d 100644 (file)
@@ -16,8 +16,8 @@ public:
     delete m;
 
     //Start the computation
-    CkPrintf("Running Hello on %d processors for %d elements\n",
-            CkNumPes(),nElements);
+    CkPrintf("Running Hello on %d processors for %d elements, physical nodes=%d\n",
+            CkNumPes(),nElements,CmiNumPhysicalNodes());
     mainProxy = thisProxy;
 
     CProxy_Hello arr = CProxy_Hello::ckNew(nElements);
index 630751e28a755695b00baa2bd051d25c579c34e6..8c52a47ffaaa283686abb9f198944fcc30561ab2 100644 (file)
@@ -9,8 +9,8 @@
 #include <stdlib.h>
 #include <converse.h>
 
-enum {nCycles = 1000};
-enum { maxMsgSize = 1 << 14 };
+enum {nCycles = 10};
+enum { maxMsgSize = 1 << 20 };
 
 CpvDeclare(int,msgSize);
 CpvDeclare(int,cycleNum);
@@ -20,7 +20,7 @@ CpvDeclare(int,node1Handler);
 CpvStaticDeclare(double,startTime);
 CpvStaticDeclare(double,endTime);
 
-#define USE_PERSISTENT     0
+#define USE_PERSISTENT     1
 
 #if USE_PERSISTENT
 PersistentHandle h;
@@ -120,7 +120,7 @@ CmiStartFn mymain()
     CpvInitialize(int,msgSize);
     CpvInitialize(int,cycleNum);
     
-    CpvAccess(msgSize)= 512 + CmiMsgHeaderSizeBytes;
+    CpvAccess(msgSize)= 8192 + CmiMsgHeaderSizeBytes;
     
     CpvInitialize(int,exitHandler);
     CpvAccess(exitHandler) = CmiRegisterHandler((CmiHandler) exitHandlerFunc);
index 67a74cd8cccf273feefd22756bc47b3de2e01e39..06d5c264c6f17170168c9dd0416055c50c2467cb 100644 (file)
@@ -196,7 +196,11 @@ void *PerAlloc(int size)
   char *ptr = (char*)res+sizeof(CmiChunkHeader);
   SIZEFIELD(ptr)=size;
   REFFIELD(ptr)=1;
-  MEMORY_REGISTER(onesided_hnd, nic_hndl,  res, size , &MEMHFIELD(ptr), &omdh, NULL, status);
+#if  CQWRITE
+  MEMORY_REGISTER(onesided_hnd, nic_hndl,  res, size , &MEMHFIELD((char*)res+sizeof(CmiChunkHeader)), &omdh, rdma_rx_cqh, status);
+#else
+  MEMORY_REGISTER(onesided_hnd, nic_hndl,  res, size , &MEMHFIELD((char*)res+sizeof(CmiChunkHeader)), &omdh, NULL, status);
+#endif
   GNI_RC_CHECK("Mem Register before post", status);
   return ptr;
 }
index 1b218a070a56684ef31b5c48f597b2c6474e0019..f02e47e1f736bbe4759e1e0fe6cc9b76e00148ea 100644 (file)
@@ -2377,14 +2377,29 @@ static void PumpCqWriteTransactions()
 
     gni_cq_entry_t          ev;
     gni_return_t            status;
-    void                    *msg;   
+    void                    *msg;  
+    int                     msg_size;
     while(1) {
         //CMI_GNI_LOCK(my_cq_lock) 
         status = GNI_CqGetEvent(rdma_rx_cqh, &ev);
         //CMI_GNI_UNLOCK(my_cq_lock)
         if(status != GNI_RC_SUCCESS) break;
         msg = (void*) ( GNI_CQ_GET_DATA(ev) & 0xFFFFFFFFFFFFL);
-
+#if CMK_PERSISTENT_COMM
+#if PRINT_SYH
+        printf(" %d CQ write event %p\n", myrank, msg);
+#endif
+        if (!IsMemHndlZero(MEMHFIELD(msg))) {
+#if PRINT_SYH
+            printf(" %d Persistent CQ write event %p\n", myrank, msg);
+#endif
+            CmiReference(msg);
+            msg_size = CmiGetMsgSize(msg);
+            CMI_CHECK_CHECKSUM(msg, msg_size);
+            handleOneRecvedMsg(msg_size, msg); 
+            continue;
+        }
+#endif
         DecreaseMsgInSend(msg);
 #if ! USE_LRTS_MEMPOOL
        // MEMORY_DEREGISTER(onesided_hnd, nic_hndl, &(((ACK_MSG *)header)->source_mem_hndl), &omdh, ((ACK_MSG *)header)->length);
@@ -2493,11 +2508,17 @@ static void PumpLocalTransactions(gni_cq_handle_t my_tx_cqh, CmiNodeLock my_cq_l
                 }
                 else {
                     CmiFree((void *)tmp_pd->local_addr);
+#if     !CQWRITE
                     MallocControlMsg(ack_msg_tmp);
                     ack_msg_tmp->source_addr = tmp_pd->remote_addr;
                     ack_msg_tmp->source_mem_hndl    = tmp_pd->remote_mem_hndl;
                     ack_msg_tmp->length  = tmp_pd->length;
                     msg_tag = PUT_DONE_TAG;
+#else
+                    sendCqWrite(inst_id, tmp_pd->remote_addr, tmp_pd->remote_mem_hndl);
+                    FreePostDesc(tmp_pd);
+                    continue;
+#endif
                 }
                 break;
 #endif
@@ -2530,7 +2551,7 @@ static void PumpLocalTransactions(gni_cq_handle_t my_tx_cqh, CmiNodeLock my_cq_l
                 else
                 {
                     msg_tag = ACK_TAG; 
-#if  !REMOTE_EVENT && !CQWRITE
+#if  !REMOTE_EVENT && !inst_id, CQWRITE
                     MallocAckMsg(ack_msg);
                     ack_msg->source_addr = tmp_pd->remote_addr;
 #endif
index 847fa00b188321a8916e996cf70fe4bb9bb495a4..f1eb41364cf24619759b377f09dd55407dfcd087 100644 (file)
@@ -8,6 +8,9 @@ all:    pgm
 pgm: $(OBJS)
        $(CHARMC) -language charm++ -o pgm $(OBJS)
 
+pgm.prj: $(OBJS)
+       $(CHARMC) -tracemode projections -language charm++ -o pgm.prj $(OBJS)
+
 cifiles: pingpong.ci
        $(CHARMC)  pingpong.ci
        touch cifiles