fix message registration bug in persistent
authorGengbin Zheng <gzheng@illinois.edu>
Mon, 19 Mar 2012 06:54:14 +0000 (23:54 -0700)
committerGengbin Zheng <gzheng@illinois.edu>
Mon, 19 Mar 2012 06:54:14 +0000 (23:54 -0700)
src/arch/gemini_gni/machine-persistent.c
src/arch/gemini_gni/machine-persistent.h
src/arch/gemini_gni/machine.c
src/arch/util/machine-common-core.c

index 2064c38e247e1d88c67776fd3cfcad11ad183f98..67a74cd8cccf273feefd22756bc47b3de2e01e39 100644 (file)
@@ -193,9 +193,12 @@ void *PerAlloc(int size)
   if (0 != posix_memalign(&res, 64, size))
       CmiAbort("PerAlloc: failed to allocate memory.");
   //printf("[%d] PerAlloc %p. \n", myrank, res);
-  MEMORY_REGISTER(onesided_hnd, nic_hndl,  res, size , &MEMHFIELD((char*)res+sizeof(CmiChunkHeader)), &omdh, NULL, status);
+  char *ptr = (char*)res+sizeof(CmiChunkHeader);
+  SIZEFIELD(ptr)=size;
+  REFFIELD(ptr)=1;
+  MEMORY_REGISTER(onesided_hnd, nic_hndl,  res, size , &MEMHFIELD(ptr), &omdh, NULL, status);
   GNI_RC_CHECK("Mem Register before post", status);
-  return (char*)res+sizeof(CmiChunkHeader);
+  return ptr;
 }
                                                                                 
 void PerFree(char *msg)
index cc6f2918dda1dd52aa1971e0547f70d0a4a04deb..cfe872fda72ad6006a8d7428acf1b55009ee78b1 100644 (file)
@@ -12,7 +12,7 @@
 
 #define PERSIST_BUFFERS_NUM             1
 
-#define PERSIST_SEQ                     0xFFFFFFFFFFFF
+#define PERSIST_SEQ                     0xFFFFFFF
 
 typedef struct  _PersistentBuf {
   void *destAddress;
index 713711898f7d51ffa8ba30fb642f89ac91bd2c5e..1b218a070a56684ef31b5c48f597b2c6474e0019 100644 (file)
@@ -2047,6 +2047,9 @@ static void PumpNetworkSmsg()
                 CmiReference(msg);
                 CMI_CHECK_CHECKSUM(msg, size);
                 handleOneRecvedMsg(size, msg); 
+#if PRINT_SYH
+                printf("[%d] PUT_DONE_TAG hand over one message, size: %d. \n", myrank, size);
+#endif
                 break;
             }
 #endif
@@ -2070,7 +2073,7 @@ static void PumpNetworkSmsg()
                      }
             }               // end switch
 #if PRINT_SYH
-            printf("[%d] from %d after switch request for Large msg is received, messageid: tag=%d\n", myrank, inst_id, msg_tag);
+            printf("[%d] from %d after switch request for smsg is received, messageid: tag=%d\n", myrank, inst_id, msg_tag);
 #endif
             smsg_recv_count ++;
             msg_tag = GNI_SMSG_ANY_TAG;
@@ -2116,12 +2119,9 @@ static gni_return_t  registerMessage(void *msg, int size, int seqno, gni_mem_han
 
 #if CMK_PERSISTENT_COMM
         // persistent message is always registered
-    if (seqno == PERSIST_SEQ) {
-        if (!IsMemHndlZero(MEMHFIELD(msg))) {
-            *memh = MEMHFIELD(msg);
-            return GNI_RC_SUCCESS;
-        }
-        CmiAbort("registerMessage> persistent memory should already be registered.");
+    if (!IsMemHndlZero(MEMHFIELD(msg))) {
+        *memh = MEMHFIELD(msg);
+        return GNI_RC_SUCCESS;
     }
 #endif
     if(seqno == 0)
@@ -2496,6 +2496,7 @@ static void PumpLocalTransactions(gni_cq_handle_t my_tx_cqh, CmiNodeLock my_cq_l
                     MallocControlMsg(ack_msg_tmp);
                     ack_msg_tmp->source_addr = tmp_pd->remote_addr;
                     ack_msg_tmp->source_mem_hndl    = tmp_pd->remote_mem_hndl;
+                    ack_msg_tmp->length  = tmp_pd->length;
                     msg_tag = PUT_DONE_TAG;
                 }
                 break;
@@ -2732,6 +2733,9 @@ static void  SendRdmaMsg()
 #if MACHINE_DEBUG_LOG
                 buffered_recv_msg += register_size;
                 MACHSTATE(8, "GO request from buffered\n"); 
+#endif
+#if PRINT_SYH
+                printf("[%d] SendRdmaMsg: post succeed. seqno: %d\n", myrank, pd->cqwrite_value);
 #endif
             }else           // cannot post
             {
@@ -2740,6 +2744,9 @@ static void  SendRdmaMsg()
 #else
                 pre = ptr;
                 ptr = ptr->next;
+#endif
+#if PRINT_SYH
+                printf("[%d] SendRdmaMsg: post failed. seqno: %d\n", myrank, pd->cqwrite_value);
 #endif
                 break;
             }
index 24f49374aa5fa754f3fa6807ac08eb2781976675..ea713be0aa9baaf71d3317df9be615aa240d3499 100644 (file)
@@ -526,7 +526,7 @@ void CmiFreeSendFn(int destPE, int size, char *msg) {
     } else {
 #if CMK_PERSISTENT_COMM
         if (CpvAccess(phs)) {
-          if (size > 8192) {
+          if (size > 1024) {
             CmiAssert(CpvAccess(curphs) < CpvAccess(phsSize));
             int destNode = CmiNodeOf(destPE);
             CMI_DEST_RANK(msg) = CmiRankOf(destPE);