CmiAlloc calls LrtsAlloc
authorGengbin Zheng <gzheng@illinois.edu>
Thu, 4 Aug 2011 02:50:31 +0000 (19:50 -0700)
committerGengbin Zheng <gzheng@illinois.edu>
Thu, 4 Aug 2011 02:50:31 +0000 (19:50 -0700)
src/arch/gemini_gni/machine.c
src/arch/util/machine-lrts.h
src/conv-core/convcore.c

index 665b990cf44731062290f54fae062c1f8dc2d221..20612184277e5665e8be72cc766fd413a9a70bea 100644 (file)
@@ -148,9 +148,6 @@ static MSG_LIST *buffered_fma_tail = 0;
 /* functions  */
 
 
-static void* LrtsAlloc(int);
-static void  LrtsFree(void*);
-
 static void* LrtsAllocRegister(int n_bytes, gni_mem_handle_t* mem_hndl);
 
 static void*  aligned_memory_alloc(size_t size, size_t alignment)
@@ -516,7 +513,7 @@ static void PumpNetworkMsgs()
         if(msg_tag == data_tag)
         {
             msg_nbytes = *(int*)header;
-            msg_data = LrtsAlloc(msg_nbytes);
+            msg_data = CmiAlloc(msg_nbytes);
             memcpy(msg_data, (char*)header+sizeof(int), msg_nbytes);
             handleOneRecvedMsg(msg_nbytes, msg_data);
         }else if(msg_tag == control_tag)
@@ -562,7 +559,7 @@ static void PumpNetworkMsgs()
             CmiFree((void*)request_msg->source_addr);
             //CmiPrintf("++++## release ACK msg is received on PE:%d message size=%d\n", myrank, request_msg->length);
         }else{
-            CmiPrintf("weird tag problne\n");
+            CmiPrintf("weird tag problem\n");
             CmiAbort("Unknown tag\n");
         }
         GNI_SmsgRelease(ep_hndl_array[inst_id]);
@@ -614,7 +611,7 @@ static void PumpLocalTransactions()
            status = GNI_PostCqWrite(ep_hndl_array[inst_id], &ack_pd);
            GNI_RC_CHECK("Ack Post by CQWrite ", status);
            */
-           // CmiPrintf("\nPE:%d Received large message by get , sizefield=%d, length=%d, addr=%p\n", myrank, SIZEFIELD((void*)tmp_pd->local_addr), tmp_pd->length, tmp_pd->remote_addr); 
+            //CmiPrintf("\nPE:%d Received large message by get , sizefield=%d, length=%d, addr=%p\n", myrank, SIZEFIELD((void*)tmp_pd->local_addr), tmp_pd->length, tmp_pd->remote_addr); 
            //CmiPrintf("\n+PE:%d Received large message by get , sizefield=%d, length=%d, addr=%p\n", myrank, remote_length , tmp_pd->length, (void*)remote_addr); 
            MallocControlMsg(ack_msg_tmp);
            ack_msg_tmp->source = myrank;
@@ -923,23 +920,37 @@ static void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID)
     PRINT_INFO("\nDone with LrtsInit")
 }
 
-static void* LrtsAlloc(int n_bytes)
+#define ALIGNBUF                64
+
+void* LrtsAlloc(int n_bytes, int header)
 {
     if(n_bytes <= SMSG_PER_MSG)
     {
-        return CmiAlloc(n_bytes);
-    }else if(n_bytes < LRTS_GNI_RDMA_THRESHOLD)
+        int totalsize = n_bytes+header;
+        return malloc(totalsize);
+/*
+    }else if(n_bytes <= LRTS_GNI_RDMA_THRESHOLD)
     {
-        return CmiAlloc(n_bytes);
+        return malloc(n_bytes);
+*/
     }else 
     {
-        return memalign(64, n_bytes);
+        CmiAssert(header <= ALIGNBUF);
+        char *res = memalign(ALIGNBUF, n_bytes+ALIGNBUF);
+        return res + ALIGNBUF - header;
     }
 }
 
-static void  LrtsFree(void *msg)
+void  LrtsFree(void *msg)
 {
+    int size = SIZEFIELD((char*)msg+sizeof(CmiChunkHeader));
+    if (size <= SMSG_PER_MSG)
+      free(msg);
+    else
+      free((char*)msg + sizeof(CmiChunkHeader) - ALIGNBUF);
+/*
     CmiFree(msg);
+*/
 }
 
 static void* LrtsAllocRegister(int n_bytes, gni_mem_handle_t* mem_hndl)
index b901b65f20072546790c161b9a5953185423ba92..67c6bb866ae620fadd76ee00612c80c373b656d0 100644 (file)
@@ -17,5 +17,6 @@ void LrtsExit();
 /* ### End of Machine-running Related Functions ### */
 void LrtsPostNonLocal();
 
-void* LrtsAlloc(int);
+void* LrtsAlloc(int, int);
+void  LrtsFree(void*);
 #endif
index f9ca9d4ea32ee2a977e2d54e18755435b761983d..887eec3017fc450ce1272448c08e42e40e82ba8a 100644 (file)
@@ -213,6 +213,10 @@ CpvDeclare(void *, CkGridObject);
 CpvDeclare(void *, CsdGridQueue);
 #endif
 
+#if CMK_CRAYXE
+void* LrtsAlloc(int, int);
+void  LrtsFree(void*);
+#endif
 
 /*****************************************************************************
  *
@@ -2763,6 +2767,8 @@ void *CmiAlloc(int size)
   res = (char*) arena_malloc(size+sizeof(CmiChunkHeader));
 #elif CMK_USE_IBVERBS | CMK_USE_IBUD
   res = (char *) infi_CmiAlloc(size+sizeof(CmiChunkHeader));
+#elif CMK_CRAYXE
+  res =(char *) LrtsAlloc(size, sizeof(CmiChunkHeader));
 #elif CONVERSE_POOL
   res =(char *) CmiPoolAlloc(size+sizeof(CmiChunkHeader));
 #else
@@ -2857,6 +2863,8 @@ void CmiFree(void *blk)
       }
 #endif
     infi_CmiFree(BLKSTART(parentBlk));
+#elif CMK_CRAYXE
+    LrtsFree(BLKSTART(parentBlk));
 #elif CONVERSE_POOL
     CmiPoolFree(BLKSTART(parentBlk));
 #else