CmiDirect/CkDirect changes
authorPritish Jetley <pjetley2@illinois.edu>
Wed, 26 Mar 2008 22:44:11 +0000 (22:44 +0000)
committerPritish Jetley <pjetley2@illinois.edu>
Wed, 26 Mar 2008 22:44:11 +0000 (22:44 +0000)
src/arch/net/machine-ibverbs.c
src/ck-core/charm.h
src/ck-core/ckdirect.h [new file with mode: 0644]
src/conv-core/cmidirect.h [new file with mode: 0644]
tests/charm++/pingpong/Makefile
tests/charm++/pingpong/pingpong.C

index 1938dd4976dd7e3a1a5ccfaeac5e183953e0d9ac..73775bd66490d3cdc6d28a105175e9c853153639 100644 (file)
@@ -1123,7 +1123,7 @@ static inline void processAsyncEvents(){
        
 }
 
-void pollCmiDirectQ();
+static void pollCmiDirectQ();
 
 static inline  void CommunicationServer_nolock(int toBuffer) {
        int processed;
@@ -2247,8 +2247,6 @@ send and receive data from the middle of his arrays without any copying on eithe
 side
 *********************************************************************************************/
 
-#define MAXHANDLES 512
-
 struct infiDirectRequestPacket{
        int senderProc;
        int handle;
@@ -2257,16 +2255,9 @@ struct infiDirectRequestPacket{
        int senderBufSize;
 };
 
-static struct infiDirectUserHandle{
-       int handle;
-       int senderNode;
-       int recverNode;
-       void *recverBuf;
-       int recverBufSize;
-       char recverKey[32];
-       double initialValue;
-};
+#include "cmidirect.h"
 
+#define MAXHANDLES 512
 typedef struct {
        int id;
        void *buf;
@@ -2289,8 +2280,15 @@ typedef struct directPollingQNodeStruct {
        struct directPollingQNodeStruct *next;
 } directPollingQNode;
 
+// data structures 
+
 directPollingQNode *headDirectPollingQ=NULL,*tailDirectPollingQ=NULL;
 
+static infiDirectHandleTable **sendHandleTable=NULL;
+static infiDirectHandleTable **recvHandleTable=NULL;
+
+static int *recvHandleCount=NULL;
+
 void addHandleToPollingQ(infiDirectHandle *handle){
        directPollingQNode *newNode = malloc(sizeof(directPollingQNode));
        newNode->handle = handle;
@@ -2322,12 +2320,6 @@ infiDirectHandle *removeHandleFromPollingQ(){
        return retHandle;
 }*/
 
-
-static infiDirectHandleTable **sendHandleTable=NULL;
-static infiDirectHandleTable **recvHandleTable=NULL;
-
-static int *recvHandleCount=NULL;
-
 static inline infiDirectHandleTable **createHandleTable(){
        infiDirectHandleTable **table = malloc(_Cmi_numnodes*sizeof(infiDirectHandleTable *));
        int i;
@@ -2424,10 +2416,12 @@ struct infiDirectUserHandle CmiDirect_createHandle(int senderNode,void *recvBuf,
 /****
  To be called on the sender to attach the sender's buffer to this handle
 ******/
-void CmiDirect_assocLocalBuffer(int recverNode,struct infiDirectUserHandle *userHandle,void *sendBuf,int sendBufSize){
+void CmiDirect_assocLocalBuffer(struct infiDirectUserHandle *userHandle,void *sendBuf,int sendBufSize){
        int tableIdx,idx;
        int i;
        int handle = userHandle->handle;
+       int recverNode  = userHandle->recverNode;
+
        infiDirectHandleTable *table;
 
        if(sendHandleTable == NULL){
@@ -2575,11 +2569,6 @@ void CmiDirect_put(struct infiDirectUserHandle *userHandle){
                {
                        packet->size = sizeof(struct infiDirectRequestPacket);
                        packet->buf = (char *)(table->handles[idx].packet);
-                       packet->header.code = INFIDIRECT_REQUEST;
-                       packet->header.nodeNo = _Cmi_mynode;
-                       packet->ogm = NULL;
-
-                       node = nodes_by_pe[recverProc];
                        struct ibv_mr *packetKey = METADATAFIELD((void *)table->handles[idx].packet)->key;
                        EnqueuePacket(node,packet,sizeof(struct infiDirectRequestPacket),packetKey);
                }*/
@@ -2587,7 +2576,7 @@ void CmiDirect_put(struct infiDirectUserHandle *userHandle){
 
 };
 
-/**** need not be called the first tiem *********/
+/**** need not be called the first time *********/
 void CmiDirect_ready(struct infiDirectUserHandle *userHandle){
        int handle = userHandle->handle;
        int tableIdx,idx,i;
@@ -2608,7 +2597,7 @@ void CmiDirect_ready(struct infiDirectUserHandle *userHandle){
 }
 
 
-int receivedDirectMessage(infiDirectHandle *handle){
+static int receivedDirectMessage(infiDirectHandle *handle){
        int index = handle->size - sizeof(double);
        double *lastDouble = (double *)(((char *)handle->buf)+index);
        if(*lastDouble == handle->userHandle.initialValue){
@@ -2621,7 +2610,7 @@ int receivedDirectMessage(infiDirectHandle *handle){
 }
 
 
-void pollCmiDirectQ(){
+static void pollCmiDirectQ(){
        directPollingQNode *ptr = headDirectPollingQ, *prevPtr=NULL;
        while(ptr != NULL){
                if(receivedDirectMessage(ptr->handle)){
index 0aad081877809788cdcd4b5a7f06601a8e9f117c..5b669ec0578c52ad85f93f72d65396beeedc56c0 100644 (file)
@@ -55,6 +55,7 @@ extern "C" {
 #define CkError                 CmiError
 #define CkAbort                 CmiAbort
 #define CkAssert                CmiAssert
+
 extern void  CkExit(void);
 extern char **CkGetArgv(void);
 extern int  CkGetArgc(void);
diff --git a/src/ck-core/ckdirect.h b/src/ck-core/ckdirect.h
new file mode 100644 (file)
index 0000000..5a159fb
--- /dev/null
@@ -0,0 +1,13 @@
+#ifndef _CKDIRECT_H_
+#define _CKDIRECT_H_
+#include "cmidirect.h"
+#include "charm++.h"
+
+#define CkDirect_createHandle CmiDirect_createHandle
+#define CkDirect_assocLocalBuffer CmiDirect_assocLocalBuffer
+#define CkDirect_put CmiDirect_put
+#define CkDirect_ready CmiDirect_ready
+
+PUPbytes(infiDirectUserHandle)
+
+#endif
diff --git a/src/conv-core/cmidirect.h b/src/conv-core/cmidirect.h
new file mode 100644 (file)
index 0000000..96c959e
--- /dev/null
@@ -0,0 +1,46 @@
+#ifndef _CMIDIRECT_H_
+#define _CMIDIRECT_H_
+/* This file provides an interface for users to the CmiDirect functionality.
+
+*/
+
+
+/* handle type definition */
+struct infiDirectUserHandle{
+       int handle;
+       int senderNode;
+       int recverNode;
+       void *recverBuf;
+       int recverBufSize;
+       char recverKey[32];
+       double initialValue;
+};
+
+
+/* functions */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/**
+ To be called on the receiver to create a handle and return its number
+**/
+struct infiDirectUserHandle CmiDirect_createHandle(int senderNode,void *recvBuf, int recvBufSize, void (*callbackFnPtr)(void *), void *callbackData,double initialValue);
+
+/****
+ To be called on the sender to attach the sender's buffer to this handle
+******/
+void CmiDirect_assocLocalBuffer(struct infiDirectUserHandle *userHandle,void *sendBuf,int sendBufSize);
+
+/****
+To be called on the sender to do the actual data transfer
+******/
+void CmiDirect_put(struct infiDirectUserHandle *userHandle);
+
+/**** Should not be called the first time *********/
+void CmiDirect_ready(struct infiDirectUserHandle *userHandle);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
index da28692a9d7875dd00cd1f39ff40301ca3d457a1..a19b018f4afeb82f6987c12d41321aa71c8cc396 100644 (file)
@@ -1,3 +1,4 @@
+SRC=../../../src
 CHARMC=../../../bin/charmc $(OPTS)
 
 OBJS = pingpong.o
@@ -14,7 +15,7 @@ clean:
        rm -f *.decl.h *.def.h conv-host *.o pgm charmrun
 
 pingpong.o: pingpong.C
-       $(CHARMC) -c pingpong.C
+       $(CHARMC) -I$(SRC)/conv-core pingpong.C
 
 test: all
        @echo "Intra-processor Pingpong.."
index 2e4dc25fa753b28464d1db94509d45047cca7017..dfba6fd7eb68b1c056a4c4a334136f7895e4377c 100644 (file)
@@ -3,20 +3,13 @@
 
 #define NITER 1000
 #define PAYLOAD 100
-#ifdef CMK_USE_IBVERBS 
-
 
+#ifdef CMK_USE_IBVERBS 
 extern "C" {
-  struct infiDirectUserHandle CmiDirect_createHandle(int senderNode,void *recvBuf, int recvBufSize, void (*callbackFnPtr)(void *), void *callbackData,double initialValue);
-  //  int CmiDirect_createHandle(int senderProc,void *recvBuf, int recvBufSize, void (*callbackFnPtr)(void *), void *callbackData);
-
-  void CmiDirect_assocLocalBuffer(int recverNode,struct infiDirectUserHandle *userHandle,void *sendBuf,int sendBufSize);
-  //  void CmiDirect_assocLocalBuffer(int recverProc,int handle,void *sendBuf,int sendBufSize);
-  void CmiDirect_put(struct infiDirectUserHandle *userHandle);
-  void CmiDirect_ready(struct infiDirectUserHandle *userHandle);
-    //void CmiDirect_put(int recverProc,int handle);
+#include "cmidirect.h"
 }
 #endif
+
 class Fancy
 {
   char _str[12];
@@ -35,17 +28,6 @@ class CkArrayIndexFancy : public CkArrayIndex {
 };
 
 #include "pingpong.decl.h"
-#ifdef CMK_USE_IBVERBS 
-struct infiDirectUserHandle{
-       int handle;
-       int senderNode;
-       int recverNode;
-       void *recverBuf;
-       int recverBufSize;
-       char recverKey[32];
-       double initialValue;
-};
-#endif
 class PingMsg : public CMessage_PingMsg
 {
   public:
@@ -240,7 +222,7 @@ public:
 #ifdef CMK_USE_IBVERBS 
     struct infiDirectUserHandle *_shandle=(struct infiDirectUserHandle *) ptr;
     shandle=*_shandle;
-    CmiDirect_assocLocalBuffer(nbr,&shandle,sbuff,payload);
+    CmiDirect_assocLocalBuffer(&shandle,sbuff,payload);
 #endif
   }
   void start(void)