added warm up for multiple pair pingpong
[charm.git] / examples / converse / pingpong / pingpong_multipairs.C
1
2 /**********************************************************
3       Converse Ping-pong to test the message latency and bandwidth
4       Modified from Milind's ping-pong
5       
6       Sameer Kumar 02/07/05
7 ***************************************************/
8
9 #include <stdlib.h>
10 #include <converse.h>
11
12 enum {nCycles =4004};
13 enum {skip =4};
14 enum { maxMsgSize = 1 << 22 };
15
16 CpvDeclare(int,recvNum);
17 CpvDeclare(double,sumTime);
18 CpvDeclare(int,msgSize);
19 CpvDeclare(int,cycleNum);
20 CpvDeclare(int,exitHandler);
21 CpvDeclare(int,reduceHandler);
22 CpvDeclare(int,startRingHandler);
23 CpvDeclare(int,node0Handler);
24 CpvDeclare(int,node1Handler);
25 CpvStaticDeclare(double,startTime);
26 CpvStaticDeclare(double,endTime);
27
28 #define         HALF        CmiNumPes()/2
29 #define USE_PERSISTENT     1
30
31 #if USE_PERSISTENT
32 PersistentHandle h;
33 #endif
34
35 // Start the pingpong for each message size
36 void startRing(char* msg_2)
37 {
38   CpvAccess(cycleNum) = -1;
39
40   CmiFree(msg_2);
41   //Increase message in powers of 4. Also add a converse header to that
42   CpvAccess(msgSize) = (CpvAccess(msgSize)-CmiMsgHeaderSizeBytes)*2 + 
43       CmiMsgHeaderSizeBytes;
44
45   char *msg = (char *)CmiAlloc(CpvAccess(msgSize));
46   *((int *)(msg+CmiMsgHeaderSizeBytes)) = CpvAccess(msgSize);
47   CmiSetHandler(msg,CpvAccess(node0Handler));
48   CmiSyncSendAndFree(CmiMyPe(), CpvAccess(msgSize), msg);
49 }
50
51 void reduceHandlerFunc(char *msg)
52 {
53    CpvAccess(recvNum) += 1;
54    CpvAccess(sumTime) += *((double*)(msg+(CmiMsgHeaderSizeBytes)));
55    if(CpvAccess(recvNum) == HALF)
56    {
57        double us_time = (CpvAccess(sumTime))/(2.*(nCycles-skip)*HALF)*1e6;
58        CmiPrintf("%d\t\t  %.2lf   %.2f\n",
59            CpvAccess(msgSize)-CmiMsgHeaderSizeBytes,
60            us_time, (CpvAccess(msgSize)-CmiMsgHeaderSizeBytes)/us_time);
61        CpvAccess(recvNum) = 0;
62    
63        if (CpvAccess(msgSize) < maxMsgSize)
64        {
65            for(int i=0; i<HALF; i++)
66            {
67                void *sendmsg = CmiAlloc(CmiMsgHeaderSizeBytes);
68                CmiSetHandler(sendmsg,CpvAccess(startRingHandler));
69                CmiSyncSendAndFree(i, CmiMsgHeaderSizeBytes, sendmsg);
70            }
71        }
72        else {
73            //exit
74            void *sendmsg = CmiAlloc(CmiMsgHeaderSizeBytes);
75            CmiSetHandler(sendmsg,CpvAccess(exitHandler));
76            CmiSyncBroadcastAllAndFree(CmiMsgHeaderSizeBytes,sendmsg);
77        }
78    }
79    CmiFree(msg);
80 }
81 //the pingpong has finished, record message time
82 void ringFinished(char *msg)
83 {
84   CmiFree(msg);
85
86   double elaps_time = CpvAccess(endTime)-CpvAccess(startTime);
87   //Print the time for that message size
88   //CmiPrintf("\t\t  %.2lf\n", 
89              //(1e6*(CpvAccess(endTime)-CpvAccess(startTime)))/(2.*nCycles));
90   //Have we finished all message sizes?
91   int mysize = CmiMsgHeaderSizeBytes+sizeof(double);
92   void *sendmsg = CmiAlloc(mysize);
93   *((double*)((char*)sendmsg+CmiMsgHeaderSizeBytes)) = elaps_time;
94   CmiSetHandler(sendmsg,CpvAccess(reduceHandler));
95   CmiSyncSendAndFree(0, mysize, sendmsg);
96 }
97
98 //We finished for all message sizes. Exit now
99 CmiHandler exitHandlerFunc(char *msg)
100 {
101     CmiFree(msg);
102     CsdExitScheduler();
103     return 0;
104 }
105
106
107 //Handler on Node 0
108 CmiHandler node0HandlerFunc(char *msg)
109 {
110     CpvAccess(cycleNum)++;
111     if(CpvAccess(cycleNum)== skip)
112         CpvAccess(startTime) = CmiWallTimer();
113
114     if (CpvAccess(cycleNum) == nCycles) {
115         CpvAccess(endTime) = CmiWallTimer();
116         ringFinished(msg);
117     }
118     else {
119         CmiSetHandler(msg,CpvAccess(node1Handler));
120         *((int *)(msg+CmiMsgHeaderSizeBytes)) = CpvAccess(msgSize);
121         
122 #if USE_PERSISTENT
123         CmiUsePersistentHandle(&h, 1);
124 #endif
125         CmiSyncSendAndFree(CmiMyPe()+HALF,CpvAccess(msgSize),msg);
126 #if USE_PERSISTENT
127         CmiUsePersistentHandle(NULL, 0);
128 #endif
129     }
130     return 0;
131 }
132
133 CmiHandler node1HandlerFunc(char *msg)
134 {
135     CpvAccess(msgSize) = *((int *)(msg+CmiMsgHeaderSizeBytes));
136     CmiSetHandler(msg,CpvAccess(node0Handler));
137     
138 #if USE_PERSISTENT
139     CmiUsePersistentHandle(&h, 1);
140 #endif
141     CmiSyncSendAndFree(CmiMyPe()-HALF,CpvAccess(msgSize),msg);
142 #if USE_PERSISTENT
143     CmiUsePersistentHandle(NULL, 0);
144 #endif
145     return 0;
146 }
147
148
149 //Converse main. Initialize variables and register handlers
150 CmiStartFn mymain()
151 {
152     CpvInitialize(int,msgSize);
153     CpvInitialize(int,recvNum);
154     CpvInitialize(double,sumTime);
155     CpvInitialize(int,cycleNum);
156     CpvAccess(recvNum) = 0; 
157     CpvAccess(sumTime) = 0; 
158     CpvAccess(msgSize)= 4 + CmiMsgHeaderSizeBytes;
159     
160     CpvInitialize(int,reduceHandler);
161     CpvAccess(reduceHandler) = CmiRegisterHandler((CmiHandler) reduceHandlerFunc);
162     CpvInitialize(int,exitHandler);
163     CpvAccess(exitHandler) = CmiRegisterHandler((CmiHandler) exitHandlerFunc);
164     CpvInitialize(int,startRingHandler);
165     CpvAccess(startRingHandler) = CmiRegisterHandler((CmiHandler) startRing);
166     CpvInitialize(int,node0Handler);
167     CpvAccess(node0Handler) = CmiRegisterHandler((CmiHandler) node0HandlerFunc);
168     CpvInitialize(int,node1Handler);
169     CpvAccess(node1Handler) = CmiRegisterHandler((CmiHandler) node1HandlerFunc);
170     
171     CpvInitialize(double,startTime);
172     CpvInitialize(double,endTime);
173     
174     int otherPe = CmiMyPe() ^ 1;
175     
176 #if USE_PERSISTENT
177     h = CmiCreatePersistent(otherPe, maxMsgSize+1024);
178 #endif
179     if(CmiMyPe() == 0)
180     {
181         CmiPrintf("Multiple pair send/recv\n bytes \t\t latency(us)\t bandwidth(MBytes/sec)\n");
182     }
183     if (CmiMyPe() <CmiNumPes()/2)
184     {
185         void *sendmsg = CmiAlloc(CmiMsgHeaderSizeBytes);
186         CmiSetHandler(sendmsg,CpvAccess(startRingHandler));
187         CmiSyncSendAndFree(CmiMyPe(), CmiMsgHeaderSizeBytes, sendmsg);
188    }    
189     return 0;
190 }
191
192 int main(int argc,char *argv[])
193 {
194     ConverseInit(argc,argv,(CmiStartFn)mymain,0,0);
195     return 0;
196 }