fix parallel sending bug
[charm.git] / examples / charm++ / PMEMimic / PMEMimic_Parallel / PMEMimic.C
1 #include <stdio.h>
2 #include "PMEMimic.decl.h"
3
4 /*readonly*/ CProxy_Main mainProxy;
5 /*readonly*/ 
6
7 int     N;
8 int     grid_x;
9 int     grid_y;
10 int     grid_z;
11 int     max_iter;
12 int     pes_per_node;
13 int     grain_size;
14 CProxy_PMEPencil_X pme_x;
15 CProxy_PMEPencil_Y pme_y;
16 CProxy_PMEPencil_Z pme_z;
17
18 class DataMsg : public CMessage_DataMsg
19 {
20 public:
21     int phrase;
22     char data[2048];
23
24 };
25
26 class PMEMap : public CkArrayMap 
27
28     int offset;
29 public: 
30     PMEMap(int off) { offset = off;} 
31     PMEMap(CkMigrateMessage *m){} 
32     int registerArray(CkArrayIndex& numElements,CkArrayID aid) { 
33         return 0; 
34     } 
35     int procNum(int /*arrayHdl*/,const CkArrayIndex &idx) { 
36         int penum;
37         int *index =  (int *)idx.data();
38         int node_index =  index[0]*grid_x + index[1];
39         penum = node_index * pes_per_node + index[2];
40         return penum; 
41     } 
42 }; 
43
44
45 /*mainchare*/
46 class Main : public CBase_Main
47 {
48     double startTimer;
49     int done_pme, iteration;
50 public:
51
52     Main(CkArgMsg* m)
53     {
54         //Process command-line arguments
55  
56         grid_x = grid_y = grid_z = 10;
57         max_iter = 100;
58         pes_per_node = 3;
59         if(m->argc > 1)
60         {
61             pes_per_node = atoi(m->argv[1]);;
62             grid_x = grid_y = grid_z = atoi(m->argv[2]);
63             max_iter = atoi(m->argv[3]);
64         }
65         grain_size = grid_x/pes_per_node;
66         delete m;
67
68     //Start the computation
69       CkPrintf("Running PMEMimic on %d processors for %d elements\n",
70           CkNumPes(), grid_x);
71       mainProxy = thisProxy;
72
73       CProxy_PMEMap myMap_x=CProxy_PMEMap::ckNew(0); 
74       CkArrayOptions opts_x(grid_y, grid_z, pes_per_node); 
75       opts_x.setMap(myMap_x);
76
77       CProxy_PMEMap myMap_y=CProxy_PMEMap::ckNew(1); 
78       CkArrayOptions opts_y(grid_x, grid_z, pes_per_node); 
79       opts_y.setMap(myMap_y);
80
81       CProxy_PMEMap myMap_z=CProxy_PMEMap::ckNew(2); 
82       CkArrayOptions opts_z(grid_x, grid_y, pes_per_node); 
83       opts_z.setMap(myMap_z);
84
85       pme_x = CProxy_PMEPencil_X::ckNew(0, opts_x);
86       pme_y = CProxy_PMEPencil_Y::ckNew(1, opts_y);
87       pme_z = CProxy_PMEPencil_Z::ckNew(2, opts_z);
88
89       done_pme=0;
90       startTimer = CmiWallTimer();
91       pme_x.start();
92       
93     };
94
95     void done()
96     {
97         done_pme++;
98         if(done_pme == grid_x*grid_x)
99         {
100             done_pme = 0;
101
102             CkPrintf("PME(%d, %d, %d) on %d PEs, %d iteration, avg time:%f(ms)\n", grid_x, grid_y, grid_z, CkNumPes(), max_iter, (CmiWallTimer()-startTimer)/max_iter*1000);
103             CkExit();
104         }
105     }
106 };
107
108 /*array [1D]*/
109 class PMEPencil_X : public CBase_PMEPencil_X
110 {
111     int PME_index;
112     int buffered_num, buffered_phrase;
113     int recv_nums, iteration;
114 public:
115   PMEPencil_X(int i)
116   {
117       PME_index = i;
118       recv_nums = 0;
119       iteration = 0;
120       buffered_num = 0;
121   }
122   PMEPencil_X(CkMigrateMessage *m) {}
123
124   void start()
125   {
126    //thisindex.x thisindex.y
127     // x (yz)(x), y(x, z)(y)
128     int yindex = thisIndex.x/grain_size;
129     for(int x=0; x<grain_size; x++)
130     {
131       DataMsg *msg= new DataMsg;
132       msg->phrase = 1;
133       //CmiPrintf("g=%d(%d,%d,%d)==>(%d, %d,%d)\n", grain_size, thisIndex.x, thisIndex.y, thisIndex.z, x+thisIndex.z*grain_size, thisIndex.y, yindex);
134       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
135     }
136   }
137   void recvTrans(DataMsg *msg_recv)
138   {
139     int expect_num, index;
140     expect_num = grid_x/pes_per_node;
141     index = msg_recv->phrase;
142
143     if(msg_recv->phrase != PME_index)
144     {
145         buffered_num++;
146         buffered_phrase = msg_recv->phrase;
147         delete msg_recv;
148         return;
149     }
150     recv_nums++;
151     if(recv_nums == expect_num)
152     {
153         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
154         if(index == 0  ) //x (y,z) to y(x,z)
155         {
156             iteration++;
157             if(iteration == max_iter)
158             {
159                 mainProxy.done();
160                 return;
161             }
162             int yindex = thisIndex.x/grain_size;
163             for(int x=0; x<grain_size; x++)
164             {
165                 DataMsg *msg= new DataMsg;
166                 msg->phrase = msg_recv->phrase+1;
167                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
168             }
169         }else if(index == 1) //y(x,z) send to z(x,y)
170         {
171             int zindex = thisIndex.y/grain_size;
172             for(int y=0; y<grain_size; y++)
173             {
174                 DataMsg *msg= new DataMsg;
175                 msg->phrase = msg_recv->phrase+1;
176                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
177             }
178             PME_index = 3;
179             recv_nums = buffered_num;
180         }else if(index == 2) //Z(x,y) send to y(x,z)
181         {
182             int yindex = thisIndex.y/grain_size;
183             for(int z=0; z<grain_size; z++)
184             {
185                 DataMsg *msg= new DataMsg;
186                 msg->phrase = msg_recv->phrase+1;
187                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
188             }
189         } else if(index == 3) //y(x,z) to x(y,z)
190         {
191             int xindex = thisIndex.x/grain_size;
192             for(int y=0; y<grain_size; y++)
193             {
194                 DataMsg *msg= new DataMsg;
195                 msg->phrase = 0;
196                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
197             }
198             PME_index = 1;
199             recv_nums = buffered_num;
200         }
201         recv_nums = 0;
202     }
203     delete msg_recv;
204   }
205 };
206 /*array [1D]*/
207 class PMEPencil_Y : public CBase_PMEPencil_Y
208 {
209     int PME_index;
210     int buffered_num, buffered_phrase;
211     int recv_nums, iteration;
212 public:
213   PMEPencil_Y(int i)
214   {
215       PME_index = i;
216       recv_nums = 0;
217       iteration = 0;
218       buffered_num = 0;
219   }
220   PMEPencil_Y(CkMigrateMessage *m) {}
221
222   void start()
223   {
224    //thisindex.x thisindex.y
225     // x (yz)(x), y(x, z)(y)
226     int yindex = thisIndex.x/grain_size;
227     for(int x=0; x<grain_size; x++)
228     {
229       DataMsg *msg= new DataMsg;
230       msg->phrase = 1;
231       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
232     }
233   }
234   void recvTrans(DataMsg *msg_recv)
235   {
236     int expect_num, index;
237     expect_num = grid_x/pes_per_node;
238     index = msg_recv->phrase;
239
240     if(msg_recv->phrase != PME_index)
241     {
242         buffered_num++;
243         buffered_phrase = msg_recv->phrase;
244         delete msg_recv;
245         return;
246     }
247     recv_nums++;
248     if(recv_nums == expect_num)
249     {
250         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
251         if(index == 0  ) //x (y,z) to y(x,z)
252         {
253             iteration++;
254             if(iteration == max_iter)
255             {
256                 mainProxy.done();
257                 return;
258             }
259             int yindex = thisIndex.x/grain_size;
260             for(int x=0; x<grain_size; x++)
261             {
262                 DataMsg *msg= new DataMsg;
263                 msg->phrase = msg_recv->phrase+1;
264                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
265             }
266         }else if(index == 1) //y(x,z) send to z(x,y)
267         {
268             int zindex = thisIndex.y/grain_size;
269             for(int y=0; y<grain_size; y++)
270             {
271                 DataMsg *msg= new DataMsg;
272                 msg->phrase = msg_recv->phrase+1;
273                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
274             }
275             PME_index = 3;
276             recv_nums = buffered_num;
277         }else if(index == 2) //Z(x,y) send to y(x,z)
278         {
279             int yindex = thisIndex.y/grain_size;
280             for(int z=0; z<grain_size; z++)
281             {
282                 DataMsg *msg= new DataMsg;
283                 msg->phrase = msg_recv->phrase+1;
284                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
285             }
286         } else if(index == 3) //y(x,z) to x(y,z)
287         {
288             int xindex = thisIndex.x/grain_size;
289             for(int y=0; y<grain_size; y++)
290             {
291                 DataMsg *msg= new DataMsg;
292                 msg->phrase = 0;
293                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
294             }
295             PME_index = 1;
296             recv_nums = buffered_num;
297         }
298         recv_nums = 0;
299     }
300     delete msg_recv;
301   }
302 };
303
304 /*array [1D]*/
305 class PMEPencil_Z : public CBase_PMEPencil_Z
306 {
307     int PME_index;
308     int buffered_num, buffered_phrase;
309     int recv_nums, iteration;
310 public:
311   PMEPencil_Z(int i)
312   {
313       PME_index = i;
314       recv_nums = 0;
315       iteration = 0;
316       buffered_num = 0;
317   }
318   PMEPencil_Z(CkMigrateMessage *m) {}
319
320   void start()
321   {
322    //thisindex.x thisindex.y
323     // x (yz)(x), y(x, z)(y)
324     int yindex = thisIndex.x/grain_size;
325     for(int x=0; x<grain_size; x++)
326     {
327       DataMsg *msg= new DataMsg;
328       msg->phrase = 1;
329       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
330     }
331   }
332   void recvTrans(DataMsg *msg_recv)
333   {
334     int expect_num, index;
335     expect_num = grid_x/pes_per_node;
336     index = msg_recv->phrase;
337
338     if(msg_recv->phrase != PME_index)
339     {
340         buffered_num++;
341         buffered_phrase = msg_recv->phrase;
342         delete msg_recv;
343         return;
344     }
345     recv_nums++;
346     if(recv_nums == expect_num)
347     {
348         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
349         if(index == 0  ) //x (y,z) to y(x,z)
350         {
351             iteration++;
352             if(iteration == max_iter)
353             {
354                 mainProxy.done();
355                 return;
356             }
357             int yindex = thisIndex.x/grain_size;
358             for(int x=0; x<grain_size; x++)
359             {
360                 DataMsg *msg= new DataMsg;
361                 msg->phrase = msg_recv->phrase+1;
362                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
363             }
364         }else if(index == 1) //y(x,z) send to z(x,y)
365         {
366             int zindex = thisIndex.y/grain_size;
367             for(int y=0; y<grain_size; y++)
368             {
369                 DataMsg *msg= new DataMsg;
370                 msg->phrase = msg_recv->phrase+1;
371                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
372             }
373             PME_index = 3;
374             recv_nums = buffered_num;
375         }else if(index == 2) //Z(x,y) send to y(x,z)
376         {
377             int yindex = thisIndex.y/grain_size;
378             for(int z=0; z<grain_size; z++)
379             {
380                 DataMsg *msg= new DataMsg;
381                 msg->phrase = msg_recv->phrase+1;
382                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
383             }
384         } else if(index == 3) //y(x,z) to x(y,z)
385         {
386             int xindex = thisIndex.x/grain_size;
387             for(int y=0; y<grain_size; y++)
388             {
389                 DataMsg *msg= new DataMsg;
390                 msg->phrase = 0;
391                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
392             }
393             PME_index = 1;
394             recv_nums = buffered_num;
395         }
396         recv_nums = 0;
397     }
398     delete msg_recv;
399   }
400 };
401
402
403 #include "PMEMimic.def.h"