added initial section multicast.
[charm.git] / examples / charm++ / PMEMimic / PMEMimic_Parallel / PMEMimic.C
1 #include <stdio.h>
2 #include "PMEMimic.decl.h"
3 #include "ckmulticast.h"
4
5 /*readonly*/ CProxy_Main mainProxy;
6 /*readonly*/ 
7
8 int     N;
9 int     grid_x;
10 int     grid_y;
11 int     grid_z;
12 int     max_iter;
13 int     pes_per_node;
14 int     grain_size;
15 CProxy_PMEPencil_X pme_x;
16 CProxy_PMEPencil_Y pme_y;
17 CProxy_PMEPencil_Z pme_z;
18
19 CkGroupID mCastGrpId;
20
21 class DataMsg : public CkMcastBaseMsg, public CMessage_DataMsg
22 {
23 public:
24     int phrase;
25     char data[2048];
26
27 };
28
29 class PMEMap : public CkArrayMap 
30
31     int offset;
32 public: 
33     PMEMap(int off) { offset = off;} 
34     PMEMap(CkMigrateMessage *m){} 
35     int registerArray(CkArrayIndex& numElements,CkArrayID aid) { 
36         return 0; 
37     } 
38     int procNum(int /*arrayHdl*/,const CkArrayIndex &idx) { 
39         int penum;
40         int *index =  (int *)idx.data();
41         int node_index =  index[0]*grid_x + index[1];
42         penum = node_index * pes_per_node + index[2];
43         return penum; 
44     } 
45 }; 
46
47
48 /*mainchare*/
49 class Main : public CBase_Main
50 {
51     double startTimer;
52     int done_pme, iteration;
53 public:
54
55     Main(CkArgMsg* m)
56     {
57         //Process command-line arguments
58  
59         grid_x = grid_y = grid_z = 10;
60         max_iter = 100;
61         pes_per_node = 3;
62         if(m->argc > 1)
63         {
64             pes_per_node = atoi(m->argv[1]);;
65             grid_x = grid_y = grid_z = atoi(m->argv[2]);
66             max_iter = atoi(m->argv[3]);
67         }
68         grain_size = grid_x/pes_per_node;
69         delete m;
70
71     //Start the computation
72       CkPrintf("Running PMEMimic on %d processors for %d elements\n",
73           CkNumPes(), grid_x);
74       mainProxy = thisProxy;
75
76       CProxy_PMEMap myMap_x=CProxy_PMEMap::ckNew(0); 
77       CkArrayOptions opts_x(grid_y, grid_z, pes_per_node); 
78       opts_x.setMap(myMap_x);
79
80       CProxy_PMEMap myMap_y=CProxy_PMEMap::ckNew(1); 
81       CkArrayOptions opts_y(grid_x, grid_z, pes_per_node); 
82       opts_y.setMap(myMap_y);
83
84       CProxy_PMEMap myMap_z=CProxy_PMEMap::ckNew(2); 
85       CkArrayOptions opts_z(grid_x, grid_y, pes_per_node); 
86       opts_z.setMap(myMap_z);
87
88       pme_x = CProxy_PMEPencil_X::ckNew(0, opts_x);
89       pme_y = CProxy_PMEPencil_Y::ckNew(1, opts_y);
90       pme_z = CProxy_PMEPencil_Z::ckNew(2, opts_z);
91
92       mCastGrpId = CProxy_CkMulticastMgr::ckNew();
93       CkMulticastMgr *mg = CProxy_CkMulticastMgr(mCastGrpId).ckLocalBranch();
94
95       CProxySection_PMEPencil_X mcp_x[grid_y][grid_z];
96
97       for (int i=0; i<grid_y; i++)
98         for (int j=0; j<grid_z; j++) {
99           mcp_x[i][j] = CProxySection_PMEPencil_X::ckNew(pme_x, i, i, 1, j, j, 1,  0, pes_per_node-1, 1);
100           mcp_x[i][j].ckSectionDelegate(mg);
101           CkCallback *cb = new CkCallback(CkIndex_PMEPencil_X::cb_client(NULL), CkArrayIndex3D(i,j,0), pme_x);
102           mg->setReductionClient(mcp_x[i][j], cb);
103       }
104
105       done_pme=0;
106       startTimer = CmiWallTimer();
107       pme_x.start();
108       
109     };
110
111     void done()
112     {
113         done_pme++;
114         if(done_pme == grid_x*grid_x)
115         {
116             done_pme = 0;
117
118             CkPrintf("PME(%d, %d, %d) on %d PEs, %d iteration, avg time:%f(ms)\n", grid_x, grid_y, grid_z, CkNumPes(), max_iter, (CmiWallTimer()-startTimer)/max_iter*1000);
119             CkExit();
120         }
121     }
122 };
123
124 /*array [1D]*/
125 class PMEPencil_X : public CBase_PMEPencil_X
126 {
127     int PME_index;
128     int buffered_num, buffered_phrase;
129     int recv_nums, iteration;
130 public:
131   PMEPencil_X(int i)
132   {
133       PME_index = i;
134       recv_nums = 0;
135       iteration = 0;
136       buffered_num = 0;
137   }
138   PMEPencil_X(CkMigrateMessage *m) {}
139
140   void start()
141   {
142    //thisindex.x thisindex.y
143     // x (yz)(x), y(x, z)(y)
144     int yindex = thisIndex.x/grain_size;
145     for(int x=0; x<grain_size; x++)
146     {
147       DataMsg *msg= new DataMsg;
148       msg->phrase = 1;
149       //CmiPrintf("g=%d(%d,%d,%d)==>(%d, %d,%d)\n", grain_size, thisIndex.x, thisIndex.y, thisIndex.z, x+thisIndex.z*grain_size, thisIndex.y, yindex);
150       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
151     }
152   }
153   void recvTrans(DataMsg *msg_recv)
154   {
155     int expect_num, index;
156     expect_num = grid_x/pes_per_node;
157     index = msg_recv->phrase;
158
159     if(msg_recv->phrase != PME_index)
160     {
161         buffered_num++;
162         buffered_phrase = msg_recv->phrase;
163         delete msg_recv;
164         return;
165     }
166     recv_nums++;
167     if(recv_nums == expect_num)
168     {
169         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
170         if(index == 0  ) //x (y,z) to y(x,z)
171         {
172             iteration++;
173             if(iteration == max_iter)
174             {
175                 mainProxy.done();
176                 return;
177             }
178             int yindex = thisIndex.x/grain_size;
179             for(int x=0; x<grain_size; x++)
180             {
181                 DataMsg *msg= new DataMsg;
182                 msg->phrase = msg_recv->phrase+1;
183                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
184             }
185         }else if(index == 1) //y(x,z) send to z(x,y)
186         {
187             int zindex = thisIndex.y/grain_size;
188             for(int y=0; y<grain_size; y++)
189             {
190                 DataMsg *msg= new DataMsg;
191                 msg->phrase = msg_recv->phrase+1;
192                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
193             }
194             PME_index = 3;
195             recv_nums = buffered_num;
196         }else if(index == 2) //Z(x,y) send to y(x,z)
197         {
198             int yindex = thisIndex.y/grain_size;
199             for(int z=0; z<grain_size; z++)
200             {
201                 DataMsg *msg= new DataMsg;
202                 msg->phrase = msg_recv->phrase+1;
203                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
204             }
205         } else if(index == 3) //y(x,z) to x(y,z)
206         {
207             int xindex = thisIndex.x/grain_size;
208             for(int y=0; y<grain_size; y++)
209             {
210                 DataMsg *msg= new DataMsg;
211                 msg->phrase = 0;
212                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
213             }
214             PME_index = 1;
215             recv_nums = buffered_num;
216         }
217         recv_nums = 0;
218     }
219     delete msg_recv;
220   }
221   void cb_client(CkReductionMsg *msg) {
222   }
223 };
224
225 /*array [1D]*/
226 class PMEPencil_Y : public CBase_PMEPencil_Y
227 {
228     int PME_index;
229     int buffered_num, buffered_phrase;
230     int recv_nums, iteration;
231 public:
232   PMEPencil_Y(int i)
233   {
234       PME_index = i;
235       recv_nums = 0;
236       iteration = 0;
237       buffered_num = 0;
238   }
239   PMEPencil_Y(CkMigrateMessage *m) {}
240
241   void start()
242   {
243    //thisindex.x thisindex.y
244     // x (yz)(x), y(x, z)(y)
245     int yindex = thisIndex.x/grain_size;
246     for(int x=0; x<grain_size; x++)
247     {
248       DataMsg *msg= new DataMsg;
249       msg->phrase = 1;
250       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
251     }
252   }
253   void recvTrans(DataMsg *msg_recv)
254   {
255     int expect_num, index;
256     expect_num = grid_x/pes_per_node;
257     index = msg_recv->phrase;
258
259     if(msg_recv->phrase != PME_index)
260     {
261         buffered_num++;
262         buffered_phrase = msg_recv->phrase;
263         delete msg_recv;
264         return;
265     }
266     recv_nums++;
267     if(recv_nums == expect_num)
268     {
269         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
270         if(index == 0  ) //x (y,z) to y(x,z)
271         {
272             iteration++;
273             if(iteration == max_iter)
274             {
275                 mainProxy.done();
276                 return;
277             }
278             int yindex = thisIndex.x/grain_size;
279             for(int x=0; x<grain_size; x++)
280             {
281                 DataMsg *msg= new DataMsg;
282                 msg->phrase = msg_recv->phrase+1;
283                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
284             }
285         }else if(index == 1) //y(x,z) send to z(x,y)
286         {
287             int zindex = thisIndex.y/grain_size;
288             for(int y=0; y<grain_size; y++)
289             {
290                 DataMsg *msg= new DataMsg;
291                 msg->phrase = msg_recv->phrase+1;
292                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
293             }
294             PME_index = 3;
295             recv_nums = buffered_num;
296         }else if(index == 2) //Z(x,y) send to y(x,z)
297         {
298             int yindex = thisIndex.y/grain_size;
299             for(int z=0; z<grain_size; z++)
300             {
301                 DataMsg *msg= new DataMsg;
302                 msg->phrase = msg_recv->phrase+1;
303                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
304             }
305         } else if(index == 3) //y(x,z) to x(y,z)
306         {
307             int xindex = thisIndex.x/grain_size;
308             for(int y=0; y<grain_size; y++)
309             {
310                 DataMsg *msg= new DataMsg;
311                 msg->phrase = 0;
312                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
313             }
314             PME_index = 1;
315             recv_nums = buffered_num;
316         }
317         recv_nums = 0;
318     }
319     delete msg_recv;
320   }
321   void cb_client(CkReductionMsg *msg) {
322   }
323 };
324
325 /*array [1D]*/
326 class PMEPencil_Z : public CBase_PMEPencil_Z
327 {
328     int PME_index;
329     int buffered_num, buffered_phrase;
330     int recv_nums, iteration;
331 public:
332   PMEPencil_Z(int i)
333   {
334       PME_index = i;
335       recv_nums = 0;
336       iteration = 0;
337       buffered_num = 0;
338   }
339   PMEPencil_Z(CkMigrateMessage *m) {}
340
341   void start()
342   {
343    //thisindex.x thisindex.y
344     // x (yz)(x), y(x, z)(y)
345     int yindex = thisIndex.x/grain_size;
346     for(int x=0; x<grain_size; x++)
347     {
348       DataMsg *msg= new DataMsg;
349       msg->phrase = 1;
350       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
351     }
352   }
353   void recvTrans(DataMsg *msg_recv)
354   {
355     int expect_num, index;
356     expect_num = grid_x/pes_per_node;
357     index = msg_recv->phrase;
358
359     if(msg_recv->phrase != PME_index)
360     {
361         buffered_num++;
362         buffered_phrase = msg_recv->phrase;
363         delete msg_recv;
364         return;
365     }
366     recv_nums++;
367     if(recv_nums == expect_num)
368     {
369         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
370         if(index == 0  ) //x (y,z) to y(x,z)
371         {
372             iteration++;
373             if(iteration == max_iter)
374             {
375                 mainProxy.done();
376                 return;
377             }
378             int yindex = thisIndex.x/grain_size;
379             for(int x=0; x<grain_size; x++)
380             {
381                 DataMsg *msg= new DataMsg;
382                 msg->phrase = msg_recv->phrase+1;
383                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
384             }
385         }else if(index == 1) //y(x,z) send to z(x,y)
386         {
387             int zindex = thisIndex.y/grain_size;
388             for(int y=0; y<grain_size; y++)
389             {
390                 DataMsg *msg= new DataMsg;
391                 msg->phrase = msg_recv->phrase+1;
392                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
393             }
394             PME_index = 3;
395             recv_nums = buffered_num;
396         }else if(index == 2) //Z(x,y) send to y(x,z)
397         {
398             int yindex = thisIndex.y/grain_size;
399             for(int z=0; z<grain_size; z++)
400             {
401                 DataMsg *msg= new DataMsg;
402                 msg->phrase = msg_recv->phrase+1;
403                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
404             }
405         } else if(index == 3) //y(x,z) to x(y,z)
406         {
407             int xindex = thisIndex.x/grain_size;
408             for(int y=0; y<grain_size; y++)
409             {
410                 DataMsg *msg= new DataMsg;
411                 msg->phrase = 0;
412                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
413             }
414             PME_index = 1;
415             recv_nums = buffered_num;
416         }
417         recv_nums = 0;
418     }
419     delete msg_recv;
420   }
421   void cb_client(CkReductionMsg *msg) {
422   }
423 };
424
425
426 #include "PMEMimic.def.h"