minor change
[charm.git] / examples / charm++ / PMEMimic / PMEMimic_Parallel / PMEMimic.C
1 #include <stdio.h>
2 #include "PMEMimic.decl.h"
3 #include "ckmulticast.h"
4
5 /*readonly*/ CProxy_Main mainProxy;
6 /*readonly*/ 
7
8 int     N;
9 int     grid_x;
10 int     grid_y;
11 int     grid_z;
12 int     max_iter;
13 int     pes_per_node;
14 int     grain_size;
15 CProxy_PMEPencil_X pme_x;
16 CProxy_PMEPencil_Y pme_y;
17 CProxy_PMEPencil_Z pme_z;
18
19 CkGroupID mCastGrpId;
20
21 class DataMsg : public CkMcastBaseMsg, public CMessage_DataMsg
22 {
23 public:
24     int phrase;
25     char data[2048];
26
27 };
28
29 class PMEMap : public CkArrayMap 
30
31     int offset;
32 public: 
33     PMEMap(int off) { offset = off;} 
34     PMEMap(CkMigrateMessage *m){} 
35     int registerArray(CkArrayIndex& numElements,CkArrayID aid) { 
36         return 0; 
37     } 
38     int procNum(int /*arrayHdl*/,const CkArrayIndex &idx) { 
39         int penum;
40         int *index =  (int *)idx.data();
41         int node_index =  index[0]*grid_x + index[1];
42         penum = node_index * pes_per_node + index[2];
43         return penum; 
44     } 
45 }; 
46
47
48 /*mainchare*/
49 class Main : public CBase_Main
50 {
51     double startTimer;
52     int done_pme, iteration;
53 public:
54
55     Main(CkArgMsg* m)
56     {
57         //Process command-line arguments
58  
59         grid_x = grid_y = grid_z = 10;
60         max_iter = 100;
61         pes_per_node = 3;
62         if(m->argc > 1)
63         {
64             pes_per_node = atoi(m->argv[1]);;
65             grid_x = grid_y = grid_z = atoi(m->argv[2]);
66             max_iter = atoi(m->argv[3]);
67         }
68         grain_size = grid_x/pes_per_node;
69         delete m;
70
71     //Start the computation
72       CkPrintf("Running PMEMimic on %d processors for %d elements\n",
73           CkNumPes(), grid_x);
74       mainProxy = thisProxy;
75
76       CProxy_PMEMap myMap_x=CProxy_PMEMap::ckNew(0); 
77       CkArrayOptions opts_x(grid_y, grid_z, pes_per_node); 
78       opts_x.setMap(myMap_x);
79
80       CProxy_PMEMap myMap_y=CProxy_PMEMap::ckNew(1); 
81       CkArrayOptions opts_y(grid_x, grid_z, pes_per_node); 
82       opts_y.setMap(myMap_y);
83
84       CProxy_PMEMap myMap_z=CProxy_PMEMap::ckNew(2); 
85       CkArrayOptions opts_z(grid_x, grid_y, pes_per_node); 
86       opts_z.setMap(myMap_z);
87
88       pme_x = CProxy_PMEPencil_X::ckNew(0, opts_x);
89       pme_y = CProxy_PMEPencil_Y::ckNew(1, opts_y);
90       pme_z = CProxy_PMEPencil_Z::ckNew(2, opts_z);
91
92       mCastGrpId = CProxy_CkMulticastMgr::ckNew();
93       CkMulticastMgr *mg = CProxy_CkMulticastMgr(mCastGrpId).ckLocalBranch();
94
95       CProxySection_PMEPencil_X mcp_x[grid_y][grid_z];
96
97       for (int i=0; i<grid_y; i++)
98         for (int j=0; j<grid_z; j++) {
99           mcp_x[i][j] = CProxySection_PMEPencil_X::ckNew(pme_x, i, i, 1, j, j, 1,  0, pes_per_node-1, 1);
100           mcp_x[i][j].ckSectionDelegate(mg);
101           CkCallback *cb = new CkCallback(CkIndex_PMEPencil_X::cb_client(NULL), CkArrayIndex3D(i,j,0), pme_x);
102           mg->setReductionClient(mcp_x[i][j], cb);
103       }
104
105       done_pme=0;
106       startTimer = CmiWallTimer();
107       pme_x.start();
108       
109     };
110
111     void done()
112     {
113         done_pme++;
114         if(done_pme == grid_x*grid_x)
115         {
116             done_pme = 0;
117
118             CkPrintf("PME(%d, %d, %d) on %d PEs, %d iteration, avg time:%f(ms)\n", grid_x, grid_y, grid_z, CkNumPes(), max_iter, (CmiWallTimer()-startTimer)/max_iter*1000);
119             CkExit();
120         }
121     }
122 };
123
124 /*array [1D]*/
125 class PMEPencil_X : public CBase_PMEPencil_X
126 {
127     int PME_index;
128     int buffered_num, buffered_phrase;
129     int recv_nums, iteration;
130     CkSectionInfo sid;
131 public:
132   PMEPencil_X(int i)
133   {
134       PME_index = i;
135       recv_nums = 0;
136       iteration = 0;
137       buffered_num = 0;
138
139       if (thisIndex.z == 0) {
140           CkMulticastMgr *mg = CProxy_CkMulticastMgr(mCastGrpId).ckLocalBranch();
141           CProxySection_PMEPencil_X mcp_x;
142
143           mcp_x = CProxySection_PMEPencil_X::ckNew(pme_x, thisIndex.x, thisIndex.x, 1, thisIndex.y, thisIndex.y, 1,  0, pes_per_node-1, 1);
144           mcp_x.ckSectionDelegate(mg);
145           CkCallback *cb = new CkCallback(CkIndex_PMEPencil_X::cb_client(NULL), CkArrayIndex3D(thisIndex.x,thisIndex.y,0), pme_x);
146           mg->setReductionClient(mcp_x, cb);
147       }
148   }
149   PMEPencil_X(CkMigrateMessage *m) {}
150
151   void start()
152   {
153    //thisindex.x thisindex.y
154     // x (yz)(x), y(x, z)(y)
155     int yindex = thisIndex.x/grain_size;
156     for(int x=0; x<grain_size; x++)
157     {
158       DataMsg *msg= new DataMsg;
159       msg->phrase = 1;
160       //CmiPrintf("g=%d(%d,%d,%d)==>(%d, %d,%d)\n", grain_size, thisIndex.x, thisIndex.y, thisIndex.z, x+thisIndex.z*grain_size, thisIndex.y, yindex);
161       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
162     }
163   }
164   void recvTrans(DataMsg *msg_recv)
165   {
166     int expect_num, index;
167     expect_num = grid_x/pes_per_node;
168     index = msg_recv->phrase;
169
170     CkGetSectionInfo(sid, msg_recv);
171
172     if(msg_recv->phrase != PME_index)
173     {
174         buffered_num++;
175         buffered_phrase = msg_recv->phrase;
176         delete msg_recv;
177         return;
178     }
179     recv_nums++;
180     if(recv_nums == expect_num)
181     {
182         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
183         if(index == 0  ) //x (y,z) to y(x,z)
184         {
185             iteration++;
186             if(iteration == max_iter)
187             {
188                 mainProxy.done();
189                 return;
190             }
191             int yindex = thisIndex.x/grain_size;
192             for(int x=0; x<grain_size; x++)
193             {
194                 DataMsg *msg= new DataMsg;
195                 msg->phrase = msg_recv->phrase+1;
196                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
197             }
198         }else if(index == 1) //y(x,z) send to z(x,y)
199         {
200             int zindex = thisIndex.y/grain_size;
201             for(int y=0; y<grain_size; y++)
202             {
203                 DataMsg *msg= new DataMsg;
204                 msg->phrase = msg_recv->phrase+1;
205                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
206             }
207             PME_index = 3;
208             recv_nums = buffered_num;
209         }else if(index == 2) //Z(x,y) send to y(x,z)
210         {
211             int yindex = thisIndex.y/grain_size;
212             for(int z=0; z<grain_size; z++)
213             {
214                 DataMsg *msg= new DataMsg;
215                 msg->phrase = msg_recv->phrase+1;
216                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
217             }
218         } else if(index == 3) //y(x,z) to x(y,z)
219         {
220             int xindex = thisIndex.x/grain_size;
221             for(int y=0; y<grain_size; y++)
222             {
223                 DataMsg *msg= new DataMsg;
224                 msg->phrase = 0;
225                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
226             }
227             PME_index = 1;
228             recv_nums = buffered_num;
229         }
230         recv_nums = 0;
231         CkMulticastMgr *mg = CProxy_CkMulticastMgr(mCastGrpId).ckLocalBranch();
232         mg->contribute(0, NULL,CkReduction::nop, sid);
233     }
234     delete msg_recv;
235   }
236   void cb_client(CkReductionMsg *msg) {
237   }
238 };
239
240 /*array [1D]*/
241 class PMEPencil_Y : public CBase_PMEPencil_Y
242 {
243     int PME_index;
244     int buffered_num, buffered_phrase;
245     int recv_nums, iteration;
246 public:
247   PMEPencil_Y(int i)
248   {
249       PME_index = i;
250       recv_nums = 0;
251       iteration = 0;
252       buffered_num = 0;
253   }
254   PMEPencil_Y(CkMigrateMessage *m) {}
255
256   void start()
257   {
258    //thisindex.x thisindex.y
259     // x (yz)(x), y(x, z)(y)
260     int yindex = thisIndex.x/grain_size;
261     for(int x=0; x<grain_size; x++)
262     {
263       DataMsg *msg= new DataMsg;
264       msg->phrase = 1;
265       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
266     }
267   }
268   void recvTrans(DataMsg *msg_recv)
269   {
270     int expect_num, index;
271     expect_num = grid_x/pes_per_node;
272     index = msg_recv->phrase;
273
274     if(msg_recv->phrase != PME_index)
275     {
276         buffered_num++;
277         buffered_phrase = msg_recv->phrase;
278         delete msg_recv;
279         return;
280     }
281     recv_nums++;
282     if(recv_nums == expect_num)
283     {
284         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
285         if(index == 0  ) //x (y,z) to y(x,z)
286         {
287             iteration++;
288             if(iteration == max_iter)
289             {
290                 mainProxy.done();
291                 return;
292             }
293             int yindex = thisIndex.x/grain_size;
294             for(int x=0; x<grain_size; x++)
295             {
296                 DataMsg *msg= new DataMsg;
297                 msg->phrase = msg_recv->phrase+1;
298                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
299             }
300         }else if(index == 1) //y(x,z) send to z(x,y)
301         {
302             int zindex = thisIndex.y/grain_size;
303             for(int y=0; y<grain_size; y++)
304             {
305                 DataMsg *msg= new DataMsg;
306                 msg->phrase = msg_recv->phrase+1;
307                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
308             }
309             PME_index = 3;
310             recv_nums = buffered_num;
311         }else if(index == 2) //Z(x,y) send to y(x,z)
312         {
313             int yindex = thisIndex.y/grain_size;
314             for(int z=0; z<grain_size; z++)
315             {
316                 DataMsg *msg= new DataMsg;
317                 msg->phrase = msg_recv->phrase+1;
318                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
319             }
320         } else if(index == 3) //y(x,z) to x(y,z)
321         {
322             int xindex = thisIndex.x/grain_size;
323             for(int y=0; y<grain_size; y++)
324             {
325                 DataMsg *msg= new DataMsg;
326                 msg->phrase = 0;
327                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
328             }
329             PME_index = 1;
330             recv_nums = buffered_num;
331         }
332         recv_nums = 0;
333     }
334     delete msg_recv;
335   }
336   void cb_client(CkReductionMsg *msg) {
337   }
338 };
339
340 /*array [1D]*/
341 class PMEPencil_Z : public CBase_PMEPencil_Z
342 {
343     int PME_index;
344     int buffered_num, buffered_phrase;
345     int recv_nums, iteration;
346 public:
347   PMEPencil_Z(int i)
348   {
349       PME_index = i;
350       recv_nums = 0;
351       iteration = 0;
352       buffered_num = 0;
353   }
354   PMEPencil_Z(CkMigrateMessage *m) {}
355
356   void start()
357   {
358    //thisindex.x thisindex.y
359     // x (yz)(x), y(x, z)(y)
360     int yindex = thisIndex.x/grain_size;
361     for(int x=0; x<grain_size; x++)
362     {
363       DataMsg *msg= new DataMsg;
364       msg->phrase = 1;
365       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
366     }
367   }
368   void recvTrans(DataMsg *msg_recv)
369   {
370     int expect_num, index;
371     expect_num = grid_x/pes_per_node;
372     index = msg_recv->phrase;
373
374     if(msg_recv->phrase != PME_index)
375     {
376         buffered_num++;
377         buffered_phrase = msg_recv->phrase;
378         delete msg_recv;
379         return;
380     }
381     recv_nums++;
382     if(recv_nums == expect_num)
383     {
384         //CkPrintf("[%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
385         if(index == 0  ) //x (y,z) to y(x,z)
386         {
387             iteration++;
388             if(iteration == max_iter)
389             {
390                 mainProxy.done();
391                 return;
392             }
393             int yindex = thisIndex.x/grain_size;
394             for(int x=0; x<grain_size; x++)
395             {
396                 DataMsg *msg= new DataMsg;
397                 msg->phrase = msg_recv->phrase+1;
398                 pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
399             }
400         }else if(index == 1) //y(x,z) send to z(x,y)
401         {
402             int zindex = thisIndex.y/grain_size;
403             for(int y=0; y<grain_size; y++)
404             {
405                 DataMsg *msg= new DataMsg;
406                 msg->phrase = msg_recv->phrase+1;
407                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
408             }
409             PME_index = 3;
410             recv_nums = buffered_num;
411         }else if(index == 2) //Z(x,y) send to y(x,z)
412         {
413             int yindex = thisIndex.y/grain_size;
414             for(int z=0; z<grain_size; z++)
415             {
416                 DataMsg *msg= new DataMsg;
417                 msg->phrase = msg_recv->phrase+1;
418                 pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
419             }
420         } else if(index == 3) //y(x,z) to x(y,z)
421         {
422             int xindex = thisIndex.x/grain_size;
423             for(int y=0; y<grain_size; y++)
424             {
425                 DataMsg *msg= new DataMsg;
426                 msg->phrase = 0;
427                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
428             }
429             PME_index = 1;
430             recv_nums = buffered_num;
431         }
432         recv_nums = 0;
433     }
434     delete msg_recv;
435   }
436   void cb_client(CkReductionMsg *msg) {
437   }
438 };
439
440
441 #include "PMEMimic.def.h"