examples/charm++: Always use CkWallTimer, not CmiWallTimer (tested)
[charm.git] / examples / charm++ / PMEMimic / PMEMimic_Parallel / PMEMimic.C
1 #include <stdio.h>
2 #include "PMEMimic.decl.h"
3 #include "ckmulticast.h"
4
5 /*readonly*/ CProxy_Main mainProxy;
6 /*readonly*/ 
7 #define  YH_DEBUG 0
8 int     N;
9 int     grid_x;
10 int     grid_y;
11 int     grid_z;
12 int     max_iter;
13 int     pes_per_node_type;
14 int     pes_per_node;
15 int     grain_size;
16 CProxy_PMEPencil_X pme_x;
17 CProxy_PMEPencil_Y pme_y;
18 CProxy_PMEPencil_Z pme_z;
19
20
21 class DataMsg : public CMessage_DataMsg
22 {
23 public:
24     int phrase;
25     char data[2048];
26
27 };
28
29 class PMEMap : public CkArrayMap 
30
31     int offset;
32 public: 
33     PMEMap(int off) { 
34         if(pes_per_node == pes_per_node_type) offset = 0;
35         else offset = off*pes_per_node_type;} 
36     PMEMap(CkMigrateMessage *m){} 
37     int registerArray(CkArrayIndex& numElements,CkArrayID aid) { 
38         return 0; 
39     } 
40     int procNum(int /*arrayHdl*/,const CkArrayIndex &idx) { 
41         int penum;
42         int *index =  (int *)idx.data();
43         int node_index =  index[0]*grid_x + index[1];
44         penum = node_index * pes_per_node + index[2]+offset;
45         return penum; 
46     } 
47 }; 
48
49
50 /*mainchare*/
51 class Main : public CBase_Main
52 {
53     double nextPhraseTimer;
54     int done_pme, iteration;
55 public:
56
57     Main(CkArgMsg* m)
58     {
59         //Process command-line arguments
60  
61         grid_x = grid_y = grid_z = 10;
62         max_iter = 100;
63         pes_per_node_type = 3;
64         if(m->argc > 1)
65         {
66             pes_per_node = atoi(m->argv[1]);
67             pes_per_node_type = atoi(m->argv[2]);
68             pes_per_node_type = pes_per_node/pes_per_node_type; // 1 or 3
69             grid_x = grid_y = grid_z = atoi(m->argv[3]);
70             max_iter = atoi(m->argv[4]);
71         }
72         grain_size = grid_x/(pes_per_node_type);
73         delete m;
74       CkPrintf("exec pes_per_node_type(must be 3 times) grid_x iteration\n");
75     //Start the computation
76       CkPrintf("Running PMEMimic on %d processors for %d elements\n",
77           CkNumPes(), grid_x);
78       mainProxy = thisProxy;
79
80       CProxy_PMEMap myMap_x=CProxy_PMEMap::ckNew(0); 
81       CkArrayOptions opts_x(grid_y, grid_z, pes_per_node_type); 
82       opts_x.setMap(myMap_x);
83
84       CProxy_PMEMap myMap_y=CProxy_PMEMap::ckNew(1); 
85       CkArrayOptions opts_y(grid_x, grid_z, pes_per_node_type); 
86       opts_y.setMap(myMap_y);
87
88       CProxy_PMEMap myMap_z=CProxy_PMEMap::ckNew(2); 
89       CkArrayOptions opts_z(grid_x, grid_y, pes_per_node_type); 
90       opts_z.setMap(myMap_z);
91
92       pme_x = CProxy_PMEPencil_X::ckNew(0, opts_x);
93       pme_y = CProxy_PMEPencil_Y::ckNew(1, opts_y);
94       pme_z = CProxy_PMEPencil_Z::ckNew(2, opts_z);
95
96       done_pme=0;
97       nextPhraseTimer = CkWallTimer();
98       pme_x.nextPhrase();
99       
100     };
101
102     void done()
103     {
104         done_pme++;
105         if(done_pme == grid_x*grid_x)
106         {
107             done_pme = 0;
108
109             CkPrintf("\nPME(%d, %d, %d) on %d PEs (%d pes/node)(%d nodes)\n %d iteration, average time:%f(ms)\n", grid_x, grid_y, grid_z, CkNumPes(), pes_per_node, CkNumPes()/pes_per_node, max_iter, (CkWallTimer()-nextPhraseTimer)/max_iter*1000);
110             CkExit();
111         }
112     }
113 };
114
115 /*array [3D]*/
116 class PMEPencil_X : public CBase_PMEPencil_X
117 {
118     int recv_nums, iteration;
119     int barrier_num;
120     int phrase ;
121     int expect_num;
122 public:
123   PMEPencil_X(int i)
124   {
125       recv_nums = 0;
126       iteration = 0;
127       barrier_num = 0;
128       phrase = 1;
129       expect_num = grid_x/pes_per_node_type;
130   }
131   PMEPencil_X(CkMigrateMessage *m) {}
132
133   void nextPhrase()
134   {
135    //thisindex.x thisindex.y
136     // x (yz)(x), y(x, z)(y)
137     int yindex = thisIndex.x/grain_size;
138     for(int x=0; x<grain_size; x++)
139     {
140       DataMsg *msg= new DataMsg;
141       msg->phrase = phrase;
142 #if     YH_DEBUG
143       CmiPrintf("X==>y %d (%d,%d,%d)==>(%d, %d,%d)\n", grain_size, thisIndex.x, thisIndex.y, thisIndex.z, x+thisIndex.z*grain_size, thisIndex.y, yindex);
144 #endif
145       pme_y(x+thisIndex.z*grain_size, thisIndex.y, yindex ).recvTrans(msg);  
146     }
147   }
148   void recvTrans(DataMsg *msg_recv)
149   {
150     recv_nums++;
151     if(recv_nums == expect_num)
152     {
153 #if     YH_DEBUG
154         CkPrintf("X [%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
155 #endif
156         phrase = msg_recv->phrase+1;
157         pme_x(thisIndex.x, thisIndex.y, 0).reducePencils();
158         recv_nums = 0;
159     }
160     delete msg_recv;
161   }
162   void reducePencils() {
163     barrier_num++;
164     if(barrier_num == pes_per_node_type)
165     {
166         iteration++;
167         if(iteration == max_iter)
168         {
169             mainProxy.done();
170             return;
171         }
172        for(int i=0; i<pes_per_node_type; i++)
173        {
174             pme_x(thisIndex.x, thisIndex.y, i).nextPhrase();
175        }
176        barrier_num = 0;
177     }
178   }
179 };
180
181 /*array [3D]*/
182 class PMEPencil_Y : public CBase_PMEPencil_Y
183 {
184     int PME_index;
185     int buffered_num, buffered_phrase;
186     int recv_nums, iteration;
187     int barrier_num;
188     int phrase ;
189     int expect_num;
190 public:
191   PMEPencil_Y(int i)
192   {
193       PME_index = 1;
194       recv_nums = 0;
195       iteration = 0;
196       buffered_num = 0;
197       barrier_num = 0;
198       phrase = 1;
199       expect_num = grid_x/pes_per_node_type;
200   }
201   PMEPencil_Y(CkMigrateMessage *m) {}
202
203   void nextPhrase()
204   {
205       if(phrase == 1) //y(x,z) send to z(x,y)
206       {
207           int zindex = thisIndex.y/grain_size;
208             for(int y=0; y<grain_size; y++)
209             {
210                 DataMsg *msg= new DataMsg;
211                 msg->phrase = phrase+1;
212                 pme_z(thisIndex.x, y+thisIndex.z*grain_size, zindex).recvTrans(msg); 
213 #if     YH_DEBUG
214                 CmiPrintf("y==>Z %d (%d,%d,%d)==>(%d, %d,%d)\n", grain_size, thisIndex.x, thisIndex.y, thisIndex.z, thisIndex.x, y+thisIndex.z*grain_size, zindex);
215 #endif
216             }
217             PME_index = 3; 
218         } else if(phrase == 3) //y(x,z) to x(y,z)
219         {
220             int xindex = thisIndex.x/grain_size;
221             for(int y=0; y<grain_size; y++)
222             {
223                 DataMsg *msg= new DataMsg;
224                 msg->phrase = 0;
225                 pme_x(y+grain_size*thisIndex.z, thisIndex.y, xindex).recvTrans(msg); 
226 #if     YH_DEBUG
227                 CmiPrintf("y==>X %d (%d,%d,%d)==>(%d, %d,%d)\n", grain_size, thisIndex.x, thisIndex.y, thisIndex.z, y+grain_size*thisIndex.z, thisIndex.y, xindex);
228 #endif
229             }
230             PME_index = 1;
231         }
232   }
233   void recvTrans(DataMsg *msg_recv)
234   {
235     
236     if(msg_recv->phrase != PME_index)
237     {
238         buffered_num++;
239         buffered_phrase = msg_recv->phrase;
240         delete msg_recv;
241         return;
242     }
243     recv_nums++;
244     if(recv_nums == expect_num)
245     {
246 #if     YH_DEBUG
247         CkPrintf("Y [%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
248 #endif
249         phrase = msg_recv->phrase;
250         pme_y(thisIndex.x, thisIndex.y, 0).reducePencils();
251         recv_nums = buffered_num;
252         buffered_num = 0;
253     }
254     delete msg_recv;
255   }
256   void reducePencils() {
257     barrier_num++;
258     if(barrier_num == pes_per_node_type)
259     {
260        for(int i=0; i<pes_per_node_type; i++)
261        {
262             pme_y(thisIndex.x, thisIndex.y, i).nextPhrase();
263        }
264        barrier_num = 0;
265     }
266   }
267 };
268
269 /*array [3D]*/
270 class PMEPencil_Z : public CBase_PMEPencil_Z
271 {
272     int recv_nums, iteration;
273     int barrier_num;
274     int phrase; 
275     int expect_num;
276 public:
277   PMEPencil_Z(int i)
278   {
279       recv_nums = 0;
280       iteration = 0;
281       barrier_num = 0;
282       phrase = 1;
283       expect_num = grid_x/pes_per_node_type;
284   }
285   
286   PMEPencil_Z(CkMigrateMessage *m) {}
287
288   void nextPhrase()
289   {
290    //thisindex.x thisindex.y
291     // Z , y(x, z)(y)
292     int yindex = thisIndex.y/grain_size;
293     for(int z=0; z<grain_size; z++)
294     {
295         DataMsg *msg= new DataMsg;
296         msg->phrase = phrase+1;
297         pme_y(thisIndex.x, z+thisIndex.z*grain_size, yindex).recvTrans(msg); 
298 #if     YH_DEBUG
299         CmiPrintf("Z==>Y %d (%d,%d,%d)==>(%d, %d,%d)\n", grain_size, thisIndex.x, thisIndex.y, thisIndex.z, thisIndex.x, z+thisIndex.z*grain_size, yindex);
300 #endif
301     }
302   }
303   void recvTrans(DataMsg *msg_recv)
304   {
305     recv_nums++;
306     if(recv_nums == expect_num)
307     {
308 #if     YH_DEBUG
309         CkPrintf(" Z [%d, %d, %d] phrase %d, iter=%d\n", thisIndex.x, thisIndex.y, thisIndex.z, msg_recv->phrase, iteration);
310 #endif
311         phrase = msg_recv->phrase;
312         pme_z(thisIndex.x, thisIndex.y, 0).reducePencils();
313         recv_nums = 0;
314     }
315     delete msg_recv;
316   }
317   void reducePencils() {
318     barrier_num++;
319     if(barrier_num == pes_per_node_type)
320     {
321        for(int i=0; i<pes_per_node_type; i++)
322        {
323             pme_z(thisIndex.x, thisIndex.y, i).nextPhrase();
324        }
325        barrier_num = 0;
326     }
327   }
328 };
329
330 #include "PMEMimic.def.h"