Merge branch 'charm' of charmgit:charm into charm
[charm.git] / src / conv-com / graphrouter.C
1 /*****************************************************************************
2  * $Source$
3  * $Author$
4  * $Date$
5  * $Revision$
6  *****************************************************************************/
7 /**
8    @addtogroup ConvComlibRouter
9    @{
10    @file This file although built, has no usage in the entire system!!!
11 */
12
13 #include "graphrouter.h"
14 #include "hypercubetopology.h"
15
16 #define PERSISTENT_BUFSIZE 65536
17
18 #define gmap(pe) {if (gpes) pe=gpes[pe];}
19
20 GraphRouter::GraphRouter(int n, int me, Strategy *parent) : Router(parent) {
21     init(n, me, new HypercubeTopology(n, me));
22 }
23
24 void GraphRouter::init(int n, int me, TopologyDescriptor *tp)
25 {  
26     NumPes=n;
27     MyPe=me;
28     gpes=NULL;
29     this->tp = tp;
30     
31     PeGraph = new PeTable(NumPes);
32     pesToSend = new int[NumPes];
33     nstages = tp->getNumStages() + 1;
34     currentIteration = 0;
35     
36     stageComplete = new int[nstages];
37     recvExpected = new int[nstages];
38     recvCount = new int[nstages];
39
40     memset(stageComplete, 0, nstages * sizeof(int));
41     memset(recvCount, 0, nstages * sizeof(int));
42     
43     for(int count = 1; count < nstages; count++)
44         recvExpected[count] = tp->getNumMessagesExpected(count);
45     
46     curStage = 0;
47     ComlibPrintf("me=%d NUMPES=%d nstages=%d\n", MyPe, n, nstages);
48 }
49
50 GraphRouter::~GraphRouter()
51 {
52     delete PeGraph;
53     delete pesToSend;
54     delete tp;
55     delete [] stageComplete;
56     delete [] recvExpected;
57     delete [] recvCount;
58     delete [] neighborPeList;
59 }
60
61 void GraphRouter::NumDeposits(comID, int num)
62 {
63 }
64
65 void GraphRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
66 {
67     int npe=NumPes;
68     int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
69     for (int i=0;i<npe;i++) destpes[i]=i;
70     EachToManyMulticast(id, size, msg, npe, destpes, more);
71 }
72
73 void GraphRouter::sendMessages(comID id, int cur_stage){
74     int nsteps = tp->getNumSteps(cur_stage);
75     int nextpe = 0, npestosend = 0;
76     
77     for(int stepcount = 0; stepcount < nsteps; stepcount ++){
78         tp->getPesToSend(stepcount, cur_stage, npestosend, pesToSend, nextpe);
79         
80         gmap(nextpe);
81         ComlibPrintf("%d:sending to %d for %d pes in stage %d\n", MyPe, nextpe, npestosend, cur_stage);
82
83         int len;
84 #if CMK_COMLIB_USE_VECTORIZE
85         PTvectorlist newmsg;
86         newmsg=PeGraph->ExtractAndVectorize(id, cur_stage + 1, npestosend, 
87                                        pesToSend);
88 #else
89         char *newmsg;
90         newmsg=PeGraph->ExtractAndPack(id, cur_stage + 1, npestosend, 
91                                        pesToSend, &len);
92 #endif
93 #if CMK_PERSISTENT_COM
94         if(len < PERSISTENT_BUFSIZE)
95             if(currentIteration % 2)
96                 CmiUsePersistentHandle(&handlerArrayOdd[cur_stage], 1);
97             else
98                 CmiUsePersistentHandle(&handlerArrayEven[cur_stage], 1);
99 #endif          
100         
101         if (newmsg) {
102             if(cur_stage < nstages - 2)
103                 CmiSetHandler(newmsg, CkpvAccess(RouterRecvHandle));
104             else
105                 CmiSetHandler(newmsg, CkpvAccess(RouterProcHandle));
106 #if CMK_COMLIB_USE_VECTORIZE
107             CmiSyncVectorSendAndFree(nextpe, -newmsg->count, newmsg->sizes, newmsg->msgs);
108 #else
109             CmiSyncSendAndFree(nextpe, len, newmsg);
110 #endif
111         }
112         else {
113             SendDummyMsg(id, nextpe, cur_stage + 1);
114         }
115         
116 #if CMK_PERSISTENT_COM
117         if(len < PERSISTENT_BUFSIZE)
118             CmiUsePersistentHandle(NULL, 0);
119 #endif          
120     }
121 }
122
123 void GraphRouter::EachToManyMulticast(comID id, int size, void *msg, 
124                                       int numpes, int *destpes, int more)
125 {
126     PeGraph->InsertMsgs(numpes, destpes, size, msg);
127     if (more) return;
128
129     ComlibPrintf("All messages received %d\n", MyPe);
130     sendMessages(id, 0);
131
132     curStage = 1;
133
134     int stage_itr;
135     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
136         if(stageComplete[stage_itr]){
137             sendMessages(id, stage_itr);
138             stageComplete[stage_itr] = 0;
139         }
140         else break;
141     }
142     curStage = stage_itr;
143     if(curStage == nstages - 1)
144         ProcManyMsg(id, NULL);
145     else 
146         PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
147 }
148
149 void GraphRouter::RecvManyMsg(comID id, char *msg)
150 {
151     int stage = 0;
152     stage = PeGraph->UnpackAndInsert(msg);
153     
154     recvCount[stage] ++;
155     if (recvCount[stage] == recvExpected[stage]) {
156         ComlibPrintf("%d recvcount=%d recvexpected = %d stage=%d\n", MyPe, recvCount[stage], recvExpected[stage], stage);
157         
158         recvCount[stage] = 0;
159         stageComplete[stage] = 1;
160     }
161     
162     int stage_itr;
163     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
164         if(stageComplete[stage_itr]){
165             sendMessages(id, stage_itr);
166             stageComplete[stage_itr] = 0;
167         }
168         else break;
169     }
170     curStage = stage_itr;
171     if(curStage == nstages - 1)
172         ProcManyMsg(id, NULL);
173     else 
174         PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
175 }
176
177 void GraphRouter::DummyEP(comID id, int stage)
178 {
179     if(stage < nstages - 1) {
180         recvCount[stage] ++;
181         if (recvCount[stage] == recvExpected[stage]) {
182             ComlibPrintf("%d DUMMY recvcount=%d recvexpected = %d\n", MyPe, recvCount[stage], recvExpected[stage]);
183             recvCount[stage] = 0;
184             stageComplete[stage] = 1;
185         }
186
187         int stage_itr;
188         for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
189             if(stageComplete[stage_itr]){
190                 sendMessages(id, stage_itr);
191                 stageComplete[stage] = 0;
192             }
193             else break;
194         }
195         curStage = stage_itr;
196         if(curStage == nstages - 1)
197             ProcManyMsg(id, NULL);
198         else 
199             PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
200     }
201     else 
202         ProcManyMsg(id, NULL);
203 }
204
205 void GraphRouter:: ProcManyMsg(comID id, char *m)
206 {
207     int stage = nstages - 1;
208     if(m) {
209         PeGraph->UnpackAndInsert(m);
210         recvCount[stage] ++;
211     }
212
213     if(recvCount[stage] == recvExpected[stage]) {
214         ComlibPrintf("%d proc many msg %d\n", MyPe, stage);
215         stageComplete[stage] = 1;
216     }
217     else 
218         return;
219     
220     if(curStage != nstages -1)
221         return;
222
223     currentIteration ++;
224     recvCount[stage] = 0;
225     PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
226     
227     PeGraph->Purge();
228     curStage = 0;
229     Done(id);
230 }
231
232 Router * newgraphobject(int n, int me, Strategy *strat)
233 {
234     ComlibPrintf("In create graph router \n");
235     Router *obj = new GraphRouter(n, me, strat);
236     return(obj);
237 }
238
239 void GraphRouter :: SetMap(int *pes)
240 {
241     gpes=pes;
242
243 #if CMK_PERSISTENT_COM
244     numNeighbors=0;
245     neighborPeList = new int[NumPes];
246
247     tp->getNeighbors(numNeighbors, neighborPeList);
248     handlerArrayOdd = new PersistentHandle[numNeighbors];
249     handlerArrayEven = new PersistentHandle[numNeighbors];
250
251     //Persistent handlers for all the neighbors
252     int pcount = 0;
253     for (pcount = 0; pcount < numNeighbors; pcount++) {
254         int dest = neighborPeList[pcount];
255         gmap(dest);
256         ComlibPrintf("%d:Creating Persistent Buffer of size %d at %d\n", MyPe,
257                      PERSISTENT_BUFSIZE, dest);
258         handlerArrayOdd[pcount] = CmiCreatePersistent(dest, 
259                                                       PERSISTENT_BUFSIZE);
260         ComlibPrintf("%d:Creating Even Persistent Buffer of size %d at %d\n",
261                      MyPe, PERSISTENT_BUFSIZE, dest);
262         handlerArrayEven[pcount] = CmiCreatePersistent(dest, 
263                                                        PERSISTENT_BUFSIZE);
264     }
265 #endif
266 }
267
268 /*@}*/