fixing for ft
[charm.git] / src / conv-com / graphrouter.C
1 /**
2    @addtogroup ConvComlibRouter
3    @{
4    @file This file although built, has no usage in the entire system!!!
5 */
6
7 #include "graphrouter.h"
8 #include "hypercubetopology.h"
9
10 #define PERSISTENT_BUFSIZE 65536
11
12 #define gmap(pe) {if (gpes) pe=gpes[pe];}
13
14 GraphRouter::GraphRouter(int n, int me, Strategy *parent) : Router(parent) {
15     init(n, me, new HypercubeTopology(n, me));
16 }
17
18 void GraphRouter::init(int n, int me, TopologyDescriptor *tp)
19 {  
20     NumPes=n;
21     MyPe=me;
22     gpes=NULL;
23     this->tp = tp;
24     
25     PeGraph = new PeTable(NumPes);
26     pesToSend = new int[NumPes];
27     nstages = tp->getNumStages() + 1;
28     currentIteration = 0;
29     
30     stageComplete = new int[nstages];
31     recvExpected = new int[nstages];
32     recvCount = new int[nstages];
33
34     memset(stageComplete, 0, nstages * sizeof(int));
35     memset(recvCount, 0, nstages * sizeof(int));
36     
37     for(int count = 1; count < nstages; count++)
38         recvExpected[count] = tp->getNumMessagesExpected(count);
39     
40     curStage = 0;
41     ComlibPrintf("me=%d NUMPES=%d nstages=%d\n", MyPe, n, nstages);
42 }
43
44 GraphRouter::~GraphRouter()
45 {
46     delete PeGraph;
47     delete pesToSend;
48     delete tp;
49     delete [] stageComplete;
50     delete [] recvExpected;
51     delete [] recvCount;
52     delete [] neighborPeList;
53 }
54
55 void GraphRouter::NumDeposits(comID, int num)
56 {
57 }
58
59 void GraphRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
60 {
61     int npe=NumPes;
62     int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
63     for (int i=0;i<npe;i++) destpes[i]=i;
64     EachToManyMulticast(id, size, msg, npe, destpes, more);
65 }
66
67 void GraphRouter::sendMessages(comID id, int cur_stage){
68     int nsteps = tp->getNumSteps(cur_stage);
69     int nextpe = 0, npestosend = 0;
70     
71     for(int stepcount = 0; stepcount < nsteps; stepcount ++){
72         tp->getPesToSend(stepcount, cur_stage, npestosend, pesToSend, nextpe);
73         
74         gmap(nextpe);
75         ComlibPrintf("%d:sending to %d for %d pes in stage %d\n", MyPe, nextpe, npestosend, cur_stage);
76
77         int len;
78 #if CMK_COMLIB_USE_VECTORIZE
79         PTvectorlist newmsg;
80         newmsg=PeGraph->ExtractAndVectorize(id, cur_stage + 1, npestosend, 
81                                        pesToSend);
82 #else
83         char *newmsg;
84         newmsg=PeGraph->ExtractAndPack(id, cur_stage + 1, npestosend, 
85                                        pesToSend, &len);
86 #endif
87 #if CMK_PERSISTENT_COM
88         if(len < PERSISTENT_BUFSIZE)
89             if(currentIteration % 2)
90                 CmiUsePersistentHandle(&handlerArrayOdd[cur_stage], 1);
91             else
92                 CmiUsePersistentHandle(&handlerArrayEven[cur_stage], 1);
93 #endif          
94         
95         if (newmsg) {
96             if(cur_stage < nstages - 2)
97                 CmiSetHandler(newmsg, CkpvAccess(RouterRecvHandle));
98             else
99                 CmiSetHandler(newmsg, CkpvAccess(RouterProcHandle));
100 #if CMK_COMLIB_USE_VECTORIZE
101             CmiSyncVectorSendAndFree(nextpe, -newmsg->count, newmsg->sizes, newmsg->msgs);
102 #else
103             CmiSyncSendAndFree(nextpe, len, newmsg);
104 #endif
105         }
106         else {
107             SendDummyMsg(id, nextpe, cur_stage + 1);
108         }
109         
110 #if CMK_PERSISTENT_COM
111         if(len < PERSISTENT_BUFSIZE)
112             CmiUsePersistentHandle(NULL, 0);
113 #endif          
114     }
115 }
116
117 void GraphRouter::EachToManyMulticast(comID id, int size, void *msg, 
118                                       int numpes, int *destpes, int more)
119 {
120     PeGraph->InsertMsgs(numpes, destpes, size, msg);
121     if (more) return;
122
123     ComlibPrintf("All messages received %d\n", MyPe);
124     sendMessages(id, 0);
125
126     curStage = 1;
127
128     int stage_itr;
129     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
130         if(stageComplete[stage_itr]){
131             sendMessages(id, stage_itr);
132             stageComplete[stage_itr] = 0;
133         }
134         else break;
135     }
136     curStage = stage_itr;
137     if(curStage == nstages - 1)
138         ProcManyMsg(id, NULL);
139     else 
140         PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
141 }
142
143 void GraphRouter::RecvManyMsg(comID id, char *msg)
144 {
145     int stage = 0;
146     stage = PeGraph->UnpackAndInsert(msg);
147     
148     recvCount[stage] ++;
149     if (recvCount[stage] == recvExpected[stage]) {
150         ComlibPrintf("%d recvcount=%d recvexpected = %d stage=%d\n", MyPe, recvCount[stage], recvExpected[stage], stage);
151         
152         recvCount[stage] = 0;
153         stageComplete[stage] = 1;
154     }
155     
156     int stage_itr;
157     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
158         if(stageComplete[stage_itr]){
159             sendMessages(id, stage_itr);
160             stageComplete[stage_itr] = 0;
161         }
162         else break;
163     }
164     curStage = stage_itr;
165     if(curStage == nstages - 1)
166         ProcManyMsg(id, NULL);
167     else 
168         PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
169 }
170
171 void GraphRouter::DummyEP(comID id, int stage)
172 {
173     if(stage < nstages - 1) {
174         recvCount[stage] ++;
175         if (recvCount[stage] == recvExpected[stage]) {
176             ComlibPrintf("%d DUMMY recvcount=%d recvexpected = %d\n", MyPe, recvCount[stage], recvExpected[stage]);
177             recvCount[stage] = 0;
178             stageComplete[stage] = 1;
179         }
180
181         int stage_itr;
182         for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
183             if(stageComplete[stage_itr]){
184                 sendMessages(id, stage_itr);
185                 stageComplete[stage] = 0;
186             }
187             else break;
188         }
189         curStage = stage_itr;
190         if(curStage == nstages - 1)
191             ProcManyMsg(id, NULL);
192         else 
193             PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
194     }
195     else 
196         ProcManyMsg(id, NULL);
197 }
198
199 void GraphRouter:: ProcManyMsg(comID id, char *m)
200 {
201     int stage = nstages - 1;
202     if(m) {
203         PeGraph->UnpackAndInsert(m);
204         recvCount[stage] ++;
205     }
206
207     if(recvCount[stage] == recvExpected[stage]) {
208         ComlibPrintf("%d proc many msg %d\n", MyPe, stage);
209         stageComplete[stage] = 1;
210     }
211     else 
212         return;
213     
214     if(curStage != nstages -1)
215         return;
216
217     currentIteration ++;
218     recvCount[stage] = 0;
219     PeGraph->ExtractAndDeliverLocalMsgs(MyPe, container);
220     
221     PeGraph->Purge();
222     curStage = 0;
223     Done(id);
224 }
225
226 Router * newgraphobject(int n, int me, Strategy *strat)
227 {
228     ComlibPrintf("In create graph router \n");
229     Router *obj = new GraphRouter(n, me, strat);
230     return(obj);
231 }
232
233 void GraphRouter :: SetMap(int *pes)
234 {
235     gpes=pes;
236
237 #if CMK_PERSISTENT_COM
238     numNeighbors=0;
239     neighborPeList = new int[NumPes];
240
241     tp->getNeighbors(numNeighbors, neighborPeList);
242     handlerArrayOdd = new PersistentHandle[numNeighbors];
243     handlerArrayEven = new PersistentHandle[numNeighbors];
244
245     //Persistent handlers for all the neighbors
246     int pcount = 0;
247     for (pcount = 0; pcount < numNeighbors; pcount++) {
248         int dest = neighborPeList[pcount];
249         gmap(dest);
250         ComlibPrintf("%d:Creating Persistent Buffer of size %d at %d\n", MyPe,
251                      PERSISTENT_BUFSIZE, dest);
252         handlerArrayOdd[pcount] = CmiCreatePersistent(dest, 
253                                                       PERSISTENT_BUFSIZE);
254         ComlibPrintf("%d:Creating Even Persistent Buffer of size %d at %d\n",
255                      MyPe, PERSISTENT_BUFSIZE, dest);
256         handlerArrayEven[pcount] = CmiCreatePersistent(dest, 
257                                                        PERSISTENT_BUFSIZE);
258     }
259 #endif
260 }
261
262 /*@}*/