added a variable higherLevel and a method deliverer to be used to uniform the
[charm.git] / src / conv-com / graphrouter.C
1 /*****************************************************************************
2  * $Source$
3  * $Author$
4  * $Date$
5  * $Revision$
6  *****************************************************************************/
7
8 #include "graphrouter.h"
9 #include "hypercubetopology.h"
10
11 #define gmap(pe) {if (gpes) pe=gpes[pe];}
12
13 GraphRouter::GraphRouter(int n, int me){
14     init(n, me, new HypercubeTopology(n, me));
15 }
16
17 void GraphRouter::init(int n, int me, TopologyDescriptor *tp)
18 {  
19     NumPes=n;
20     MyPe=me;
21     gpes=NULL;
22     this->tp = tp;
23     
24     PeGraph = new PeTable(NumPes);
25     pesToSend = new int[NumPes];
26     nstages = tp->getNumStages() + 1;
27     currentIteration = 0;
28     
29     stageComplete = new int[nstages];
30     recvExpected = new int[nstages];
31     recvCount = new int[nstages];
32
33     memset(stageComplete, 0, nstages * sizeof(int));
34     memset(recvCount, 0, nstages * sizeof(int));
35     
36     for(int count = 1; count < nstages; count++)
37         recvExpected[count] = tp->getNumMessagesExpected(count);
38     
39     curStage = 0;
40     ComlibPrintf("me=%d NUMPES=%d nstages=%d\n", MyPe, n, nstages);
41 }
42
43 GraphRouter::~GraphRouter()
44 {
45     delete PeGraph;
46     delete pesToSend;
47     delete tp;
48     delete [] stageComplete;
49     delete [] recvExpected;
50     delete [] recvCount;
51     delete [] neighborPeList;
52 }
53
54 void GraphRouter::NumDeposits(comID, int num)
55 {
56 }
57
58 void GraphRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
59 {
60     int npe=NumPes;
61     int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
62     for (int i=0;i<npe;i++) destpes[i]=i;
63     EachToManyMulticast(id, size, msg, npe, destpes, more);
64 }
65
66 void GraphRouter::sendMessages(comID id, int cur_stage){
67     int nsteps = tp->getNumSteps(cur_stage);
68     int nextpe = 0, npestosend = 0;
69     
70     for(int stepcount = 0; stepcount < nsteps; stepcount ++){
71         tp->getPesToSend(stepcount, cur_stage, npestosend, pesToSend, nextpe);
72         
73         gmap(nextpe);
74         ComlibPrintf("%d:sending to %d for %d pes in stage %d\n", MyPe, nextpe, npestosend, cur_stage);
75
76         int len;
77         char *newmsg;
78         newmsg=PeGraph->ExtractAndPack(id, cur_stage + 1, npestosend, 
79                                        pesToSend, &len);
80         
81 #if CMK_PERSISTENT_COMM
82         if(len < PERSISTENT_BUFSIZE)
83             if(currentIteration % 2)
84                 CmiUsePersistentHandle(&handlerArrayOdd[cur_stage], 1);
85             else
86                 CmiUsePersistentHandle(&handlerArrayEven[cur_stage], 1);
87 #endif          
88         
89         if (newmsg) {
90             if(cur_stage < nstages - 2)
91                 CmiSetHandler(newmsg, CkpvAccess(RecvHandle));
92             else
93                 CmiSetHandler(newmsg, CkpvAccess(ProcHandle));
94
95             CmiSyncSendAndFree(nextpe, len, newmsg);
96         }
97         else {
98             SendDummyMsg(id, nextpe, cur_stage + 1);
99         }
100         
101 #if CMK_PERSISTENT_COMM
102         if(len < PERSISTENT_BUFSIZE)
103             CmiUsePersistentHandle(NULL, 0);
104 #endif          
105     }
106 }
107
108 void GraphRouter::EachToManyMulticast(comID id, int size, void *msg, 
109                                       int numpes, int *destpes, int more)
110 {
111     PeGraph->InsertMsgs(numpes, destpes, size, msg);
112     if (more) return;
113
114     ComlibPrintf("All messages received %d\n", MyPe);
115     sendMessages(id, 0);
116
117     curStage = 1;
118
119     int stage_itr;
120     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
121         if(stageComplete[stage_itr]){
122             sendMessages(id, stage_itr);
123             stageComplete[stage_itr] = 0;
124         }
125         else break;
126     }
127     curStage = stage_itr;
128     if(curStage == nstages - 1)
129         ProcManyMsg(id, NULL);
130     else 
131         PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
132 }
133
134 void GraphRouter::RecvManyMsg(comID id, char *msg)
135 {
136     int stage = 0;
137     stage = PeGraph->UnpackAndInsert(msg);
138     
139     recvCount[stage] ++;
140     if (recvCount[stage] == recvExpected[stage]) {
141         ComlibPrintf("%d recvcount=%d recvexpected = %d stage=%d\n", MyPe, recvCount[stage], recvExpected[stage], stage);
142         
143         recvCount[stage] = 0;
144         stageComplete[stage] = 1;
145     }
146     
147     int stage_itr;
148     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
149         if(stageComplete[stage_itr]){
150             sendMessages(id, stage_itr);
151             stageComplete[stage_itr] = 0;
152         }
153         else break;
154     }
155     curStage = stage_itr;
156     if(curStage == nstages - 1)
157         ProcManyMsg(id, NULL);
158     else 
159         PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
160 }
161
162 void GraphRouter::DummyEP(comID id, int stage)
163 {
164     if(stage < nstages - 1) {
165         recvCount[stage] ++;
166         if (recvCount[stage] == recvExpected[stage]) {
167             ComlibPrintf("%d DUMMY recvcount=%d recvexpected = %d\n", MyPe, recvCount[stage], recvExpected[stage]);
168             recvCount[stage] = 0;
169             stageComplete[stage] = 1;
170         }
171
172         int stage_itr;
173         for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
174             if(stageComplete[stage_itr]){
175                 sendMessages(id, stage_itr);
176                 stageComplete[stage] = 0;
177             }
178             else break;
179         }
180         curStage = stage_itr;
181         if(curStage == nstages - 1)
182             ProcManyMsg(id, NULL);
183         else 
184             PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
185     }
186     else 
187         ProcManyMsg(id, NULL);
188 }
189
190 void GraphRouter:: ProcManyMsg(comID id, char *m)
191 {
192     int stage = nstages - 1;
193     if(m) {
194         PeGraph->UnpackAndInsert(m);
195         recvCount[stage] ++;
196     }
197
198     if(recvCount[stage] == recvExpected[stage]) {
199         ComlibPrintf("%d proc many msg %d\n", MyPe, stage);
200         stageComplete[stage] = 1;
201     }
202     else 
203         return;
204     
205     if(curStage != nstages -1)
206         return;
207
208     currentIteration ++;
209     recvCount[stage] = 0;
210     PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
211     
212     PeGraph->Purge();
213     curStage = 0;
214     Done(id);
215 }
216
217 Router * newgraphobject(int n, int me)
218 {
219     ComlibPrintf("In create graph router \n");
220     Router *obj = new GraphRouter(n, me);
221     return(obj);
222 }
223
224 void GraphRouter :: SetMap(int *pes)
225 {
226     gpes=pes;
227
228 #if CMK_PERSISTENT_COMM
229     numNeighbors=0;
230     neighborPeList = new int[NumPes];
231
232     tp->getNeighbors(numNeighbors, neighborPeList);
233     handlerArrayOdd = new PersistentHandle[numNeighbors];
234     handlerArrayEven = new PersistentHandle[numNeighbors];
235
236     //Persistent handlers for all the neighbors
237     int pcount = 0;
238     for (pcount = 0; pcount < numNeighbors; pcount++) {
239         int dest = neighborPeList[pcount];
240         gmap(dest);
241         ComlibPrintf("%d:Creating Persistent Buffer of size %d at %d\n", MyPe,
242                      PERSISTENT_BUFSIZE, dest);
243         handlerArrayOdd[pcount] = CmiCreatePersistent(dest, 
244                                                       PERSISTENT_BUFSIZE);
245         ComlibPrintf("%d:Creating Even Persistent Buffer of size %d at %d\n",
246                      MyPe, PERSISTENT_BUFSIZE, dest);
247         handlerArrayEven[pcount] = CmiCreatePersistent(dest, 
248                                                        PERSISTENT_BUFSIZE);
249     }
250 #endif
251 }