new implementation of AllToAll routines. now an option is present to enable
[charm.git] / src / conv-com / graphrouter.C
1 /*****************************************************************************
2  * $Source$
3  * $Author$
4  * $Date$
5  * $Revision$
6  *****************************************************************************/
7
8 #include "graphrouter.h"
9 #include "hypercubetopology.h"
10
11 #define gmap(pe) {if (gpes) pe=gpes[pe];}
12
13 GraphRouter::GraphRouter(int n, int me){
14     init(n, me, new HypercubeTopology(n, me));
15 }
16
17 void GraphRouter::init(int n, int me, TopologyDescriptor *tp)
18 {  
19     NumPes=n;
20     MyPe=me;
21     gpes=NULL;
22     this->tp = tp;
23     
24     PeGraph = new PeTable(NumPes);
25     pesToSend = new int[NumPes];
26     nstages = tp->getNumStages() + 1;
27     currentIteration = 0;
28     
29     stageComplete = new int[nstages];
30     recvExpected = new int[nstages];
31     recvCount = new int[nstages];
32
33     memset(stageComplete, 0, nstages * sizeof(int));
34     memset(recvCount, 0, nstages * sizeof(int));
35     
36     for(int count = 1; count < nstages; count++)
37         recvExpected[count] = tp->getNumMessagesExpected(count);
38     
39     curStage = 0;
40     ComlibPrintf("me=%d NUMPES=%d nstages=%d\n", MyPe, n, nstages);
41 }
42
43 GraphRouter::~GraphRouter()
44 {
45     delete PeGraph;
46     delete pesToSend;
47     delete tp;
48     delete [] stageComplete;
49     delete [] recvExpected;
50     delete [] recvCount;
51     delete [] neighborPeList;
52 }
53
54 void GraphRouter::NumDeposits(comID, int num)
55 {
56 }
57
58 void GraphRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
59 {
60     int npe=NumPes;
61     int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
62     for (int i=0;i<npe;i++) destpes[i]=i;
63     EachToManyMulticast(id, size, msg, npe, destpes, more);
64 }
65
66 void GraphRouter::sendMessages(comID id, int cur_stage){
67     int nsteps = tp->getNumSteps(cur_stage);
68     int nextpe = 0, npestosend = 0;
69     
70     for(int stepcount = 0; stepcount < nsteps; stepcount ++){
71         tp->getPesToSend(stepcount, cur_stage, npestosend, pesToSend, nextpe);
72         
73         gmap(nextpe);
74         ComlibPrintf("%d:sending to %d for %d pes in stage %d\n", MyPe, nextpe, npestosend, cur_stage);
75
76         int len;
77 #if CMK_COMMLIB_USE_VECTORIZE
78         PTvectorlist newmsg;
79         newmsg=PeGraph->ExtractAndVectorize(id, cur_stage + 1, npestosend, 
80                                        pesToSend);
81 #else
82         char *newmsg;
83         newmsg=PeGraph->ExtractAndPack(id, cur_stage + 1, npestosend, 
84                                        pesToSend, &len);
85 #endif
86 #if CMK_PERSISTENT_COMM
87         if(len < PERSISTENT_BUFSIZE)
88             if(currentIteration % 2)
89                 CmiUsePersistentHandle(&handlerArrayOdd[cur_stage], 1);
90             else
91                 CmiUsePersistentHandle(&handlerArrayEven[cur_stage], 1);
92 #endif          
93         
94         if (newmsg) {
95             if(cur_stage < nstages - 2)
96                 CmiSetHandler(newmsg, CkpvAccess(RecvHandle));
97             else
98                 CmiSetHandler(newmsg, CkpvAccess(ProcHandle));
99 #if CMK_COMMLIB_USE_VECTORIZE
100             CmiSyncVectorSendAndFree(nextpe, -newmsg->count, newmsg->sizes, newmsg->msgs);
101 #else
102             CmiSyncSendAndFree(nextpe, len, newmsg);
103 #endif
104         }
105         else {
106             SendDummyMsg(id, nextpe, cur_stage + 1);
107         }
108         
109 #if CMK_PERSISTENT_COMM
110         if(len < PERSISTENT_BUFSIZE)
111             CmiUsePersistentHandle(NULL, 0);
112 #endif          
113     }
114 }
115
116 void GraphRouter::EachToManyMulticast(comID id, int size, void *msg, 
117                                       int numpes, int *destpes, int more)
118 {
119     PeGraph->InsertMsgs(numpes, destpes, size, msg);
120     if (more) return;
121
122     ComlibPrintf("All messages received %d\n", MyPe);
123     sendMessages(id, 0);
124
125     curStage = 1;
126
127     int stage_itr;
128     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
129         if(stageComplete[stage_itr]){
130             sendMessages(id, stage_itr);
131             stageComplete[stage_itr] = 0;
132         }
133         else break;
134     }
135     curStage = stage_itr;
136     if(curStage == nstages - 1)
137         ProcManyMsg(id, NULL);
138     else 
139         PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
140 }
141
142 void GraphRouter::RecvManyMsg(comID id, char *msg)
143 {
144     int stage = 0;
145     stage = PeGraph->UnpackAndInsert(msg);
146     
147     recvCount[stage] ++;
148     if (recvCount[stage] == recvExpected[stage]) {
149         ComlibPrintf("%d recvcount=%d recvexpected = %d stage=%d\n", MyPe, recvCount[stage], recvExpected[stage], stage);
150         
151         recvCount[stage] = 0;
152         stageComplete[stage] = 1;
153     }
154     
155     int stage_itr;
156     for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
157         if(stageComplete[stage_itr]){
158             sendMessages(id, stage_itr);
159             stageComplete[stage_itr] = 0;
160         }
161         else break;
162     }
163     curStage = stage_itr;
164     if(curStage == nstages - 1)
165         ProcManyMsg(id, NULL);
166     else 
167         PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
168 }
169
170 void GraphRouter::DummyEP(comID id, int stage)
171 {
172     if(stage < nstages - 1) {
173         recvCount[stage] ++;
174         if (recvCount[stage] == recvExpected[stage]) {
175             ComlibPrintf("%d DUMMY recvcount=%d recvexpected = %d\n", MyPe, recvCount[stage], recvExpected[stage]);
176             recvCount[stage] = 0;
177             stageComplete[stage] = 1;
178         }
179
180         int stage_itr;
181         for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
182             if(stageComplete[stage_itr]){
183                 sendMessages(id, stage_itr);
184                 stageComplete[stage] = 0;
185             }
186             else break;
187         }
188         curStage = stage_itr;
189         if(curStage == nstages - 1)
190             ProcManyMsg(id, NULL);
191         else 
192             PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
193     }
194     else 
195         ProcManyMsg(id, NULL);
196 }
197
198 void GraphRouter:: ProcManyMsg(comID id, char *m)
199 {
200     int stage = nstages - 1;
201     if(m) {
202         PeGraph->UnpackAndInsert(m);
203         recvCount[stage] ++;
204     }
205
206     if(recvCount[stage] == recvExpected[stage]) {
207         ComlibPrintf("%d proc many msg %d\n", MyPe, stage);
208         stageComplete[stage] = 1;
209     }
210     else 
211         return;
212     
213     if(curStage != nstages -1)
214         return;
215
216     currentIteration ++;
217     recvCount[stage] = 0;
218     PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
219     
220     PeGraph->Purge();
221     curStage = 0;
222     Done(id);
223 }
224
225 Router * newgraphobject(int n, int me)
226 {
227     ComlibPrintf("In create graph router \n");
228     Router *obj = new GraphRouter(n, me);
229     return(obj);
230 }
231
232 void GraphRouter :: SetMap(int *pes)
233 {
234     gpes=pes;
235
236 #if CMK_PERSISTENT_COMM
237     numNeighbors=0;
238     neighborPeList = new int[NumPes];
239
240     tp->getNeighbors(numNeighbors, neighborPeList);
241     handlerArrayOdd = new PersistentHandle[numNeighbors];
242     handlerArrayEven = new PersistentHandle[numNeighbors];
243
244     //Persistent handlers for all the neighbors
245     int pcount = 0;
246     for (pcount = 0; pcount < numNeighbors; pcount++) {
247         int dest = neighborPeList[pcount];
248         gmap(dest);
249         ComlibPrintf("%d:Creating Persistent Buffer of size %d at %d\n", MyPe,
250                      PERSISTENT_BUFSIZE, dest);
251         handlerArrayOdd[pcount] = CmiCreatePersistent(dest, 
252                                                       PERSISTENT_BUFSIZE);
253         ComlibPrintf("%d:Creating Even Persistent Buffer of size %d at %d\n",
254                      MyPe, PERSISTENT_BUFSIZE, dest);
255         handlerArrayEven[pcount] = CmiCreatePersistent(dest, 
256                                                        PERSISTENT_BUFSIZE);
257     }
258 #endif
259 }