Cleaned up the code and added code to handle the new topologies like Irregular mesh...
[charm.git] / src / ck-ldb / TopoCentLB.C
1 /**************************************************************************
2 ** Amit Sharma (asharma6@uiuc.edu)
3 ** November 23, 2004
4 **
5 ** This is a topology conscious load balancer.
6 ** It migrates objects to new processors based on the topology in which the processors are connected.
7 ****************************************************************************/
8
9 #include <math.h>
10 #include <stdlib.h>
11 #include "charm++.h"
12 #include "cklists.h"
13 #include "CentralLB.h"
14
15 #include "TopoCentLB.decl.h"
16
17 #include "TopoCentLB.h"
18
19 #define alpha PER_MESSAGE_SEND_OVERHEAD_DEFAULT  /*Startup time per message, seconds*/
20 #define beta PER_BYTE_SEND_OVERHEAD_DEFAULT     /*Long-message time per byte, seconds*/
21 #define DEG_THRES 0.50
22
23 //#define MAX_EDGE
24 //#define RAND_COMM
25 #define make_mapping 0
26
27 CreateLBFunc_Def(TopoCentLB,"Balance objects based on the network topology");
28
29
30 /*static void lbinit (void)
31 {
32   LBRegisterBalancer ("TopoCentLB",
33                       CreateTopoCentLB,
34                       AllocateTopoCentLB,
35                       "Balance objects based on the network topology");
36 }*/
37
38
39 TopoCentLB::TopoCentLB(const CkLBOptions &opt) : CentralLB (opt)
40 {
41   lbname = "TopoCentLB";
42   if (CkMyPe () == 0) {
43     CkPrintf ("[%d] TopoCentLB created\n",CkMyPe());
44   }
45 }
46
47
48 CmiBool TopoCentLB::QueryBalanceNow (int _step)
49 {
50   return CmiTrue;
51 }
52
53 TopoCentLB::~TopoCentLB(){
54         if(partgraph)   delete partgraph;
55         if(topo) delete topo;
56 }
57
58 /*This routine partitions the task graph minimizing the communication and balancing the object load on all partitions*/
59 /*It uses METIS library to accomplish that*/
60 void TopoCentLB::computePartitions(CentralLB::LDStats *stats,int count,int *newmap)
61 {
62         
63   int numobjs = stats->n_objs;
64         int i, j, m;
65
66   // allocate space for the computing data
67   double *objtime = new double[numobjs];
68   int *objwt = new int[numobjs];
69   int *origmap = new int[numobjs];
70   LDObjHandle *handles = new LDObjHandle[numobjs];
71   
72         for(i=0;i<numobjs;i++) {
73     objtime[i] = 0.0;
74     objwt[i] = 0;
75     origmap[i] = 0;
76   }
77
78   //Prepare compute loads for METIS library
79   for (i=0; i<stats->n_objs; i++) {
80     LDObjData &odata = stats->objData[i];
81     if (!odata.migratable) 
82       CmiAbort("MetisLB doesnot dupport nonmigratable object.\n");
83     int frompe = stats->from_proc[i];
84     origmap[i] = frompe;
85     objtime[i] = odata.wallTime*stats->procs[frompe].pe_speed;
86     handles[i] = odata.handle;
87   }
88
89   // to convert the weights on vertices to integers
90   double max_objtime = objtime[0];
91   for(i=0; i<numobjs; i++) {
92     if(max_objtime < objtime[i])
93       max_objtime = objtime[i];
94   }
95         int maxobj=0;
96         int totalwt=0;
97   double ratio = 1000.0/max_objtime;
98   for(i=0; i<numobjs; i++) {
99       objwt[i] = (int)(objtime[i]*ratio);
100                         if(maxobj<objwt[i])
101                                 maxobj=objwt[i];
102                         totalwt+=objwt[i];
103   }
104         
105   int **comm = new int*[numobjs];
106   for (i=0; i<numobjs; i++) {
107     comm[i] = new int[numobjs];
108     for (j=0; j<numobjs; j++)  {
109       comm[i][j] = 0;
110     }
111   }
112
113   //Prepare communication for METIS library
114   const int csz = stats->n_comm;
115   for(i=0; i<csz; i++) {
116       LDCommData &cdata = stats->commData[i];
117       //if(cdata.from_proc() || cdata.receiver.get_type() != LD_OBJ_MSG)
118         //continue;
119                         if(!cdata.from_proc() && cdata.receiver.get_type() == LD_OBJ_MSG){
120         int senderID = stats->getHash(cdata.sender);
121         int recverID = stats->getHash(cdata.receiver.get_destObj());
122         CmiAssert(senderID < numobjs);
123         CmiAssert(recverID < numobjs);
124                                 comm[senderID][recverID] += cdata.messages;
125         comm[recverID][senderID] += cdata.messages;
126                                 //Use bytes or messages -- do i include messages for objlist too...??
127                         }
128                         else if (cdata.receiver.get_type() == LD_OBJLIST_MSG) {
129                                 //CkPrintf("in objlist..\n");
130         int nobjs;
131         LDObjKey *objs = cdata.receiver.get_destObjs(nobjs);
132         int senderID = stats->getHash(cdata.sender);
133         for (j=0; j<nobjs; j++) {
134            int recverID = stats->getHash(objs[j]);
135            if((senderID == -1)||(recverID == -1))
136               if (_lb_args.migObjOnly()) continue;
137               else CkAbort("Error in search\n");
138            comm[senderID][recverID] += cdata.messages;
139            comm[recverID][senderID] += cdata.messages;
140         }
141                         }
142                 }
143
144 // ignore messages sent from an object to itself
145   for (i=0; i<numobjs; i++)
146     comm[i][i] = 0;
147
148   // construct the graph in CSR format
149   int *xadj = new int[numobjs+1];
150   int numedges = 0;
151   for(i=0;i<numobjs;i++) {
152     for(j=0;j<numobjs;j++) {
153       if(comm[i][j] != 0)
154         numedges++;
155     }
156   }
157   int *adjncy = new int[numedges];
158   int *edgewt = new int[numedges];
159         int factor = 10;
160   xadj[0] = 0;
161   int count4all = 0;
162   for (i=0; i<numobjs; i++) {
163     for (j=0; j<numobjs; j++) { 
164       if (comm[i][j] != 0) { 
165         adjncy[count4all] = j;
166         edgewt[count4all++] = comm[i][j]/factor;
167       }
168     }
169     xadj[i+1] = count4all;
170   }
171
172   //Call METIS routine
173   int wgtflag = 3; // Weights both on vertices and edges
174   int numflag = 0; // C Style numbering
175   int options[5];
176   int edgecut;
177   int sameMapFlag = 1;
178
179         options[0] = 0;
180
181   if (count < 1) {
182     CkPrintf("error: Number of Pe less than 1!");
183   }
184   else if (count == 1) {
185         for(m=0;m<numobjs;m++) 
186                         newmap[i] = origmap[i];
187         sameMapFlag = 1;
188   }
189   else {
190         sameMapFlag = 0;
191                 /*
192         if (count > 8)
193                         METIS_PartGraphKway(&numobjs, xadj, adjncy, objwt, edgewt, 
194                             &wgtflag, &numflag, &count, options, 
195                             &edgecut, newmap);
196           else
197                         METIS_PartGraphRecursive(&numobjs, xadj, adjncy, objwt, edgewt, 
198                                  &wgtflag, &numflag, &count, options, 
199                                  &edgecut, newmap);
200                 */
201                 METIS_PartGraphRecursive(&numobjs, xadj, adjncy, objwt, edgewt,
202                                  &wgtflag, &numflag, &count, options,
203                                  &edgecut, newmap);
204   }
205          
206  
207   //Debugging code: Checking load on each partition
208   if(_lb_args.debug() >=2){
209           int total=0;
210           int *chkwt = new int[count];
211           for(i=0;i<count;i++)
212                   chkwt[i]=0;
213           for(i=0;i<numobjs;i++){
214                 chkwt[newmap[i]] += objwt[i];
215                   total += objwt[i];
216           }
217           for(i=0;i<count;i++)
218                   CkPrintf("%d -- %d\n",i,chkwt[i]);
219           CkPrintf("Totalwt of all partitions after call to METIS:%d, Avg is %d\n",total,total/count);
220   }
221
222   //Clean up all the variables allocated in this routine
223   for(i=0;i<numobjs;i++)
224     delete[] comm[i];
225   delete[] comm;
226   delete[] objtime;
227   delete[] xadj;
228   delete[] adjncy;
229   delete[] objwt;
230   delete[] edgewt;
231         delete[] handles;
232   delete[] origmap;
233
234 }
235
236 int TopoCentLB::findMaxObjs(int *map,int totalobjs,int count)
237 {
238         int *max_num = new int[count];
239         int i;
240         int maxobjs=0;
241         
242         for(i=0;i<count;i++)
243                 max_num[i]=0;
244                 
245         for(i=0;i<totalobjs;i++)
246                 max_num[map[i]]++;
247         
248         for(i=0;i<count;i++)
249                 if(max_num[i]>maxobjs)
250                         maxobjs = max_num[i];
251         
252         delete[] max_num;
253
254         return maxobjs;
255 }
256
257 void TopoCentLB::Heapify(HeapNode *heap, int node, int heapSize)
258 {
259   int left = 2*node+1;
260   int right = 2*node+2;
261   int xchange;
262         
263   if (left < heapSize && (heap[left].key > heap[node].key))
264     xchange = left;
265   else 
266                 xchange = node;
267   
268   if (right < heapSize && (heap[right].key > heap[xchange].key))
269     xchange = right;
270
271   if (xchange != node) {
272     HeapNode tmp;
273     tmp = heap[node];
274     heap[node] = heap[xchange];
275     heap[xchange] = tmp;
276                 heapMapping[heap[node].node]=node;
277                 heapMapping[heap[xchange].node]=xchange;
278     Heapify(heap,xchange,heapSize);
279   }    
280 }
281
282
283 TopoCentLB::HeapNode TopoCentLB::extractMax(HeapNode *heap,int *heapSize){
284
285         if(*heapSize < 1)
286                 CmiAbort("Empty Heap passed to extractMin!\n");
287
288         HeapNode max = heap[0];
289         heap[0] = heap[*heapSize-1];
290         heapMapping[heap[0].node]=0;
291         *heapSize = *heapSize - 1;
292         Heapify(heap,0,*heapSize);
293         return max;
294 }
295
296 void TopoCentLB::BuildHeap(HeapNode *heap,int heapSize){
297         for(int i=heapSize/2; i >= 0; i--)
298                 Heapify(heap,i,heapSize);
299 }
300
301 void TopoCentLB :: increaseKey(HeapNode *heap,int i,double wt){
302         int val;
303         if(wt != -1.00){
304                 #ifdef MAX_EDGE
305                         if(wt>heap[i].key)
306                                 heap[i].key = wt;
307                 #else
308                         heap[i].key += wt;
309                 #endif
310         }
311         int parent = (i-1)/2;
312         
313         if(heap[parent].key >= heap[i].key)
314                 return;
315         else {
316                 HeapNode tmp = heap[parent];
317                 heap[parent] = heap[i];
318                 heap[i] = tmp;
319                 heapMapping[heap[parent].node]=parent;
320                 heapMapping[heap[i].node]=i;
321                 increaseKey(heap,parent,-1.00);
322         }
323 }
324
325 /*This routine implements the algorithm used to produce the partition-processor mapping*/
326 /*The algorithm uses an idea similar to the standard MST algorithm*/
327 void TopoCentLB :: calculateMST(PartGraph *partgraph,LBTopology *topo,int *proc_mapping,int max_comm_part){
328
329         int *inHeap;
330         double *keys;
331         int count = partgraph->n_nodes;
332         int i=0,j=0;
333
334   //Arrays needed for keeping information
335         inHeap = new int[partgraph->n_nodes];
336         keys = new double[partgraph->n_nodes];
337
338         int *assigned_procs = new int[count];
339
340         hopCount = new double*[count];
341         for(i=0;i<count;i++){
342                 proc_mapping[i]=-1;
343                 assigned_procs[i]=0;
344                 hopCount[i] = new double[count];
345                 for(j=0;j<count;j++)
346                         hopCount[i][j] = 0;
347         }
348
349   //Call a topology routine to fill up hopCount
350         topo->get_pairwise_hop_count(hopCount);
351         
352   int max_neighbors = topo->max_neighbors();
353         
354         HeapNode *heap = new HeapNode[partgraph->n_nodes];
355         heapMapping = new int[partgraph->n_nodes];
356         
357         int heapSize = 0;
358
359         for(i=0;i<partgraph->n_nodes;i++){
360                 heap[i].key = 0.00;
361                 heap[i].node = i;
362                 keys[i] = 0.00;
363                 inHeap[i] = 1;
364                 heapMapping[i]=i;
365         }
366
367         //Assign the max comm partition first
368         heap[max_comm_part].key = 1.00;
369         
370         heapSize = partgraph->n_nodes;
371         BuildHeap(heap,heapSize);
372
373         int k=0,comm_cnt=0,m=0,n=0;
374         int *commParts = new int[partgraph->n_nodes];
375         
376         //srand(count);
377
378         while(heapSize > 0){
379
380     /****Phase1: Extracting appropriate partition from heap****/
381
382                 HeapNode max = extractMax(heap,&heapSize);
383                 inHeap[max.node] = 0;
384
385                 for(i=0;i<partgraph->n_nodes;i++){
386                         commParts[i]=-1;
387                         PartGraph::Edge wt = partgraph->edges[max.node][i];
388                         if(wt == 0)
389                                 continue;
390                         if(inHeap[i]){
391                         #ifdef MAX_EDGE
392                                 if(wt>keys[i])
393                                         keys[i]=wt;
394                         #else
395                                 keys[i] += wt;
396                         #endif
397         /*This part has been COMMENTED out for optimizing the code: we handle the updation using heapMapping*/
398         /*array instead of searching for node in the heap everytime*/
399
400                                 //Update in the heap too
401                                 //First, find where this node is..in the heap
402                                 /*for(j=0;j<heapSize;j++)
403                                         if(heap[j].node == i)
404                                                 break;
405                                 if(j==heapSize)
406                                         CmiAbort("Some error in heap...\n");*/
407                                 increaseKey(heap,heapMapping[i],wt);
408                         }
409                 }
410                  
411     /*Phase2: ASSIGNING partition to processor*/
412                 
413     //Special case
414                 if(heapSize == partgraph->n_nodes-1){ //Assign max comm partition to 0th proc in the topology
415                         proc_mapping[max.node]=0;
416                         assigned_procs[0]=1;
417                         continue;
418                 }
419                 
420                 m=0;
421                 n=0;
422                 comm_cnt=0;
423
424                 int pot_proc=-1;
425                 double min_cost=-1;
426                 int min_cost_index=-1;
427                 double cost=0;
428                 int p=0;
429                 //int q=0;
430
431                 for(k=0;k<partgraph->n_nodes;k++){
432                         if(!inHeap[k] && partgraph->edges[k][max.node]){
433                                 commParts[comm_cnt]=k;
434                                 comm_cnt++;
435                         }
436                 }
437
438     //Optimized the loop by commenting out the get_hop_count code and getting all the hop counts initially
439                 for(m=0;m<count;m++){
440                         if(!assigned_procs[m]){
441                                 cost=0;
442                                 for(p=0;p<comm_cnt;p++){
443                                         //if(!hopCount[proc_mapping[commParts[p]]][m])
444                                                 //hopCount[proc_mapping[commParts[p]]][m]=topo->get_hop_count(proc_mapping[commParts[p]],m);
445                                         cost += hopCount[proc_mapping[commParts[p]]][m]*partgraph->edges[commParts[p]][max.node];
446                                 }
447                                 if(min_cost==-1 || cost<min_cost){
448                                         min_cost=cost;
449                                         min_cost_index=m;
450                                 }
451                         }
452                 }
453
454                 proc_mapping[max.node]=min_cost_index;
455                 assigned_procs[min_cost_index]=1;
456         }
457
458         //clear up memory
459         delete[] inHeap;
460         delete[] keys;
461         delete[] assigned_procs;
462         delete[] heap;
463         delete[] commParts;
464 }
465
466
467 void TopoCentLB :: work(CentralLB::LDStats *stats,int count)
468 {
469   int proc;
470   int obj;
471   int dest;
472   int i,j;
473         LDObjData *odata;
474         
475         if (_lb_args.debug() >= 2) {
476     CkPrintf("In TopoCentLB Strategy...\n");
477   }
478   
479   // Make sure that there is at least one available processor.
480   for (proc = 0; proc < count; proc++) {
481     if (stats->procs[proc].available) {
482       break;
483     }
484   }
485         if (proc == count) {
486     CmiAbort ("TopoCentLB: no available processors!");
487   }
488
489   
490   removeNonMigratable(stats,count);
491
492         int *newmap = new int[stats->n_objs];
493
494
495         if(make_mapping)
496                 computePartitions(stats,count,newmap);
497         else{
498                 //mapping taken from previous algo
499                 for(i=0;i<stats->n_objs;i++){
500       newmap[i]=stats->from_proc[i];
501     }
502         }
503
504   //Debugging Code
505   if(_lb_args.debug() >=2){
506           CkPrintf("Map obtained from partitioning:\n");
507     for(i=0;i<stats->n_objs;i++)
508                   CkPrintf(" %d,%d ",i,newmap[i]);
509   }
510
511         int max_objs = findMaxObjs(newmap,stats->n_objs,count);
512         
513         partgraph = new PartGraph(count,max_objs);
514
515         //Fill up the partition graph - first fill the nodes and then, the edges
516
517         for(i=0;i<stats->n_objs;i++)
518         {
519                 PartGraph::Node* n = &partgraph->nodes[newmap[i]];
520                 n->obj_list[n->num_objs]=i;
521                 n->num_objs++;
522         }
523
524         int *addedComm=new int[count];
525   
526   stats->makeCommHash();
527   
528         int max_comm_part=-1;
529         
530         double max_comm=0;
531
532   //Try putting random amount of communication on the partition graph edges to see if things work fine
533   //This also checks the running time of the algorithm since number of edges is high than in a practical scenario
534         #ifdef RAND_COMM
535         for(i=0;i<count;i++){
536                 for(j=i+1;j<count;j++){
537                         int val;
538                         if(rand()%5==0)
539                                 val=0;
540                         else
541                                 val= rand()%1000;
542                                 
543                         partgraph->edges[i][j] = val;
544                         partgraph->edges[j][i] = val;
545                         
546                         partgraph->nodes[i].comm += val;
547                         partgraph->nodes[j].comm += val;
548                         
549                         if(partgraph->nodes[i].comm > max_comm){
550                                 max_comm = partgraph->nodes[i].comm;
551                                 max_comm_part = i;
552                         }
553                         if(partgraph->nodes[j].comm > max_comm){
554                                 max_comm = partgraph->nodes[j].comm;
555                                 max_comm_part = j;
556                         }
557                 }
558         }
559         #else
560   //Adding communication to the partition graph edges
561         for(i=0;i<stats->n_comm;i++)
562         {
563                 //DO I consider other comm too....i.e. to or from a processor
564                 LDCommData &cdata = stats->commData[i];
565                 if(!cdata.from_proc() && cdata.receiver.get_type() == LD_OBJ_MSG){
566         int senderID = stats->getHash(cdata.sender);
567         int recverID = stats->getHash(cdata.receiver.get_destObj());
568                         CmiAssert(senderID < stats->n_objs);
569                 CmiAssert(recverID < stats->n_objs);
570                 
571                         if(newmap[senderID]==newmap[recverID])
572                                 continue;
573         
574                         if(partgraph->edges[newmap[senderID]][newmap[recverID]] == 0){
575                                 partgraph->nodes[newmap[senderID]].degree++;
576                                 partgraph->nodes[newmap[recverID]].degree++;
577                         }
578                 
579                         partgraph->edges[newmap[senderID]][newmap[recverID]] += cdata.bytes;
580                         partgraph->edges[newmap[recverID]][newmap[senderID]] += cdata.bytes;
581                         
582                         partgraph->nodes[newmap[senderID]].comm += cdata.bytes;
583                         partgraph->nodes[newmap[recverID]].comm += cdata.bytes;
584
585       //Keeping track of maximum communiacting partition
586                         if(partgraph->nodes[newmap[senderID]].comm > max_comm){
587                                 max_comm = partgraph->nodes[newmap[senderID]].comm;
588                                 max_comm_part = newmap[senderID];
589                         }
590                         if(partgraph->nodes[newmap[recverID]].comm > max_comm){
591                                 max_comm = partgraph->nodes[newmap[recverID]].comm;
592                                 max_comm_part = newmap[recverID];
593                         }
594                 }
595                 else if(cdata.receiver.get_type() == LD_OBJLIST_MSG) {
596                         int nobjs;
597         LDObjKey *objs = cdata.receiver.get_destObjs(nobjs);
598       int senderID = stats->getHash(cdata.sender);
599                         for(j=0;j<count;j++)
600                                 addedComm[j]=0;
601                         for (j=0; j<nobjs; j++) {
602         int recverID = stats->getHash(objs[j]);
603         if((senderID == -1)||(recverID == -1))
604                 if (_lb_args.migObjOnly()) continue;
605                 else CkAbort("Error in search\n");
606                                         
607                                 if(newmap[senderID]==newmap[recverID])
608                                         continue;
609         
610                                 if(partgraph->edges[newmap[senderID]][newmap[recverID]] == 0){
611                                         partgraph->nodes[newmap[senderID]].degree++;
612                                         partgraph->nodes[newmap[recverID]].degree++;
613                                 }
614
615         //Communication added only once for a message sent to many objects on a single processor
616                                 if(!addedComm[newmap[recverID]]){
617                                         partgraph->edges[newmap[senderID]][newmap[recverID]] += cdata.bytes;
618                                         partgraph->edges[newmap[recverID]][newmap[senderID]] += cdata.bytes;
619         
620                                         partgraph->nodes[newmap[senderID]].comm += cdata.bytes;
621                                         partgraph->nodes[newmap[recverID]].comm += cdata.bytes;
622
623                                         if(partgraph->nodes[newmap[senderID]].comm > max_comm){
624                                                 max_comm = partgraph->nodes[newmap[senderID]].comm;
625                                                 max_comm_part = newmap[senderID];
626                                         }
627                                         if(partgraph->nodes[newmap[recverID]].comm > max_comm){
628                                                 max_comm = partgraph->nodes[newmap[recverID]].comm;
629                                                 max_comm_part = newmap[recverID];
630                                         }
631                                         //bytesComm[newmap[senderID]][newmap[recverID]] += cdata.bytes;
632                                         //bytesComm[newmap[recverID]][newmap[senderID]] += cdata.bytes;
633                                         addedComm[newmap[recverID]]=1;
634                                 }
635       }
636                 }
637
638         }
639         #endif
640         
641         int *proc_mapping = new int[count];
642         
643         delete [] addedComm;
644                 
645         LBtopoFn topofn;
646         int max_neighbors=0;
647         int *neigh;
648         int num_neigh=0;
649
650   //Parsing the command line input for getting the processor topology
651         char *lbcopy = strdup(_lbtopo);
652   char *ptr = strchr(lbcopy, ':');
653   if (ptr!=NULL)
654     ptr = strtok(lbcopy, ":");
655         else
656                 ptr=lbcopy;
657
658         topofn = LBTopoLookup(ptr);
659   if (topofn == NULL) {
660     char str[1024];
661     CmiPrintf("TopoCentLB> Fatal error: Unknown topology: %s. Choose from:\n", ptr);
662     printoutTopo();
663     sprintf(str, "TopoCentLB> Fatal error: Unknown topology: %s", ptr);
664     CmiAbort(str);
665   }
666   
667         topo = topofn(count);
668
669   //Call the core routine to produce the partition processor mapping
670         calculateMST(partgraph,topo,proc_mapping,max_comm_part);
671         //Returned partition graph is a Maximum Spanning Tree -- converted in above function itself
672
673   //Debugging code: Result of mapping partition graph onto processor graph
674         if (_lb_args.debug()>1) {
675           CkPrintf("Resultant mapping..(partition,processor)\n");
676           for(i=0;i<count;i++)
677                   CkPrintf("%d,%d\n",i,proc_mapping[i]);
678   }
679
680   //Store the result in the load balancing database
681         int pe;
682         PartGraph::Node* n;
683         for(i=0;i<count;i++){
684                 pe = proc_mapping[i];
685                 n = &partgraph->nodes[i];
686                 for(j=0;j<n->num_objs;j++){
687                         stats->to_proc[n->obj_list[j]] = pe;
688                         if (_lb_args.debug()>1) 
689         CkPrintf("[%d] Obj %d migrating from %d to %d\n", CkMyPe(),n->obj_list[j],stats->from_proc[n->obj_list[j]],pe);
690                 }
691         }
692
693         delete[] newmap;
694         delete[] proc_mapping;
695         //Delete hopCount
696         for(i=0;i<count;i++)
697                 delete[] hopCount[i];
698
699         delete[] hopCount;
700         delete[] heapMapping;
701 }
702
703 #include "TopoCentLB.def.h"