The new version of comlib! This version passed "make test" in charm/tests on order...
authorIsaac Dooley <idooley2@illinois.edu>
Thu, 28 May 2009 20:15:15 +0000 (20:15 +0000)
committerIsaac Dooley <idooley2@illinois.edu>
Thu, 28 May 2009 20:15:15 +0000 (20:15 +0000)
111 files changed:
src/ck-com/AAMLearner.C
src/ck-com/AAMLearner.h
src/ck-com/AAPLearner.C
src/ck-com/AAPLearner.h
src/ck-com/BroadcastStrategy.C
src/ck-com/BroadcastStrategy.h
src/ck-com/ComlibArrayListener.C
src/ck-com/ComlibArrayListener.h
src/ck-com/ComlibLearner.h
src/ck-com/ComlibManager.C
src/ck-com/ComlibManager.ci
src/ck-com/ComlibManager.h
src/ck-com/ComlibModuleInterface.C [new file with mode: 0644]
src/ck-com/ComlibSectionInfo.C
src/ck-com/ComlibSectionInfo.h
src/ck-com/ComlibStats.C
src/ck-com/ComlibStats.h
src/ck-com/ComlibStrategy.C
src/ck-com/ComlibStrategy.h
src/ck-com/DirectMulticastStrategy.C
src/ck-com/DirectMulticastStrategy.h
src/ck-com/DummyStrategy.C
src/ck-com/DummyStrategy.h
src/ck-com/EachToManyMulticastStrategy.C
src/ck-com/EachToManyMulticastStrategy.h
src/ck-com/KDirectMulticastStrategy.C
src/ck-com/KDirectMulticastStrategy.h
src/ck-com/MPIStrategy.C
src/ck-com/MPIStrategy.h
src/ck-com/MsgPacker.C
src/ck-com/MsgPacker.h
src/ck-com/MultiRingMulticast.C
src/ck-com/MultiRingMulticast.h
src/ck-com/MulticastStrategy.C [new file with mode: 0644]
src/ck-com/MulticastStrategy.h [new file with mode: 0644]
src/ck-com/NodeMulticast.C
src/ck-com/NodeMulticast.h
src/ck-com/PipeBroadcastStrategy.C
src/ck-com/PipeBroadcastStrategy.h
src/ck-com/PrioStreaming.C
src/ck-com/PrioStreaming.h
src/ck-com/RectMulticastStrategy.C
src/ck-com/RectMulticastStrategy.h
src/ck-com/RingMulticastStrategy.C
src/ck-com/RingMulticastStrategy.h
src/ck-com/comlib.h
src/ck-core/charm++.h
src/ck-core/ckarray.C
src/ck-core/ckarray.ci
src/ck-core/ckarray.h
src/ck-core/ckcallback.C
src/ck-core/cklocation.C
src/ck-core/cklocation.h
src/ck-core/cksection.h
src/ck-core/envelope.h
src/ck-cp/controlPoints.C
src/ck-ldb/TopoCentLB.h
src/ck-ldb/TopoLB.h
src/ck-perf/trace-bluegene.C
src/conv-com/3dgridrouter.C
src/conv-com/3dgridrouter.h
src/conv-com/MeshStreamingStrategy.C [new file with mode: 0644]
src/conv-com/MeshStreamingStrategy.h [new file with mode: 0644]
src/conv-com/StreamingStrategy.C [new file with mode: 0644]
src/conv-com/StreamingStrategy.h [new file with mode: 0644]
src/conv-com/convcomlib.h
src/conv-com/convcomlibmanager.C
src/conv-com/convcomlibmanager.h
src/conv-com/convcomlibstrategy.C
src/conv-com/convcomlibstrategy.h
src/conv-com/dummyconversestrategy.C
src/conv-com/dummyconversestrategy.h
src/conv-com/graphrouter.C
src/conv-com/graphrouter.h
src/conv-com/gridrouter.C
src/conv-com/gridrouter.h
src/conv-com/hypercuberouter.C [new file with mode: 0644]
src/conv-com/hypercuberouter.h [new file with mode: 0644]
src/conv-com/hypercubetopology.C
src/conv-com/hypercubetopology.h
src/conv-com/nocomlib.c [new file with mode: 0644]
src/conv-com/petable.C
src/conv-com/petable.h
src/conv-com/pipebroadcastconverse.C
src/conv-com/pipebroadcastconverse.h
src/conv-com/pipelinestrategy.C
src/conv-com/pipelinestrategy.h
src/conv-com/prefixrouter.C
src/conv-com/prefixrouter.h
src/conv-com/router.h
src/conv-com/routerstrategy.C
src/conv-com/routerstrategy.h
src/conv-com/treerouter.C
src/conv-com/treerouter.h
src/conv-core/convcore.c
src/conv-core/converse.h
src/conv-core/memory.c
src/libs/ck-libs/ampi/ampi.C
src/libs/ck-libs/ampi/ampiimpl.h
src/libs/ck-libs/cache/CkCache.h
src/libs/ck-libs/datatransfer/parallelsurfacetransfer.C
src/libs/ck-libs/datatransfer/paralleltransfer.C
src/libs/ck-libs/pose/gvt.C
src/libs/ck-libs/pose/pose.C
src/libs/ck-libs/pose/pose.h
src/libs/ck-libs/pose/sim.C
src/libs/ck-libs/tcharm/tcharm.C
src/libs/ck-libs/tcharm/tcharm.h
src/scripts/Make.depends
src/scripts/Makefile
src/scripts/charmc

index 72a13dd2722283a2d8f42d5e8c00ac24d4d63f5b..967eca875a1b9f769f07e7d273d9824f1dbee4a6 100644 (file)
-#include "AAMLearner.h"
-#include "ComlibManager.h"
-
-#include "EachToManyMulticastStrategy.h"
-//#include "RingMulticastStrategy.h"
-
-AAMLearner::AAMLearner() {
-   init();
-}
-
-void AAMLearner::init() {
-    alpha = ALPHA;
-    beta = BETA;
-    gamma = GAMMA;
-}
-
-Strategy *AAMLearner::optimizePattern(Strategy *strat, 
-                                           ComlibGlobalStats &stats) {
-    CharmStrategy *in_strat = (CharmStrategy *)strat;
-    double npes;              //, *pelist;
-    CharmStrategy *ostrat = NULL;
-
-    double degree = 0, msgsize = 0, nmsgs = 0;
-    stats.getAverageStats(strat->getInstance(), msgsize, nmsgs, 
-                          degree, npes);
-
-    double dcost = computeDirect(npes, msgsize, degree);
-    double mcost = computeMesh(npes, msgsize, degree);
-    double gcost = computeGrid(npes, msgsize, degree);
-    double hcost = computeHypercube(npes, msgsize, degree);
-    double mincost = min4(dcost, mcost, gcost, hcost);
-
-    int minstrat = -1;
-    if(in_strat->getType() == ARRAY_STRATEGY) {
-        CkArrayID said, daid;
-        CkArrayIndexMax *sidxlist, *didxlist;
-        int nsrc, ndest;
+// #ifdef filippo
+
+// #include "AAMLearner.h"
+// #include "ComlibManager.h"
+
+// #include "EachToManyMulticastStrategy.h"
+// //#include "RingMulticastStrategy.h"
+
+// AAMLearner::AAMLearner() {
+//    init();
+// }
+
+// void AAMLearner::init() {
+//     alpha = ALPHA;
+//     beta = BETA;
+//     gamma = GAMMA;
+// }
+
+// Strategy *AAMLearner::optimizePattern(Strategy *strat, 
+//                                            ComlibGlobalStats &stats) {
+//     CharmStrategy *in_strat = (CharmStrategy *)strat;
+//     double npes;              //, *pelist;
+//     CharmStrategy *ostrat = NULL;
+
+//     double degree = 0, msgsize = 0, nmsgs = 0;
+//     stats.getAverageStats(strat->getInstance(), msgsize, nmsgs, 
+//                           degree, npes);
+
+//     double dcost = computeDirect(npes, msgsize, degree);
+//     double mcost = computeMesh(npes, msgsize, degree);
+//     double gcost = computeGrid(npes, msgsize, degree);
+//     double hcost = computeHypercube(npes, msgsize, degree);
+//     double mincost = min4(dcost, mcost, gcost, hcost);
+
+//     int minstrat = -1;
+//     if(in_strat->getType() == ARRAY_STRATEGY) {
+//         CkArrayID said, daid;
+//         CkArrayIndexMax *sidxlist, *didxlist;
+//         int nsrc, ndest;
         
-        in_strat->ainfo.getSourceArray(said, sidxlist, nsrc);
-        in_strat->ainfo.getDestinationArray(daid, didxlist, ndest);
+//         in_strat->ainfo.getSourceArray(said, sidxlist, nsrc);
+//         in_strat->ainfo.getDestinationArray(daid, didxlist, ndest);
                
-        if(dcost == mincost) 
-            minstrat = USE_DIRECT;        
+//         if(dcost == mincost) 
+//             minstrat = USE_DIRECT;        
         
-        else if(mcost == mincost) 
-            minstrat = USE_MESH;                
-        else if(gcost == mincost) 
-            minstrat = USE_GRID;
-        else if(hcost == mincost) 
-            minstrat = USE_HYPERCUBE;               
-
-        //CkPrintf("Choosing router %d, %g, %g, %g\n", minstrat, 
-        //       mcost, hcost, dcost);
+//         else if(mcost == mincost) 
+//             minstrat = USE_MESH;                
+//         else if(gcost == mincost) 
+//             minstrat = USE_GRID;
+//         else if(hcost == mincost) 
+//             minstrat = USE_HYPERCUBE;               
+
+//         //CkPrintf("Choosing router %d, %g, %g, %g\n", minstrat, 
+//         //       mcost, hcost, dcost);
         
-        //if(minstrat != USE_DIRECT) {
-        ostrat = new EachToManyMulticastStrategy
-            (minstrat, said, daid,
-             nsrc, sidxlist, ndest,
-             didxlist);
+//         //if(minstrat != USE_DIRECT) {
+//         ostrat = new EachToManyMulticastStrategy
+//             (minstrat, said, daid,
+//              nsrc, sidxlist, ndest,
+//              didxlist);
         
-        ostrat->setMulticast();
+//         ostrat->setMulticast();
 
-        /*
-          }        
-          else {
-          ostrat = new RingMulticastStrategy(said, daid);
+//         /*
+//           }        
+//           else {
+//           ostrat = new RingMulticastStrategy(said, daid);
           
-          }
-        */
+//           }
+//         */
         
-        ostrat->setInstance(in_strat->getInstance());
-        ((EachToManyMulticastStrategy *)ostrat)->enableLearning();
-    }
-    else
-        CkAbort("Groups Not Implemented Yet\n");
+//         ostrat->setInstance(in_strat->getInstance());
+//         ((EachToManyMulticastStrategy *)ostrat)->enableLearning();
+//     }
+//     else
+//         CkAbort("Groups Not Implemented Yet\n");
 
-    //Group strategy implement later, foo bar !!
+//     //Group strategy implement later, foo bar !!
     
-    return ostrat;
-}
-
-//P = number of processors, m = msgsize, d = degree
-double AAMLearner::computeDirect(double P, double m, double d) {
-    double cost = 0.0;
-    cost = d * alpha;
-    cost += d * m * beta;
+//     return ostrat;
+// }
+
+// //P = number of processors, m = msgsize, d = degree
+// double AAMLearner::computeDirect(double P, double m, double d) {
+//     double cost = 0.0;
+//     cost = d * alpha;
+//     cost += d * m * beta;
     
-    return cost;
-}
-
-/******************* CHECK EQUATIONS FOR AAM ***********/
-//P = number of processors, m = msgsize, d = degree
-double AAMLearner::computeMesh(double P, double m, double d) {
-    double cost = 0.0;
-    cost = 2 * sqrt((double) P) * alpha;
-    cost += d * m * (beta + gamma);
+//     return cost;
+// }
+
+// /******************* CHECK EQUATIONS FOR AAM ***********/
+// //P = number of processors, m = msgsize, d = degree
+// double AAMLearner::computeMesh(double P, double m, double d) {
+//     double cost = 0.0;
+//     cost = 2 * sqrt((double) P) * alpha;
+//     cost += d * m * (beta + gamma);
     
-    return cost;
-}
+//     return cost;
+// }
 
-//P = number of processors, m = msgsize, d = degree
-double AAMLearner::computeHypercube(double P, double m, double d) {
+// //P = number of processors, m = msgsize, d = degree
+// double AAMLearner::computeHypercube(double P, double m, double d) {
 
-    if(P == 0)
-        return 0;
+//     if(P == 0)
+//         return 0;
 
-    double cost = 0.0;
-    double log_2_P = log(P)/log(2.0);
+//     double cost = 0.0;
+//     double log_2_P = log(P)/log(2.0);
     
-    cost = log_2_P * alpha;
-    cost += d * m * (beta + gamma);
+//     cost = log_2_P * alpha;
+//     cost += d * m * (beta + gamma);
 
-    return cost;
-}
+//     return cost;
+// }
 
-//P = number of processors, m = msgsize, d = degree
-double AAMLearner::computeGrid(double P, double m, double d) {
+// //P = number of processors, m = msgsize, d = degree
+// double AAMLearner::computeGrid(double P, double m, double d) {
 
-    double cost = 0.0;
-    cost = 3 * cubeRoot((double) P) * alpha;
-    cost += d * m * (beta + gamma);
+//     double cost = 0.0;
+//     cost = 3 * cubeRoot((double) P) * alpha;
+//     cost += d * m * (beta + gamma);
     
-    return cost;
-}
+//     return cost;
+// }
 
+// #endif
index bca091cddef9ec75e0ba944080af21139683ece5..baba344515b1ff6a2174668510819dbbf36fa871 100644 (file)
@@ -1,3 +1,9 @@
+/**
+   @addtogroup CharmComlib
+   @{
+*/
+
+
 
 #ifndef AAMLEARNER_H
 #define AAMLEARNER_H
@@ -33,3 +39,4 @@ class AAMLearner : public ComlibLearner {
 
 
 #endif
+/*@}*/
index ea516fbb9e464826c87cd31b56935dadbf9a82a4..066ea54957f06372815deaa3a82620bf0686ab7e 100644 (file)
-#include "AAPLearner.h"
-#include "ComlibManager.h"
-#include "EachToManyMulticastStrategy.h"
-
-#define max(a,b) ((a > b) ? a : b)
-
-AAPLearner::AAPLearner() {
-    init();
-}
-
-void AAPLearner::init() {
-    alpha = ALPHA;
-    beta = BETA;
-}
-
-Strategy *AAPLearner::optimizePattern(Strategy *strat, 
-                                           ComlibGlobalStats &stats) {
-    CharmStrategy *in_strat = (CharmStrategy *)strat;
-    double npes;              //, *pelist;
-    CharmStrategy *ostrat = NULL;
-
-    /*
-      if(in_strat->getType() == ARRAY_STRATEGY) {
-      in_strat->ainfo.getCombinedPeList(pelist, npes);
-      }
+// #ifdef filippo
+
+// #include "AAPLearner.h"
+// #include "ComlibManager.h"
+// #include "EachToManyMulticastStrategy.h"
+
+// #define max(a,b) ((a > b) ? a : b)
+
+// AAPLearner::AAPLearner() {
+//     init();
+// }
+
+// void AAPLearner::init() {
+//     alpha = ALPHA;
+//     beta = BETA;
+// }
+
+// Strategy *AAPLearner::optimizePattern(Strategy *strat, 
+//                                            ComlibGlobalStats &stats) {
+//     CharmStrategy *in_strat = (CharmStrategy *)strat;
+//     double npes;              //, *pelist;
+//     CharmStrategy *ostrat = NULL;
+
+//     /*
+//       if(in_strat->getType() == ARRAY_STRATEGY) {
+//       in_strat->ainfo.getCombinedPeList(pelist, npes);
+//       }
       
-      if(in_strat->getType() == GROUP_STRATEGY) {
-      CkGroupID gid;
-      //Convert to combined pelist
-      in_strat->ginfo.getSourceGroup(gid, pelist, npes);
-      }
-    */
-
-    double degree = 0, msgsize = 0, nmsgs = 0;
-    stats.getAverageStats(strat->getInstance(), msgsize, nmsgs, degree, npes);
-
-    double dcost = computeDirect(npes, msgsize, degree);
-    double mcost = computeMesh(npes, msgsize, degree);
-    double gcost = computeGrid(npes, msgsize, degree);
-    double hcost = computeHypercube(npes, msgsize, degree);
-    double mincost = min4(dcost, mcost, gcost, hcost);
-
-    int minstrat = USE_MESH;
-    if(dcost == mincost) 
-        minstrat = USE_DIRECT;
-    else if(mcost == mincost)                     
-        minstrat = USE_MESH;                
-    else if(gcost == mincost) 
-        minstrat = USE_GRID;
-    else if(hcost == mincost) 
-        minstrat = USE_HYPERCUBE;
-
-    //CkPrintf("Choosing router %d, %g, %g, %g, %g; %g : %g,%g,%g\n", minstrat, 
-    //       mcost, hcost, gcost, dcost, mincost, npes, msgsize, degree);
+//       if(in_strat->getType() == GROUP_STRATEGY) {
+//       CkGroupID gid;
+//       //Convert to combined pelist
+//       in_strat->ginfo.getSourceGroup(gid, pelist, npes);
+//       }
+//     */
+
+//     double degree = 0, msgsize = 0, nmsgs = 0;
+//     stats.getAverageStats(strat->getInstance(), msgsize, nmsgs, degree, npes);
+
+//     double dcost = computeDirect(npes, msgsize, degree);
+//     double mcost = computeMesh(npes, msgsize, degree);
+//     double gcost = computeGrid(npes, msgsize, degree);
+//     double hcost = computeHypercube(npes, msgsize, degree);
+//     double mincost = min4(dcost, mcost, gcost, hcost);
+
+//     int minstrat = USE_MESH;
+//     if(dcost == mincost) 
+//         minstrat = USE_DIRECT;
+//     else if(mcost == mincost)                     
+//         minstrat = USE_MESH;                
+//     else if(gcost == mincost) 
+//         minstrat = USE_GRID;
+//     else if(hcost == mincost) 
+//         minstrat = USE_HYPERCUBE;
+
+//     //CkPrintf("Choosing router %d, %g, %g, %g, %g; %g : %g,%g,%g\n", minstrat, 
+//     //       mcost, hcost, gcost, dcost, mincost, npes, msgsize, degree);
     
-    if(in_strat->getType() == ARRAY_STRATEGY) {
-        CkArrayID said, daid;
-        CkArrayIndexMax *sidxlist, *didxlist;
-        int nsrc, ndest;
+//     if(in_strat->getType() == ARRAY_STRATEGY) {
+//         CkArrayID said, daid;
+//         CkArrayIndexMax *sidxlist, *didxlist;
+//         int nsrc, ndest;
         
-        in_strat->ainfo.getSourceArray(said, sidxlist, nsrc);
-        in_strat->ainfo.getDestinationArray(daid, didxlist, ndest);
+//         in_strat->ainfo.getSourceArray(said, sidxlist, nsrc);
+//         in_strat->ainfo.getDestinationArray(daid, didxlist, ndest);
                 
-        ostrat = new EachToManyMulticastStrategy
-            (minstrat, said, daid,
-             nsrc, sidxlist, ndest,
-             didxlist);
-
-        ostrat->setInstance(in_strat->getInstance());
-        ((EachToManyMulticastStrategy *) ostrat)->enableLearning();
-    }
+//         ostrat = new EachToManyMulticastStrategy
+//             (minstrat, said, daid,
+//              nsrc, sidxlist, ndest,
+//              didxlist);
+
+//         ostrat->setInstance(in_strat->getInstance());
+//         ((EachToManyMulticastStrategy *) ostrat)->enableLearning();
+//     }
     
-    //Group strategy implement later, foo bar !!
-    if(in_strat->getType() == GROUP_STRATEGY) {
-        CkGroupID gid;
-        int src_npes, *src_pelist;
-        int dest_npes, *dest_pelist;
-        in_strat->ginfo.getSourceGroup(gid, src_pelist, src_npes);
-        in_strat->ginfo.getDestinationGroup(gid, dest_pelist, dest_npes); 
-
-        ostrat = new EachToManyMulticastStrategy
-            (minstrat, src_npes, src_pelist, dest_npes, dest_pelist);
-        ((EachToManyMulticastStrategy *) ostrat)->enableLearning();
-        ostrat->setInstance(in_strat->getInstance());
-    }
-
-    return ostrat;
-}
-
-//P = number of processors, m = msgsize, d = degree
-double AAPLearner::computeDirect(double P, double m, double d) {
-    double cost1, cost2;
-
-    /*  //Old equations do not model bursts 
-      cost = d * alpha;
-      cost += d * m * beta;
-    */
-
-    cost1 = (d-1) * ALPHA_NIC1 + alpha + m * beta + d * m * GAMMA_NIC; 
-    cost2 = alpha + d * ALPHA_NIC2 +  d * m * beta + m * GAMMA_NIC;
+//     //Group strategy implement later, foo bar !!
+//     if(in_strat->getType() == GROUP_STRATEGY) {
+//         CkGroupID gid;
+//         int src_npes, *src_pelist;
+//         int dest_npes, *dest_pelist;
+//         in_strat->ginfo.getSourceGroup(gid, src_pelist, src_npes);
+//         in_strat->ginfo.getDestinationGroup(gid, dest_pelist, dest_npes); 
+
+//         ostrat = new EachToManyMulticastStrategy
+//             (minstrat, src_npes, src_pelist, dest_npes, dest_pelist);
+//         ((EachToManyMulticastStrategy *) ostrat)->enableLearning();
+//         ostrat->setInstance(in_strat->getInstance());
+//     }
+
+//     return ostrat;
+// }
+
+// //P = number of processors, m = msgsize, d = degree
+// double AAPLearner::computeDirect(double P, double m, double d) {
+//     double cost1, cost2;
+
+//     /*  //Old equations do not model bursts 
+//       cost = d * alpha;
+//       cost += d * m * beta;
+//     */
+
+//     cost1 = (d-1) * ALPHA_NIC1 + alpha + m * beta + d * m * GAMMA_NIC; 
+//     cost2 = alpha + d * ALPHA_NIC2 +  d * m * beta + m * GAMMA_NIC;
     
-    return max(cost1, cost2); 
-}
+//     return max(cost1, cost2); 
+// }
 
-//P = number of processors, m = msgsize, d = degree
-double AAPLearner::computeMesh(double P, double m, double d) {
+// //P = number of processors, m = msgsize, d = degree
+// double AAPLearner::computeMesh(double P, double m, double d) {
 
-    double cost1, cost2;
+//     double cost1, cost2;
 
-    /* old equation 
-    cost = 2 * sqrt((double) P) * alpha;
-    cost += 2 * d * m * beta;
-    */
+//     /* old equation 
+//     cost = 2 * sqrt((double) P) * alpha;
+//     cost += 2 * d * m * beta;
+//     */
 
-    double sqrt_p = ceil(sqrt((double) P));
+//     double sqrt_p = ceil(sqrt((double) P));
 
-    cost1 = 2 * (sqrt_p - 2) * ALPHA_NIC1 + 2 * alpha + 2 * m * beta
-      + 2 * d * m * GAMMA_NIC; 
-    cost2 = 2 * alpha + 2 * (sqrt_p - 2) * ALPHA_NIC2 + 2 * d * m * beta + 2 * m *GAMMA_NIC;
+//     cost1 = 2 * (sqrt_p - 2) * ALPHA_NIC1 + 2 * alpha + 2 * m * beta
+//       + 2 * d * m * GAMMA_NIC; 
+//     cost2 = 2 * alpha + 2 * (sqrt_p - 2) * ALPHA_NIC2 + 2 * d * m * beta + 2 * m *GAMMA_NIC;
     
-    return max(cost1, cost2) + d * ALPHA_CHARM; 
-}
+//     return max(cost1, cost2) + d * ALPHA_CHARM; 
+// }
 
-//P = number of processors, m = msgsize, d = degree
-double AAPLearner::computeHypercube(double P, double m, double d) {
+// //P = number of processors, m = msgsize, d = degree
+// double AAPLearner::computeHypercube(double P, double m, double d) {
 
-    //Temporarily disabling hypercube
-    return 100;
+//     //Temporarily disabling hypercube
+//     return 100;
 
-    if(P == 0)
-        return 0;
+//     if(P == 0)
+//         return 0;
 
-    double cost = 0.0;
-    double log_2_P = log(P)/log(2.0);
+//     double cost = 0.0;
+//     double log_2_P = log(P)/log(2.0);
     
-    if(d >= P/2) {
-      cost = log_2_P * alpha;
-      cost += (P/2) * log_2_P * m * (beta + GAMMA_NIC + GAMMA_MEM);
-    }
-    else {
-      cost = log_2_P * alpha;
-      cost += log_2_P * d * m * (beta + GAMMA_NIC + GAMMA_MEM);
-    }
+//     if(d >= P/2) {
+//       cost = log_2_P * alpha;
+//       cost += (P/2) * log_2_P * m * (beta + GAMMA_NIC + GAMMA_MEM);
+//     }
+//     else {
+//       cost = log_2_P * alpha;
+//       cost += log_2_P * d * m * (beta + GAMMA_NIC + GAMMA_MEM);
+//     }
     
-    return cost + d * ALPHA_CHARM;
-}
-
-//P = number of processors, m = msgsize, d = degree
-double AAPLearner::computeGrid(double P, double m, double d) {
-    double cost1, cost2 = 0.0;
-    /*
-      cost = 3 * cubeRoot((double) P) * alpha;
-      cost += 3 * d * m * beta;
-    */
-
-    double cbrt_p = ceil(cubeRoot((double) P));
-
-    cost1 = 3 * (cbrt_p - 2) * ALPHA_NIC1 + 3 * alpha + 3 * m * beta +
-      3 * d *m * GAMMA_NIC;  
-    cost2 = 3 * alpha + 3 * (cbrt_p - 2) * ALPHA_NIC2 + 3 * d * m * beta + 3 * m *GAMMA_NIC; 
+//     return cost + d * ALPHA_CHARM;
+// }
+
+// //P = number of processors, m = msgsize, d = degree
+// double AAPLearner::computeGrid(double P, double m, double d) {
+//     double cost1, cost2 = 0.0;
+//     /*
+//       cost = 3 * cubeRoot((double) P) * alpha;
+//       cost += 3 * d * m * beta;
+//     */
+
+//     double cbrt_p = ceil(cubeRoot((double) P));
+
+//     cost1 = 3 * (cbrt_p - 2) * ALPHA_NIC1 + 3 * alpha + 3 * m * beta +
+//       3 * d *m * GAMMA_NIC;  
+//     cost2 = 3 * alpha + 3 * (cbrt_p - 2) * ALPHA_NIC2 + 3 * d * m * beta + 3 * m *GAMMA_NIC; 
     
-    return max(cost1, cost2) + d * ALPHA_CHARM;
-}
+//     return max(cost1, cost2) + d * ALPHA_CHARM;
+// }
 
+// #endif
index 70654f9b6ffcaf31edf24670052d90ea2978bff4..c391f00d44b857e80e8e75f3745d911ba94616c5 100644 (file)
@@ -1,3 +1,7 @@
+/**
+   @addtogroup CharmComlib
+   @{
+*/
 
 #ifndef AAPLEARNER_H
 #define AAPLEARNER_H
@@ -48,3 +52,5 @@ class AAPLearner : public ComlibLearner {
 
 
 #endif
+
+/*@}*/
index 985ba6c8d3edc50234b3284a80d8803d79ed5b96..96435d92a9c956212884bb5cc211c8b4a95f8848 100644 (file)
@@ -1,16 +1,16 @@
-
-//Broadcast strategy for charm++ programs using the net version
-//This strategy implements a tree based broadcast
-//Developed by Sameer Kumar 04/10/04
-
-//Extend for array sections later
+/**
+   @addtogroup ComlibCharmStrategy
+   @{
+   @file
+*/
 
 #include "BroadcastStrategy.h"
-#include "ComlibManager.h"
+//#include "ComlibManager.h"
 
 CkpvExtern(CkGroupID, cmgrID);
 extern int sfactor;
 
+/*
 static void recv_bcast_handler(void *msg) {
     CmiMsgHeaderExt *conv_header = (CmiMsgHeaderExt *) msg;
     int instid = conv_header->stratid;
@@ -20,7 +20,7 @@ static void recv_bcast_handler(void *msg) {
     
     bstrat->handleMessage((char *)msg);    
 }
-
+*/
 
 //Initialize the hypercube variables
 void BroadcastStrategy::initHypercube() {
@@ -32,7 +32,7 @@ void BroadcastStrategy::initHypercube() {
 //Constructor, 
 //Can read spanning factor from command line
 BroadcastStrategy::BroadcastStrategy(int topology) : 
-    CharmStrategy(), _topology(topology) {
+    Strategy(), CharmStrategy(), _topology(topology) {
     spanning_factor = DEFAULT_BROADCAST_SPANNING_FACTOR;
     if(sfactor > 0)
         spanning_factor = sfactor;
@@ -44,8 +44,9 @@ BroadcastStrategy::BroadcastStrategy(int topology) :
 //Array Constructor
 //Can read spanning factor from command line
 BroadcastStrategy::BroadcastStrategy(CkArrayID aid, int topology) : 
-    CharmStrategy(), _topology(topology) {
+    Strategy(), CharmStrategy(), _topology(topology) {
         
+       CkAbort("BroadcastStrategy currently works only for groups");
     setType(ARRAY_STRATEGY);
     ainfo.setDestinationArray(aid);
     
@@ -82,16 +83,17 @@ void BroadcastStrategy::insertMessage(CharmMessageHolder *cmsg){
 }
 
 //not implemented here because no bracketing is required for this strategy
-void BroadcastStrategy::doneInserting(){
-}
-
+//void BroadcastStrategy::doneInserting(){
+//}
 
+/*
 //register the converse handler to recieve the broadcast message
 void BroadcastStrategy::beginProcessing(int nelements) {
     handlerId = CkRegisterHandler((CmiHandler)recv_bcast_handler);
 }
+*/
 
-void BroadcastStrategy::handleMessage(char *msg) {
+void BroadcastStrategy::handleMessage(void *msg) {
     if(_topology == USE_TREE)
         handleTree(msg);
     else if(_topology == USE_HYPERCUBE) 
@@ -99,7 +101,7 @@ void BroadcastStrategy::handleMessage(char *msg) {
     else CkAbort("Unknown Topology");
 }
 
-void BroadcastStrategy::handleTree(char *msg){
+void BroadcastStrategy::handleTree(void *msg){
     
     envelope *env = (envelope *)msg;
     CmiMsgHeaderExt *conv_header = (CmiMsgHeaderExt *) msg;
@@ -109,7 +111,7 @@ void BroadcastStrategy::handleTree(char *msg){
     
     CkAssert(startpe>=0 && startpe < CkNumPes());
     
-    CmiSetHandler(msg, handlerId);
+    CmiSetHandler(msg, CkpvAccess(comlib_handler));
     
     conv_header->stratid = getInstance();
     
@@ -131,18 +133,20 @@ void BroadcastStrategy::handleTree(char *msg){
 
         CkAssert(p>=0 && p < CkNumPes() && p != CkMyPe());
 
-        CmiSyncSend(p, size, msg);
+        CmiSyncSend(p, size, (char*)msg);
     }
 
-    if(getType() == GROUP_STRATEGY)
+    if(getType() == GROUP_STRATEGY) {
+      ComlibPrintf("BroadcastStrategy: delivering message\n");
         CkSendMsgBranch(env->getEpIdx(), EnvToUsr(env), CkMyPe(), 
                         env->getGroupNum());
+    }
     else if(getType() == ARRAY_STRATEGY)
         ainfo.localBroadcast(env);        
 }
 
 
-void BroadcastStrategy::handleHypercube(char *msg){
+void BroadcastStrategy::handleHypercube(void *msg){
     envelope *env = (envelope *)msg;
 
     CmiMsgHeaderExt *conv_header = (CmiMsgHeaderExt *) msg;
@@ -155,7 +159,7 @@ void BroadcastStrategy::handleHypercube(char *msg){
     //CkPrintf("In hypercube %d, %d\n", (int)logp, curcycle); 
     
     /* assert(startpe>=0 && startpe<_Cmi_numpes); */
-    CmiSetHandler(msg, handlerId);
+    CmiSetHandler(msg, CkpvAccess(comlib_handler));
 
     conv_header->stratid = getInstance();
 
@@ -180,12 +184,14 @@ void BroadcastStrategy::handleHypercube(char *msg){
         }     
         
         if(p < CkNumPes())
-            CmiSyncSendFn(p, size, msg);                    
+            CmiSyncSendFn(p, size, (char*)msg);                    
     }
     
-    if(getType() == GROUP_STRATEGY)
+    if(getType() == GROUP_STRATEGY) {
+      ComlibPrintf("BroadcastStrategy: delivering message\n");
         CkSendMsgBranch(env->getEpIdx(), EnvToUsr(env), CkMyPe(), 
                         env->getGroupNum());
+    }
     else if(getType() == ARRAY_STRATEGY)
         ainfo.localBroadcast(env);        
 }
@@ -193,9 +199,12 @@ void BroadcastStrategy::handleHypercube(char *msg){
 
 //Pack the group id and the entry point of the user message
 void BroadcastStrategy::pup(PUP::er &p){
+    Strategy::pup(p);
     CharmStrategy::pup(p);    
 
     p | spanning_factor;
     p | _topology;
     p | logp;
 }
+
+/*@}*/
index adb54b689604b909b59b94353a009c9e3f5e58d9..af8fdace97ca559f8d087251c621a6b518fa5651 100644 (file)
@@ -1,3 +1,9 @@
+/**
+   @addtogroup ComlibCharmStrategy
+   @{
+   @file
+*/
+
 #ifndef BRAODCAST_STRATEGY
 #define BRAODCAST_STRATEGY
 
 
 #include "ComlibManager.h"
 
-//Broadcast strategy for charm++ programs using the net version
-//This stategy will wonly work for groups.
-//This strategy implements a tree based broadcast
-//I will extent it for arrays later.
-//Developed by Sameer Kumar 04/10/04
+/**
+   Broadcast strategy for charm++ programs using the net version.
+   This stategy will only work for groups.
+   This strategy implements a tree based broadcast
+
+   Developed by Sameer Kumar 04/10/04
+
+   @warning This strategy works only in particular situations and is not
+   generic. Its usage is adviced against.
 
-class BroadcastStrategy : public CharmStrategy {
+*/
+
+class BroadcastStrategy : public Strategy, public CharmStrategy {
 
     int _topology;         //Topology to use Tree or Hypercube
 
-    int handlerId;          //broadcast handler id
+    //int handlerId;          //broadcast handler id
     int spanning_factor;    //the spanning factor of the tree
 
     double logp;       //ceil of log of number of processors
 
     void initHypercube();      
 
-    void handleTree(char *msg);
-    void handleHypercube(char *msg);    
+    void handleTree(void *msg);
+    void handleHypercube(void *msg);    
 
  public:
     BroadcastStrategy(int topology = USE_HYPERCUBE);
     BroadcastStrategy(CkArrayID aid, int topology = USE_HYPERCUBE);
-    BroadcastStrategy(CkMigrateMessage *){}
+    BroadcastStrategy(CkMigrateMessage *m): Strategy(m), CharmStrategy(m) {}
+    void insertMessage(MessageHolder *msg) {insertMessage((CharmMessageHolder*)msg);};
     void insertMessage(CharmMessageHolder *msg);
-    void doneInserting();
+    //void doneInserting();
 
-    void handleMessage(char *msg);
-    void beginProcessing(int nelements);
+    void handleMessage(void *msg);
+    //void beginProcessing(int nelements);
 
     virtual void pup(PUP::er &p);
     PUPable_decl(BroadcastStrategy);
 };
 #endif
+
+/*@}*/
index 9b4f94a3fb668f34d745f0036c0c95adc019040a..b6effc8371b953bddd2b3722e6393903b4ca1adb 100644 (file)
-#include "charm++.h"
-#include "envelope.h"
-#include "ckhashtable.h"
-
-ComlibArrayListener::ComlibArrayListener () 
-    : CkArrayListener(0){ //Carry 1 int for the sid, not used yet
-    nElements = 0;
-    ht = new CkHashtableT<CkArrayIndexMax, CkArrayIndexMax *>;
-    //    CkPrintf("Creating Array Listener\n");
-}
-
-ComlibArrayListener::ComlibArrayListener (CkMigrateMessage *m)
-    :CkArrayListener(m) {
-    nElements = 0;
-    ht = new CkHashtableT<CkArrayIndexMax, CkArrayIndexMax *>;
-}
-
-void ComlibArrayListener::pup(PUP::er &p) {}
-
-void ComlibArrayListener::ckElementCreating(ArrayElement *elt){
-    addElement(elt, CmiFalse);
-    //CkPrintf("[%d] Element Created\n", CkMyPe());
-}
-
-void ComlibArrayListener::ckElementDied(ArrayElement *elt){
-    deleteElement(elt, CmiFalse);
-}
-
-void ComlibArrayListener::ckElementLeaving(ArrayElement *elt){
-    deleteElement(elt, CmiTrue);
-}
-
-CmiBool ComlibArrayListener::ckElementArriving(ArrayElement *elt){
-    addElement(elt, CmiTrue);
-    return CmiTrue;
-}
-
-void ComlibArrayListener::addElement(ArrayElement *elt, 
-                                     CmiBool migration_flag){
-    if(nElements == 0)
-        thisArrayID = elt->ckGetArrayID();
-
-    ht->put(elt->thisIndexMax) = &(elt->thisIndexMax);
-    //elt->thisIndexMax.print();
-    nElements ++;
-
-    if(!migration_flag) {
-        for(int count = 0; count < strategyList.length(); count++){
-            CharmStrategy *strategy = (CharmStrategy *)
-                strategyList[count]->strategy;
-            if(isRegistered(elt, strategy)) {
-                strategyList[count]->numElements ++;
-            }
-        }   
-    }
-}
-
-void ComlibArrayListener::deleteElement(ArrayElement *elt, 
-                                        CmiBool migration_flag){
-    ht->remove(elt->thisIndexMax);
-    nElements --;
-    
-    if(!migration_flag) {
-        for(int count = 0; count < strategyList.length(); count++){
-            CharmStrategy *strategy = (CharmStrategy *)
-                strategyList[count]->strategy;
-            if(isRegistered(elt, strategy)) {
-                strategyList[count]->numElements --;
-            }
-        }   
-    }
-}
-
-int ComlibArrayListener::isRegistered(ArrayElement *elt, 
-                                      CharmStrategy *strat){
-    CkArrayIndexMax idx = elt->thisIndexMax;
-
-    CkArrayID st_aid;
-    int st_nelements;
-    CkArrayIndexMax *st_elem;
-    strat->ainfo.getSourceArray(st_aid, st_elem, st_nelements);
-
-    if(st_nelements < 0)
-        CkAbort("Not an Array Strategy\n");
-    
-    if(st_nelements == 0)
-        return 1;   
-
-    for(int count = 0; count < st_nelements; count ++)
-        if(st_elem[count].compare(idx))
-            return 1;
-
-    return 0;
-}
-//Assumes strategy is already present in the strategy table   
-void ComlibArrayListener::registerStrategy(StrategyTableEntry *stable_entry) {
-    strategyList.insertAtEnd(stable_entry);
-
-    CharmStrategy *strat = (CharmStrategy *) stable_entry->strategy;
-
-    CkArrayID st_aid;
-    int st_nelements;
-    CkArrayIndexMax *st_elem;
-    strat->ainfo.getSourceArray(st_aid, st_elem, st_nelements);
-
-    if(st_nelements == 0) {//All elements of array in strategy
-        stable_entry->numElements += nElements;
-        /*
-        CkHashtableIterator *ht_iterator = ht->iterator();
-        ht_iterator->seekStart();
-        while(ht_iterator->hasNext()){
-            CkArrayIndexMax *idx;
-            ht_iterator->next((void **)&idx);
-            stable_entry->strategy->insertLocalIndex(*idx);       
-        }
-        */
-    }
-    else if (st_nelements > 0){ //Only some elements belong to strategy
-        for(int count = 0; count < st_nelements; count ++)
-            if(ht->get(st_elem[count]) != NULL) {
-                stable_entry->numElements ++;
-            }
-    }
-    else 
-        CkAbort("NOT an Array Strategy\n");
-
-}
-
-void ComlibArrayListener::getLocalIndices(CkVec<CkArrayIndexMax> &vec){
-    
-    CkHashtableIterator *ht_iterator = ht->iterator();
-    ht_iterator->seekStart();
-    while(ht_iterator->hasNext()){
-        CkArrayIndexMax *idx;
-        ht_iterator->next((void **)&idx);
-        vec.insertAtEnd(*idx);       
-    }
-}
-
-
+///**
+//   @addtogroup CharmComlib
+//*/
+///*@{*/
+//
+///** @file */
+//
+////#include "charm++.h"
+////#include "envelope.h"
+////#include "ckhashtable.h"
+//#include "ComlibArrayListener.h"
+//#include "ComlibManager.h"
+//
+///*
+//ComlibArrayListener::ComlibArrayListener () 
+//    : CkArrayListener(0){ //Carry 1 int for the sid, not used yet
+//    nElements = 0;
+//    ht = new CkHashtableT<CkArrayIndexMax, CkArrayIndexMax *>;
+//    //    CkPrintf("Creating Array Listener\n");
+//}
+//*/
+//
+//ComlibArrayListener::ComlibArrayListener(CkArrayID &id) : CkArrayListener(0) {
+//  setupFinished = 0;
+//  thisArrayID = id;
+//  ComlibPrintf("[%d] Creating ComlibArrayListener for array %d\n",CkMyPe(),((CkGroupID)id).idx);
+//}
+//
+//ComlibArrayListener::ComlibArrayListener (CkMigrateMessage *m)
+//    :CkArrayListener(m) {
+//  /*
+//    nElements = 0;
+//    ht = new CkHashtableT<CkArrayIndexMax, CkArrayIndexMax *>;
+//  */
+//}
+//
+//void ComlibArrayListener::pup(PUP::er &p) {
+//  ComlibPrintf("[%d] ComlibArrayListener pupping for %s, why?!?\n",CkMyPe(),
+//               p.isPacking()?"packing":(p.isUnpacking()?"unpacking":"sizing"));
+//}
+//
+//void ComlibArrayListener::ckEndInserting() {
+//     CkAssert(0);
+//     CkAssert(setupFinished==0);
+//
+//     ComlibPrintf("[%d] ComlibArrayListener::ckEndInserting\n",CkMyPe());
+//     CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+//     for (int i=0; i<userIDs.size(); ++i) {
+//             cgproxy[0].bracketedFinishSetup(userIDs[i]);
+//     }
+//
+//     setupFinished = 1;
+//  
+//}
+//
+//void ComlibArrayListener::ckElementCreating(ArrayElement *elt){
+//  ComlibPrintf("[%d] ComlibArrayListener::ckElementCreating\n",CkMyPe());
+//  for (int i=0; i<users.size(); ++i) {
+//    users[i]->newElement(thisArrayID, elt->ckGetArrayIndex());
+//  }
+//  //addElement(elt, CmiFalse);
+//    //CkPrintf("[%d] Element Created\n", CkMyPe());
+//}
+///*
+//void ComlibArrayListener::ckElementDied(ArrayElement *elt){
+//    deleteElement(elt, CmiFalse);
+//}
+//
+//void ComlibArrayListener::ckElementLeaving(ArrayElement *elt){
+//    deleteElement(elt, CmiTrue);
+//}
+//
+//CmiBool ComlibArrayListener::ckElementArriving(ArrayElement *elt){
+//    addElement(elt, CmiTrue);
+//    return CmiTrue;
+//}
+//
+//void ComlibArrayListener::addElement(ArrayElement *elt, 
+//                                     CmiBool migration_flag){
+//    if(nElements == 0)
+//        thisArrayID = elt->ckGetArrayID();
+//
+//    ht->put(elt->thisIndexMax) = &(elt->thisIndexMax);
+//    //elt->thisIndexMax.print();
+//    nElements ++;
+//
+//    if(!migration_flag) {
+//        for(int count = 0; count < strategyList.length(); count++){
+//            CharmStrategy *strategy = (CharmStrategy *)
+//                strategyList[count]->strategy;
+//            if(isRegistered(elt, strategy)) {
+//                strategyList[count]->numElements ++;
+//            }
+//        }   
+//    }
+//}
+//
+//void ComlibArrayListener::deleteElement(ArrayElement *elt, 
+//                                        CmiBool migration_flag){
+//    ht->remove(elt->thisIndexMax);
+//    nElements --;
+//    
+//    if(!migration_flag) {
+//        for(int count = 0; count < strategyList.length(); count++){
+//            CharmStrategy *strategy = (CharmStrategy *)
+//                strategyList[count]->strategy;
+//            if(isRegistered(elt, strategy)) {
+//                strategyList[count]->numElements --;
+//            }
+//        }   
+//    }
+//}
+//
+//int ComlibArrayListener::isRegistered(ArrayElement *elt, 
+//                                      CharmStrategy *strat){
+//    CkArrayIndexMax idx = elt->thisIndexMax;
+//
+//    CkArrayID st_aid;
+//    int st_nelements;
+//    CkArrayIndexMax *st_elem;
+//    strat->ainfo.getSourceArray(st_aid, st_elem, st_nelements);
+//
+//    if(st_nelements < 0)
+//        CkAbort("Not an Array Strategy\n");
+//    
+//    if(st_nelements == 0)
+//        return 1;   
+//
+//    for(int count = 0; count < st_nelements; count ++)
+//        if(st_elem[count].compare(idx))
+//            return 1;
+//
+//    return 0;
+//}
+// 
+////Assumes strategy is already present in the strategy table   
+//void ComlibArrayListener::registerStrategy(StrategyTableEntry *stable_entry) {
+//    strategyList.insertAtEnd(stable_entry);
+//
+//    CharmStrategy *strat = (CharmStrategy *) stable_entry->strategy;
+//
+//    CkArrayID st_aid;
+//    int st_nelements;
+//    CkArrayIndexMax *st_elem;
+//    strat->ainfo.getSourceArray(st_aid, st_elem, st_nelements);
+//
+//    if(st_nelements == 0) {//All elements of array in strategy
+//        stable_entry->numElements += nElements;
+//
+////         CkHashtableIterator *ht_iterator = ht->iterator();
+////         ht_iterator->seekStart();
+////         while(ht_iterator->hasNext()){
+////             CkArrayIndexMax *idx;
+////             ht_iterator->next((void **)&idx);
+////             stable_entry->strategy->insertLocalIndex(*idx);       
+////         }
+//
+//    }
+//    else if (st_nelements > 0){ //Only some elements belong to strategy
+//        for(int count = 0; count < st_nelements; count ++)
+//            if(ht->get(st_elem[count]) != NULL) {
+//                stable_entry->numElements ++;
+//            }
+//    }
+//    else 
+//        CkAbort("NOT an Array Strategy\n");
+//
+//}
+//
+//void ComlibArrayListener::getLocalIndices(CkVec<CkArrayIndexMax> &vec){
+//    
+//    CkHashtableIterator *ht_iterator = ht->iterator();
+//    ht_iterator->seekStart();
+//    while(ht_iterator->hasNext()){
+//        CkArrayIndexMax *idx;
+//        ht_iterator->next((void **)&idx);
+//        vec.insertAtEnd(*idx);       
+//    }
+//}
+//*/
+//
+///*@}*/
index fc9a4a952837b6ace28bf7deda825759624657ba..7cae1b75f67e2e57e67d011fe9942d2f7a292056 100644 (file)
@@ -1,45 +1,80 @@
-#ifndef COMMLIBARRAYLISTENER_H
-#define COMMLIBARRAYLISTENER_H
 
-#include "ComlibStrategy.h"
-#include "ckhashtable.h"
-
-class ComlibArrayListener : public CkArrayListener{
-    int nElements;
-    CkArrayID thisArrayID;
-    CkVec <StrategyTableEntry *> strategyList;
-    CkHashtableT<CkArrayIndexMax, CkArrayIndexMax*> *ht;
-    
-    int isRegistered(ArrayElement *elt, CharmStrategy *astrat);
-    void addElement(ArrayElement *elt, CmiBool mogration_flag);
-    void deleteElement(ArrayElement *elt, CmiBool migration_flag);
-    
- public:
-    ComlibArrayListener();
-    ComlibArrayListener(CkMigrateMessage *);
-
-    void ckElementCreating(ArrayElement *elt);
-    void ckElementDied(ArrayElement *elt);
-    
-    void ckElementLeaving(ArrayElement *elt);
-    CmiBool ckElementArriving(ArrayElement *elt);
-    
-    //Add strategy to listening list, strategy will get an the number
-    //of array elements lying on that processor
-    void registerStrategy(StrategyTableEntry *);
-
-    //remove strategy from table, and now it will not get updates
-    //about this array
-    void unregisterStrategy(StrategyTableEntry *entry) {
-        for(size_t count = 0; count < strategyList.size(); count++)
-            if(strategyList[count] == entry)
-                strategyList.remove(count);
-    }
-
-    void getLocalIndices(CkVec<CkArrayIndexMax> &vec);
-
-    void pup(PUP::er &p);
-    PUPable_decl(ComlibArrayListener);
-};
-
-#endif
+///*@{*/
+//
+///** @file */
+//
+//#ifndef COMMLIBARRAYLISTENER_H
+//#define COMMLIBARRAYLISTENER_H
+//
+//#include "ComlibStrategy.h"
+//
+///**
+// * This class is used by the ComlibArrayInfo class to keep track of new objects
+// * when the user specification is so. Namely, a ComlibArrayInfo can register
+// * itself with this ArrayListener so that every element that for every element
+// * that is created on this processor, the ComlibArrayInfo class is notified and
+// * can take appropriate action.
+// */
+//class ComlibArrayListener : public CkArrayListener {
+//  //int nElements;
+//  int setupFinished;
+//  CkArrayID thisArrayID;
+//  /*
+//  CkVec <StrategyTableEntry *> strategyList;
+//  CkHashtableT<CkArrayIndexMax, CkArrayIndexMax*> *ht;
+//    
+//  int isRegistered(ArrayElement *elt, CharmStrategy *astrat);
+//  void addElement(ArrayElement *elt, CmiBool mogration_flag);
+//  void deleteElement(ArrayElement *elt, CmiBool migration_flag);
+//  */
+//  CkVec<ComlibArrayInfo*> users;
+//  CkVec<int> userIDs;
+//
+// public:
+//  ComlibArrayListener(CkArrayID &id);
+//  ComlibArrayListener(CkMigrateMessage *);
+//
+//  inline bool operator==(CkArrayID &a) {
+//    return (thisArrayID == a);
+//  }
+//
+//  inline bool operator==(ComlibArrayListener &l) {
+//    return operator==(l.thisArrayID);
+//  }
+//
+//  inline void registerUser(ComlibArrayInfo *ai, int stratid=0) {
+//    users.push_back(ai);
+//    userIDs.push_back(stratid);
+//  }
+//
+//  void ckEndInserting();
+//
+//  void ckElementCreating(ArrayElement *elt);
+//  //void ckElementDied(ArrayElement *elt);
+//    
+//  //void ckElementLeaving(ArrayElement *elt);
+//  //CmiBool ckElementArriving(ArrayElement *elt);
+//    
+//  //Add strategy to listening list, strategy will get an the number
+//  //of array elements lying on that processor
+//  //void registerStrategy(StrategyTableEntry *);
+//
+//  //remove strategy from table, and now it will not get updates
+//  //about this array
+//  /*
+//  void unregisterStrategy(StrategyTableEntry *entry) {
+//    for(int count = 0; count < strategyList.size(); count++)
+//      if(strategyList[count] == entry)
+//     strategyList.remove(count);
+//  }
+//
+//  void getLocalIndices(CkVec<CkArrayIndexMax> &vec);
+//  */
+//
+//  void pup(PUP::er &p);
+//  PUPable_decl(ComlibArrayListener);
+//};
+//
+//#endif
+//
+///*@}*/
index e2b7af6552e9fa39273507496a317db2a9782e66..8fd6698370a1662e27a27dec171e77e03da605ba 100644 (file)
@@ -1,36 +1,48 @@
 #ifndef COMLIBLEARNER_H
 #define COMLIBLEARNER_H
 
+
+/**
+   @addtogroup CharmComlib
+   @{
+   @file
+
+   @brief ComlibLearner
+
+*/
+
 #include "convcomlibstrategy.h"
 
+class ComlibGlobalStats;
 
-/* Communication library learner which takes a strategy or a list of
+/** Communication library learner which takes a strategy or a list of
    strategies as input along with the communication pattern of the
    objects belonging to those strategies and returns new strategies to
    replace the input strategies. These new strategies optimize the
    communication pattern. */
 
-class ComlibGlobalStats;
 class ComlibLearner {
  public:
     virtual ~ComlibLearner() {}
-    //Configures parameters of the learner. Will be called by the
-    //communication library on every processor after the second
-    //barrier of the communication library.
+    //Configures parameters of the learner. Will be called by the
+    //communication library on every processor after the second
+    //barrier of the communication library.
     virtual void init() {}
     
-    //Optimizes a specific strategy. Returns a new optimized strategy
+    //Optimizes a specific strategy. Returns a new optimized strategy
     virtual Strategy* optimizePattern(Strategy *strat, 
                                       ComlibGlobalStats &sdata){
         return NULL;
     }
     
-    //Optimizes the communication pattern of a group of strategies
-    //together
+    //Optimizes the communication pattern of a group of strategies
+    //together
     virtual Strategy** optimizePattern(Strategy **strat, 
                                        ComlibGlobalStats &sdata){
         return NULL;
     }
 };
 
+/*@}*/
+
 #endif
index 3ceacc58cc09df032dabff4c91eea2c4ce16ea0f..fb69ff07d93b4068d94a6336e731e628bfc83615 100644 (file)
 /**
-   @addtogroup Comlib
+   @addtogroup CharmComlib
+   @{
+   @file
+   Implementation of the functions in ComlibManager.h and handler for message
+   transportation.
 */
-/*@{*/
+
 #include "ComlibManager.h"
-#include "EachToManyMulticastStrategy.h"
-#include "DirectMulticastStrategy.h"
-#include "StreamingStrategy.h"
-#include "DummyStrategy.h"
-#include "MPIStrategy.h"
-#include "NodeMulticast.h"
-#include "MsgPacker.h"
-#include "RingMulticastStrategy.h"
-#include "MultiRingMulticast.h"
-#include "PipeBroadcastStrategy.h"
-#include "BroadcastStrategy.h"
-#include "MeshStreamingStrategy.h"
-#include "RectMulticastStrategy.h"
-#include "PrioStreaming.h"
+#include "comlib.h"
+#include "ck.h"
+#include "envelope.h"
 
-CkpvExtern(int, RecvdummyHandle);
 
-CkpvDeclare(int, RecvmsgHandle);
-CkpvDeclare(int, RecvCombinedShortMsgHdlrIdx);
+// We only want to print debug information for a single strategy. Otherwise we'll get incredibly confused
+#undef ComlibManagerPrintf
+//#define ComlibManagerPrintf  if(instid==1)ComlibPrintf
+#define ComlibManagerPrintf  ComlibPrintf
+
+#define getmax(a,b) ((a)>(b)?(a):(b))
+
+CkpvExtern(int, RecvdummyHandle);
 CkpvDeclare(CkGroupID, cmgrID);
-CkpvExtern(ConvComlibManager *, conv_com_ptr);
+
+/***************************************************************************
+ * Handlers section:
+ *
+ * all the handlers used by the ComlibManager to coordinate the work, and
+ * propagate the messages from one processor to another
+ ***************************************************************************/
 
 //Handler to receive array messages
+CkpvDeclare(int, RecvmsgHandle);
+
 void recv_array_msg(void *msg){
 
-    ComlibPrintf("%d:In recv_msg\n", CkMyPe());
+  //   ComlibPrintf("%d:In recv_msg\n", CkMyPe());
 
-    if(msg == NULL)
-        return;
-    
-    register envelope* env = (envelope *)msg;
-    env->setUsed(0);
-    env->getsetArrayHops()=1;
-    CkUnpackMessage(&env);
+       if(msg == NULL)
+               return;
 
-    int srcPe = env->getSrcPe();
-    int sid = ((CmiMsgHeaderExt *) env)->stratid;
+       register envelope* env = (envelope *)msg;
+       env->setUsed(0);
+       env->getsetArrayHops()=1; 
+       CkUnpackMessage(&env);
 
-    ComlibPrintf("%d: Recording receive %d, %d, %d\n", CkMyPe(), 
-             sid, env->getTotalsize(), srcPe);
+       int srcPe = env->getSrcPe();
+       int sid = ((CmiMsgHeaderExt *) env)->stratid;
 
-    RECORD_RECV_STATS(sid, env->getTotalsize(), srcPe);
-    
-    CkArray *a=(CkArray *)_localBranch(env->getsetArrayMgr());
-    //if(!comm_debug)
-    a->deliver((CkArrayMessage *)EnvToUsr(env), CkDeliver_queue);
+       //      ComlibPrintf("%d: Recording receive %d, %d, %d\n", CkMyPe(), sid, env->getTotalsize(), srcPe);
 
-    ComlibPrintf("%d:Out of recv_msg\n", CkMyPe());
-    return;
-}
+       RECORD_RECV_STATS(sid, env->getTotalsize(), srcPe);
 
-void recv_combined_array_msg(void *msg){
-    if(msg == NULL)
-        return;
-    
-    ComlibPrintf("%d:In recv_combined_array_msg\n", CkMyPe());
+       CkArray *a=(CkArray *)_localBranch(env->getsetArrayMgr());
+       
+       a->deliver((CkArrayMessage *)EnvToUsr(env), CkDeliver_queue);
 
-    MsgPacker::deliver((CombinedMessage *)msg);
+       //      ComlibPrintf("%d:Out of recv_msg\n", CkMyPe());
+       return;
 }
 
+
+
+/**
+   A debugging routine that will periodically print out the status of the message queues. 
+   The Definition is at bottom of this file.
+ */
+static void periodicDebugPrintStatus(void* ptr, double currWallTime);
+
+
+
+
+
+/***************************************************************************
+ * Initialization section:
+ *
+ * Routines used by Comlib to initialize itself and all the strategies on all
+ * the processors. Used only at the beginning of the program.
+ ***************************************************************************/
+
 ComlibManager::ComlibManager(){
-    init();
-    ComlibPrintf("In comlibmanager constructor\n");
+       init();
 }
 
 void ComlibManager::init(){
-    
-    initComlibManager();
-
-    if (CkMyRank() == 0) {
-    PUPable_reg(CharmStrategy);
-    PUPable_reg(CharmMessageHolder);
-    }
-    
-    //    comm_debug = 1;
-    
-    numStatsReceived = 0;
-    curComlibController = 0;
-    clibIteration = 0;
-    
-    strategyCreated = CmiFalse;
 
-    CkpvInitialize(comRectHashType *, com_rect_ptr); 
-    CkpvAccess(com_rect_ptr)= new comRectHashType;
+  CcdCallFnAfterOnPE((CcdVoidFn)periodicDebugPrintStatus, (void*)this, 4000, CkMyPe());
 
-    CkpvInitialize(ClibLocationTableType*, locationTable);
-    CkpvAccess(locationTable) = new CkHashtableT <ClibGlobalArrayIndex, int>;
+  if(CkNumPes() == 1 ){
+    ComlibPrintf("Doing nothing in ComlibManager::init() because we are running on 1 pe.\n");
+  } else {
 
-    CkpvInitialize(CkArrayIndexMax, cache_index);
-    CkpvInitialize(int, cache_pe);
-    CkpvInitialize(CkArrayID, cache_aid);
+       initComlibManager();
 
-    CkpvAccess(cache_index).nInts = -1;
-    CkpvAccess(cache_aid).setZero();
+       if (CkMyRank() == 0) {
+               PUPable_reg(CharmMessageHolder);
+       }
 
-    CkpvInitialize(int, RecvmsgHandle);
-    CkpvAccess(RecvmsgHandle) =CkRegisterHandler((CmiHandler)recv_array_msg);
+       numStatsReceived = 0;
+       curComlibController = 0;
+       clibIteration = 0;
 
-    bcast_pelist = new int [CkNumPes()];
-    for(int brcount = 0; brcount < CkNumPes(); brcount++)
-        bcast_pelist[brcount] = brcount;
+       CkpvInitialize(comRectHashType *, com_rect_ptr); 
+       CkpvAccess(com_rect_ptr)= new comRectHashType;
 
-    CkpvInitialize(int, RecvCombinedShortMsgHdlrIdx);
-    CkpvAccess(RecvCombinedShortMsgHdlrIdx) = 
-        CkRegisterHandler((CmiHandler)recv_combined_array_msg);
-    
-    section_send_event = traceRegisterUserEvent("ArraySectionMulticast");
-    
-    npes = CkNumPes();
-    pelist = NULL;
+       CkpvInitialize(int, RecvmsgHandle);
+       CkpvAccess(RecvmsgHandle) =CkRegisterHandler((CmiHandler)recv_array_msg);
 
-    CkpvInitialize(CkGroupID, cmgrID);
-    CkpvAccess(cmgrID) = thisgroup;
+       bcast_pelist = new int [CkNumPes()];
+       for(int brcount = 0; brcount < CkNumPes(); brcount++)
+               bcast_pelist[brcount] = brcount;
 
-    dummyArrayIndex.nInts = 0;
+       section_send_event = traceRegisterUserEvent("ArraySectionMulticast");
 
-    curStratID = 0;
-    prevStratID = -1;
-    //prioEndIterationFlag = 1;
-
-    strategyTable = CkpvAccess(conv_com_ptr)->getStrategyTable();
-    
-    receivedTable = 0;
-    setupComplete = 0;
+       CkpvInitialize(CkGroupID, cmgrID);
+       CkpvAccess(cmgrID) = thisgroup;
 
-    barrierReached = 0;
-    barrier2Reached = 0;
+       dummyArrayIndex.nInts = 0;
+       converseManager = CkpvAccess(conv_com_ptr);
 
-    bcount = b2count = 0;
-    lbUpdateReceived = CmiFalse;
+       setupComplete = 0;
 
-    isRemote = 0;
-    remotePe = -1;
+       CkpvInitialize(int, migrationDoneHandlerID);
+       CkpvAccess(migrationDoneHandlerID) = 
+               CkRegisterHandler((CmiHandler) ComlibNotifyMigrationDoneHandler);
 
-    CkpvInitialize(int, migrationDoneHandlerID);
-    CkpvAccess(migrationDoneHandlerID) = 
-        CkRegisterHandler((CmiHandler) ComlibNotifyMigrationDoneHandler);
-    
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    cgproxy[curComlibController].barrier();
+       CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+       cgproxy[curComlibController].barrier();
+  }
 }
 
 //First barrier makes sure that the communication library group 
 //has been created on all processors
 void ComlibManager::barrier(){
-    ComlibPrintf("In barrier %d\n", bcount);
-    if(CkMyPe() == 0) {
-        bcount ++;
-        if(bcount == CkNumPes()){
-            bcount = 0;
-            barrierReached = 1;
-            barrier2Reached = 0;
-
-            if(strategyCreated)
-                broadcastStrategies();
-        }
-    }
+       static int bcount = 0;
+       ComlibPrintf("barrier %d\n", bcount);
+       if(CkMyPe() == 0) {
+               bcount ++;
+               if(bcount == CkNumPes()){
+                       bcount = 0;
+                       //barrierReached = 1;
+                       //barrier2Reached = 0;
+
+                       CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+                       cgproxy.resumeFromSetupBarrier();
+               }
+       }
 }
 
-//Has finished passing the strategy list to all the processors
-void ComlibManager::barrier2(){
-    if(CkMyPe() == 0) {
-        b2count ++;
-        ComlibPrintf("In barrier2 %d\n", bcount);
-        if(b2count == CkNumPes()) {
-            b2count = 0; 
-            CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-            cgproxy.resumeFromBarrier2();
-        }
-    }
+
+
+/**
+   Due to the possibility of race conditions in the initialization of charm, this
+   barrier prevents comlib from being activated before all group branches are created.
+   This function completes the initialization of the charm layer of comlib.
+
+   In this function we also call ComlibDoneCreating for the user (basically
+   triggering the broadcast of the strategies created in Main::Main. Here the
+   Main::Main has for sure executed, otherwise we will not have received
+   confirmation by all other processors.
+ */
+void ComlibManager::resumeFromSetupBarrier(){
+       ComlibPrintf("[%d] resumeFromSetupBarrier Charm group ComlibManager setup finished\n", CkMyPe());
+
+       setupComplete = 1;
+       ComlibDoneCreating();
+       sendBufferedMessagesAllStrategies();
 }
 
-//Registers a set of strategies with the communication library
-ComlibInstanceHandle ComlibManager::createInstance() {
-  
-    CkpvAccess(conv_com_ptr)->nstrats++;    
-    ComlibInstanceHandle cinst(CkpvAccess(conv_com_ptr)->nstrats -1,
-                               CkpvAccess(cmgrID));  
-    return cinst;
+
+/***************************************************************************
+ Determine whether the delegated messages should be buffered until the 
+ strategy has recovered from any error conditions and startup. Once the 
+ buffers can be flushed, ComlibManager::sendBufferedMessages() will be called
+***************************************************************************/
+bool ComlibManager::shouldBufferMessagesNow(int instid){
+  StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+  return (!setupComplete) || myEntry->errorMode == ERROR_MODE || myEntry->errorMode == CONFIRM_MODE || myEntry->bufferOutgoing;
 }
 
-void ComlibManager::registerStrategy(int pos, CharmStrategy *strat) {
-    
-    strategyCreated = true;
 
-    ListOfStrategies.enq(strat);
-    strat->setInstance(pos);
+/***************************************************************************
+ Calls ComlibManager::sendBufferedMessages for each strategy.
+***************************************************************************/
+void ComlibManager::sendBufferedMessagesAllStrategies(){
+  int nstrats = converseManager->getNumStrats();
+  for(int i=0;i<nstrats;i++){
+    sendBufferedMessages(i);
+  }
 }
 
-//End of registering function, if barriers have been reached send them over
-void ComlibManager::broadcastStrategies() {
-    if(!barrierReached)
-      return;    
 
-    lbUpdateReceived = CmiFalse;
-    barrierReached = 0;
+/***************************************************************************
+   Send all the buffered messages once startup has completed, and we have 
+   recovered from any errors have recovered.
+***************************************************************************/
+void ComlibManager::sendBufferedMessages(int instid){
+  StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+
+  if(! shouldBufferMessagesNow(instid) && delayMessageSendBuffer[instid].size() > 0){
+    ComlibManagerPrintf("[%d] sendBufferedMessages Sending %d buffered messages for instid=%d\n", CkMyPe(), delayMessageSendBuffer[instid].size(), instid);
+
+
+    for (std::set<CharmMessageHolder*>::iterator iter = delayMessageSendBuffer[instid].begin(); iter != delayMessageSendBuffer[instid].end(); ++iter) {
+      CharmMessageHolder* cmsg = *iter;
+         
+      switch(cmsg->type){
+           
+      case CMH_ARRAYSEND:
+       //          CkAbort("CMH_ARRAYSEND unimplemented");
+       CkpvAccess(conv_com_ptr)->insertMessage(cmsg, instid);
+       CkpvAccess(conv_com_ptr)->doneInserting(instid);
+       break;
+           
+      case CMH_GROUPSEND:
+       CkAbort("CMH_GROUPSEND unimplemented");
+       break;
+           
+      case CMH_ARRAYBROADCAST:
+      case CMH_ARRAYSECTIONSEND: 
+      case CMH_GROUPBROADCAST:
+       // Multicast/broadcast to an array or a section:
+       cmsg->sec_id = cmsg->copy_of_sec_id;
+       CkpvAccess(conv_com_ptr)->insertMessage(cmsg, instid);
+       CkpvAccess(conv_com_ptr)->doneInserting(instid);
+       break;
+           
+      default:
+       CkAbort("Unknown cmsg->type was found in buffer of delayed messages\n");
+      }
+         
+    }
 
-    ComlibPrintf("Sending Strategies %d, %d\n", 
-                 CkpvAccess(conv_com_ptr)->nstrats, 
-                 ListOfStrategies.length());
+    delayMessageSendBuffer[instid].clear();
+  } 
+  else {
+    ComlibManagerPrintf("sendBufferedMessages is not flushing buffered messages for strategy %d because myEntry->errorMode=%d && setupComplete = %d && myEntry->discoveryMode = %d && myEntry->bufferOutgoing = %d\n", instid, (int)myEntry->errorMode, (int)setupComplete, (int)myEntry->discoveryMode, (int)myEntry->bufferOutgoing);  
+  }
 
-    StrategyWrapper sw;
-    sw.total_nstrats = CkpvAccess(conv_com_ptr)->nstrats;
 
-    if(ListOfStrategies.length() > 0) {
-        int len = ListOfStrategies.length();
-        sw.s_table = new Strategy* [len];
-        sw.nstrats = len;
-        
-        for (int count=0; count < len; count++)
-            sw.s_table[count] = ListOfStrategies.deq();
-    }
-    else {
-        sw.nstrats = 0;
-        sw.s_table = 0;
-    }
 
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    cgproxy.receiveTable(sw, *CkpvAccess(locationTable));
 }
 
-//Called when the array/group element starts sending messages
-void ComlibManager::beginIteration(){
-    //right now does not do anything might need later
-    ComlibPrintf("[%d]:In Begin Iteration %d\n", CkMyPe(), (* strategyTable)[0].elementCount);
-    //prioEndIterationFlag = 0;
-}
 
-void ComlibManager::setInstance(int instID){
 
-    curStratID = instID;
-    ComlibPrintf("[%d]:In setInstance\n", CkMyPe(), (* strategyTable)[instID].elementCount);
+/***************************************************************************
+ * Bracketed section:
+ *
+ * Routines for bracketed strategies, to call the brackets begin and end, and to
+ * restore count errors after some objects migrate.
+ * 
+ * This function must only be called after all messages from the previous 
+ * iteration have been delivered. This is the application's responsibility to 
+ * ensure. The iteration values provided must increase monotonically.
+ ***************************************************************************/
+
+/// Called when the array/group element starts sending messages
+void ComlibManager::beginIteration(int instid, int iteration){
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+                           
+       ComlibManagerPrintf("[%d] beginIteration iter=%d lastKnownIteration=%d  %s %s %s\n", CkMyPe(), iteration, myEntry->lastKnownIteration, myEntry->errorModeString(),  myEntry->errorModeServerString(),  myEntry->discoveryModeString() );
+
+
+       if(iteration > myEntry->lastKnownIteration){
+             // Verify & update errorModeServer:
+             if(CkMyPe()==0){
+               CkAssert(myEntry->errorModeServer == NORMAL_MODE_SERVER || 
+                        myEntry->errorModeServer == STARTUP_MODE_SERVER ||  
+                        myEntry->errorModeServer == ERROR_FIXED_MODE_SERVER);
+               myEntry->errorModeServer = NORMAL_MODE_SERVER;
+             } else {
+               CkAssert(myEntry->errorModeServer == NON_SERVER_MODE_SERVER || 
+                        myEntry->errorModeServer == STARTUP_MODE_SERVER);
+               myEntry->errorModeServer = NON_SERVER_MODE_SERVER;
+             }
+             // Verify & update errorMode:
+             CkAssert(myEntry->errorMode == ERROR_FIXED_MODE || myEntry->errorMode == NORMAL_MODE  );
+             myEntry->errorMode = NORMAL_MODE;   
+
+             myEntry->lastKnownIteration = iteration;
+             myEntry->nBeginItr = 1; // we are the first time to be called this iteration
+             myEntry->nEndItr = 0;
+             myEntry->nProcSync = 0;
+             myEntry->totalEndCounted = 0;
+             myEntry->discoveryMode = NORMAL_DISCOVERY_MODE;
+             myEntry->nEndSaved = 0;
+             
+             
+             ComlibManagerPrintf("[%d] beginIteration Starting Next Iteration ( # %d )\n", CkMyPe(), iteration);
+       } else {
+               myEntry->nBeginItr++;
+               ComlibManagerPrintf("[%d] beginIteration continuing iteration # %d\n", CkMyPe(), iteration);
+       }
+       
+       
+       // We need to check for error conditions here as well as EndIteration.
+       // This will ensure that if we are the processor that detects this error, 
+       // we won't deliver at least this message until the strategy is fixed
+       if (myEntry->nBeginItr > myEntry->numElements) {
+         ComlibManagerPrintf("[%d] beginIteration BUFFERING OUTGOING\n",CkMyPe());                     
+         myEntry->bufferOutgoing = 1;
+       }
+       
 }
 
-//called when the array elements has finished sending messages
-void ComlibManager::endIteration(){
-    //    prioEndIterationFlag = 1;
-    prevStratID = -1;
-    
-    ComlibPrintf("[%d]:In End Iteration(%d) %d, %d\n", CkMyPe(), 
-                 curStratID, 
-                 (* strategyTable)[curStratID].elementCount, 
-                 (* strategyTable)[curStratID].numElements);
-
-    if(isRemote) {
-        isRemote = 0;
-        sendRemote();
-        return;
-    }
 
-    if(!receivedTable) {
-        (* strategyTable)[curStratID].nEndItr++;
-        return;
-    }        
-    
-    (* strategyTable)[curStratID].elementCount++;
-    int count = 0;
-    
-    if((* strategyTable)[curStratID].elementCount == (* strategyTable)[curStratID].numElements) {
-        
-        ComlibPrintf("[%d]:In End Iteration %d\n", CkMyPe(), (* strategyTable)[curStratID].elementCount);
-        
-        if(barrier2Reached) {      
-           (* strategyTable)[curStratID].strategy->doneInserting();
-        }
-       else (* strategyTable)[curStratID].call_doneInserting = 1;
+/** Called by user program when each element has finished sending its messages.
+
+    If no errors are detected, ConvComlibManager::doneInserting() is called on the 
+    underlying converse strategy. If an error is detected, then ConvComlibManager::doneInserting
+    is NOT called. This likely causes the the underlying converse strategy to buffer the 
+    messages until we recover from the error mode, although we have buffered at least some of
+    the messages, so the user program cannot run ahead and start the next iteration.
+
+*/
+
+void ComlibManager::endIteration(int instid, int step){ 
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
        
-        (* strategyTable)[curStratID].elementCount = 0;
-    }
-    ComlibPrintf("After EndIteration\n");
-}
+       // migration is not allowed between begin and end of iteration, 
+       // so each element must call end iteration on the same processor 
+       // as it called begin iteration
+       CkAssert(myEntry->nEndItr <= myEntry->nBeginItr);
+       CkAssert(step == myEntry->lastKnownIteration);
 
-//receive the list of strategies
-//Insert the strategies into the strategy table in converse comm lib.
-//CkpvAccess(conv_com_ptr) points to the converse commlib instance
-void ComlibManager::receiveTable(StrategyWrapper &sw, 
-                                 CkHashtableT<ClibGlobalArrayIndex, int> 
-                                 &htable)
-{
 
-    ComlibPrintf("[%d] In receiveTable %d, ite=%d\n", CkMyPe(), sw.nstrats, 
-                 clibIteration);
+       myEntry->nEndItr++;
+       
+       ComlibManagerPrintf("[%d] endIteration called\n",CkMyPe());
+       
+       
+       if (myEntry->bufferOutgoing) {
+         // If migration was detected in ComlibManager::beginIteration and hence messages have been buffered:
+         CkAssert(delayMessageSendBuffer[instid].size() > 0);
+         CProxy_ComlibManager myProxy(thisgroup);
+         myProxy[CkMyPe()].bracketedStartErrorRecoveryProcess(instid, step);
+       } 
+       else if(myEntry->nEndItr == myEntry->numElements) {
+         // If all the objects have called beginIteration and endIteration and no errors were detected
+         CkAssert(converseManager->isReady(instid));
+         converseManager->doneInserting(instid);
+       }
+       
+}
 
-    receivedTable = 1;
 
-    delete CkpvAccess(locationTable);
-    //Delete all records in it too !!!!!!!!!!
+/** Start recovery of errors. 
+    This entry method calls itself repeatedly if the underlying 
+    converse strategy is not yet ready.
 
-    CkpvAccess(locationTable) =  NULL;
+    If the PE is already in error mode, then only the difference 
+    in the counts will be contributed.
+*/
+void ComlibManager::bracketedStartErrorRecoveryProcess(int instid, int step){  
+  CProxy_ComlibManager myProxy(thisgroup);
+  StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);     
 
-    CkpvAccess(locationTable) = new 
-        CkHashtableT<ClibGlobalArrayIndex, int>;
+  
 
-    CkHashtableIterator *ht_iterator = htable.iterator();
-    ht_iterator->seekStart();
-    while(ht_iterator->hasNext()){
-        ClibGlobalArrayIndex *idx;
-        int *pe;
-        pe = (int *)ht_iterator->next((void **)&idx);
-        
-        ComlibPrintf("[%d] HASH idx %d on %d\n", CkMyPe(), 
-                     idx->idx.data()[0], *pe);
+  if(converseManager->isReady(instid)){
+    ComlibManagerPrintf("[%d] bracketedStartErrorRecoveryProcess() %s %s %s\n", CkMyPe(), myEntry->errorModeString(),  myEntry->errorModeServerString(),  myEntry->discoveryModeString() );
 
-        CkpvAccess(locationTable)->put(*idx) = *pe;       
+    CkAssert(myEntry->strategy != NULL);
+    CkAssert(myEntry->errorMode == NORMAL_MODE || myEntry->errorMode == ERROR_MODE);
+         
+    if (!myEntry->strategy->isBracketed()) {
+      CkPrintf("[%d] endIteration called unecessarily for a non-bracketed strategy\n", CkMyPe());
+      return;
     }
-    
-    //Reset cached array element index. Location table may have changed
-    CkpvAccess(cache_index).nInts = -1;
-    CkpvAccess(cache_aid).setZero();
-
-    CkArrayID st_aid;
-    int st_nelements;
-    CkArrayIndexMax *st_elem;
-    int temp_curstratid = curStratID;
-
-    CkpvAccess(conv_com_ptr)->nstrats = sw.total_nstrats;
-    clib_stats.setNstrats(sw.total_nstrats);
-
-    //First recreate strategies
-    int count = 0;
-    for(count = 0; count < sw.nstrats; count ++) {
-        CharmStrategy *cur_strategy = (CharmStrategy *)sw.s_table[count];
-        
-        //set the instance to the current count
-        //currently all strategies are being copied to all processors
-        //later strategies will be selectively copied
-        
-        //location of this strategy table entry in the strategy table
-        int loc = cur_strategy->getInstance();
-        
-        if(loc >= MAX_NUM_STRATS)
-            CkAbort("Strategy table is full \n");
-
-        CharmStrategy *old_strategy;
-
-        //If this is a learning decision and the old strategy has to
-        //be gotten rid of, finalize it here.
-        if((old_strategy = 
-            (CharmStrategy *)CkpvAccess(conv_com_ptr)->getStrategy(loc)) 
-           != NULL) {
-            old_strategy->finalizeProcessing();
-            
-            //Unregister from array listener if array strategy
-            if(old_strategy->getType() == ARRAY_STRATEGY) {
-                ComlibArrayInfo &as = ((CharmStrategy *)cur_strategy)->ainfo;
-                as.getSourceArray(st_aid, st_elem, st_nelements);
-
-                (* strategyTable)[loc].numElements = 0;
-                if(!st_aid.isZero()) {
-                    ComlibArrayListener *calistener = CkArrayID::
-                        CkLocalBranch(st_aid)->getComlibArrayListener();
-                    
-                    calistener->unregisterStrategy(&((*strategyTable)[loc]));
-                }
-            }
-        }
-        
-        //Insert strategy, frees an old strategy and sets the
-        //strategy_table entry to point to the new one
-        CkpvAccess(conv_com_ptr)->insertStrategy(cur_strategy, loc);
-        
-        ComlibPrintf("[%d] Inserting_strategy \n", CkMyPe());       
-
-        if(cur_strategy->getType() == ARRAY_STRATEGY &&
-           cur_strategy->isBracketed()){ 
-
-            ComlibPrintf("Inserting Array Listener\n");
-            
-            ComlibArrayInfo &as = ((CharmStrategy *)cur_strategy)->ainfo;
-            as.getSourceArray(st_aid, st_elem, st_nelements);
-            
-            (* strategyTable)[loc].numElements = 0;
-            if(!st_aid.isZero()) {            
-                ComlibArrayListener *calistener = 
-                    CkArrayID::CkLocalBranch(st_aid)->getComlibArrayListener();
-                
-                calistener->registerStrategy(&((* strategyTable)[loc]));
-            }
-        }                      
-        else { //if(cur_strategy->getType() == GROUP_STRATEGY){
-         (* strategyTable)[loc].numElements = 1;
-        }
-        
-        (* strategyTable)[loc].elementCount = 0;
-        cur_strategy->beginProcessing((* strategyTable)[loc].numElements); 
+       
+    if (myEntry->errorMode == NORMAL_MODE) {
+      ComlibManagerPrintf("[%d] bracketedStartErrorRecoveryProcess()\n", CkMyPe());
+      myEntry->nEndSaved = myEntry->nEndItr;
+      myProxy[0].bracketedReceiveCount(instid, CkMyPe(), myEntry->nEndSaved, 1, step);
+      myEntry->errorMode = ERROR_MODE;
+      bracketedStartDiscovery(instid);
+    } else {
+      // Send the updated count
+      int update = myEntry->nEndItr - myEntry->nEndSaved;
+      if (update > 0) {
+       //      ComlibManagerPrintf("bracketedStartErrorRecoveryProcess sending update to bracketedReceiveCount\n");
+       CProxy_ComlibManager(thisgroup)[0].bracketedReceiveCount(instid, CkMyPe(), update, 0, step);
+       myEntry->nEndSaved = myEntry->nEndItr;
+      }
+      
     }
+  } else {
+    ComlibManagerPrintf("[%d] bracketedStartErrorRecoveryProcess() REENQUEUE\n", CkMyPe() );
+    // Re-enqueue myself because we can't start error recovery process until converse strategy is ready
+    myProxy[CkMyPe()].bracketedStartErrorRecoveryProcess(instid, step);
+  }
+}
 
-    //Resume all end iterarions. Newer strategies may have more 
-    //or fewer elements to expect for!!
-    for(count = 0; count < CkpvAccess(conv_com_ptr)->nstrats; count++) {
-        ComlibPrintf("[%d] endIteration from receiveTable %d, %d\n", 
-                     CkMyPe(), count,
-                     (* strategyTable)[count].nEndItr);
-                         
-        curStratID = count;
-        for(int itr = 0; itr < (* strategyTable)[count].nEndItr; itr++) 
-            endIteration();  
-        
-        (* strategyTable)[count].nEndItr = 0;        
-    }           
-    
-    curStratID = temp_curstratid;
-    ComlibPrintf("receivedTable %d\n", sw.nstrats);
-    
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    cgproxy[curComlibController].barrier2();
+
+/// Invoked on all processors to inform that an error has been detected.
+void ComlibManager::bracketedErrorDetected(int instid, int step) {
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       
+       bracketedCatchUpPE(instid,step);
+       CkAssert(step == myEntry->lastKnownIteration);
+       CkAssert(myEntry->errorMode == NORMAL_MODE || myEntry->errorMode == ERROR_MODE);
+
+       ComlibManagerPrintf("[%d] bracketedErrorDetected()\n", CkMyPe());
+
+       if (myEntry->errorMode == NORMAL_MODE) {
+         // save the value we are sending to bracketedReceiveCount
+         myEntry->nEndSaved = myEntry->nEndItr; // save the value we are sending to bracketedReceiveCount
+         CProxy_ComlibManager(thisgroup)[0].bracketedReceiveCount(instid, CkMyPe(), myEntry->nEndSaved, 1, step);
+         bracketedStartDiscovery(instid);
+         myEntry->errorMode = ERROR_MODE;
+
+       } else { // ERROR_MODE
+         // If we have an update count, send it
+         int update = myEntry->nEndItr - myEntry->nEndSaved;
+         ComlibManagerPrintf("bracketedErrorDetected update=%d\n", update);
+         if (update > 0) {
+           ComlibManagerPrintf("bracketedErrorDetected sending update to bracketedReceiveCount\n");
+           CProxy_ComlibManager(thisgroup)[0].bracketedReceiveCount(instid, CkMyPe(), update, 0, step);
+           myEntry->nEndSaved = myEntry->nEndItr;
+         }
+       }
+       
 }
 
-void ComlibManager::resumeFromBarrier2(){
+/// Invoked on all processors. After processor 0 has a count match, it sends out
+/// a broadcast on this entry method to get a confirmation from all others.
+void ComlibManager::bracketedConfirmCount(int instid) {
+        StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       CkAssert(myEntry->errorMode == ERROR_MODE);
+       myEntry->errorMode = CONFIRM_MODE;
+       CProxy_ComlibManager myProxy(thisgroup);
+       ComlibManagerPrintf("[%d] bracketedConfirmCount\n", CkMyPe());
+       myProxy[0].bracketedCountConfirmed(instid, myEntry->nEndSaved, myEntry->lastKnownIteration);
+}
+
+/// Invoked on processor 0 upon a request to confirm the count. If the count is
+/// correct a "NewPeList" is sent out, otherwise the error mode is returned with
+/// "ErrorDetected"
+void ComlibManager::bracketedCountConfirmed(int instid, int count, int step) {
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       CkAssert(CkMyPe() == 0);
+       // Advance PE0 to current step if we had no local objects
+       bracketedCatchUpPE(instid, step);
+       CkAssert(myEntry->errorModeServer == CONFIRM_MODE_SERVER);
+       CkAssert(step == myEntry->lastKnownIteration);
+
+       myEntry->total += count;
+       
+       ComlibManagerPrintf("[%d] bracketedCountConfirmed\n", CkMyPe());
+       
+       if (++myEntry->peConfirmCounter == CkNumPes()) {
+               myEntry->peConfirmCounter = 0;
+
+               CkAssert(myEntry->total == myEntry->totalEndCounted);
+                 
+               CProxy_ComlibManager(thisgroup).bracketedReceiveNewCount(instid);
+               myEntry->errorModeServer = ERROR_FIXED_MODE_SERVER;
+                 
+               myEntry->total = 0;     
+       }
 
-    if(!receivedTable) 
-        //Application called atsync inbetween, receiveTable 
-        //and resumeFromBarrier2. This will only happen when there
-        //is no element on this processor and an element arrived, 
-        //leading to a resumeFromSync and hence an AtSync in comlib.
-        return; //A new resumeFromBarrier2 is on its way
-    
-    setupComplete = 1;
-
-    barrier2Reached = 1;
-    barrierReached = 0;
-
-    clibIteration ++;
-
-    ComlibPrintf("[%d] Barrier 2 reached nstrats = %d, ite = %d\n", 
-                 CkMyPe(), CkpvAccess(conv_com_ptr)->nstrats, clibIteration);
-
-    for (int count = 0; count < CkpvAccess(conv_com_ptr)->nstrats; 
-         count ++) {
-        if (!(* strategyTable)[count].tmplist.isEmpty()) {
-            
-            while (!(* strategyTable)[count].tmplist.isEmpty()) {
-                CharmMessageHolder *cmsg = (CharmMessageHolder *) 
-                    (* strategyTable)[count].tmplist.deq();
-                
-                if((*strategyTable)[count].strategy->getType() == 
-                   ARRAY_STRATEGY) {
-                    if(cmsg->dest_proc >= 0) {
-                        envelope *env  = UsrToEnv(cmsg->getCharmMessage());
-                        cmsg->dest_proc = getLastKnown
-                            (env->getsetArrayMgr(),
-                             env->getsetArrayIndex());
-                    }
-                    //else multicast
-                }                                
-                
-                if(cmsg->dest_proc == CkMyPe()) {
-                    CmiSyncSendAndFree(CkMyPe(), cmsg->size,
-                                       (char *)
-                                       UsrToEnv(cmsg->getCharmMessage()));
-                    delete cmsg;
-                }
-                else
-                    (* strategyTable)[count].strategy->insertMessage(cmsg);
-            }
-        }
-        
-        if ((* strategyTable)[count].call_doneInserting) {
-            (* strategyTable)[count].call_doneInserting = 0;
-            ComlibPrintf("[%d] Calling done inserting \n", CkMyPe());
-            (* strategyTable)[count].strategy->doneInserting();
-        }
-    }
-    
-    ComlibPrintf("[%d] After Barrier2\n", CkMyPe());
 }
 
-extern int _charmHandlerIdx;
 
-void ComlibManager::ArraySend(CkDelegateData *pd,int ep, void *msg, 
-                              const CkArrayIndexMax &idx, CkArrayID a){
+
+/** Update the state for a PE that was likely lacking any local objects.
+    If no local objects exist, noone will call begin iteration, and hence,
+    the lastKnownIteration value will be old. We need to advance to the 
+    current iteration before we can proceed.
+*/
+void ComlibManager::bracketedCatchUpPE(int instid, int step){
+  StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+  CkAssert(step >= myEntry->lastKnownIteration);
+  if(step > myEntry->lastKnownIteration){
     
-    if(pd != NULL) {
-        ComlibInstanceHandle *ci = (ComlibInstanceHandle *)pd;
-        setInstance(ci->_instid);
+    myEntry->total = 0;
+    myEntry->lastKnownIteration = step;
+    myEntry->nBeginItr = 0;
+    myEntry->nEndItr = 0;
+    myEntry->nProcSync = 0;
+    myEntry->totalEndCounted = 0;
+    myEntry->nEndSaved = 0;
+    myEntry->errorMode = NORMAL_MODE;
+    myEntry->discoveryMode = NORMAL_DISCOVERY_MODE;
+
+    if(CkMyPe()==0){
+      CkAssert(myEntry->errorModeServer == NORMAL_MODE_SERVER || myEntry->errorModeServer == ERROR_FIXED_MODE_SERVER);
+      ComlibManagerPrintf("[%d] NORMAL_MODE_SERVER *******************AAAAAAAAAAAA**********************\n", CkMyPe());
+      myEntry->errorModeServer = NORMAL_MODE_SERVER;
+    } else {
+      CkAssert(myEntry->errorModeServer == NON_SERVER_MODE_SERVER || myEntry->errorModeServer == STARTUP_MODE_SERVER);
+      myEntry->errorModeServer == NON_SERVER_MODE_SERVER;
     }
     
-    ComlibPrintf("[%d] In Array Send\n", CkMyPe());
-    
-    CkArrayIndexMax myidx = idx;
-    int dest_proc = getLastKnown(a, myidx); 
-    
-    //Reading from two hash tables is a big overhead
-    //int amgr_destpe = CkArrayID::CkLocalBranch(a)->lastKnown(myidx);
-    
-    register envelope * env = UsrToEnv(msg);
-    
-    env->getsetArrayMgr()=a;
-    env->getsetArraySrcPe()=CkMyPe();
-    env->getsetArrayEp()=ep;
-    env->getsetArrayHops()=0;
-    env->getsetArrayIndex()=idx;
-    env->setUsed(0);
-    ((CmiMsgHeaderExt *)env)->stratid = curStratID;
-
-    //RECORD_SEND_STATS(curStratID, env->getTotalsize(), dest_proc);
-
-    if(isRemote) {
-        CkPackMessage(&env);        
-        CharmMessageHolder *cmsg = new 
-            CharmMessageHolder((char *)msg, dest_proc);
-        
-        remoteQ.enq(cmsg);
-        return;
-    }
     
-    //With migration some array messages may be directly sent Also no
-    //message processing should happen before the comlib barriers have
-    //gone through
-    if(dest_proc == CkMyPe() && setupComplete){  
-        CkArray *amgr = (CkArray *)_localBranch(a);
-        amgr->deliver((CkArrayMessage *)msg, CkDeliver_queue);
-        
-        return;
-    }
+  }
+}
 
-    CkPackMessage(&env);
-    CmiSetHandler(env, CkpvAccess(RecvmsgHandle));        
-    
-    //get rid of the new.
-    CharmMessageHolder *cmsg = new 
-        CharmMessageHolder((char *)msg, dest_proc);
-    
-    ComlibPrintf("[%d] Before Insert on strat %d received = %d\n", CkMyPe(), curStratID, setupComplete);
-    
-    if (setupComplete)        
-        (* strategyTable)[curStratID].strategy->insertMessage(cmsg);
-    else 
-        (* strategyTable)[curStratID].tmplist.enq(cmsg);
-    
-    //CmiPrintf("After Insert\n");
+/// Invoked on processor 0 when a processor sends a count of how many elements
+/// have already deposited. This number is incremental, and it refers to the
+/// previous one sent by that processor. Processor 0 stores temporarily all the
+/// numbers it receives. When a match happens, processor 0 switches to "confirm"
+/// mode and send out a request for confirmation to all other processors.
+/// The final parameter, "step", is used so that if  no objects exist on a processor,
+/// then the counts sent by the processor will be tagged with an old timestep, and can 
+/// safely be ignored. If no objects are on a PE, then beginIteration will never
+/// be called there.
+void ComlibManager::bracketedReceiveCount(int instid, int pe, int count, int isFirst, int step) {
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       CkAssert(CkMyPe() == 0);
+       CkAssert(myEntry->errorModeServer == NORMAL_MODE_SERVER || myEntry->errorModeServer == ERROR_MODE_SERVER );
+
+       ComlibManagerPrintf("[%d] bracketedReceiveCount step=%d \n", CkMyPe(), step);
+
+       
+       // Advance PE0 to current step if we had no local objects
+       bracketedCatchUpPE(instid, step);
+
+       // Encountering a message from an old step
+       CkAssert(step == myEntry->lastKnownIteration);
+
+       myEntry->totalEndCounted += count;
+
+       ComlibManagerPrintf("[%d] bracketedReceiveCount step=%d totalEndCounted=%d count=%d\n", CkMyPe(), step, myEntry->totalEndCounted, count);
+
+       
+       myEntry->nProcSync += isFirst; // isFirst is 1 the first time a processor send a message,
+       CkAssert(myEntry->nProcSync <= CkNumPes());
+       
+       if (myEntry->errorModeServer == NORMAL_MODE_SERVER) { 
+               // first time this is called
+               CkAssert(myEntry->nProcSync == 1);
+               CProxy_ComlibManager(thisgroup).bracketedErrorDetected(instid, step);
+               myEntry->errorModeServer = ERROR_MODE_SERVER;
+               ComlibManagerPrintf("[%d] bracketedReceiveCount first time\n", CkMyPe());
+       } else { // ERROR_MODE
+       
+         CharmStrategy* s = dynamic_cast<CharmStrategy*>(myEntry->strategy);
+         ComlibArrayInfo ainfo = s->ainfo;
+         int totalsrc =  ainfo.getTotalSrc() ;
+         
+         if(myEntry->nProcSync == CkNumPes() && myEntry->totalEndCounted == totalsrc) {
+           // ok, we received notifications from all PEs and all objects have called endIteration
+           myEntry->errorModeServer = CONFIRM_MODE_SERVER; 
+           ComlibManagerPrintf("[%d] bracketedReceiveCount errorModeServer is now CONFIRM_MODE calling bracketedConfirmCount totalsrc=%d\n", CkMyPe(), (int)totalsrc);
+           CProxy_ComlibManager(thisgroup).bracketedConfirmCount(instid);
+                       
+         }
+       }
+               
 }
 
 
-#include "qd.h"
-//CkpvExtern(QdState*, _qd);
+/// Invoked on all processors. After the count has been checked and it matches
+/// the number of emements involved in the bracketed operation, processor 0
+/// sends a broadcast to acknowledge the success. 
+/// The strategy is disabled until a new Pe list is received.
+void ComlibManager::bracketedReceiveNewCount(int instid) {
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       CkAssert(myEntry->errorMode == CONFIRM_MODE);
 
-void ComlibManager::GroupSend(CkDelegateData *pd,int ep, void *msg, int onPE, CkGroupID gid){
-    
+       myEntry->errorMode = ERROR_FIXED_MODE;
+       
+       myEntry->nEndItr -= myEntry->nEndSaved;
+       myEntry->nBeginItr -= myEntry->nEndSaved;
 
-    if(pd != NULL) {
-        ComlibInstanceHandle *ci = (ComlibInstanceHandle *)pd;
-        setInstance(ci->_instid);
-    }
+       myEntry->nEndSaved = 0;
 
-    int dest_proc = onPE;
+       bracketedFinalBarrier(instid);
+}
 
-    ComlibPrintf("Send Data %d %d %d\n", CkMyPe(), dest_proc, 
-                 UsrToEnv(msg)->getTotalsize());
 
-    register envelope * env = UsrToEnv(msg);
-    if(dest_proc == CkMyPe() && setupComplete){
-        _SET_USED(env, 0);
-        CkSendMsgBranch(ep, msg, dest_proc, gid);
-        return;
-    }
-    
-    ((CmiMsgHeaderExt *)env)->stratid = curStratID;
-    CpvAccess(_qd)->create(1);
 
-    env->setMsgtype(ForBocMsg);
-    env->setEpIdx(ep);
-    env->setGroupNum(gid);
-    env->setSrcPe(CkMyPe());
-    env->setUsed(0);
 
-    CkPackMessage(&env);
-    CmiSetHandler(env, _charmHandlerIdx);
+/// Invoked on all processors. When all processors have discovered all elements
+/// involved in the operation, processor 0 uses this method to broadcast the
+/// entire processor list to all. Currently the discovery process HAS to go
+/// together with the counting process, otherwise the strategy is updated at an
+/// undetermined state. In future it may be useful to implement the discovery
+/// process alone.
+void ComlibManager::bracketedReceiveNewPeList(int instid, int *count) {
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       CkAssert(myEntry->discoveryMode == STARTED_DISCOVERY_MODE);
+       myEntry->discoveryMode = FINISHED_DISCOVERY_MODE;
 
-    CharmMessageHolder *cmsg = new CharmMessageHolder((char *)msg, dest_proc); 
-    //get rid of the new.
-    
-    if(setupComplete)
-        (* strategyTable)[curStratID].strategy->insertMessage(cmsg);
-    else {
-        (* strategyTable)[curStratID].tmplist.enq(cmsg);
-    }
+       myEntry->strategy->bracketedUpdatePeKnowledge(count);
+       
+       ComlibManagerPrintf("[%d] bracketedReceiveNewPeList Updating numElements\n", CkMyPe());
+       ComlibArrayInfo *myInfo = &dynamic_cast<CharmStrategy*>(myEntry->strategy)->ainfo;
+       CkAssert((unsigned long)myInfo > 0x1000);
+       myEntry->numElements = myInfo->getLocalSrc();
+       
+       ComlibManagerPrintf("[%d] delayMessageSendBuffer[%d].size()=%d\n",CkMyPe(), instid, delayMessageSendBuffer[instid].size() );
+       ComlibManagerPrintf("[%d] delayMessageSendBuffer[%d].size()=%d\n", CkMyPe(), instid, delayMessageSendBuffer[instid].size());
+               
+       bracketedFinalBarrier(instid);
 }
 
-void ComlibManager::ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a){
-    ComlibPrintf("[%d] Array Broadcast \n", CkMyPe());
 
-    if(pd != NULL) {
-        ComlibInstanceHandle *ci = (ComlibInstanceHandle *)pd;
-        setInstance(ci->_instid);
-    }
-    
-    //Broken, add the processor list here.
+/** Start a barrier phase where all processes will enter it once both 
+    the counting and discovery processes complete.
+
+    ComlibManager::bracketedReceiveNewPeList calls this method. 
+    ComlibManager::bracketedReceiveNewPeList is called as an array broadcast to thisProxy, 
+    so every PE will call this method.
+ */
+void ComlibManager::bracketedFinalBarrier(int instid) {
+  StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+  ComlibManagerPrintf("[%d] ComlibManager::bracketedFinalBarrier %s %s %s\n", CkMyPe(), myEntry->errorModeString(),  myEntry->errorModeServerString(),  myEntry->discoveryModeString() );
+
+
+  if (myEntry->discoveryMode == FINISHED_DISCOVERY_MODE &&  myEntry->errorMode == ERROR_FIXED_MODE) {    
+    CProxy_ComlibManager(thisgroup)[0].bracketedReleaseCount(instid);
+  }
+}
 
-    register envelope * env = UsrToEnv(m);
-    env->getsetArrayMgr()=a;
-    env->getsetArraySrcPe()=CkMyPe();
-    env->getsetArrayEp()=ep;
-    env->getsetArrayHops()=0;
-    env->getsetArrayIndex()= dummyArrayIndex;
-    ((CmiMsgHeaderExt *)env)->stratid = curStratID;
 
-    CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
+/** 
+    Once all PEs report here, we will allow them to release the buffered messages from this iteration
+ */
+void ComlibManager::bracketedReleaseCount(int instid) {
+    ComlibManagerPrintf("[%d] ComlibManager::bracketedReleaseCount\n", CkMyPe());
+
+    StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+    CkAssert(CkMyPe() == 0);
+    CkAssert(myEntry->errorModeServer == ERROR_FIXED_MODE_SERVER);
     
-    //RECORD_SENDM_STATS(curStratID, env->getTotalsize(), dest_proc);
+    myEntry->numBufferReleaseReady++;
+    if(myEntry->numBufferReleaseReady == CkNumPes()) {
+      myEntry->errorModeServer = NORMAL_MODE_SERVER;
+      CProxy_ComlibManager(thisgroup).bracketedReleaseBufferedMessages(instid);
+      myEntry->numBufferReleaseReady = 0;
+    }
+}
 
-    CharmMessageHolder *cmsg = new 
-        CharmMessageHolder((char *)m, IS_BROADCAST);
-    cmsg->npes = 0;
-    cmsg->pelist = NULL;
-    cmsg->sec_id = NULL;
+/** 
+    Release any buffered messages.
+ */
+void ComlibManager::bracketedReleaseBufferedMessages(int instid) {
+  ComlibManagerPrintf("[%d] ComlibManager::bracketedReleaseBufferedMessages\n", CkMyPe());
 
-    multicast(cmsg);
+  StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+  
+  myEntry->bufferOutgoing = 0;
+  sendBufferedMessages(instid);
+  
+  converseManager->doneInserting(instid);
 }
 
-void ComlibManager::ArraySectionSend(CkDelegateData *pd,int ep, void *m, 
-                                     CkArrayID a, CkSectionID &s, int opts) {
 
-#ifndef CMK_OPTIMIZE
-    traceUserEvent(section_send_event);
-#endif
 
-    if(pd != NULL) {
-        ComlibInstanceHandle *ci = (ComlibInstanceHandle *)pd;
-        setInstance(ci->_instid);
-    }
-    
-    ComlibPrintf("[%d] Array Section Send \n", CkMyPe());
 
-    register envelope * env = UsrToEnv(m);
-    env->getsetArrayMgr()=a;
-    env->getsetArraySrcPe()=CkMyPe();
-    env->getsetArrayEp()=ep;
-    env->getsetArrayHops()=0;
-    env->getsetArrayIndex()= dummyArrayIndex;
-    ((CmiMsgHeaderExt *)env)->stratid = curStratID;
 
-    CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
-    
-    env->setUsed(0);    
-    CkPackMessage(&env);
-    
-    //Provide a dummy dest proc as it does not matter for mulitcast 
-    CharmMessageHolder *cmsg = new CharmMessageHolder((char *)m,
-                                                      IS_SECTION_MULTICAST);
-    cmsg->npes = 0;
-    cmsg->sec_id = &s;
-
-    CkSectionInfo minfo;
-    minfo.type = COMLIB_MULTICAST_MESSAGE;
-    minfo.sInfo.cInfo.instId = curStratID;
-    minfo.sInfo.cInfo.status = COMLIB_MULTICAST_ALL;  
-    minfo.sInfo.cInfo.id = 0; 
-    minfo.pe = CkMyPe();
-    ((CkMcastBaseMsg *)m)->_cookie = minfo;    
-    //    s.npes = 0;
-    //s.pelist = NULL;
-
-    multicast(cmsg);
+
+/** Called when an error is discovered. Such errors occur when it is observed
+    that an array element has migrated to a new PE. Specifically if a strategy
+    on some PE determines that it has received messages for more array elements
+    than the strategy knew were local, then some array element must have 
+    migrated to the PE. The strategy instance detecting the error will call this
+    method to initiate a global update operation that determines the locations
+    of all array elements.
+
+    This is called on each PE by the bracketedErrorDetected() method. 
+
+    Each array element previously located on this PE is examined to determine if
+    it is still here. If the array element has migrated away, then the
+    bracketedDiscover() method is called on the new PE.
+
+*/
+void ComlibManager::bracketedStartDiscovery(int instid) {
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       CkAssert(myEntry->discoveryMode == NORMAL_DISCOVERY_MODE);
+       myEntry->discoveryMode = STARTED_DISCOVERY_MODE;
+       ComlibArrayInfo *myInfo = &dynamic_cast<CharmStrategy*>(myEntry->strategy)->ainfo;
+                               
+       ComlibManagerPrintf("[%d] bracketedStartDiscovery\n", CkMyPe());
+
+       int countSrc = 0;
+       int countDest = 0;
+       CkArrayID aid;
+       CkArrayIndexMax *idx;
+       int nelem;
+       CkArray *a;
+       CProxy_ComlibManager myProxy(thisgroup);
+
+       if (myInfo->isSourceArray()) {
+               myInfo->getSourceArray(aid, idx, nelem);
+               myInfo->resetSource();
+               a = (CkArray*)_localBranch(aid);
+               for (int i=0; i<nelem; ++i) {
+                       int pe = a->lastKnown(idx[i]);
+//                     ComlibManagerPrintf("[%d] bracketedStartDiscovery src Index %d was lastKnown at pe %d\n", CkMyPe(), idx[i].data()[0], pe);
+                       if (pe == CkMyPe()) {
+                               countSrc++;
+                               myInfo->addSource(idx[i]);
+                       }
+                       else {
+                               myProxy[pe].bracketedDiscover(instid, aid, idx[i], true);
+                       }
+               }
+               delete[] idx;
+       }
+
+       if (myInfo->isDestinationArray()) {
+               myInfo->getDestinationArray(aid, idx, nelem);
+               myInfo->resetDestination();
+               a = (CkArray*)_localBranch(aid);
+               for (int i=0; i<nelem; ++i) {
+                       int pe = a->lastKnown(idx[i]);
+//                     ComlibManagerPrintf("[%d] bracketedStartDiscovery dest Index %d was lastKnown at pe %d\n", CkMyPe(), idx[i].data()[0], pe);
+                       if (pe == CkMyPe()) {
+                               countDest++;
+                               myInfo->addDestination(idx[i]);
+                       }
+                       else {
+                               myProxy[pe].bracketedDiscover(instid, aid, idx[i], false);
+                       }
+               }
+               delete[] idx;
+       }
+
+       if (countSrc > 0 || countDest > 0) {
+               // contribute only if we have something
+               myProxy[0].bracketedContributeDiscovery(instid, CkMyPe(), countSrc, countDest, myEntry->lastKnownIteration);
+       }
+
 }
 
-void ComlibManager::GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g) {
-    
-    if(pd != NULL) {
-        ComlibInstanceHandle *ci = (ComlibInstanceHandle *)pd;
-        setInstance(ci->_instid);
-    }
-    
-    register envelope * env = UsrToEnv(m);
-    
-    CpvAccess(_qd)->create(1);
 
-    env->setMsgtype(ForBocMsg);
-    env->setEpIdx(ep);
-    env->setGroupNum(g);
-    env->setSrcPe(CkMyPe());
-    env->setUsed(0);
-    ((CmiMsgHeaderExt *)env)->stratid = curStratID;
 
-    CkPackMessage(&env);
-    CmiSetHandler(env, _charmHandlerIdx);
-    
-    //Provide a dummy dest proc as it does not matter for mulitcast 
-    CharmMessageHolder *cmsg = new CharmMessageHolder((char *)m,IS_BROADCAST);
-    
-    cmsg->npes = 0;
-    cmsg->pelist = NULL;
+/** Determine the PE of a given array index. This will be called for any 
+    array element by the previous owner PE.
+
+    If the index is local to the processor, then record this in the local 
+    strategy. Also send a message to PE 0 informing PE 0 that the array
+    element was located.
 
-    multicast(cmsg);
+    If the array element is not found locally, call this method on the last 
+    known location for the element.
+
+*/
+void ComlibManager::bracketedDiscover(int instid, CkArrayID aid, CkArrayIndexMax &idx, int isSrc) {
+       ComlibManagerPrintf("[%d] bracketedDiscover\n", CkMyPe());
+       CkArray *a = (CkArray *)_localBranch(aid);
+       int pe = a->lastKnown(idx);
+       CProxy_ComlibManager myProxy(thisgroup);
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+
+       if (pe == CkMyPe()) {
+               // Object was found locally
+        
+               // notify PE 0
+               myProxy[0].bracketedContributeDiscovery(instid, pe, isSrc?1:0, isSrc?0:1, myEntry->lastKnownIteration);
+         
+
+               // Find local strategy instance
+
+               // ComlibArrayInfo *myInfo = &(dynamic_cast<CharmStrategy*>(getStrategy(instid))->ainfo);
+               CkAssert((unsigned long)myEntry->strategy > 0x1000);
+               ComlibArrayInfo *myInfo = &dynamic_cast<CharmStrategy*>(myEntry->strategy)->ainfo;
+               CkAssert((unsigned long)myInfo > 0x1000);
+
+               if (isSrc) {
+                       // Add the element as a source element for the strategy
+                       ComlibManagerPrintf("[%d] bracketedDiscover addSource\n", CkMyPe());
+                       CkAssert((unsigned long)myInfo > 0x1000);
+                       myInfo->addSource(idx);
+
+                       ComlibManagerPrintf("[%d] bracketedDiscover updating numElements\n", CkMyPe());
+                       myEntry->numElements = myInfo->getLocalSrc();           
+               }
+               else {
+                       // Add the element as a Destination element for the strategy
+                       ComlibManagerPrintf("[%d] bracketedDiscover addDestination\n", CkMyPe());
+                       myInfo->addDestination(idx);
+               }
+       } else {
+               ComlibManagerPrintf("Keep On Forwarding*********************\n");
+               // forward to next potential processor
+               myProxy[pe].bracketedDiscover(instid, aid, idx, isSrc);
+       }
 }
 
-void ComlibManager::multicast(CharmMessageHolder *cmsg) {
 
-    register envelope * env = UsrToEnv(cmsg->getCharmMessage());    
-    ComlibPrintf("[%d]: In multicast\n", CkMyPe());
 
-    env->setUsed(0);    
-    CkPackMessage(&env);
+/** On PE 0, record the notifications from all PEs about the actual locations of each
+    array element. Count the number of elements discovered. Once the new location of 
+    every elements has been discovered, broadcast the new locations to all PEs by 
+    invoking bracketedReceiveNewPeList.
+*/
+void ComlibManager::bracketedContributeDiscovery(int instid, int pe, int nsrc, int ndest, int step) {
+       CkAssert(CkMyPe() == 0);
+       ComlibManagerPrintf("[%d] bracketedContributeDiscovery pe=%d nsrc=%d ndest=%d step=%d\n", CkMyPe(), pe, nsrc, ndest, step);
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       if (myEntry->peList == 0) {
+               myEntry->peList = new int[CkNumPes()+2];
+               // peList[CkNumPes()] keeps the sum of all source objects discovered
+               // peList[CkNumPes()+1] keeps the sum of all destination objects discovered
+               for (int i=0; i<CkNumPes()+2; ++i) myEntry->peList[i]=0;
+               ComlibManagerPrintf("[%d] bracketedContributeDiscovery zeroing new peList\n", CkMyPe());
+       }
+       myEntry->peList[CkNumPes()] += nsrc;
+       myEntry->peList[CkNumPes()+1] += ndest;
+       // update the count for the sender processor. peList[i] is:
+       // 0 if proc "i" has no objects,
+       // 1 if proc "i" has only source objects,
+       // 2 if proc "i" has only destination objects,
+       // 3 if proc "i" has both source and destination objects
+       // the following code maintains the property of peList!
+       if (nsrc > 0) myEntry->peList[pe] |= 1;
+       if (ndest > 0) myEntry->peList[pe] |= 2;
+
+       ComlibArrayInfo *myInfo = &dynamic_cast<CharmStrategy*>(myEntry->strategy)->ainfo;
+       CkAssert((unsigned long)myInfo > 0x1000);
 
-    //Will be used to detect multicast message for learning
-    
-    if (setupComplete)
-       (* strategyTable)[curStratID].strategy->insertMessage(cmsg);
-    else {
-       ComlibPrintf("Enqueuing message in tmplist at %d\n", curStratID);
-        (* strategyTable)[curStratID].tmplist.enq(cmsg);
-    }
+       
+//     ComlibManagerPrintf("[%d] bracketedContributeDiscovery myEntry->peList[CkNumPes()]=%d  myInfo->getTotalSrc()=%d\n", CkMyPe(), myEntry->peList[CkNumPes()], myInfo->getTotalSrc());
+//     ComlibManagerPrintf("[%d] bracketedContributeDiscovery myEntry->peList[CkNumPes()+1]=%d  myInfo->getTotalDest()=%d\n", CkMyPe(), myEntry->peList[CkNumPes()+1], myInfo->getTotalDest());
+//             
+       if (myEntry->peList[CkNumPes()] == myInfo->getTotalSrc() &&
+                       myEntry->peList[CkNumPes()+1] == myInfo->getTotalDest()) {
+               // discovery process finished, broadcast the new pe list
+
+               CProxy_ComlibManager myProxy(thisgroup);
+               
+               ComlibManagerPrintf("[%d] bracketedContributeDiscovery calling bracketedReceiveNewPeList %d/%d, %d/%d\n",       
+                               CkMyPe(), 
+                               myEntry->peList[CkNumPes()], myInfo->getTotalSrc(), 
+                               myEntry->peList[CkNumPes()+1], myInfo->getTotalDest() );
+               
+               printPeList("bracketedContributeDiscovery peList=", myEntry->peList);
+               myProxy.bracketedReceiveNewPeList(instid, myEntry->peList);
+               delete myEntry->peList;
+               myEntry->peList = NULL;
+       } else {
+               ComlibManagerPrintf("[%d] bracketedContributeDiscovery NOT calling bracketedReceiveNewPeList %d/%d, %d/%d\n", 
+                               CkMyPe(), 
+                               myEntry->peList[CkNumPes()], myInfo->getTotalSrc(), 
+                               myEntry->peList[CkNumPes()+1], myInfo->getTotalDest() );
+       }
 
-    ComlibPrintf("After multicast\n");
 }
 
-//Collect statistics from all the processors, also gets the list of
-//array elements on each processor.
-void ComlibManager::collectStats(ComlibLocalStats &stat, int pe, 
-                                 CkVec<ClibGlobalArrayIndex> &idx_vec) {
-    
-    ComlibPrintf("%d: Collecting stats %d\n", CkMyPe(), numStatsReceived);
 
-    numStatsReceived ++;
-    clib_gstats.updateStats(stat, pe);
-    
-    for(int count = 0; count < idx_vec.length(); count++) {
-        int old_pe = CkpvAccess(locationTable)->get(idx_vec[count]);
-        
-        ComlibPrintf("Adding idx %d to %d\n", idx_vec[count].idx.data()[0], 
-                     pe);
-        
-        CkpvAccess(locationTable)->put(idx_vec[count]) = pe + CkNumPes();
-    }        
-
-    //Reset cached array element index. Location table may have changed
-    CkpvAccess(cache_index).nInts = -1;
-    CkpvAccess(cache_aid).setZero();
-
-    if(numStatsReceived == CkNumPes()) {
-        numStatsReceived = 0;
-
-        for(int count = 0; count < CkpvAccess(conv_com_ptr)->nstrats; 
-            count++ ){
-            Strategy* strat = CkpvAccess(conv_com_ptr)->getStrategy(count);
-            if(strat->getType() > CONVERSE_STRATEGY) {
-                CharmStrategy *cstrat = (CharmStrategy *)strat;
-                ComlibLearner *learner = cstrat->getLearner();
-                CharmStrategy *newstrat = NULL;
-                                
-                if(learner != NULL) {
-                    ComlibPrintf("Calling Learner\n");
-                    newstrat = (CharmStrategy *)learner->optimizePattern
-                        (strat, clib_gstats);
-                    
-                    if(newstrat != NULL)
-                        ListOfStrategies.enq(newstrat);
-                    else
-                        ListOfStrategies.enq(cstrat);
-                }
-            }
-        }
-        barrierReached = 1;
-        
-        //if(lbUpdateReceived) {
-        //lbUpdateReceived = CmiFalse;
-        broadcastStrategies();
-        //}
-    }
+void ComlibManager::printPeList(char* note, int *count){
+       
+       char *buf = (char*)malloc(1024*64);
+       sprintf(buf, "[%d] %s ", CkMyPe(), note);
+       for(int i=0;i<CkNumPes();i++){
+               switch (count[i]){
+               case 0:
+                       sprintf(buf+strlen(buf), " %d:no ", i, count[i]);
+                       break;
+               case 1:
+                       sprintf(buf+strlen(buf), " %d:source ", i, count[i]);
+                       break;
+               case 2:
+                       sprintf(buf+strlen(buf), " %d:dest ", i, count[i]);
+                       break;
+               case 3:
+                       sprintf(buf+strlen(buf), " %d:both ", i, count[i]);
+                       break;
+               }
+       }
+       
+       sprintf(buf+strlen(buf), ", all source objects discovered =  %d, all destination objects discovered = %d",  count[CkNumPes()] ,  count[CkNumPes()+1] );
+               
+       ComlibPrintf("%s\n", buf);
 }
 
-void ComlibManager::setRemote(int remote_pe){
+/***************************************************************************
+ * Delegation framework section:
+ *
+ * Reimplementation of the main routines needed for the delegation framework:
+ * this routines will be called when a message is sent through a proxy which has
+ * been associated with comlib.
+ ***************************************************************************/
+
+extern int _charmHandlerIdx;
+void msg_prepareSend_noinline(CkArrayMessage *msg, int ep,CkArrayID aid);
 
-    ComlibPrintf("Setting remote flag on\n");
 
-    remotePe = remote_pe;
-    isRemote = 1;
-}
+/** 
+    Handle messages sent via ArraySend. These are single point to point messages 
+    to array elements.
 
+    This method should not optimize direct sends in the case where buffering occurs
 
-void ComlibManager::receiveRemoteSend(CkQ<CharmMessageHolder *> &rq, 
-                                      int strat_id) {
-    setInstance(strat_id);
-    
-    int nmsgs = rq.length();
+ */
+void ComlibManager::ArraySend(CkDelegateData *pd,int ep, void *msg, 
+               const CkArrayIndexMax &idx, CkArrayID a){
 
-    ComlibPrintf("%d: Receiving remote message\n", CkMyPe());
+       CkAssert(pd != NULL);
+       ComlibDelegateData *ci = static_cast<ComlibDelegateData *>(pd);
+       int instid = ci->getID();
+       CkAssert(instid != 0);
 
-    for(int count = 0; count < nmsgs; count++) {
-        char *msg = rq.deq()->getCharmMessage();
-        envelope *env = UsrToEnv(msg);
-        
-        ArraySend(NULL, env->getsetArrayEp(), msg, env->getsetArrayIndex(), 
-                  env->getsetArrayMgr());
-    }
+       // Reading from two hash tables is a big overhead
+       CkArray *amgr = CkArrayID::CkLocalBranch(a);
+       int dest_proc = amgr->lastKnown(idx);
+
+       register envelope * env = UsrToEnv(msg);
+       msg_prepareSend_noinline((CkArrayMessage*)msg, ep, a);
+
+       env->getsetArrayIndex()=idx;
+       env->setUsed(0);
+       ((CmiMsgHeaderExt *)env)->stratid = instid;
+
+       CkPackMessage(&env);
+       CmiSetHandler(env, CkpvAccess(RecvmsgHandle));        
+       
+       CharmMessageHolder *cmsg = new CharmMessageHolder((char *)msg, dest_proc, CMH_ARRAYSEND);
+       
+       if(shouldBufferMessagesNow(instid)){
+         delayMessageSendBuffer[instid].insert(cmsg);
+         ComlibManagerPrintf("[%d] ComlibManager::ArraySend BUFFERED OUTGOING: now buffer contains %d messages\n",CkMyPe(), delayMessageSendBuffer[instid].size() );
+       } else {
+         ComlibPrintf("ComlibManager::ArraySend NOT BUFFERING inserting message into strategy %d\n",instid);
+         
+         if(dest_proc == CkMyPe()){  
+           // Directly send if object is local
+           amgr->deliver((CkArrayMessage *)msg, CkDeliver_queue);
+           return;
+         } else {
+           // Send through converse level strategy if non-local
+           converseManager->insertMessage(cmsg, instid);
+         }
+         
+       }
 
-    endIteration();
 }
 
-void ComlibManager::sendRemote(){
-    
-    int nmsgs = remoteQ.length();
 
-    //if(nmsgs == 0)
-    //  return;
+#include "qd.h"
+//CkpvExtern(QdState*, _qd);
 
-    ComlibPrintf("%d: Sending remote message \n", CkMyPe());
+void ComlibManager::GroupSend(CkDelegateData *pd,int ep, void *msg, int onPE, CkGroupID gid){
 
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); 
-    cgproxy[remotePe].receiveRemoteSend(remoteQ, curStratID);
-    
-    for(int count = 0; count < nmsgs; count++) {
-        CharmMessageHolder *cmsg = remoteQ.deq();
-        CkFreeMsg(cmsg->getCharmMessage());
-        delete cmsg;
-    }
-}
+       CkAssert(pd != NULL);
+       ComlibDelegateData *ci = static_cast<ComlibDelegateData *>(pd);
+       int instid = ci->getID();
 
+       int dest_proc = onPE;
 
-void ComlibManager::AtSync() {
+       ComlibPrintf("Send Data %d %d %d\n", CkMyPe(), dest_proc, 
+                       UsrToEnv(msg)->getTotalsize());
 
-    //comm_debug = 1;
-    ComlibPrintf("[%d] In ComlibManager::Atsync, controller %d, ite %d\n", CkMyPe(), curComlibController, clibIteration);
-
-    barrier2Reached = 0;
-    receivedTable = 0;
-    setupComplete = 0;
-    barrierReached = 0;
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-
-    int pos = 0;
-
-    CkVec<ClibGlobalArrayIndex> gidx_vec;
-
-    CkVec<CkArrayID> tmp_vec;
-    for(int count = 0; count < CkpvAccess(conv_com_ptr)->nstrats; 
-        count ++) {
-        if((* strategyTable)[count].strategy->getType() 
-           == ARRAY_STRATEGY) {
-            CharmStrategy *cstrat = (CharmStrategy*)
-                ((* strategyTable)[count].strategy);
-            
-            CkArrayID src, dest;
-            CkArrayIndexMax *elements;
-            int nelem;
-            
-            cstrat->ainfo.getSourceArray(src, elements, nelem);
-            cstrat->ainfo.getDestinationArray(dest, elements, nelem);
-
-            CmiBool srcflag = CmiFalse;
-            CmiBool destflag = CmiFalse;
-            
-            if(src == dest || dest.isZero())
-                destflag = CmiTrue;
-
-            if(src.isZero())
-                srcflag = CmiTrue;                        
-
-            for(pos = 0; pos < tmp_vec.size(); pos++) {
-                if(tmp_vec[pos] == src)
-                    srcflag = CmiTrue;
-
-                if(tmp_vec[pos] == dest)
-                    destflag = CmiTrue;
-
-                if(srcflag && destflag)
-                    break;
-            }
-
-            if(!srcflag)
-                tmp_vec.insertAtEnd(src);
-
-            if(!destflag)
-                tmp_vec.insertAtEnd(dest);
-        }
-        
-        //cant do it here, done in receiveTable
-        //if((* strategyTable)[count].strategy->getType() > CONVERSE_STRATEGY)
-        //  (* strategyTable)[count].reset();
-    }
+       register envelope * env = UsrToEnv(msg);
+       if(dest_proc == CkMyPe()){
+               _SET_USED(env, 0);
+               CkSendMsgBranch(ep, msg, dest_proc, gid);
+               return;
+       }
 
-    for(pos = 0; pos < tmp_vec.size(); pos++) {
-        CkArrayID aid = tmp_vec[pos];
+       ((CmiMsgHeaderExt *)env)->stratid = instid;
+       CpvAccess(_qd)->create(1);
 
-        ComlibArrayListener *calistener = 
-            CkArrayID::CkLocalBranch(aid)->getComlibArrayListener();
+       env->setMsgtype(ForBocMsg);
+       env->setEpIdx(ep);
+       env->setGroupNum(gid);
+       env->setSrcPe(CkMyPe());
+       env->setUsed(0);
 
-        CkVec<CkArrayIndexMax> idx_vec;
-        calistener->getLocalIndices(idx_vec);
+       CkPackMessage(&env);
+       CmiSetHandler(env, _charmHandlerIdx);
 
-        for(int idx_count = 0; idx_count < idx_vec.size(); idx_count++) {
-            ClibGlobalArrayIndex gindex;
-            gindex.aid = aid;
-            gindex.idx = idx_vec[idx_count];
+       CharmMessageHolder *cmsg = new CharmMessageHolder((char *)msg, dest_proc, CMH_GROUPSEND); 
+
+       if(shouldBufferMessagesNow(instid)){
+         ComlibPrintf("ComlibManager::GroupSend Buffering message for %d\n",instid);
+         delayMessageSendBuffer[instid].insert(cmsg);
+       } else {
+         ComlibPrintf("ComlibManager::GroupSend inserting message into strategy %d\n",instid);
+         converseManager->insertMessage(cmsg, instid);
+       }
 
-            gidx_vec.insertAtEnd(gindex);
-        }
-    }
 
-    cgproxy[curComlibController].collectStats(clib_stats, CkMyPe(), gidx_vec);
-    clib_stats.reset();
 }
 
-#include "lbdb.h"
-#include "CentralLB.h"
+void ComlibManager::ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a){
+       ComlibPrintf("[%d] Array Broadcast \n", CkMyPe());
 
-/******** FOO BAR : NEEDS to be consistent with array manager *******/
-void LDObjID2IdxMax (LDObjid ld_id, CkArrayIndexMax &idx) {
-    if(OBJ_ID_SZ < CK_ARRAYINDEX_MAXLEN)
-        CkAbort("LDB OBJ ID smaller than array index\n");
-    
-    //values higher than CkArrayIndexMax should be 0
-    for(int count = 0; count < CK_ARRAYINDEX_MAXLEN; count ++) {
-        idx.data()[count] = ld_id.id[count];
-    }
-    idx.nInts = 1;
-}
+       CkAssert(pd != NULL);
+       ComlibDelegateData *ci = static_cast<ComlibDelegateData *>(pd);
+       int instid = ci->getID();
 
-void ComlibManager::lbUpdate(LBMigrateMsg *msg) {
-    for(int count = 0; count < msg->n_moves; count ++) {
-        MigrateInfo m = msg->moves[count];
-
-        CkArrayID aid; CkArrayIndexMax idx;
-        aid = CkArrayID(m.obj.omhandle.id.id);
-        LDObjID2IdxMax(m.obj.id, idx);
-
-        ClibGlobalArrayIndex cid; 
-        cid.aid = aid;
-        cid.idx = idx;
-        
-        int pe = CkpvAccess(locationTable)->get(cid);
-
-        //Value exists in the table, so update it
-        if(pe != 0) {
-            pe = m.to_pe + CkNumPes();
-            CkpvAccess(locationTable)->getRef(cid) = pe;
-        }
-        //otherwise we dont care about these objects
-    }   
-
-    lbUpdateReceived = CmiTrue;
-    if(barrierReached) {
-        broadcastStrategies();
-        barrierReached = 0;
-    }
+       register envelope * env = UsrToEnv(m);
+       msg_prepareSend_noinline((CkArrayMessage*)m, ep, a);
+
+       env->getsetArrayIndex()= dummyArrayIndex;
+       ((CmiMsgHeaderExt *)env)->stratid = instid;
+
+       CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
+
+       CharmMessageHolder *cmsg = new CharmMessageHolder((char *)m, IS_BROADCAST, CMH_ARRAYBROADCAST);
+       cmsg->npes = 0;
+       cmsg->sec_id = NULL;
+       cmsg->array_id = a;
 
-    CkFreeMsg(msg);
+       multicast(cmsg, instid);
 }
 
-CkDelegateData* ComlibManager::ckCopyDelegateData(CkDelegateData *data) {
-    ComlibInstanceHandle *inst = new ComlibInstanceHandle
-        (*((ComlibInstanceHandle *)data));
-    return inst;
+void ComlibManager::ArraySectionSend(CkDelegateData *pd,int ep, void *m, 
+               CkArrayID a, CkSectionID &s, int opts) {
+
+       CkAssert(pd != NULL);
+       ComlibDelegateData *ci = static_cast<ComlibDelegateData *>(pd);
+       int instid = ci->getID();
+
+       ComlibPrintf("[%d] Array Section Send \n", CkMyPe());
+
+
+       //Provide a dummy dest proc as it does not matter for mulitcast 
+       CharmMessageHolder *cmsg = new CharmMessageHolder((char *)m, IS_SECTION_MULTICAST, CMH_ARRAYSECTIONSEND);
+       cmsg->npes = 0;
+       cmsg->sec_id = &s;
+       cmsg->array_id = a;
+       
+
+       msg_prepareSend_noinline((CkArrayMessage*)m, ep, a);
+       
+       register envelope * env = UsrToEnv(m);
+       env->getsetArrayIndex()= dummyArrayIndex;
+       ((CmiMsgHeaderExt *)env)->stratid = instid;
+       
+       CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
+       
+       env->setUsed(0);
+       CkPackMessage(&env);
+       
+       
+       /// @TODO why are CkSectionInfo needed since it doesn't seem to have
+       /// useful information?
+       CkSectionInfo minfo;
+       minfo.type = COMLIB_MULTICAST_MESSAGE;
+       minfo.sInfo.cInfo.instId = ci->getID();
+       //minfo.sInfo.cInfo.status = COMLIB_MULTICAST_ALL;  
+       minfo.sInfo.cInfo.id = 0; 
+       minfo.pe = CkMyPe();
+       ((CkMcastBaseMsg *)env)->_cookie = minfo;    
+       
+       multicast(cmsg, instid);
 }
 
+void ComlibManager::GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g) {
 
-CkDelegateData * ComlibManager::DelegatePointerPup(PUP::er &p,
-                                                   CkDelegateData *pd) {
+       CkAssert(pd != NULL);
+       ComlibDelegateData *ci = static_cast<ComlibDelegateData *>(pd);
+       int instid = ci->getID();
+       CkAssert(instid!=0);
 
-    if(pd == NULL)
-        return NULL;
+       register envelope * env = UsrToEnv(m);
 
-    CmiBool to_pup = CmiFalse;
+       CpvAccess(_qd)->create(1);
 
-    ComlibInstanceHandle *inst; 
-    if(!p.isUnpacking()) {
-        inst = (ComlibInstanceHandle *) pd;       
-        if(pd != NULL)
-            to_pup = CmiTrue;
-    }
-    else 
-        //Call migrate constructor
-        inst = new ComlibInstanceHandle();
+       env->setMsgtype(ForBocMsg);
+       env->setEpIdx(ep);
+       env->setGroupNum(g);
+       env->setSrcPe(CkMyPe());
+       env->setUsed(0);
+       ((CmiMsgHeaderExt *)env)->stratid = instid;
 
-    p | to_pup;
+       CkPackMessage(&env);
+       CmiSetHandler(env, _charmHandlerIdx);
 
-    if(to_pup)
-        inst->pup(p);
-    return inst;
-}    
+       //Provide a dummy dest proc as it does not matter for mulitcast 
+       CharmMessageHolder *cmsg = new CharmMessageHolder((char *)m,IS_BROADCAST, CMH_GROUPBROADCAST);
 
-void ComlibDelegateProxy(CProxy *proxy){
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    proxy->ckDelegate(cgproxy.ckLocalBranch(), NULL);
-}
+       cmsg->npes = 0;
+       //cmsg->pelist = NULL;
 
-void ComlibAssociateProxy(ComlibInstanceHandle *cinst, CProxy &proxy) {
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    proxy.ckDelegate(cgproxy.ckLocalBranch(), cinst);
+       multicast(cmsg, instid);
 }
 
-void ComlibAssociateProxy(CharmStrategy *strat, CProxy &proxy) {
-    ComlibInstanceHandle *cinst = new ComlibInstanceHandle
-        (CkGetComlibInstance());
 
-    cinst->setStrategy(strat);
-    
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    proxy.ckDelegate(cgproxy.ckLocalBranch(), cinst);
-} 
-
-ComlibInstanceHandle ComlibRegister(CharmStrategy *strat) {
-    ComlibInstanceHandle cinst(CkGetComlibInstance());
-    cinst.setStrategy(strat);
-    return cinst;
+/** 
+    Multicast the message with the specified strategy(instid).
+    This method is used in: ArrayBroadcast, ArraySectionSend, and GroupBroadcast
+ */
+void ComlibManager::multicast(CharmMessageHolder *cmsg, int instid) {
+        CkAssert(instid != 0);
+       register envelope * env = UsrToEnv(cmsg->getCharmMessage());
+
+#if DEBUG
+       StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+       ComlibPrintf("[%d] multicast setupComplete=%d errorMode=%d errorModeServer=%d\n", CkMyPe(), setupComplete, myEntry->errorMode, myEntry->errorModeServer);
+#endif
+       env->setUsed(0);
+       CkPackMessage(&env);
+       
+       if(shouldBufferMessagesNow(instid)){
+         cmsg->saveCopyOf_sec_id();
+         delayMessageSendBuffer[instid].insert(cmsg);
+       } else {
+         converseManager->insertMessage(cmsg, instid);
+       }
+       
 }
 
-void ComlibBegin(CProxy &proxy) {
-    ComlibInstanceHandle *cinst = (ComlibInstanceHandle *)proxy.ckDelegatedPtr();
-    cinst->beginIteration();
-} 
 
-void ComlibEnd(CProxy &proxy) {
-    ComlibInstanceHandle *cinst = (ComlibInstanceHandle *)proxy.ckDelegatedPtr();
-    cinst->endIteration();
+void ComlibManager::printDiagnostics(int instid){
+       ComlibManagerPrintf("[%d]     delayMessageSendBuffer.size()=%d\n", CkMyPe(), delayMessageSendBuffer[instid].size());
 }
 
-ComlibInstanceHandle CkCreateComlibInstance(){
-    return CkGetComlibInstance();
+
+void ComlibManager::printDiagnostics(){
+  
+
+  std::map<ComlibInstanceHandle, std::set<CharmMessageHolder*> >::iterator iter;
+  for(iter = delayMessageSendBuffer.begin(); iter != delayMessageSendBuffer.end(); ++iter){
+    int instid = iter->first;
+    int size = iter->second.size();
+    
+    if(size>0){
+      CkPrintf("[%d] delayMessageSendBuffer[instid=%d] contains %d messages\n", CkMyPe(), instid, size);
+      
+      if(! shouldBufferMessagesNow(instid)){
+       CkPrintf("[%d] printDiagnostics: No messages should be still in delayMessageSendBuffer[instid=%d]\n", CkMyPe(), instid);
+      } else {
+       CkPrintf("[%d] printDiagnostics: Messages still ought to be delayed in delayMessageSendBuffer[instid=%d]\n", CkMyPe(), instid);
+      }
+      
+      StrategyTableEntry *myEntry = converseManager->getStrategyTable(instid);
+      CkPrintf("[%d] printDiagnostics[instid=%d] setupComplete=%d errorMode=%d errorModeServer=%d discoveryMode=%d bufferOutgoing=%d\n", (int)CkMyPe(), (int)instid, (int)setupComplete, (int)myEntry->errorMode, (int)myEntry->errorModeServer, (int)myEntry->discoveryMode, (int)myEntry->bufferOutgoing);
+    }
+  }
+  
+  ConvComlibManager * conv_mgr = CkpvAccess(conv_com_ptr);
+  
+  conv_mgr->printDiagnostics();
+  
 }
 
-ComlibInstanceHandle CkGetComlibInstance() {
-    if(CkMyPe() != 0)
-        CkAbort("Comlib Instance can only be created on Processor 0");
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));    
-    return (cgproxy.ckLocalBranch())->createInstance();
+
+
+CkDelegateData* ComlibManager::ckCopyDelegateData(CkDelegateData *data) {
+       //ComlibDelegateData *inst = static_cast<ComlibDelegateData *>(data);
+       data->ref();
+       return data;
+       //return (new ComlibDelegateData(inst->getID()));
 }
 
-ComlibInstanceHandle CkGetComlibInstance(int id) {
-    ComlibInstanceHandle cinst(id, CkpvAccess(cmgrID));
-    return cinst;
+CkDelegateData* ComlibManager::DelegatePointerPup(PUP::er &p,
+               CkDelegateData *pd) {
+       if (!p.isUnpacking() && pd == NULL) CkAbort("Tryed to pup a null ComlibDelegateData!\n");
+       //CmiBool isNotNull = pd!=NULL ? CmiTrue : CmiFalse;
+       ComlibDelegateData *inst = static_cast<ComlibDelegateData *>(pd);
+
+       // call a migration constructor
+       if (p.isUnpacking()) inst = new ComlibDelegateData((CkMigrateMessage*)0);
+       inst->pup(p);
+       /*
+  p | isNotNull;
+  if (isNotNull) {
+    if (p.isUnpacking()) inst = new ComlibInstanceHandle();
+    inst->pup(p);
+  }
+        */
+       return inst;
 }
 
-void ComlibDoneCreating(){
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    (cgproxy.ckLocalBranch())->broadcastStrategies();
+
+//Collect statistics from all the processors, also gets the list of
+//array elements on each processor.
+void ComlibManager::collectStats(ComlibLocalStats &stat, int pe) {
 }
 
-char *router;
-int sfactor=0;
 
-class ComlibManagerMain: public Chare {
-public:
-    ComlibManagerMain(CkArgMsg *msg) {
-        
-        if(CkMyPe() == 0 && msg !=  NULL)
-            CmiGetArgString(msg->argv, "+strategy", &router);         
-
-        if(CkMyPe() == 0 && msg !=  NULL)
-            CmiGetArgInt(msg->argv, "+spanning_factor", &sfactor);
-        
-        CProxy_ComlibManager::ckNew();
-    }
-};
+/// @TODO: eliminate AtSync and move toward anytime migration!
+void ComlibManager::AtSync() {
 
-//Called by user code
-ComlibInstanceHandle::ComlibInstanceHandle() : CkDelegateData() {
-    _instid = -1;
-    _dmid.setZero();
-    _srcPe = -1;
-    toForward = 0;
 }
 
-//Called by user code
-ComlibInstanceHandle::ComlibInstanceHandle(const ComlibInstanceHandle &h) 
-    : CkDelegateData() {
-    _instid = h._instid;
-    _dmid = h._dmid;
-    toForward = h.toForward;        
 
-    ComlibPrintf("In Copy Constructor\n");
-    _srcPe = h._srcPe;
+/***************************************************************************
+ * User section:
+ *
+ * Interface available to the user to interact with the ComlibManager
+ ***************************************************************************/
+
+/** Permanently associate the given proxy with comlib, to use the instance
+    represented by cinst. All messages sent through the proxy will be forwarded
+    by comlib. */
+void ComlibAssociateProxy(ComlibInstanceHandle cinst, CProxy &proxy) {
+  if(CkNumPes() > 1){
+       CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+       proxy.ckDelegate(cgproxy.ckLocalBranch(), new ComlibDelegateData(cinst));
+  } else {
+    ComlibPrintf("Doing nothing in ComlibAssociateProxy because we have only 1 pe\n"); 
+  }
+}
 
-    reset();
-    ref();
+/** Permanently assiciate the given proxy with comlib, to use the strategy
+    represented by strat. All messages sent through the proxy will be forwarded
+    by comlib. */
+void ComlibAssociateProxy(Strategy *strat, CProxy &proxy) {
+       ComlibAssociateProxy(strat->getHandle(), proxy);
+}  
+
+/* OLD DESCRIPTION! Register a strategy to the comlib framework, and return a
+    handle to be used in the future with ComlibAssociateProxy to associate the
+    strategy with a proxy. If one strategy is registered more than once, the
+    handles will be different, but they has to be considered as aliases. */
+
+/** Provided only for backward compatibility. A strategy is registered at the
+    converse level as soon as it is created (see Strategy constructor). This
+    function just retrieve a handle from the strategy itself. */
+ComlibInstanceHandle ComlibRegister(Strategy *strat) {
+       return strat->getHandle();
 }
 
-ComlibInstanceHandle& ComlibInstanceHandle::operator=(const ComlibInstanceHandle &h) {
-    _instid = h._instid;
-    _dmid = h._dmid;
-    toForward = h.toForward;
-    _srcPe = h._srcPe;
-    
-    reset();
-    ref();
-    return *this;
+/** Call beginIteration on the ComlibManager with the instance handle for the strategy associated with the proxy. 
+ *   If no strategy has been associated with the proxy, nothing is done in this function.
+ */
+void ComlibBegin(CProxy &proxy, int iteration) {
+  if(CkNumPes() > 1){
+       ComlibDelegateData *cinst = static_cast<ComlibDelegateData *>(proxy.ckDelegatedPtr());
+       if(cinst==NULL)
+               return;
+       CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+       (cgproxy.ckLocalBranch())->beginIteration(cinst->getID(), iteration);
+  } else {
+    ComlibPrintf("Doing nothing in ComlibBegin because we have only 1 pe");  
+  }
 }
 
-//Called by the communication library
-ComlibInstanceHandle::ComlibInstanceHandle(int instid, CkGroupID dmid){
-    _instid = instid;
-    _dmid   = dmid;
-    _srcPe  = -1;
-    toForward = 0;
+/** Call endIteration on the ComlibManager with the instance handle for the strategy associated with the proxy. 
+ *   If no strategy has been associated with the proxy, nothing is done in this function.
+ */
+void ComlibEnd(CProxy &proxy, int iteration) {
+  if(CkNumPes() > 1){
+       ComlibDelegateData *cinst = static_cast<ComlibDelegateData *>(proxy.ckDelegatedPtr());
+       if(cinst==NULL)
+               return;
+       CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+       (cgproxy.ckLocalBranch())->endIteration(cinst->getID(), iteration);
+  } else {
+    ComlibPrintf("Doing nothing in ComlibEnd because we have only 1 pe");  
+  }
 }
 
-void ComlibInstanceHandle::beginIteration() { 
-    CProxy_ComlibManager cgproxy(_dmid);
+char *routerName;
+int sfactor=0;
 
-    ComlibPrintf("Instance Handle beginIteration %d, %d, %d\n", CkMyPe(), _srcPe, _instid);
 
-    //User forgot to make the instance handle a readonly or pass it
-    //into the constructor of an array and is using it directly from
-    //Main :: main
-    if(_srcPe == -1) {
-        //ComlibPrintf("Warning:Instance Handle needs to be a readonly or a private variable of an array element\n");
-        _srcPe = CkMyPe();
-    }
+/** A mainchare, used to initialize the comlib framework at the program startup.
+    Its main purpose is to create the ComlibManager group. */
+class ComlibManagerMain {
+public:
+       ComlibManagerMain(CkArgMsg *msg) {
 
-    if(_srcPe != CkMyPe() && toForward) {
-        (cgproxy.ckLocalBranch())->setRemote(_srcPe);
-    }
+               if(CkMyPe() == 0 && msg !=  NULL)
+                       CmiGetArgString(msg->argv, "+strategy", &routerName);         
 
-    (cgproxy.ckLocalBranch())->setInstance(_instid);
-    (cgproxy.ckLocalBranch())->beginIteration();   
-}
+               if(CkMyPe() == 0 && msg !=  NULL)
+                       CmiGetArgInt(msg->argv, "+spanning_factor", &sfactor);
 
-void ComlibInstanceHandle::endIteration() {
-    CProxy_ComlibManager cgproxy(_dmid);
-    (cgproxy.ckLocalBranch())->endIteration();
-}
+               CProxy_ComlibManager::ckNew();
+       }
+};
 
-void ComlibInstanceHandle::setStrategy(CharmStrategy *s) {
-    toForward = s->getForwardOnMigration();
-    CProxy_ComlibManager cgproxy(_dmid);
-    (cgproxy.ckLocalBranch())->registerStrategy(_instid, s);
-}
+/***************************************************************************
+ * ComlibInstanceHandle section:
+ *
+ * Implementation of the functions defined in the ComlibInstanceHandle class.
+ ***************************************************************************/
 
-CharmStrategy *ComlibInstanceHandle::getStrategy() {
-    if(_instid < 0) 
-        return NULL;    
-    
-    CProxy_ComlibManager cgproxy(_dmid);
-    return (cgproxy.ckLocalBranch())->getStrategy(_instid);
+ComlibDelegateData::ComlibDelegateData(int instid) : CkDelegateData(), _instid(instid) {
+       ComlibManagerPrintf("[%d] Constructing ComlibDelegateData\n", CkMyPe());
+       ref();
 }
 
-CkGroupID ComlibInstanceHandle::getComlibManagerID() {return _dmid;}    
 
 void ComlibInitSectionID(CkSectionID &sid){
-    
-    sid._cookie.type = COMLIB_MULTICAST_MESSAGE;
-    sid._cookie.pe = CkMyPe();
-    
-    sid._cookie.sInfo.cInfo.id = 0;    
-    sid.npes = 0;
-    sid.pelist = NULL;
-}
 
-void ComlibResetSectionProxy(CProxySection_ArrayBase *sproxy) {
-    CkSectionID &sid = sproxy->ckGetSectionID();
-    ComlibInitSectionID(sid);
-    sid._cookie.sInfo.cInfo.status = 0;
+       sid._cookie.type = COMLIB_MULTICAST_MESSAGE;
+       sid._cookie.pe = CkMyPe();
+
+       sid._cookie.sInfo.cInfo.id = 0;    
+       sid.npes = 0;
+       sid.pelist = NULL;
 }
 
-// for backward compatibility - for old name commlib
+
+/** For backward compatibility - for old name commlib. The function
+    _registercomlib() is generated by the translator. */
 void _registercommlib(void)
 {
-  static int _done = 0; 
-  if(_done) 
-      return; 
-  _done = 1;
-  _registercomlib();
+       static int _done = 0; 
+       if(_done) 
+               return; 
+       _done = 1;
+       _registercomlib();
 }
 
-void ComlibAtSyncHandler(void *msg) {
-    CmiFree(msg);
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    ComlibManager *cmgr_ptr = cgproxy.ckLocalBranch();
-    if(cmgr_ptr)
-        cmgr_ptr->AtSync();    
-}
 
 void ComlibNotifyMigrationDoneHandler(void *msg) {
-    CmiFree(msg);
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    ComlibManager *cmgr_ptr = cgproxy.ckLocalBranch();
-    if(cmgr_ptr)
-        cmgr_ptr->AtSync();    
+       CmiFree(msg);
+       CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+       ComlibManager *cmgr_ptr = cgproxy.ckLocalBranch();
+       if(cmgr_ptr)
+               cmgr_ptr->AtSync();    
 }
 
 
-void ComlibLBMigrationUpdate(LBMigrateMsg *msg) {
-    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    (cgproxy.ckLocalBranch())->lbUpdate(msg);
+
+
+
+
+static void periodicDebugPrintStatus(void* ptr, double currWallTime){
+  CkPrintf("[%d] periodicDebugPrintStatus()\n", CkMyPe());
+
+  ComlibManager *mgr = (ComlibManager*)ptr;
+  mgr->printDiagnostics();
+
+  CcdCallFnAfterOnPE((CcdVoidFn)periodicDebugPrintStatus, ptr, 4000, CkMyPe());
+
 }
 
+
+
+
+
 #include "comlib.def.h"
 
 /*@}*/
index fbf72f0fdc9f2eb5cf42245e88075da6156c499a..e359b9b05b7075bb0c2f0cdc35ec28042272521c 100644 (file)
@@ -4,23 +4,39 @@ module comlib {
   message ComlibDummyMsg;
   //  message PrioMsg;
   message ComlibMulticastMsg {
-       ComlibMulticastIndexCount indicesCount[];
-       CkArrayIndexMax indices[];
-       char usrMsg[];
+    ComlibMulticastIndexCount indicesCount[];
+    CkArrayIndexMax indices[];
+    char usrMsg[];
   };
 
   group [migratable] ComlibManager {
     entry ComlibManager();
 
     entry void barrier(void);
-    entry void barrier2(void);
-    entry void resumeFromBarrier2(void);
-    entry void receiveTable(StrategyWrapper sw, 
-       CkHashtableT<ClibGlobalArrayIndex, int>);
-
-    entry void receiveRemoteSend(CkQ<CharmMessageHolder *> &remoteQ, int id);
-    entry void collectStats(ComlibLocalStats s, int src, 
-       CkVec<ClibGlobalArrayIndex>);
+    entry void resumeFromSetupBarrier();
+    //entry void barrier2(void);
+    //entry void resumeFromBarrier2(void);
+    //entry void receiveTable(StrategyWrapper sw, 
+    //                     CkHashtableT<ClibGlobalArrayIndex, int>);
+
+//    entry void bracketedFinishSetup(int instid);
+    entry void bracketedReceiveCount(int instid, int pe, int count, int isFirst, int step);
+    entry void bracketedStartErrorRecoveryProcess(int instid, int step);        
+    entry void bracketedErrorDetected(int instid, int step);
+    entry void bracketedConfirmCount(int instid);
+    entry void bracketedCountConfirmed(int instid, int count, int step);
+    entry void bracketedReceiveNewCount(int instid);
+    entry void bracketedDiscover(int instid, CkArrayID aid, CkArrayIndexMax &idx, int isSrc);
+    entry void bracketedContributeDiscovery(int instid, int pe, int nsrc, int ndest, int step);
+    entry void bracketedReceiveNewPeList(int instid, int count[CkNumPes()]);
+    entry void bracketedReleaseCount(int instid);
+    entry void bracketedReleaseBufferedMessages(int instid);
+
+
+
+    //entry void receiveRemoteSend(CkQ<CharmMessageHolder *> &remoteQ, int id);
+    entry void collectStats(ComlibLocalStats s, int src);
+    //                     CkVec<ClibGlobalArrayIndex>);
   }
 
   mainchare ComlibManagerMain {
@@ -30,21 +46,24 @@ module comlib {
   //PUPable CharmStrategy;
   //PUPable MessageHolder;
   //PUPable CharmMessageHolder;
+ //  PUPable ComlibArrayListener;
 
-  PUPable DummyStrategy;
-  PUPable NodeMulticast;
-  PUPable StreamingStrategy;
-  PUPable MPIStrategy;
-  PUPable DirectMulticastStrategy;
-  PUPable RectMulticastStrategy;
   PUPable EachToManyMulticastStrategy; 
+  PUPable DirectMulticastStrategy;
   PUPable RingMulticastStrategy;       
-  PUPable PipeBroadcastStrategy;
-  PUPable MeshStreamingStrategy;
-  PUPable PrioStreaming;
+  PUPable MultiRingMulticastStrategy;
+  PUPable RectMulticastStrategy;
   PUPable BroadcastStrategy;
-  PUPable MultiRingMulticast;
+  PUPable PrioStreaming;
+  PUPable PipeBroadcastStrategy;
 
+// PUPable DummyStrategy;
+//  PUPable NodeMulticast;
+//  PUPable StreamingStrategy;
+//  PUPable MPIStrategy;
+//  PUPable MeshStreamingStrategy;
+  
   //Strategy pupable defined in the array manager code, along with the
   //array listener code
 };
index 57a8eca7b15db9bd4408b64ff8bb762a0517470e..5d0bb073fd83521da010c3e47bb96de1a5a22ede 100644 (file)
-#ifndef COMMLIBMANAGER_H
-#define COMMLIBMANAGER_H
+#ifndef COMLIBMANAGER_H
+#define COMLIBMANAGER_H
+
 
 /**
-   @defgroup Comlib  Charm++ Communication Optimization Framework 
+   @{
+   
+   @file
+
+   @brief Charm ComlibManager
+   Contains the classes ComlibManager (charm++ delegated group) and
+   ComlibDelegateData (its delegated data). It also declares the user
+   interface to the framework, all beginning with the prefix "Comlib".
+
+
+   @defgroup Comlib  Communication Optimization Framework
+   @brief A communication optimization framework.
+
+   Comlib is a framework for delegating converse and charm++ level communications to optimizing strategies which can use appropriate topological routers.
+
+   The comlib framework underwent a large refactoring that was committed into CVS in February 2009. The framework was extricated from ck-core. Various files and classes were renamed, and many bugs were fixed. A new system for bracketed strategies was introduced. 
+
+   Bracketed communication strategies, such as those for all-to-all communication, now have knowledge that is updated on demand. If errors are detected in the knowledge base (eg. a chare array element is no longer located on some PE), the strategies enter an error mode. After the knowledge has been updated, the strategies will exit the error mode and send messages along the new optimized paths. While in the error states, the strategies send existing messages directly, and buffer new messages.
+
+
+<h2>Usage Restrictions</h2>
+Strategies should be created in a MainChare's Main method (hence on PE 0). Proxies can be created later and these can be delegated to the existing strategies. 
+
+  
+<h2>Startup</h2>
+
+   The startup of Comlib happens asynchronously. The <i>module comlib</i> does not currently use any initproc routine. The <i>mainchare ComlibManagerMain</i> has a constructor that runs along with all other mainchares while charm++ messages are not activated at startup. This constructor simply sets a few variables from command line arguments, and then creates the <i>ComlibManager group </i>. After all mainchares have run (in no determinable order), then the charm++ system will release all charm++ messages. 
+
+   At this point the user program will continue asynchronously with the comlib startup.
+  
+   Then ComlibManager::ComlibManager() calls ComlibManager::init(), sets up a few variables, and then calls ComlibManager::barrier(). After barrier() has been called by all PEs, it calls ComlibManager::resumeFromSetupBarrier(). 
+
+   ComlibManager::resumeFromSetupBarrier() completes the initialization of the charm layer of comlib after all group branches are created. It is guaranteed that Main::Main has already run at this point, so all strategies created there can be broadcast.  This function calls ComlibDoneCreating() and then it sends all messages that were buffered (in unCompletedSetupBuffer).
+
+   ComlibDoneCreating() will do nothing on all PE != 0. On PE 0, it will call ConvComlibManager::doneCreating(). The strategy table will broadcast at this point.
+
+   The process for broadcasting the strategy table is as follows (see convcomlibmanager.C):
+   <ol>
+   <li>the strategies are inserted on processor 0 (and possibly in other
+   processors with the same order. The strategies are marked as "new"
+   <li>when ConvComlibManager::doneCreating() is called, processor 0 broadcasts all the new
+   strategies to all the processors, and marks them as "inSync"  
+   <li>when a processor receives a table it updates its personal table with the
+   incoming, it marks all the strategies just arrived as "inSync", and it
+   sends an acknowledgement back to processor 0.   
+   <li>when an acknowledgement is received by processor 0, a counter is
+   decremented. When it reaches 0, all the "inSync" strategies are switched
+   to status "ready" and they can start working. All the messages in the
+   tmplist are delivered. The sync is broadcasted.   
+   <li>when an acknowledgement is received by a processor other than 0, all the
+   "inSync" strategies are switched to "ready" and the messages in tmplist
+   are delivered.   
+   <li>in order to prevent two consecutive table broadcasts to interfere with
+   each other, an additional acknowledgement is sent back by each processor
+   to processor 0 to allow a new table update to happen.
+   </ol>
+
+<h2>Startup: Buffering of Messages</h2>
+
+   Because the startup of Comlib happens asynchronously. Thus, if a user program sends messages through a comlib strategy, and the strategy has not yet started up completely, then the messages may be delayed in one of two queues. 
+   
+<ol>
+   <li>CkQ<MessageHolder*> tmplist; found in convcomlibstrategy.h buffers converse level messages when the converse strategies are not ready.
+   <li>std::map<ComlibInstanceHandle, std::set<CharmMessageHolder*> > ComlibManager::delayMessageSendBuffer in ComlibManager.h buffers charm level messages at startup before ComlibManager::resumeFromSetupBarrier() or while a strategy is in an error state. Messages are flushed from here once both the startup has finished and the strategy is not in an error state. The messages are flushed from one place: ComlibManager::sendBufferedMessages().
+</ol>
+  
+
+
+
+<h2>Bracketed Strategies</h2>
+<h3>Usage of Bracketed Strategies</h3>
+Bracketed strategies have the following usage pattern. For each iteration of the program: 
+<ol>
+<li>Each source object calls ComlibManager::beginIteration(int instid, int iteration)
+<li>Each source object invokes one or more entry method(s) on the delegated proxy
+<li>Each source object then calls ComlibManager::endIteration().
+</ol>
+
+<h3>Restrictions on Bracketed Strategies</h3>
+<ol>
+<li>The user application is not allowed to call beginIteration for iteration n until all messages from iteration n-1 have been received.
+<li>Migrations of elements are not allowed between when they call ComlibManager::beginIteration and the associated ComlibManager::endIteration for the same iteration.
+</ol>
+
+<h3>Detecting migrations in Bracketed Strategies</h3>
+The instance of each strategy on each PE maintains a list of the local array elements, and the last known iteration value. The current implementation only detects migrations when a PE gains a net positive number of migrated objects. All of the objects on that PE will call ComlibManager::beginIteration. Because the strategy knows how many elements were previously on the PE, it will detect more calls to ComlibManager::beginIteration than its previous element count. At this point, the future messages for the strategy will be enqueued in a buffer (ComlibManager::delayMessageSendBuffer). Once ComlibManager::endIteration() is called, the error recovery protocol will be started. All PEs will cause any objects that have migrated away to report back to PE 0, which updates a list of object locations. Once all PEs and migrated objects have reported back to PE 0, the updated PE list will be broadcast to all PEs, and the strategy will be enabled again. At this point any buffered messages will be released. The subsequent iteration of the application should then be optimized.
+
+If two objects swap places between two PEs, the current implementation does not detect this change. In the future ComlibManager::beginIteration should compare the object to the list of known local objects, and start buffering messages and correcting this error condition.
+
+
+
+   
+   @defgroup ConvComlib  Converse Communication Optimization Framework 
+   @ingroup Comlib
+   @brief Framework for delegating converse level communication to Comlib.
+
+
+   @defgroup CharmComlib  Charm++ Communication Optimization Framework 
+   @ingroup Comlib
+   @brief Framework for delegating Charm++ level communication to Comlib. 
+
+
+   @defgroup ConvComlibRouter Converse level message Routers
+   @ingroup ConvComlib
+   @ingroup Comlib
+   @brief Routers used by converse strategies to route messages in certain topologies : grid, hypercube, etc.
+
+   Each router inherits from Router. 
+
+
+   @defgroup ComlibCharmStrategy Strategies for use in Charm++
+   @ingroup CharmComlib
+   @ingroup Comlib
+   @brief Communication optimizing strategies for use in Charm++ programs.
+
+   These strategies are used in Charm++ programs, by creating proxies that are then associated with a strategy. The future method invocations on the proxy are then handled by the strategy. 
+
+
+   @defgroup ComlibConverseStrategy Strategies for use in converse
+   @ingroup ConvComlib
+   @ingroup Comlib
+   @brief Communication optimizing strategies for use in converse programs or in other comlib strategies.
+
 */
-/*@{*/
 
-#include "charm++.h"
-#include "cksection.h"
-#include "envelope.h"
-#include "convcomlib.h"
+
+
 #include <math.h>
+#include <map>
+#include <set>
 
-#include "charm++.h"
 #include "convcomlibmanager.h"
+#include "ComlibStrategy.h"
+#include "ComlibArrayListener.h"
+#include "cksection.h"
 
 #define CHARM_MPI 0 
 #define MAX_NSTRAT 1024
                              //learning framework will discover 
                              //the appropriate strategy, not completely 
                              //implemented
-PUPbytes(comID);
-
 #include "ComlibStats.h"
 
 #include "comlib.decl.h"
 
+CkpvExtern(CkGroupID, cmgrID);
 ///Dummy message to be sent in case there are no messages to send. 
 ///Used by only the EachToMany strategy!
 class ComlibDummyMsg: public CMessage_ComlibDummyMsg {
     int dummy;
 };
 
-/*
-//Priority message to call end iteration
-class PrioMsg: public CMessage_PrioMsg {
- public:
-    int instID;
-};
-*/
+
 
 /**
- * Structure used to hold a count of the indeces associated to each pe in a multicast message.
+ * Structure used to hold a count of the indeces associated to each pe in a
+ * multicast message.
  */
+// TO BE MOVED TO MULTICAST SPECIFIC
 struct ComlibMulticastIndexCount {
   int pe;
   int count;
@@ -66,6 +186,8 @@ inline int indexCountCompare(const void *a, const void *b) {
   return 0;
 }
 
+/** Used to setup a multicast tree in SectionInfo and MulticastStrategies */
+// TO BE MOVED TO MULTICAST SPECIFIC
 class ComlibMulticastMsg : public CkMcastBaseMsg, 
                public CMessage_ComlibMulticastMsg {
     
@@ -76,105 +198,65 @@ class ComlibMulticastMsg : public CkMcastBaseMsg,
     char *usrMsg;        
 };
 
-class ComlibManager;
 
-/** An Instance of the communication library. It can be seen as the proxy given
-    by ComlibManager upon registration of a strategy to the framework.
- */
-class ComlibInstanceHandle : public CkDelegateData {
- private:    
-    int _instid;
-    CkGroupID _dmid;
-    int _srcPe;
-    int toForward;
-
- public:
-    ComlibInstanceHandle();
-    ComlibInstanceHandle(const ComlibInstanceHandle &h);
-    ComlibInstanceHandle(int instid, CkGroupID dmid);    
-   
-    ComlibInstanceHandle &operator=(const ComlibInstanceHandle &h);
+void ComlibAssociateProxy(ComlibInstanceHandle cinst, CProxy &proxy);
+void ComlibAssociateProxy(Strategy *strat, CProxy &proxy); 
 
-    void setForwardingOnMigration(){toForward = 1;} 
-    void beginIteration();
-    void endIteration();
-    
-    CkGroupID getComlibManagerID();
-    void setStrategy(CharmStrategy *);
-    CharmStrategy *getStrategy();        
-    int getSourcePe() {return _srcPe;}
+/// @deprecated
+ComlibInstanceHandle ComlibRegister(Strategy *strat);
 
-    void setSourcePe() {_srcPe = CkMyPe();}
+void ComlibBegin(CProxy &proxy, int iteration);    
+void ComlibEnd(CProxy &proxy, int iteration);    
 
-    friend class ComlibManager;
-    void pup(PUP::er &p) {
+/** The behaviour has changed from the previous version:
+    while before this function was used to reset a proxy after a migration,
+    now it is used to reset a proxy before it is reassociated with another strategy. 
+ */
+inline void ComlibResetSectionProxy(CProxySection_ArrayBase &sproxy) {
+  sproxy.ckGetSectionInfo().sInfo.cInfo.id = 0;
+}
 
-        if(p.isUnpacking())
-             reset();        
 
-        p | _instid;
-        p | _dmid;
-        p | _srcPe;
-        p | toForward;
-    }
-};
+void ComlibInitSectionID(CkSectionID &sid);
 
 class LBMigrateMsg;
 
-/** The main group doing the management of all the system. It takes care of
-    holding all the registered strategies, calling them when a message arrives,
-    and modifying them when needed by the learning framework. It installed
-    itself as a delegated class from CkDelegateMgr, overwriting the standard
-    path of message sending in charm.
+/** The main group doing the management of all the charm system. It takes care
+    of calling the strategies when a message arrives, and modifying them when
+    needed by the learning framework. It relies on ConvComlibManager for the
+    processor operations involved. It installes itself as a delegated class from
+    CkDelegateMgr, overwriting the standard path of message sending in charm.
  */
 class ComlibManager: public CkDelegateMgr {
-    friend class ComlibInstanceHandle;
+  //friend class ComlibInstanceHandle;
 
     int *bcast_pelist;  //Pelist passed to all broadcast operations
 
+    /// Used to register and record events into projection
     int section_send_event;
 
-    int remotePe;
-    CmiBool isRemote;
-    CmiBool strategyCreated;
-
-    int npes;
-    int *pelist;
-
     CkArrayIndexMax dummyArrayIndex;
 
-    //For compatibility and easier use!
-    int strategyID; //<Identifier of the strategy
-
-    ///Pointer to the converse comm lib strategy table
-    StrategyTable *strategyTable;
+    /// Pointer to the converse comlib object, for efficiency over calling CkpvAccess
+    ConvComlibManager *converseManager;
 
-    CkQ<CharmStrategy *> ListOfStrategies; //temporary list of strategies
-    
-    CkQ<CharmMessageHolder *> remoteQ;  //list of remote messages
-                                        //after the object has
-                                        //migrated
-
-    //The number of strategies created by the user
-    //int nstrats; //now part of conv comlib
-    
-    int curStratID, prevStratID;      
-    //Number of strategies created by the user.
 
-    //flags
-    int receivedTable, setupComplete, barrierReached, barrier2Reached;
-    CmiBool lbUpdateReceived;
-
-    int bcount , b2count;
-    //int totalMsgCount, totalBytes, nIterations;
-
-    ComlibArrayListener *alistener;
+    /// Lists of messages whose delivery will be postponed until the comlib strategy has been fully setup, and 
+    /// the strategy has exited an error state
+    /// The map key is a comlib instance handle
+    /// The map value is a set of messages
+    std::map<ComlibInstanceHandle, std::set<CharmMessageHolder*> > delayMessageSendBuffer;
+   
+    /// Different than 0 once this group has been created on all processors
+    int setupComplete;
+  
     int prioEndIterationFlag;
 
     ComlibGlobalStats clib_gstats; 
-    int    numStatsReceived;
+    int numStatsReceived;
 
-    int curComlibController;   //Processor where strategies are  recreated
+    int curComlibController;   //Processor where strategies are created
     int clibIteration;         //Number of such learning iterations,
                                //each of which is triggered by a
                                //loadbalancing operation
@@ -182,23 +264,27 @@ class ComlibManager: public CkDelegateMgr {
     void init(); ///Initialization function
 
     //charm_message for multicast for a section of that group
-    void multicast(CharmMessageHolder *cmsg); //charm message holder here
+    void multicast(CharmMessageHolder *cmsg, int instid); //charm message holder here
     //void multicast(void *charm_msg, int npes, int *pelist);
 
-    //The following funtions can be accessed only from ComlibInstanceHandle
-    void beginIteration();     ///Notify begining of a bracket with
-                               ///strategy identifier
+    //The following funtions can be accessed only from the user provided hooks
+    friend void ComlibBegin(CProxy&, int iteration);
+    friend void ComlibEnd(CProxy&, int iteration);
 
-    void endIteration();       ///Notify end, endIteration must be
-                               ///called if a beginIteration is
-                               ///called. Otherwise end of the entry
-                               ///method is assumed to be the end of
-                               ///the bracket.
+    ///Notify begining of a bracket with strategy identifier
+    void beginIteration(int instid, int iteration);
+    
+    ///Notify end, endIteration must be called if a beginIteration is called.
+    ///Otherwise end of the entry method is assumed to be the end of the
+    ///bracket.
+    void endIteration(int instid, int iteration);
     
-    void setInstance(int id); 
+    void printPeList(char* note, int *peList);
 
-    //void prioEndIteration(PrioMsg *pmsg);
-    void registerStrategy(int pos, CharmStrategy *s);
+    
+    bool shouldBufferMessagesNow(int instid);
+    void sendBufferedMessages(int instid);
+    void sendBufferedMessagesAllStrategies();
 
  public:
 
@@ -210,39 +296,58 @@ class ComlibManager: public CkDelegateMgr {
     ComlibManager(CkMigrateMessage *m) { }
     int useDefCtor(void){ return 1; } //Use default constructor should
     //be pupped and store all the strategies.
+
+    /* Initialization routines */
     
     void barrier(void);
-    void barrier2(void);
-    void resumeFromBarrier2(void);
-
-    //Receive table of strategies.
-    void receiveTable(StrategyWrapper &sw, 
-                      CkHashtableT <ClibGlobalArrayIndex, int>&); 
+    void resumeFromSetupBarrier();
+   
+    /* The delegation framework reimplemented functions */
 
     void ArraySend(CkDelegateData *pd,int ep, void *msg, 
                    const CkArrayIndexMax &idx, CkArrayID a);
-
-    void receiveRemoteSend(CkQ<CharmMessageHolder*> &rq, int id);
-    void sendRemote();
-
     void GroupSend(CkDelegateData *pd, int ep, void *msg, int onpe, 
                    CkGroupID gid);
-    
-    virtual void ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a);
-    virtual void GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g);
-    virtual void ArraySectionSend(CkDelegateData *pd, int ep ,void *m, 
+    void ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a);
+    void GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g);
+    void ArraySectionSend(CkDelegateData *pd, int ep ,void *m, 
                                   CkArrayID a, CkSectionID &s, int opts);
+    CkDelegateData* ckCopyDelegateData(CkDelegateData *data); 
+    CkDelegateData *DelegatePointerPup(PUP::er &p,CkDelegateData *pd);
 
-    CharmStrategy *getStrategy(int instid)
-        {return (CharmStrategy *)(* strategyTable)[instid].strategy;}
-
-    StrategyTableEntry *getStrategyTableEntry(int instid)
-        {return &((*strategyTable)[instid]);}
-
-    //To create a new strategy, returns handle to the strategy table;
-    ComlibInstanceHandle createInstance();  
-    void broadcastStrategies();             //Done creating instances
-
+    inline Strategy *getStrategy(int instid)
+        {return converseManager->getStrategy(instid);}
+
+    inline StrategyTableEntry *getStrategyTableEntry(int instid)
+        {return converseManager->getStrategyTable(instid);}
+
+    // Entry methods used by bracketed strategies when there is some object
+    // migration. The comlib Manager realizes there is an error, it enters error
+    // mode by suspending the delivery of the strategy if it has already
+    // started, and it globally reduces the number of elements which already
+    // called endIteration. When this number matches the number of elements
+    // involved in the operation (or a multiple of it in case the user overlaps
+    // iterations), the system ri-deliver the doneInserting to the strategy.
+    //  void bracketedFinishSetup(int instid);
+    void bracketedCatchUpPE(int instid, int step);
+    void bracketedReceiveCount(int instid, int pe, int count, int isFirst, int step);
+    void bracketedStartErrorRecoveryProcess(int instid, int step);
+    void bracketedErrorDetected(int instid, int step);
+    void bracketedConfirmCount(int instid);
+    void bracketedCountConfirmed(int instid, int count, int step);
+    void bracketedReceiveNewCount(int instid);
+    void bracketedReceiveNewPeList(int instid, int *count);
+
+    void bracketedFinalBarrier(int instid);
+    void bracketedReleaseCount(int instid);
+    void bracketedReleaseBufferedMessages(int instid);
+
+    void bracketedStartDiscovery(int instid);
+    void bracketedDiscover(int instid, CkArrayID aid, CkArrayIndexMax &idx, int isSrc);
+    void bracketedContributeDiscovery(int instid, int pe, int nsrc, int ndest, int step);
+
+
+    // TODO: Delete the following two methods!!!!
     void AtSync();           //User program called loadbalancer
     void lbUpdate(LBMigrateMsg *); //loadbalancing updates
 
@@ -250,49 +355,53 @@ class ComlibManager: public CkDelegateMgr {
     //void learnPattern(int totalMessageCount, int totalBytes);
     //void switchStrategy(int strat);
 
-    void setRemote(int remotePe);
-
-    void collectStats(ComlibLocalStats &s, int src,CkVec<ClibGlobalArrayIndex>&);
+    //void collectStats(ComlibLocalStats &s, int src,CkVec<ClibGlobalArrayIndex>&);
+    void collectStats(ComlibLocalStats &s, int src);
 
-    //Returns the processor on which the comlib sees the array element
-    //belonging to
-    inline int getLastKnown(CkArrayID a, CkArrayIndexMax &idx) {
-        return ComlibGetLastKnown(a, idx);
-    }
+    /// Print information about the number of undelivered messages awaiting a specified strategy to be ready    
+    void printDiagnostics(int cinst);
 
-    CkDelegateData* ckCopyDelegateData(CkDelegateData *data); 
-    CkDelegateData *DelegatePointerPup(PUP::er &p,CkDelegateData *pd);
-};
+    /// Print information about the number of undelivered messages awaiting any strategy to be ready
+    void printDiagnostics();   
+       
+}; 
 
-void ComlibDelegateProxy(CProxy *proxy);
-void ComlibAssociateProxy(ComlibInstanceHandle *cinst, CProxy &proxy);
-void ComlibAssociateProxy(CharmStrategy *strat, CProxy &proxy); 
-ComlibInstanceHandle ComlibRegister(CharmStrategy *strat);
-void ComlibBegin(CProxy &proxy);    
-void ComlibEnd(CProxy &proxy);    
-
-ComlibInstanceHandle CkCreateComlibInstance();
-ComlibInstanceHandle CkGetComlibInstance();
-ComlibInstanceHandle CkGetComlibInstance(int id);
+/** This class is used by ComlibManager (the delegator manager) as its delegated
+    data. The only thing it contains is the position in the system of the
+    strategy it represents.
+ */
+class ComlibDelegateData : public CkDelegateData {
+ private:
+  int _instid; ///< Position of this instance in the strategy table
 
-void ComlibResetSectionProxy(CProxySection_ArrayBase *sproxy);
+  friend void ComlibAssociateProxy(ComlibInstanceHandle, CProxy&);
+  friend CkDelegateData *ComlibManager::ckCopyDelegateData(CkDelegateData*);
+  ComlibDelegateData(int instid);
 
-inline void ComlibResetProxy(CProxy *aproxy) {
-  ComlibInstanceHandle *handle = 
-    (ComlibInstanceHandle *) aproxy->ckDelegatedPtr();
-  handle->setSourcePe();
-}
+ public:
+  ComlibDelegateData(CkMigrateMessage *) : CkDelegateData() { ref(); }
+  
+  void beginIteration();
+  void endIteration();
 
-//Only Called when the strategies are not being created in main::main
-void ComlibDoneCreating(); 
+  /// Get the position of this instance in the strategy table
+  inline int getID() {return _instid;}
+  
+  void pup(PUP::er &p) {
+    p | _instid;
+  }
 
-void ComlibInitSectionID(CkSectionID &sid);
+};
 
 void ComlibAtSync(void *msg);
 void ComlibNotifyMigrationDoneHandler(void *msg);
 void ComlibLBMigrationUpdate(LBMigrateMsg *);
 
-#define RECORD_SEND_STATS(sid, bytes, dest) {             \
+
+#ifdef filippo
+// The old interface
+#define RECORD_SENDSTATS(sid, bytes, dest) {             \
         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));               \
         cgproxy.ckLocalBranch()->clib_stats.recordSend(sid, bytes, dest); \
 }\
@@ -312,6 +421,14 @@ void ComlibLBMigrationUpdate(LBMigrateMsg *);
         cgproxy.ckLocalBranch()->clib_stats.recordRecvM(sid, bytes, src_arr, nsrc); \
 }\
 
-/*@}*/
+#else
+// the new version of comlib does not do anything with these yet
+#define RECORD_SEND_STATS(sid, bytes, dest) /* empty */
+#define RECORD_RECV_STATS(sid, bytes, src) /* empty */
+#define RECORD_SENDM_STATS(sid, bytes, dest_arr, ndest) /* empty */
+#define RECORD_RECVM_STATS(sid, bytes, src_arr, nsrc) /* empty */
+#endif
+
+/** @} */
 
 #endif
diff --git a/src/ck-com/ComlibModuleInterface.C b/src/ck-com/ComlibModuleInterface.C
new file mode 100644 (file)
index 0000000..3318fba
--- /dev/null
@@ -0,0 +1,34 @@
+/**
+   @addtogroup CharmComlib
+   @{
+   @file
+   This file is compiled and linked into all Charm++ programs, 
+   even those that do not use -module comlib. Thus, a few functions
+   are implemented here that are referenced from parts of the charm++ 
+   core. These functions can handle both the case where the comlib
+   module is used and where it is not used.
+*/
+
+#include "ComlibStrategy.h"
+
+//calls ComlibNotifyMigrationDone(). Even compiles when -module comlib
+//is not included. Hack to make loadbalancer work without comlib
+//currently.
+CkpvDeclare(int, migrationDoneHandlerID);
+
+
+void ComlibNotifyMigrationDone() {
+    if(CkpvInitialized(migrationDoneHandlerID)) 
+        if(CkpvAccess(migrationDoneHandlerID) > 0) {
+            char *msg = (char *)CmiAlloc(CmiReservedHeaderSize);
+            CmiSetHandler(msg, CkpvAccess(migrationDoneHandlerID));
+#if CMK_BLUEGENE_CHARM
+           // bluegene charm should avoid directly calling converse
+            CmiSyncSendAndFree(CkMyPe(), CmiReservedHeaderSize, msg);
+#else
+            CmiHandleMessage(msg);
+#endif
+        }
+}
+
+/*@}*/
index 9fa27438bac20ab755521191f4ab42fd4bebbb08..c8de2036b1918ef1129db46d9b8d33c191dc0449 100644 (file)
@@ -1,35 +1,60 @@
-#include "charm++.h" // for CMK_HAS_ALLOCA_H
+/**
+   @addtogroup CharmComlib
+   @{ 
+   @file
+  
+   Implementations of classes defined in ComlibSectionInfo.h
+*/
+
+#include "ComlibManager.h"
+#include "ComlibSectionInfo.h"
+#ifdef __MINGW_H 
+#include <malloc.h> 
+#endif
+
+
+#define USE_CONTROL_POINTS 0
+
+#if USE_CONTROL_POINTS
+#include "controlPoints.h"
+#endif
 
 #if CMK_HAS_ALLOCA_H
 #include <alloca.h>
 #endif
 
-#ifdef __MINGW_H
-#include <malloc.h>
-#endif
 
-#include "ComlibManager.h"
-#include "ComlibSectionInfo.h"
 
-ComlibMulticastMsg * ComlibSectionInfo::getNewMulticastMessage(CharmMessageHolder *cmsg, int needSort){
+/// Create a new multicast message
+ComlibMulticastMsg * ComlibSectionInfo::getNewMulticastMessage(CharmMessageHolder *cmsg, int needSort, int instanceID){
     
+  cmsg->checkme();
+
     if(cmsg->sec_id == NULL || cmsg->sec_id->_nElems == 0)
         return NULL;
 
     void *m = cmsg->getCharmMessage();
     envelope *env = UsrToEnv(m);
         
-    if(cmsg->sec_id->_cookie.sInfo.cInfo.id != 0) 
-        CmiAbort("In correct section\n");
-
+    // Crate a unique identifier for section id in cmsg->sec_id
     initSectionID(cmsg->sec_id);   
 
     CkPackMessage(&env);
+
+    const CkArrayID destArrayID(env->getsetArrayMgr());
     int nRemotePes, nRemoteIndices;
     ComlibMulticastIndexCount *indicesCount;
     int *belongingList;
-    getPeCount(cmsg->sec_id->_nElems, cmsg->sec_id->_elems, nRemotePes, nRemoteIndices, indicesCount, belongingList);
-    if (nRemotePes == 0) return NULL;
+    getPeCount(cmsg->sec_id->_nElems, cmsg->sec_id->_elems, destArrayID, nRemotePes, nRemoteIndices, indicesCount, belongingList);
+
+    //     if (nRemotePes == 0) return NULL;
+
+#if 0
+    CkPrintf("nRemotePes=%d\n", nRemotePes);
+    CkPrintf("nRemoteIndices=%d\n",nRemoteIndices);
+    CkPrintf("env->getTotalsize()=%d\n", env->getTotalsize());
+    CkPrintf("cmsg->sec_id->_nElems=%d\n", cmsg->sec_id->_nElems);
+ #endif
 
     int sizes[3];
     sizes[0] = nRemotePes;
@@ -53,30 +78,30 @@ ComlibMulticastMsg * ComlibSectionInfo::getNewMulticastMessage(CharmMessageHolde
     CkArrayIndexMax **indicesPe = (CkArrayIndexMax**)alloca(nRemotePes * sizeof(CkArrayIndexMax*));
 
     if (needSort) {
-      // if we are sorting the array, then we need to fix the problem that belongingList
-      // refers to the original ordering! This is done by mapping indicesPe in a way coherent
-      // with the original ordering.
-      int previous, i, j;
-      qsort(msg->indicesCount, sizes[0], sizeof(ComlibMulticastIndexCount), indexCountCompare);
-
-      for (j=0; j<nRemotePes; ++j) if (indicesCount[j].pe == msg->indicesCount[0].pe) break;
-      indicesPe[j] = msg->indices;
-      previous = j;
-      for (i=1; i<nRemotePes; ++i) {
-       for (j=0; j<nRemotePes; ++j) if (indicesCount[j].pe == msg->indicesCount[i].pe) break;
-       indicesPe[j] = indicesPe[previous] + indicesCount[previous].count;
-        previous = j;
-      }
+       // if we are sorting the array, then we need to fix the problem that belongingList
+       // refers to the original ordering! This is done by mapping indicesPe in a way coherent
+       // with the original ordering.
+       int previous, i, j;
+       qsort(msg->indicesCount, sizes[0], sizeof(ComlibMulticastIndexCount), indexCountCompare);
+
+       for (j=0; j<nRemotePes; ++j) if (indicesCount[j].pe == msg->indicesCount[0].pe) break;
+       indicesPe[j] = msg->indices;
+       previous = j;
+       for (i=1; i<nRemotePes; ++i) {
+               for (j=0; j<nRemotePes; ++j) if (indicesCount[j].pe == msg->indicesCount[i].pe) break;
+               indicesPe[j] = indicesPe[previous] + indicesCount[previous].count;
+               previous = j;
+       }
     } else {
-      indicesPe[0] = msg->indices;
-      for (int i=1; i<nRemotePes; ++i) indicesPe[i] = indicesPe[i-1] + indicesCount[i-1].count;
+       indicesPe[0] = msg->indices;
+       for (int i=1; i<nRemotePes; ++i) indicesPe[i] = indicesPe[i-1] + indicesCount[i-1].count;
     }
 
     for (int i=0; i<cmsg->sec_id->_nElems; ++i) {
-      if (belongingList[i] >= 0) {
-       *indicesPe[belongingList[i]] = cmsg->sec_id->_elems[i];
-       indicesPe[belongingList[i]]++;
-      }
+       if (belongingList[i] >= 0) {
+               *indicesPe[belongingList[i]] = cmsg->sec_id->_elems[i];
+               indicesPe[belongingList[i]]++;
+       }
     }
     memcpy(msg->usrMsg, env, sizes[2] * sizeof(char));
     envelope *newenv = UsrToEnv(msg);
@@ -93,7 +118,6 @@ ComlibMulticastMsg * ComlibSectionInfo::getNewMulticastMessage(CharmMessageHolde
     newenv->setEvent(env->getEvent());
     newenv->setSrcPe(env->getSrcPe());
     
-    CkPackMessage(&newenv);        
     return (ComlibMulticastMsg *)EnvToUsr(newenv);
 }
 
@@ -131,26 +155,26 @@ void ComlibSectionInfo::unpack(envelope *cb_env,
       }
     else
       {
-       nLocalElems = ccmsg->indicesCount[i].count;
-
-       /*
-         CkPrintf("Unpacking: %d local elements:",nLocalElems);
-         for (int j=0; j<nLocalElems; ++j) CkPrintf(" %d",((int*)&dest_indices[j])[1]);
-         CkPrintf("\n");
-       */
-       /*
-         for(int count = 0; count < ccmsg->nIndices; count++){
-         CkArrayIndexMax idx = ccmsg->indices[count];
+    nLocalElems = ccmsg->indicesCount[i].count;
+
+    /*
+    ComlibPrintf("Unpacking: %d local elements:",nLocalElems);
+    for (int j=0; j<nLocalElems; ++j) ComlibPrintf(" %d",((int*)&dest_indices[j])[1]);
+    ComlibPrintf("\n");
+    */
+    /*
+    for(int count = 0; count < ccmsg->nIndices; count++){
+        CkArrayIndexMax idx = ccmsg->indices[count];
         
-         //This will work because. lastknown always knows if I have the
-         //element of not
-         int dest_proc = ComlibGetLastKnown(destArrayID, idx);
-         //CkArrayID::CkLocalBranch(destArrayID)->lastKnown(idx);
+        //This will work because. lastknown always knows if I have the
+        //element of not
+        int dest_proc = ComlibGetLastKnown(destArrayID, idx);
+        //CkArrayID::CkLocalBranch(destArrayID)->lastKnown(idx);
         
-         //        if(dest_proc == CkMyPe())
-         dest_indices.insertAtEnd(idx);                        
-         }
-       */
+        //        if(dest_proc == CkMyPe())
+        dest_indices.insertAtEnd(idx);                        
+    }
+    */
       }
     envelope *usrenv = (envelope *) ccmsg->usrMsg;
     env = (envelope *)CmiAlloc(usrenv->getTotalsize());
@@ -159,10 +183,11 @@ void ComlibSectionInfo::unpack(envelope *cb_env,
 
 
 void ComlibSectionInfo::processOldSectionMessage(CharmMessageHolder *cmsg) {
+  cmsg->checkme();
 
     ComlibPrintf("Process Old Section Message \n");
 
-    int cur_sec_id = ComlibSectionInfo::getSectionID(*cmsg->sec_id);
+    int cur_sec_id = cmsg->sec_id->getSectionID();
 
     //Old section id, send the id with the message
     CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)cmsg->getCharmMessage();
@@ -170,10 +195,18 @@ void ComlibSectionInfo::processOldSectionMessage(CharmMessageHolder *cmsg) {
     cbmsg->_cookie.sInfo.cInfo.status = COMLIB_MULTICAST_OLD_SECTION;
 }
 
+CkMcastBaseMsg *ComlibSectionInfo::getNewDeliveryErrorMsg(CkMcastBaseMsg *base) {
+  CkMcastBaseMsg *dest= (CkMcastBaseMsg*)CkAllocMsg(0, sizeof(CkMcastBaseMsg), 0);
+  memcpy(dest, base, sizeof(CkMcastBaseMsg));
+  dest->_cookie.sInfo.cInfo.status = COMLIB_MULTICAST_SECTION_ERROR;
+  return dest;
+}
+
 void ComlibSectionInfo::getPeList(int _nElems, 
-                                  CkArrayIndexMax *_elems, 
+                                  CkArrayIndexMax *_elems,
+                                 CkArrayID &destArrayID,
                                   int &npes, int *&pelist){
-    
+
     int length = CkNumPes();
     if(length > _nElems)    //There will not be more processors than
                             //number of elements. This is wastage of
@@ -186,9 +219,11 @@ void ComlibSectionInfo::getPeList(int _nElems,
     
     int count = 0, acount = 0;
     
+    CkArray *a = (CkArray *)_localBranch(destArrayID);
     for(acount = 0; acount < _nElems; acount++){
         
-        int p = ComlibGetLastKnown(destArrayID, _elems[acount]);
+      //int p = ComlibGetLastKnown(destArrayID, _elems[acount]);
+       int p = a->lastKnown(_elems[acount]);
         
         if(p == -1) CkAbort("Invalid Section\n");        
         for(count = 0; count < npes; count ++)
@@ -196,20 +231,58 @@ void ComlibSectionInfo::getPeList(int _nElems,
                 break;
         
         if(count == npes) {
-            pelist[npes ++] = p;
+         pelist[npes ++] = p;
         }
     }   
-
+    
     if(npes == 0) {
-        delete [] pelist;
-        pelist = NULL;
+      delete [] pelist;
+      pelist = NULL;
     }
 }
 
 
+
+inline int getPErepresentingNodeContainingPE(int pe){
+
+#if 1
+    return pe;
+
+#else
+
+#if USE_CONTROL_POINTS
+  std::vector<int> v;
+  v.push_back(1);
+  if(CkNumPes() >= 2)
+    v.push_back(2);
+  if(CkNumPes() >= 4)
+    v.push_back(4);
+  if(CkNumPes() >= 8)
+    v.push_back(8);
+  int pes_per_node = controlPoint("Number of PEs per Node", v);
+#else
+  int pes_per_node = 1;
+#endif
+
+
+    if(getenv("PE_PER_NODES") != NULL)
+       pes_per_node = CkNumPes()/atoi(getenv("PE_PER_NODES"));
+           
+    if( pes_per_node > 1 && pes_per_node <= CkNumPes() ){
+        ComlibPrintf("NODE AWARE Sending a message to a representative of the node instead of its real owner\n");
+        int newpe = pe - (pe % pes_per_node);
+        return newpe;
+    } else {
+       return pe;
+    }
+#endif    
+}
+
+
 void ComlibSectionInfo::getPeCount(int nindices, CkArrayIndexMax *idxlist, 
-                     int &npes, int &nidx,
+                     const CkArrayID &destArrayID, int &npes, int &nidx,
                      ComlibMulticastIndexCount *&counts, int *&belongs) {
+
   int count = 0;
   int i;
     
@@ -222,11 +295,29 @@ void ComlibSectionInfo::getPeCount(int nindices, CkArrayIndexMax *idxlist,
   npes = 0;
   nidx = 0;
 
+  CkArray *a = (CkArray *)_localBranch(destArrayID);
   for(i=0; i<nindices; ++i){
-    int p = ComlibGetLastKnown(destArrayID, idxlist[i]);
+    //int p = ComlibGetLastKnown(destArrayID, idxlist[i]);
+    int p = a->lastKnown(idxlist[i]);
+
+
+#define USE_NODE_AWARE 1
+#if USE_NODE_AWARE
+#warning "USING NODE AWARE SECTION INFOs"
+    ComlibPrintf("NODE AWARE old p=%d\n", p);
+  
+    // p = getPErepresentingNodeContainingPE(p);
+    
+    ComlibPrintf("NODE AWARE new p=%d\n", p);
+#endif
     
     if(p == -1) CkAbort("Invalid Section\n");        
 
+    if(p == CkMyPe()) {
+      belongs[i] = -1;
+      continue;
+    }
+
     //Collect processors
     for(count = 0; count < npes; count ++)
       if(counts[count].pe == p)
@@ -238,30 +329,28 @@ void ComlibSectionInfo::getPeCount(int nindices, CkArrayIndexMax *idxlist,
       ++npes;
     }
 
-    if(p == CkMyPe()) {
-      belongs[i] = -1;
-      continue;
-    }
-
     ++nidx;
     counts[count].count++;
     belongs[i] = count;
   }
-  //CkPrintf("section has %d procs\n",npes);
-
-  if(npes == 0) {
-    delete [] counts;
-    delete [] belongs;
-    counts = NULL;
-    belongs = NULL;
-  }
+  //ComlibPrintf("section has %d procs\n",npes);
+
+//   if(npes == 0) {
+//     delete [] counts;
+//     delete [] belongs;
+//     counts = NULL;
+//     belongs = NULL;
+//   }
 }
 
 
 void ComlibSectionInfo::getRemotePelist(int nindices, 
-                                        CkArrayIndexMax *idxlist, 
+                                        CkArrayIndexMax *idxlist,
+                                       CkArrayID &destArrayID,
                                         int &npes, int *&pelist) {
 
+       ComlibPrintf("ComlibSectionInfo::getRemotePelist()\n");
+       
     int count = 0, acount = 0;
     
     int length = CkNumPes();
@@ -283,10 +372,12 @@ void ComlibSectionInfo::getRemotePelist(int nindices,
     pelist = new int[length+1];
     npes = 0;
 
+    CkArray *a = (CkArray *)_localBranch(destArrayID);
     for(acount = 0; acount < nindices; acount++){
         
-        int p = ComlibGetLastKnown(destArrayID, idxlist[acount]);
-        if(p == CkMyPe())
+      //int p = ComlibGetLastKnown(destArrayID, idxlist[acount]);
+      int p = a->lastKnown(idxlist[acount]);
+      if(p == CkMyPe())
             continue;
         
         if(p == -1) CkAbort("Invalid Section\n");        
@@ -308,20 +399,40 @@ void ComlibSectionInfo::getRemotePelist(int nindices,
 }
 
 
-void ComlibSectionInfo::getLocalIndices(int nindices, 
-                                        CkArrayIndexMax *idxlist, 
+void ComlibSectionInfo::getLocalIndices(int nindices,
+                                        CkArrayIndexMax *idxlist,
+                                       CkArrayID &destArrayID,
                                         CkVec<CkArrayIndexMax> &idx_vec){    
-    int count = 0, acount = 0;
+       ComlibPrintf("ComlibSectionInfo::getLocalIndices()\n");
+       
+       int count = 0, acount = 0;
     idx_vec.resize(0);
     
+    CkArray *a = (CkArray *)_localBranch(destArrayID);
     for(acount = 0; acount < nindices; acount++){
-        int p = ComlibGetLastKnown(destArrayID, idxlist[acount]);
+        //int p = ComlibGetLastKnown(destArrayID, idxlist[acount]);
+        int p = a->lastKnown(idxlist[acount]);
         if(p == CkMyPe()) 
             idx_vec.insertAtEnd(idxlist[acount]);
     }
 }
 
 
-void ComlibSectionInfo::localMulticast(envelope *env){
-    ComlibArrayInfo::localMulticast(&localDestIndexVec, env);
+
+void ComlibSectionInfo::getNodeLocalIndices(int nindices,
+                                        CkArrayIndexMax *idxlist,
+                                       CkArrayID &destArrayID,
+                                        CkVec<CkArrayIndexMax> &idx_vec){    
+    int count = 0, acount = 0;
+    idx_vec.resize(0);
+    
+    CkArray *a = (CkArray *)_localBranch(destArrayID);
+    for(acount = 0; acount < nindices; acount++){
+        //int p = ComlibGetLastKnown(destArrayID, idxlist[acount]);
+        int p = a->lastKnown(idxlist[acount]);
+        if(p == CkMyPe()) 
+            idx_vec.insertAtEnd(idxlist[acount]);
+    }
 }
+
+
index bdaf2e540ac489c2f903add7998b67f3c146f302..f30d8c53b7f120537120af3adb6630f8f5fdf764 100644 (file)
@@ -1,3 +1,12 @@
+/**
+   @addtogroup CharmComlib
+   @{
+   @file
+   
+   @brief Utility classes to handle sections in comlib where they are needed
+   (basically multicast strategies).
+*/
+
 #ifndef COMLIB_SECTION_INFO
 #define COMLIB_SECTION_INFO
 
@@ -5,8 +14,8 @@
   Helper classes that help strategies manage array sections 
 ***********/
 
-/* Hash key that lets a strategy access a section id data structure
-   given the source processor and the MaxSectionId on that processor
+/** Hash key that lets a strategy access a section id data structure
+    given the source processor and the MaxSectionId on that processor
 */
 class ComlibSectionHashKey{
  public:
@@ -40,7 +49,7 @@ inline int ComlibSectionHashKey::compare(const ComlibSectionHashKey &k2) const
     return 0;
 }
 
-/*For calls to qsort*/
+/**For calls to qsort*/
 inline int intCompare(const void *a, const void *b){
     int a1 = *(int *) a;
     int b1 = *(int *) b;
@@ -71,7 +80,8 @@ inline CkHashCode ComlibSectionHashKey::staticHash(const void *v,size_t){
 /*************** End CkHashtable Functions *****************/
 
 
-
+/// Holds information about a strategy, in case it is used multiple times
+/// All this information is derived from the incoming messages
 class ComlibSectionHashObject {
  public:
     //My local indices
@@ -82,8 +92,11 @@ class ComlibSectionHashObject {
     int *pelist;
 
     void *msg;
+
+    //Flags associated with the section
+    int isOld; // 1 if the section is indeed old
     
-    ComlibSectionHashObject(): indices(0) {
+    ComlibSectionHashObject(): indices(0), isOld(0) {
         npes = 0;
         pelist = NULL;
        msg = NULL;
@@ -96,51 +109,56 @@ class ComlibSectionHashObject {
 };
 
 
-/*** Class that helps a communication library strategy manage array
-     sections
-***************/
+/** 
+    Helps a communication library strategy manage array sections, creating unique
+    identifiers for sections, and parsing messages.
 
+    The class maintains a counter for generating unique ids for each section.
+    It also provides utility functions that can extract information such as PE 
+    lists from messages.
+ */
 class ComlibSectionInfo {
-    /* Array ID of the array section */
-    CkArrayID destArrayID;
-    
-    //Unique section id for this section
-    //Will be used to access a hashtable on remote processors
-    int MaxSectionID;
-
-    //Instance ID of the strategy
-    int instanceID;
-
-    CkVec<CkArrayIndexMax> localDestIndexVec;
+  /// Maximum section id used so far. Incremented for each new CkSectionID that gets created.
+  /// Will be used (along with the source PE) to access a hashtable on remote processors, when
+  /// looking up persistent data for the section.
+  int MaxSectionID;
 
  public:
 
     ComlibSectionInfo() { 
-        destArrayID.setZero(); MaxSectionID = 1; instanceID = 0;    
+      MaxSectionID = 1;
     }
-
-    ComlibSectionInfo(CkArrayID dest, int instance){
-        destArrayID = dest;
-        instanceID = instance;
-
-        MaxSectionID = 1;
-    }
-    
+   
+    /// Create a unique identifier for the supplied CkSectionID
     inline void initSectionID(CkSectionID *sid) {
-        sid->_cookie.sInfo.cInfo.id = MaxSectionID ++;            
+      if (MaxSectionID > 65000) {
+        CkAbort("Too many sections allocated, wrapping of ints should be done!\n");
+      }
+      ComlibPrintf("[%d] ComlibSectionInfo::initSectionID: creating section number %d for proc %d\n",
+          CkMyPe(), MaxSectionID, sid->_cookie.pe);
+      sid->_cookie.sInfo.cInfo.id = MaxSectionID ++;
+      sid->_cookie.pe = CkMyPe();
     }
-
-    inline int getInstID(){ return(instanceID);}
     
     void processOldSectionMessage(CharmMessageHolder *cmsg);
 
+    /**
+     * Create a new message to be sent to the root processor of this broadcast
+     * to tell it that we missed some objects during delivery. This new message
+     * is returned, and it contains the same section id contained in the
+     * CkMcastBaseMsg passed as parameter.
+     */
+    CkMcastBaseMsg *getNewDeliveryErrorMsg(CkMcastBaseMsg *base);
+
     /**
      * Starting from a message to be sent, it generates a new message containing
      * the information about the multicast, together with the message itself.
      * The info about the multicast is contained in the field sec_id of cmsg.
      * The processors will be order by MyPe() if requested.
+     * The destination array to lookup is in the envelope of the message.
      */
-    ComlibMulticastMsg *getNewMulticastMessage(CharmMessageHolder *cmsg, int needSort);
+    ComlibMulticastMsg *getNewMulticastMessage(CharmMessageHolder *cmsg, int needSort, int instanceID);
 
     /**
      * Given a ComlibMulticastMsg arrived through the network as input (cb_env),
@@ -162,40 +180,45 @@ class ComlibSectionInfo {
      * procs.
      
      * @param nindices size of the array idxlist (input)
-     * @param idxlist array of indeces of the section (input)
-     * @param npes number of processors involved (output)
+     * @param idxlist array of indices of the section (input)
+     * @param destArrayID array ID to which the indeces refer (input)
+     * @param npes number of remote processors involved (output)
      * @param nidx number of indices that are remote (output)
      * @param counts array of associations pe-count: number of elements in proc pe (output, new'ed(CkNumPes()))
-     * @param belongs array of integers expressing association of elements with pes: belongs[i] = index in counts of the processor having index i (output, new'ed(nidx))
+     * @param belongs array of integers expressing association of elements with pes: belongs[i] = index in counts of the processor having index i (output, new'ed(nindices))
     */
-    void getPeCount(int nindices, CkArrayIndexMax *idxlist, 
+    void getPeCount(int nindices, CkArrayIndexMax *idxlist, const CkArrayID &destArrayID,
                    int &npes, int &nidx,
                    ComlibMulticastIndexCount *&counts, int *&belongs);
 
     void getPeList(envelope *cb_env, int npes, int *&pelist);
 
-    void getRemotePelist(int nindices, CkArrayIndexMax *idxlist, 
+    /**
+     * Returns the list of remote procs (therefore not including our processor)
+     * involved in the array index list.
+
+     * @param nindices size of the array idxlist (input
+     * @param idxlist array of indices of the section (input)
+     * @param destArrayID array ID to which the indeces refer (input)
+     * @param npes number of processors involved (output)
+     * @param pelist list of the processors involved (output, new'ed)
+     */
+    void getRemotePelist(int nindices, CkArrayIndexMax *idxlist, CkArrayID &destArrayID,
                          int &npes, int *&pelist);
 
-    void getPeList(int nindices, CkArrayIndexMax *idxlist, 
+    /** Returns the same list as getRemotePeList, only that it does not exclude
+       the local processor from the list if it is involved. */
+    void getPeList(int nindices, CkArrayIndexMax *idxlist, CkArrayID &destArrayID,
                    int &npes, int *&pelist);
 
-    void getLocalIndices(int nindices, CkArrayIndexMax *idxlist, 
+    void getLocalIndices(int nindices, CkArrayIndexMax *idxlist, CkArrayID &destArrayID,
+                         CkVec<CkArrayIndexMax> &idx_vec);   
+        
+    void getNodeLocalIndices(int nindices, CkArrayIndexMax *idxlist, CkArrayID &destArrayID,
                          CkVec<CkArrayIndexMax> &idx_vec);   
         
-    static inline int getSectionID(CkSectionID id) {
-        return id._cookie.sInfo.cInfo.id;
-    }
-
-    inline CkArrayID getDestArrayID() {
-        return destArrayID;
-    }
-
     void pup(PUP::er &p) {
-        p | destArrayID;
         p | MaxSectionID;
-        p | instanceID;
-        p | localDestIndexVec;
     }
 };
 
index 1aaa79d148e862f4a7e3c876c5b765ea77ae8116..22f83582aad88c81386b505241a841884bd0cf89 100644 (file)
@@ -1,3 +1,8 @@
+/**
+   @addtogroup CharmComlib
+   @{
+   @file
+*/
 
 #include "ComlibStats.h"
 #include "ComlibManager.h"
@@ -36,3 +41,5 @@ void ComlibGlobalStats::getAverageStats(int sid, double &avMsgSize,
     }
 }
 
+
+/*@}*/
index 48992b42a36734b00bcd4da15965a774d9b219e8..96d77323c5858f6791036b0c901951cbccff0183 100644 (file)
@@ -1,6 +1,14 @@
 #ifndef COMLIB_STATS_H
 #define COMLIB_STATS_H
 
+/**
+   @addtogroup CharmComlib
+   @{
+   @file
+   
+   @brief Classes for storing simple statistics about messages send and received.
+*/
+
 #include "charm++.h"
 #include "convcomlibmanager.h"
 
@@ -275,4 +283,5 @@ class ComlibGlobalStats {
   void getAverageStats(int sid, double &, double &, double &, double &);
 };
 
+/*@}*/
 #endif
index b5a2b59cb7f72d967018f9ca32a69092f14533b7..ede3d44fba6e09103cd4664a5911ac53d7d4305a 100644 (file)
@@ -1,41 +1,24 @@
+/**
+   @addtogroup CharmComlib
+   @{
+   @file
+   Implementations of ComlibStrategy.h
+*/
 
-#include "charm++.h"
-#include "envelope.h"
+#include "ComlibStrategy.h"
 
-//calls ComlibNotifyMigrationDone(). Even compiles when -module comlib
-//is not included. Hack to make loadbalancer work without comlib
-//currently.
-CkpvDeclare(int, migrationDoneHandlerID);
 
-//Class that defines the entry methods that a Charm level strategy
-//must define.  To write a new strategy inherit from this class and
-//define the virtual methods.  Every strategy can also define its own
-//constructor and have any number of arguments. Also call the parent
-//class methods in those methods.
-
-void CharmStrategy::insertMessage(MessageHolder *mh){
-    insertMessage((CharmMessageHolder *)mh);
-}
 
 void CharmStrategy::pup(PUP::er &p) {
-    Strategy::pup(p);
+  //Strategy::pup(p);
     p | nginfo;
     p | ginfo;
     p | ainfo;
-    p | forwardOnMigration;
+    //p | forwardOnMigration;
     p | mflag;
     p | onFinish;
 }
 
-CharmMessageHolder::CharmMessageHolder(char * msg, int proc) 
-    : MessageHolder((char *)UsrToEnv(msg), proc, 
-                    UsrToEnv(msg)->getTotalsize()){
-    
-    sec_id = NULL;    
-}
-
-CharmMessageHolder::~CharmMessageHolder() { 
-}
 
 void CharmMessageHolder::pup(PUP::er &p) {
 
@@ -48,7 +31,7 @@ void CharmMessageHolder::pup(PUP::er &p) {
     sec_id = NULL;
 }
 
-PUPable_def(CharmStrategy);
+//PUPable_def(CharmStrategy);
 PUPable_def(CharmMessageHolder);
 
 ComlibNodeGroupInfo::ComlibNodeGroupInfo() {
@@ -150,7 +133,7 @@ void ComlibGroupInfo::setDestinationGroup(CkGroupID gid, int *pelist,
 }
 
 void ComlibGroupInfo::getDestinationGroup(CkGroupID &gid, int *&pelist, 
-                                         int &npes){
+                                         int &npes) {
     gid = this->dgid;
     npes = ndestpes;
 
@@ -158,10 +141,28 @@ void ComlibGroupInfo::getDestinationGroup(CkGroupID &gid, int *&pelist,
     memcpy(pelist, destpelist, npes * sizeof(int));
 }
 
-void ComlibGroupInfo::getDestinationGroup(CkGroupID &gid){
+void ComlibGroupInfo::getDestinationGroup(CkGroupID &gid) {
     gid = this->dgid;
 }
 
+int *ComlibGroupInfo::getCombinedCountList() {
+  int *result = new int[CkNumPes()];
+  int i;
+  for (i=0; i<CkNumPes(); ++i) result[i] = 0;
+  if (nsrcpes != 0) {
+    for (i=0; i<nsrcpes; ++i) result[srcpelist[i]] |= 1;
+  } else {
+    for (i=0; i<CkNumPes(); ++i) result[i] |= 1;
+  }
+  if (ndestpes != 0) {
+    for (i=0; i<ndestpes; ++i) result[destpelist[i]] |= 2;
+  } else {
+    for (i=0; i<CkNumPes(); ++i) result[i] |= 2;
+  }
+  return result;
+}
+
+/*
 void ComlibGroupInfo::getCombinedPeList(int *&pelist, int &npes) {
     int count = 0;        
     pelist = 0;
@@ -191,20 +192,26 @@ void ComlibGroupInfo::getCombinedPeList(int *&pelist, int &npes) {
         }                        
     }
 }
+*/
 
 ComlibArrayInfo::ComlibArrayInfo() {
-
+       
     src_aid.setZero();
-    nSrcIndices = -1;
-    src_elements = NULL;
+    //nSrcIndices = -1;
+    //src_elements = NULL;
+    isAllSrc = 0;
+    totalSrc = 0;
     isSrcArray = 0;
 
     dest_aid.setZero();
-    nDestIndices = -1;
-    dest_elements = NULL;
+    //nDestIndices = -1;
+    //dest_elements = NULL;
+    isAllDest = 0;
+    totalDest = 0;
     isDestArray = 0;
 };
 
+/*
 ComlibArrayInfo::~ComlibArrayInfo() {
     //CkPrintf("in comlibarrayinfo destructor\n");
 
@@ -214,57 +221,100 @@ ComlibArrayInfo::~ComlibArrayInfo() {
     if(nDestIndices > 0)
         delete [] dest_elements;
 }
+*/
 
+/// Set the  source array used for this strategy. 
+/// The list of array indices should be the whole portion of the array involved in the strategy.
+/// The non-local array elements will be cleaned up inside purge() at migration of the strategy
 void ComlibArrayInfo::setSourceArray(CkArrayID aid, 
                                          CkArrayIndexMax *e, int nind){
     src_aid = aid;
-    isSrcArray = 1;    
+    isSrcArray = 1;
+    /*
     nSrcIndices = nind;
     if(nind > 0) {
         src_elements = new CkArrayIndexMax[nind];
         memcpy(src_elements, e, sizeof(CkArrayIndexMax) * nind);
     }
+    */
+    src_elements.removeAll();
+    
+    for (int i=0; i<nind; ++i) src_elements.push_back(e[i]);
+    
+    if (src_elements.size() == 0) 
+       isAllSrc = 1;
+    else 
+       isAllSrc = 0;
+    
+    totalSrc = src_elements.size();
+    
 }
 
 
 void ComlibArrayInfo::getSourceArray(CkArrayID &aid, 
                                          CkArrayIndexMax *&e, int &nind){
     aid = src_aid;
-    nind = nSrcIndices;
-    e = src_elements;
+    nind = src_elements.length();//nSrcIndices;
+    e = src_elements.getVec();//src_elements;
 }
 
 
 void ComlibArrayInfo::setDestinationArray(CkArrayID aid, 
                                           CkArrayIndexMax *e, int nind){
     dest_aid = aid;
-    isDestArray = 1;    
+    isDestArray = 1;
+    /*
     nDestIndices = nind;
     if(nind > 0) {
         dest_elements = new CkArrayIndexMax[nind];
         memcpy(dest_elements, e, sizeof(CkArrayIndexMax) * nind);
     }
+    */
+    dest_elements.removeAll();
+    for (int i=0; i<nind; ++i) dest_elements.push_back(e[i]);
+
+    if (dest_elements.size() == 0) 
+       isAllDest = 1;
+    else 
+       isAllDest = 0;
+    
+    totalDest = dest_elements.size();
+    
 }
 
 
 void ComlibArrayInfo::getDestinationArray(CkArrayID &aid, 
                                           CkArrayIndexMax *&e, int &nind){
     aid = dest_aid;
-    nind = nDestIndices;
-    e = dest_elements;
+    nind = dest_elements.length();
+    e = dest_elements.getVec();
 }
 
-
+/// @TODO fix the pup!
 //Each strategy must define his own Pup interface.
 void ComlibArrayInfo::pup(PUP::er &p){ 
     p | src_aid;
-    p | nSrcIndices;
+    //p | nSrcIndices;
     p | isSrcArray;
+    p | isAllSrc;
+    p | totalSrc;
+    p | src_elements;
     
     p | dest_aid;
-    p | nDestIndices;
+    //p | nDestIndices;
     p | isDestArray;
-    
+    p | isAllDest;
+    p | totalDest;
+    p | dest_elements;
+
+    if (p.isPacking() || p.isUnpacking()) {
+      // calling purge both during packing (at the end) and during unpacking
+      // allows this code to be executed both on processor 0 (where the object
+      // is created) and on every other processor where it arrives through PUP.
+      purge();
+    }
+
+    /*    
     if(p.isUnpacking() && nSrcIndices > 0) 
         src_elements = new CkArrayIndexMax[nSrcIndices];
     
@@ -282,8 +332,81 @@ void ComlibArrayInfo::pup(PUP::er &p){
         dest_elements = NULL;
 
     localDestIndexVec.resize(0);
+    */
+    
+}
+
+void ComlibArrayInfo::newElement(CkArrayID &id, const CkArrayIndex &idx) {
+  ComlibPrintf("ComlibArrayInfo::newElement\n");
+  if (isAllSrc && id==src_aid) src_elements.push_back(idx);
+  if (isAllDest && id==dest_aid) dest_elements.push_back(idx);
+}
+
+void ComlibArrayInfo::purge() {
+       int i;
+       CkArray *a;
+//     ComlibPrintf("[%d] ComlibArrayInfo::purge srcArray=%d (%d), destArray=%d (%d)\n",CkMyPe(),isSrcArray,isAllSrc,isDestArray,isAllDest);
+       if (isSrcArray) {
+               a = (CkArray *)_localBranch(src_aid);
+               if (isAllSrc) {
+                       // gather the information of all the elements that are currenly present here
+                       ComlibElementIterator iter(&src_elements);
+                       a->getLocMgr()->iterate(iter);
+
+                       // register to ComlibArrayListener for this array id
+//                     ComlibManagerRegisterArrayListener(src_aid, this);
+               } else {
+                       // delete all the elements of which we are not homePe
+                       for (i=src_elements.size()-1; i>=0; --i) {
+                               if (a->homePe(src_elements[i]) != CkMyPe()) { 
+                                       
+//                                     ComlibPrintf("[%d] removing home=%d src element %d  i=%d\n", CkMyPe(),a->homePe(src_elements[i]), src_elements[i].data()[0], i);
+                                       src_elements.remove(i); 
+                               }
+                       }
+               }
+       }
+       if (isDestArray) {
+               a = (CkArray *)_localBranch(dest_aid);
+               if (isAllDest) {
+                       // gather the information of all the elements that are currenly present here
+                       ComlibElementIterator iter(&dest_elements);
+                       a->getLocMgr()->iterate(iter);
+
+                       // register to ComlibArrayListener for this array id
+//                     ComlibManagerRegisterArrayListener(dest_aid, this);
+               } else {
+                       // delete all the elements of which we are not homePe
+                       for (i=dest_elements.size()-1; i>=0; --i) {
+                               if (a->homePe(dest_elements[i]) != CkMyPe()) {
+//                                     ComlibPrintf("[%d] removing home=%d dest element %d  i=%d\n", CkMyPe(), a->homePe(dest_elements[i]), dest_elements[i].data()[0], i);
+                                       dest_elements.remove(i); 
+                               }
+                       }
+               }
+       }
+}
+
+int *ComlibArrayInfo::getCombinedCountList() {
+  int *result = new int[CkNumPes()];
+  int i;
+  for (i=0; i<CkNumPes(); ++i) result[i] = 0;
+  CkArray *a = (CkArray *)_localBranch(src_aid);
+  if (src_elements.size() != 0) {
+    for (i=0; i<src_elements.size(); ++i) result[a->homePe(src_elements[i])] |= 1;
+  } else {
+    for (i=0; i<CkNumPes(); ++i) result[i] |= 1;
+  }
+  a = (CkArray *)_localBranch(dest_aid);
+  if (dest_elements.size() != 0) {
+    for (i=0; i<dest_elements.size(); ++i) result[a->homePe(dest_elements[i])] |= 2;
+  } else {
+    for (i=0; i<CkNumPes(); ++i) result[i] |= 2;
+  }
+  return result;
 }
 
+/*
 //Get the list of destination processors
 void ComlibArrayInfo::getDestinationPeList(int *&destpelist, int &ndestpes) {
     
@@ -310,11 +433,13 @@ void ComlibArrayInfo::getDestinationPeList(int *&destpelist, int &ndestpes) {
     }
 
     ndestpes = 0;
+    CkArray *amgr = CkArrayID::CkLocalBranch(dest_aid);
 
     //Find the last known processors of the array elements
     for(acount = 0; acount < nDestIndices; acount++) {
 
-        int p = ComlibGetLastKnown(dest_aid, dest_elements[acount]); 
+      //int p = ComlibGetLastKnown(dest_aid, dest_elements[acount]); 
+        int p = amgr->lastKnown(dest_elements[acount]);
         
         for(count = 0; count < ndestpes; count ++)
             if(destpelist[count] == p)
@@ -348,9 +473,12 @@ void ComlibArrayInfo::getSourcePeList(int *&srcpelist, int &nsrcpes) {
     }
 
     nsrcpes = 0;
+    CkArray *amgr = CkArrayID::CkLocalBranch(src_aid);
+
     for(acount = 0; acount < nSrcIndices; acount++) {
         
-        int p = ComlibGetLastKnown(src_aid, src_elements[acount]); 
+      //int p = ComlibGetLastKnown(src_aid, src_elements[acount]); 
+        int p = amgr->lastKnown(src_elements[acount]);
         
         for(count = 0; count < nsrcpes; count ++)
             if(srcpelist[count] == p)
@@ -394,10 +522,13 @@ void ComlibArrayInfo::getCombinedPeList(int *&pelist, int &npes) {
         if(npes == 0)
             pelist = new int[CkNumPes()];
         
+       CkArray *amgr = CkArrayID::CkLocalBranch(src_aid);
+
         //Add source processors to the destination processors
         //already obtained
         for(int acount = 0; acount < nSrcIndices; acount++) {
-            int p = ComlibGetLastKnown(src_aid, src_elements[acount]);
+         //int p = ComlibGetLastKnown(src_aid, src_elements[acount]);
+           int p = amgr->lastKnown(src_elements[acount]);
 
             for(count = 0; count < npes; count ++)
                 if(pelist[count] == p)
@@ -407,157 +538,107 @@ void ComlibArrayInfo::getCombinedPeList(int *&pelist, int &npes) {
         }                        
     }
 }
+*/
 
+/// Broadcast the message to all local elements
 void ComlibArrayInfo::localBroadcast(envelope *env) {
-    //Insert all local elements into a vector
-    if(localDestIndexVec.size()==0 && !dest_aid.isZero()) {
-        CkArray *dest_array = CkArrayID::CkLocalBranch(dest_aid);
-        
-        if(nDestIndices == 0){            
-            dest_array->getComlibArrayListener()->getLocalIndices
-                (localDestIndexVec);
-        }
-        else {
-            for(int count = 0; count < nDestIndices; count++) {
-                if(ComlibGetLastKnown(dest_aid, dest_elements[count])
-                   == CkMyPe())
-                    localDestIndexVec.insertAtEnd(dest_elements[count]);
-            }
-        }
-    }
+  int count = localMulticast(&dest_elements, env);
+  ComlibPrintf("[%d] ComlibArrayInfo::localBroadcast to %d elements (%d non local)\n",CmiMyPe(),dest_elements.size(),count);
+
+//  char buf[100000];
+//  buf[0] = '\0';
+//  for(int i=0;i<dest_elements.size();i++){
+//       sprintf(buf+strlen(buf), " %d", dest_elements[i].data()[0]);
+//  }
+//  ComlibPrintf("dest_elements = %s\n", buf);
 
-    ComlibArrayInfo::localMulticast(&localDestIndexVec, env);
 }
 
-/*
+/**
   This method multicasts the message to all the indices in vec.  It
-  also takes care to check if the entry method is readonly or not? If
+  also takes care to check if the entry method is readonly or not. If
   readonly (nokeep) the message is not copied.
 
   It also makes sure that the entry methods are logged in projections
   and that the array manager is notified about array element
   migrations.  Hence this function should be used extensively in the
-  communication library strategies */
+  communication library strategies
+
+  This method is more general than just ComlibArrayInfo dest_aid since it takes
+  the destination array id directly form the message envelope.
+
+  @return the number of destination objects which were not local (information
+  retrieved from the array/location manager)
+*/
 
 #include "register.h"
-void ComlibArrayInfo::localMulticast(CkVec<CkArrayIndexMax>*vec,
+int ComlibArrayInfo::localMulticast(CkVec<CkArrayIndexMax>*vec,
                                      envelope *env){
-
+  int count = 0;
     //Multicast the messages to all elements in vec
     int nelements = vec->size();
     if(nelements == 0) {
         CmiFree(env);
-        return;
+        return 0;
     }
 
     void *msg = EnvToUsr(env);
     int ep = env->getsetArrayEp();
     CkUnpackMessage(&env);
 
-    CkArrayID dest_aid = env->getsetArrayMgr();
+    CkArrayID destination_aid = env->getsetArrayMgr();
     env->setPacked(0);
     env->getsetArrayHops()=1;
     env->setUsed(0);
 
     CkArrayIndexMax idx;
-    
-    for(int count = 0; count < nelements-1; count ++){
-        idx = (*vec)[count];
-        //if(comm_debug) idx.print();
+
+    //ComlibPrintf("sending to %d elements\n",nelements);
+    for(int i = 0; i < nelements-1; i ++){
+        idx = (*vec)[i];
+        //if(com_debug) idx.print();
 
         env->getsetArrayIndex() = idx;
+        //ComlibPrintf("sending to: "); idx.print();
         
-        CkArray *a=(CkArray *)_localBranch(dest_aid);
+        CkArray *a=(CkArray *)_localBranch(destination_aid);
         if(_entryTable[ep]->noKeep)
-            a->deliver((CkArrayMessage *)msg, CkDeliver_inline, CK_MSG_KEEP);
+            count += a->deliver((CkArrayMessage *)msg, CkDeliver_inline, CK_MSG_KEEP);
         else {
             void *newmsg = CkCopyMsg(&msg);
-            a->deliver((CkArrayMessage *)newmsg, CkDeliver_queue);
+            count += a->deliver((CkArrayMessage *)newmsg, CkDeliver_queue);
         }
 
     }
 
     idx = (*vec)[nelements-1];
-    //if(comm_debug) idx.print();
+    //if(com_debug) idx.print();
     env->getsetArrayIndex() = idx;
+    //ComlibPrintf("sending to: "); idx.print();
     
-    CkArray *a=(CkArray *)_localBranch(dest_aid);
+    CkArray *a=(CkArray *)_localBranch(destination_aid);
     if(_entryTable[ep]->noKeep) {
-        a->deliver((CkArrayMessage *)msg, CkDeliver_inline, CK_MSG_KEEP);
+        count += a->deliver((CkArrayMessage *)msg, CkDeliver_inline, CK_MSG_KEEP);
         CmiFree(env);
     }
     else
-        a->deliver((CkArrayMessage *)msg, CkDeliver_queue);
+        count += a->deliver((CkArrayMessage *)msg, CkDeliver_queue);
+
+    return count;
 }
 
-/* Delivers a message to an array element, making sure that
-   projections is notified */
+/** Delivers a message to an array element, making sure that
+    projections is notified */
 void ComlibArrayInfo::deliver(envelope *env){
-    
+    ComlibPrintf("In ComlibArrayInfo::deliver()\n");
+               
     env->setUsed(0);
     env->getsetArrayHops()=1;
     CkUnpackMessage(&env);
     
     CkArray *a=(CkArray *)_localBranch(env->getsetArrayMgr());
-    a->deliver((CkArrayMessage *)EnvToUsr(env), CkDeliver_queue);    
+    a->deliver((CkArrayMessage *)EnvToUsr(env), CkDeliver_queue);
 }
 
-void ComlibNotifyMigrationDone() {
-    if(CkpvInitialized(migrationDoneHandlerID)) 
-        if(CkpvAccess(migrationDoneHandlerID) > 0) {
-            char *msg = (char *)CmiAlloc(CmiReservedHeaderSize);
-            CmiSetHandler(msg, CkpvAccess(migrationDoneHandlerID));
-#if CMK_BLUEGENE_CHARM
-           // bluegene charm should avoid directly calling converse
-            CmiSyncSendAndFree(CkMyPe(), CmiReservedHeaderSize, msg);
-#else
-            CmiHandleMessage(msg);
-#endif
-        }
-}
-
-
-//Stores the location of many array elements used by the
-//strategies.  Since hash table returns a reference to the object
-//and for an int that will be 0, the actual value stored is pe +
-//CkNumPes so 0 would mean processor -CkNumPes which is invalid.
-CkpvDeclare(ClibLocationTableType *, locationTable);
-
-CkpvDeclare(CkArrayIndexMax, cache_index);
-CkpvDeclare(int, cache_pe);
-CkpvDeclare(CkArrayID, cache_aid);
 
-int ComlibGetLastKnown(CkArrayID aid, CkArrayIndexMax idx) {
-    //CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
-    //return (cgproxy.ckLocalBranch())->getLastKnown(aid, idx);
-
-    if(!CkpvInitialized(locationTable)) {
-        CkAbort("Uninitialized table\n");
-    }
-    CkAssert(CkpvAccess(locationTable) != NULL);
-
-    
-    if(CkpvAccess(cache_index) == idx && CkpvAccess(cache_aid) == aid)
-        return CkpvAccess(cache_pe);
-    
-    ClibGlobalArrayIndex cidx;
-    cidx.aid = aid;
-    cidx.idx = idx;
-    int pe = CkpvAccess(locationTable)->get(cidx);
-    
-    if(pe == 0) {
-        //Array element does not exist in the table
-        
-        CkArray *array = CkArrayID::CkLocalBranch(aid);
-        pe = array->lastKnown(idx) + CkNumPes();
-        CkpvAccess(locationTable)->put(cidx) = pe;
-       //CkPrintf("LAST KNOWN %d: (%d %d %d %d) -> %d\n",CkMyPe(),((short*)&idx)[2],((short*)&idx)[3],((short*)&idx)[4],((short*)&idx)[5],pe);
-    }
-    //CkPrintf("last pe = %d \n", pe - CkNumPes());
-    
-    CkpvAccess(cache_index) = idx;
-    CkpvAccess(cache_aid) = aid;
-    CkpvAccess(cache_pe) = pe - CkNumPes();
-
-    return pe - CkNumPes();
-}
+/*@}*/
index 1c43478dff1fef7662cf3eab927cab2d47c347ab..f1a0e98f976de5c7dff27e2beb74151c392515dd 100644 (file)
@@ -1,5 +1,13 @@
-#ifndef COMMLIBSTRATEGY_H
-#define COMMLIBSTRATEGY_H
+/**
+   @addtogroup CharmComlib
+   @{
+   @file
+
+   @brief Defines CharmStrategy and CharmMessageHolder.
+*/
+
+#ifndef COMLIBSTRATEGY_H
+#define COMLIBSTRATEGY_H
 
 #include "charm++.h"
 #include "ckhashtable.h"
 
 CkpvExtern(int, migrationDoneHandlerID);
 
-//Class managing Charm++ messages in the communication library.
-//It is aware of envelopes, arrays, etc
+
+/// Defines labels for distinguishing between types of sends in a CharmMessageHolder
+enum CmhMessageType { CMH_ARRAYSEND, 
+                     CMH_GROUPSEND, 
+                     CMH_ARRAYBROADCAST, 
+                     CMH_ARRAYSECTIONSEND, 
+                     CMH_GROUPBROADCAST     };
+
+
+
+/// Class managing Charm++ messages in the communication library.
+/// It is aware of envelopes, arrays, etc
 class CharmMessageHolder : public MessageHolder{
  public:
+    /// An unused, and probably unnecessary array that was used to avoid a memory corruption that clobbers members in this class. The bug has likely been fixed.
+
+#define BUGTRAPSIZE 5000
+
+    int bug_trap[BUGTRAPSIZE];
+
+    /// The section information for an enqueued multicast message
     CkSectionID *sec_id;
 
-    CharmMessageHolder() : MessageHolder() {sec_id = NULL;}
-    CharmMessageHolder(CkMigrateMessage *m) : MessageHolder(m) {}
+    /// Saves a copy of the CkSectionID sec_id when enqueing a multicast message
+    CkSectionID *copy_of_sec_id;  
+
+    CkArrayID array_id; 
+
+    /// Type of message we are buffering
+    CmhMessageType type; 
     
-    CharmMessageHolder(char * msg, int dest_proc);
-    ~CharmMessageHolder();
+ CharmMessageHolder(char * msg, int proc, CmhMessageType t) : 
+    MessageHolder((char *)UsrToEnv(msg), UsrToEnv(msg)->getTotalsize(), proc){
+      type = t;
+      sec_id = NULL;
+      copy_of_sec_id = NULL;
+      for(int i=0;i<BUGTRAPSIZE;i++){
+       bug_trap[i] = 0;
+      }
+      
+      checkme();
+    }
+    
+    /// Verfiy that the bug_trap array has not been corrupted. Noone should ever write to that array.
+    void checkme() {
+       for(int i=0;i<BUGTRAPSIZE;i++){
+         if(bug_trap[i] != 0){
+           CkPrintf("bug_trap[%d]=%d (should be 0) bug_trap[%d] is at %p\n", i, bug_trap[i], i, &bug_trap[i]);
+           CkAbort("Corruption of CharmMessageHolder has occured\n");
+         }
+       }
+    }
+
+
+    CharmMessageHolder(CkMigrateMessage *m) : MessageHolder(m) {
+      for(int i=0;i<BUGTRAPSIZE;i++){
+       bug_trap[i] = 0;
+      }
+      checkme();
+    }
+    
+
+
+    ~CharmMessageHolder(){
+      checkme();
+    }
 
     inline char * getCharmMessage() {
         return (char *)EnvToUsr((envelope *) data);
+       checkme();
     }
     
-    virtual void pup(PUP::er &p);
-    PUPable_decl(CharmMessageHolder);
-};
+    /// Store a local copy of the sec_id, so I can use it later.
+    inline void saveCopyOf_sec_id(){
+      ComlibPrintf("[%d] saveCopyOf_sec_id sec_id=%p NULL=%d\n", CkMyPe(), sec_id, NULL);
+
+      checkme();
+
+      if(sec_id!=NULL){
+
+       ComlibPrintf("Original has values: _nElems=%d, npes=%d\n", sec_id->_nElems, sec_id->npes );
+       CkAssert(sec_id->_nElems>=0);
+       CkAssert(sec_id->npes>=0);
+
+
+       // Create a new CkSectionID, allocating its members
+       copy_of_sec_id = new CkSectionID();
+       copy_of_sec_id->_elems = new CkArrayIndexMax[sec_id->_nElems];
+       copy_of_sec_id->pelist = new int[sec_id->npes];
+       
+       // Copy in the values
+       copy_of_sec_id->_cookie = sec_id->_cookie;
+       copy_of_sec_id->_nElems = sec_id->_nElems;
+       copy_of_sec_id->npes = sec_id->npes;
+       ComlibPrintf("Copy has values: _nElems=%d, npes=%d\n", copy_of_sec_id->_nElems, copy_of_sec_id->npes );
+       for(int i=0; i<sec_id->_nElems; i++){
+         copy_of_sec_id->_elems[i] = sec_id->_elems[i];
+       }
+       for(int i=0; i<sec_id->npes; i++){
+         copy_of_sec_id->pelist[i] = sec_id->pelist[i];
+       }
+
+       // change local pointer to the new copy of the CkSectionID
+       ComlibPrintf("saving copy of sec_id into %p\n", copy_of_sec_id);
+      }
+
+      checkme();
 
+    }
 
-//Struct to store the comlib location table info
-struct ClibGlobalArrayIndex {
-    CkArrayID aid;
-    CkArrayIndexMax idx;
+    inline void freeCopyOf_sec_id(){
+    /*   if(copy_of_sec_id != NULL){ */
+/*     CkPrintf("delete %p\n", sec_id); */
+/*     delete[] copy_of_sec_id->_elems; */
+/*     delete[] copy_of_sec_id->pelist; */
+/*     delete copy_of_sec_id; */
+/*     copy_of_sec_id = NULL; */
+/*       } */
 
-    //These routines allow ClibGlobalArrayIndex to be used in
-    //  a CkHashtableT
-    CkHashCode hash(void) const;
-    static CkHashCode staticHash(const void *a,size_t);
-    int compare(const ClibGlobalArrayIndex &ind) const;
-    static int staticCompare(const void *a,const void *b,size_t);
-};
-PUPbytes(ClibGlobalArrayIndex);
-
-/*********** CkHashTable functions ******************/
-inline CkHashCode ClibGlobalArrayIndex::hash(void) const
-{
-    register CkHashCode ret = idx.hash() | (CkGroupID(aid).idx << 16);
-    return ret;
-}
-
-inline int ClibGlobalArrayIndex::compare(const ClibGlobalArrayIndex &k2) const
-{
-    if(idx == k2.idx && aid == k2.aid)
-        return 1;
-    
-    return 0;
-}
+      checkme();
 
-//ClibGlobalArrayIndex CODE
-inline int ClibGlobalArrayIndex::staticCompare(const void *k1, const void *k2, 
-                                                size_t ){
-    return ((const ClibGlobalArrayIndex *)k1)->
-        compare(*(const ClibGlobalArrayIndex *)k2);
-}
+    }
 
-inline CkHashCode ClibGlobalArrayIndex::staticHash(const void *v,size_t){
-    return ((const ClibGlobalArrayIndex *)v)->hash();
-}
 
+    virtual void pup(PUP::er &p);
+    PUPable_decl(CharmMessageHolder);
+};
 
-typedef CkHashtableT<ClibGlobalArrayIndex,int> ClibLocationTableType;
-    
-//Stores the location of many array elements used by the
-//strategies.  Since hash table returns a reference to the object
-//and for an int that will be 0, the actual value stored is pe +
-//CkNumPes so 0 would mean processor -CkNumPes which is invalid.
-CkpvExtern(ClibLocationTableType *, locationTable);
 
-CkpvExtern(CkArrayIndexMax, cache_index);
-CkpvExtern(int, cache_pe);
-CkpvExtern(CkArrayID, cache_aid);
 
-//Info classes that help bracketed streategies manage objects
-//Each info class points to a list of source (or destination) objects
-//ArrayInfo also access the array listener interface
+// Info classes that help bracketed streategies manage objects.
+// Each info class points to a list of source (or destination) objects.
+// ArrayInfo also access the array listener interface
 
 class ComlibNodeGroupInfo {
  protected:
@@ -129,88 +193,139 @@ class ComlibGroupInfo {
     void getDestinationGroup(CkGroupID &gid);
     void getDestinationGroup(CkGroupID &dgid,int *&destpelist, int &ndestpes);
 
-    void getCombinedPeList(int *&pelist, int &npes);
+    /// This routine returnes an array of size CkNumPes() where each element
+    /// follow the convention for bracketed strategies counts.
+    int *getCombinedCountList();
+    //void getCombinedPeList(int *&pelist, int &npes);
     void pup(PUP::er &p);
 };
 
 class ComlibMulticastMsg;
 
-/* Array strategy helper class.
+/** Array strategy helper class.
    Stores the source and destination arrays.
    Computes most recent processor maps of source and destinaton arrays.
    
    Array section helper functions, make use of sections easier for the
    communication library.
 */
-
 class ComlibArrayInfo {
  protected:
-    CkArrayID src_aid;
-    CkArrayIndexMax *src_elements; //src array elements
-    int nSrcIndices;              //number of source indices   
+    CkArrayID src_aid;         ///< Source Array ID
+    CkVec<CkArrayIndexMax> src_elements;     ///< local source array elements
     int isSrcArray;
+    int isAllSrc; ///< if true then all the array is involved in the operation
+    int totalSrc; ///< The total number of src elements involved in the strategy
 
     CkArrayID dest_aid;
-    CkArrayIndexMax *dest_elements; //dest array elements
-    int nDestIndices;              //number of destintation indices   
+    CkVec<CkArrayIndexMax> dest_elements; ///< destination indices
     int isDestArray;
-
-    CkVec<CkArrayIndexMax> localDestIndexVec;
+    int isAllDest; ///< if true then all the array is involved in the operation
+    int totalDest; ///< The total number of array elements involved in the strategy
     
  public:
     ComlibArrayInfo();
-    ~ComlibArrayInfo();
+    //~ComlibArrayInfo();
 
     void setSourceArray(CkArrayID aid, CkArrayIndexMax *e=0, int nind=0);
     int isSourceArray(){return isSrcArray;}
     void getSourceArray(CkArrayID &aid, CkArrayIndexMax *&e, int &nind);
-    
+    /// This operation leaks memory is the index vector is not retrieved before!
+    void resetSource() {new (&src_elements) CkVec<CkArrayIndexMax>();};
+    void addSource(CkArrayIndexMax &e) {
+       src_elements.push_back(e);
+//     ComlibPrintf("[%d] src_elements.push_back(%d)  now contains %d\n", CkMyPe(), e.data()[0], src_elements.size());
+    }
+
+    /// Get the number of source array elements
+    int getTotalSrc() {return totalSrc;}
+    int getLocalSrc() {return src_elements.size();}
+
     void setDestinationArray(CkArrayID aid, CkArrayIndexMax *e=0, int nind=0);
     int isDestinationArray(){return isDestArray;}
     void getDestinationArray(CkArrayID &aid, CkArrayIndexMax *&e, int &nind);
+    /// This operation leaks memory is the index vector is not retrieved before!
+    void resetDestination() {new (&dest_elements) CkVec<CkArrayIndexMax>();};
+    void addDestination(CkArrayIndexMax &e) {
+       dest_elements.push_back(e);
+       ComlibPrintf("[%d] dest_elements.push_back(%d)  now contains %d\n", CkMyPe(), e.data()[0], dest_elements.size());       
+    }
+    int getTotalDest() {return totalDest;}
+    int getLocalDest() {return dest_elements.size();}
+
+    void newElement(CkArrayID &id, const CkArrayIndex &idx);
 
     void localBroadcast(envelope *env);
-    static void localMulticast(CkVec<CkArrayIndexMax> *idx_vec,envelope *env);
+    static int localMulticast(CkVec<CkArrayIndexMax> *idx_vec,envelope *env);
     static void deliver(envelope *env);
 
-    void getSourcePeList(int *&pelist, int &npes);
-    void getDestinationPeList(int *&pelist, int &npes);
-    void getCombinedPeList(int *&pelist, int &npes);
+    /// This routine is called only once at the beginning and will take the list
+    /// of source and destination elements and leave only those that have homePe
+    /// here.
+    void purge();
+
+    /// This routine returnes an array of size CkNumPes() where each element
+    /// follow the convention for bracketed strategies counts.
+    int *getCombinedCountList();
+    //void getSourcePeList(int *&pelist, int &npes);
+    //void getDestinationPeList(int *&pelist, int &npes);
+    //void getCombinedPeList(int *&pelist, int &npes);
     
     void pup(PUP::er &p);
 };
 
+/** Implementation of CkLocIterator to get all the local elements from a
+    specified processor. Currently used by ComlibArrayInfo */
+class ComlibElementIterator : public CkLocIterator {
+ public:
+  CkVec<CkArrayIndexMax> *list;
+
+  ComlibElementIterator(CkVec<CkArrayIndexMax> *l) : CkLocIterator() {
+    list = l;
+  }
+
+  virtual void addLocation (CkLocation &loc) {
+    list->push_back(loc.getIndex());
+  }
+};
+
+
 
-/* All Charm++ communication library strategies should inherit from
-   this strategy. They should specify their object domain by setting
-   Strategy::type. They have three helpers predefined for them for
-   node groups, groups and arrays */
+// forward declaration of wrapper for ComlibManager::registerArrayListener
+void ComlibManagerRegisterArrayListener(CkArrayID, ComlibArrayInfo*);
 
-class CharmStrategy : public Strategy {
+/** All Charm++ communication library strategies should inherit from 
+    CharmStrategy, rather than inheriting from class Strategy (or one of its
+    subclasses). They should specify their object domain by setting
+    Strategy::type. They have three helpers predefined for them for node groups,
+    groups and arrays. */
+class CharmStrategy {
     
  protected:
-    int forwardOnMigration;
+  //int forwardOnMigration;
     ComlibLearner *learner;
     CmiBool mflag;    //Does this strategy handle point-to-point or 
     CkCallback onFinish;
 
  public:
+    //int iterationNumber;
     ComlibGroupInfo ginfo;
     ComlibNodeGroupInfo nginfo;
 
-    //The communication library array listener watches and monitors
-    //the array elements belonging to ainfo.src_aid
+    //The communication library array listener watches and monitors
+    //the array elements belonging to ainfo.src_aid
     ComlibArrayInfo ainfo;
     
-    CharmStrategy() : Strategy() {
-        setType(GROUP_STRATEGY); 
-        forwardOnMigration = 0;
-        learner = NULL;
-        mflag = CmiFalse;
+    CharmStrategy() { //: Strategy() {
+      //setType(GROUP_STRATEGY); 
+      //forwardOnMigration = 0;
+      //iterationNumber = 0;
+      learner = NULL;
+      mflag = CmiFalse;
     }
 
-    CharmStrategy(CkMigrateMessage *m) : Strategy(m){
-        learner = NULL;
+    CharmStrategy(CkMigrateMessage *m) { //: Strategy(m){
+      learner = NULL;
     }
 
     //Set flag to optimize strategy for 
@@ -231,50 +346,19 @@ class CharmStrategy : public Strategy {
       return onFinish;
     }
 
-    //Called for each message
-    //Function inserts a Charm++ message
-    virtual void insertMessage(CharmMessageHolder *msg) {
-        CkAbort("Bummer Should Not come here:CharmStrategy is abstract\n");
-    }
-
-    //Removed the virtual!
-    //Charm strategies should not use Message Holder
-    void insertMessage(MessageHolder *msg);
-    
-    //Added a new call that is called after the strategy had be
-    //created on every processor.
-    //DOES NOT exist in Converse Strategies
-    virtual void beginProcessing(int nelements){}
-
-    //Added a new call that is called after the strategy had be
-    //created on every processor.  DOES NOT exist in Converse
-    //Strategies. Called when the strategy is deactivated, possibly as
-    //a result of a learning decision
-    virtual void finalizeProcessing(){}
-
-    //Called when a message is received in the strategy handler
-    virtual void handleMessage(void *msg) {
-        CkPrintf("Warning: In CharmStrategy::handleMessage\n");
-        CkPrintf("Did you corrupt the message ????\n");
-    }
     
     ComlibLearner *getLearner() {return learner;}
     void setLearner(ComlibLearner *l) {learner = l;}
     
     virtual void pup(PUP::er &p);
-    PUPable_decl(CharmStrategy);
-
-    void setForwardOnMigration(int f) {
-        forwardOnMigration = f;
-    }
-    
-    int getForwardOnMigration() {
-        return forwardOnMigration;
-    }
 };
 
 //API calls which will be valid when communication library is not linked
 void ComlibNotifyMigrationDone();
-int ComlibGetLastKnown(CkArrayID aid, CkArrayIndexMax idx);
+//int ComlibGetLastKnown(CkArrayID aid, CkArrayIndexMax idx);
+
 
 #endif
+
+/*@}*/
index 71099f34b03b96a8138d8d355cacdfd556c46f57..df17edfa9ff488d1706d7ebbcef0d363a130f488 100644 (file)
@@ -1,5 +1,5 @@
-
 /********************************************************
+OLD DESCRIPTION
         Section multicast strategy suite. DirectMulticast and its
         derivatives, multicast messages to a section of array elements
         created on the fly. The section is invoked by calling a
 
 **********************************************/
 
-
-#include "DirectMulticastStrategy.h"
-
-CkpvExtern(CkGroupID, cmgrID);
-
-/*
-void *DMHandler(void *msg){
-    ComlibPrintf("[%d]:In CallbackHandler\n", CkMyPe());
-    DirectMulticastStrategy *nm_mgr;    
-    
-    CkMcastBaseMsg *bmsg = (CkMcastBaseMsg *)EnvToUsr((envelope *)msg);
-    int instid = bmsg->_cookie.sInfo.cInfo.instId;
-    
-    nm_mgr = (DirectMulticastStrategy *) 
-        CProxy_ComlibManager(CkpvAccess(cmgrID)).
-        ckLocalBranch()->getStrategy(instid);
-    
-    nm_mgr->handleMulticastMessage(msg);
-    return NULL;
-}
+/**
+   @addtogroup ComlibCharmStrategy
+   @{
+   @file 
 */
 
-DirectMulticastStrategy::DirectMulticastStrategy(CkArrayID aid, 
-                                                int isPersistent)
-    :  CharmStrategy() {
-
-    ainfo.setDestinationArray(aid);
-    setType(ARRAY_STRATEGY);
-
-    this->isPersistent = isPersistent;
-}
-
-//Destroy all old built routes
-DirectMulticastStrategy::~DirectMulticastStrategy() {
-    
-    ComlibPrintf("Calling Distructor\n");
-
-    if(getLearner() != NULL)
-        delete getLearner();
-        
-    CkHashtableIterator *ht_iterator = sec_ht.iterator();
-    ht_iterator->seekStart();
-    while(ht_iterator->hasNext()){
-        void **data;
-        data = (void **)ht_iterator->next();        
-        ComlibSectionHashObject *obj = (ComlibSectionHashObject *) (* data);
-        if(obj != NULL)
-            delete obj;
-    }
-}
-
-void DirectMulticastStrategy::insertMessage(CharmMessageHolder *cmsg){
-    
-    ComlibPrintf("[%d] Comlib Direct Section Multicast: insertMessage \n", 
-                 CkMyPe());   
-
-    if(cmsg->dest_proc == IS_SECTION_MULTICAST && cmsg->sec_id != NULL) { 
-        CkSectionID *sid = cmsg->sec_id;
-        int cur_sec_id = ComlibSectionInfo::getSectionID(*sid);
-        
-        if(cur_sec_id > 0) {        
-           sinfo.processOldSectionMessage(cmsg);            
-            
-            ComlibSectionHashKey 
-                key(CkMyPe(), sid->_cookie.sInfo.cInfo.id);        
-            ComlibSectionHashObject *obj = sec_ht.get(key);
-
-            if(obj == NULL)
-                CkAbort("Cannot Find Section\n");
-
-            envelope *env = UsrToEnv(cmsg->getCharmMessage());
-            localMulticast(env, obj);
-            remoteMulticast(env, obj);
-        }
-        else {
-            //New sec id, so send it along with the message
-            void *newmsg = sinfo.getNewMulticastMessage(cmsg, needSorting());
-            insertSectionID(sid);
-
-            ComlibSectionHashKey 
-                key(CkMyPe(), sid->_cookie.sInfo.cInfo.id);        
-            
-            ComlibSectionHashObject *obj = sec_ht.get(key);
-
-           if(obj == NULL)
-                CkAbort("Cannot Find Section\n");
-
-            /*
-           CkPrintf("%u: Src = %d dest:", key.hash(), CkMyPe());
-           for (int i=0; i<obj->npes; ++i)
-             CkPrintf(" %d",obj->pelist[i]);
-           CkPrintf(", map:");
-           ComlibMulticastMsg *lll = (ComlibMulticastMsg*)newmsg;
-           envelope *ppp = UsrToEnv(newmsg);
-           CkUnpackMessage(&ppp);
-           int ttt=0;
-           for (int i=0; i<lll->nPes; ++i) {
-             CkPrintf(" %d (",lll->indicesCount[i].pe);
-             for (int j=0; j<lll->indicesCount[i].count; ++j) {
-               CkPrintf(" %d",((int*)&(lll->indices[ttt]))[1]);
-               ttt++;
-             }
-             CkPrintf(" )");
-           }
-           CkPackMessage(&ppp);
-           CkPrintf("\n");
-           */
-
-            char *msg = cmsg->getCharmMessage();
-            localMulticast(UsrToEnv(msg), obj);
-            CkFreeMsg(msg);
-            
-            if (newmsg != NULL) remoteMulticast(UsrToEnv(newmsg), obj);
-        }        
-    }
-    else 
-        CkAbort("Section multicast cannot be used without a section proxy");
-
-    delete cmsg;       
-}
-
-void DirectMulticastStrategy::insertSectionID(CkSectionID *sid) {
-    
-    ComlibSectionHashKey 
-        key(CkMyPe(), sid->_cookie.sInfo.cInfo.id);
-
-    ComlibSectionHashObject *obj = NULL;    
-    obj = sec_ht.get(key);
-    
-    if(obj != NULL)
-        delete obj;
-    
-    obj = createObjectOnSrcPe(sid->_nElems, sid->_elems);
-    sec_ht.put(key) = obj;
-
-}
-
-
-ComlibSectionHashObject *
-DirectMulticastStrategy::createObjectOnSrcPe(int nindices, CkArrayIndexMax *idxlist) {
-
-    ComlibSectionHashObject *obj = new ComlibSectionHashObject();
-    
-    sinfo.getRemotePelist(nindices, idxlist, obj->npes, obj->pelist);
-    sinfo.getLocalIndices(nindices, idxlist, obj->indices);
-    
-    return obj;
-}
-
-
-ComlibSectionHashObject *
-DirectMulticastStrategy::createObjectOnIntermediatePe(int nindices,
-                                                     CkArrayIndexMax *idxlist,
-                                                     int npes,
-                                                     ComlibMulticastIndexCount *counts,
-                                                     int srcpe) {
-
-    ComlibSectionHashObject *obj = new ComlibSectionHashObject();
-        
-    obj->pelist = 0;
-    obj->npes = 0;
-
-    obj->indices.resize(0);
-    for (int i=0; i<nindices; ++i) obj->indices.insertAtEnd(idxlist[i]);
-    //sinfo.getLocalIndices(nindices, idxlist, obj->indices);
-
-    return obj;
-}
-
-
-void DirectMulticastStrategy::doneInserting(){
-    //Do nothing! Its a bracketed strategy
-}
+#include "DirectMulticastStrategy.h"
 
-extern void CmiReference(void *);
 
-//Send the multicast message the local array elements. The message is 
-//copied and sent if elements exist. 
-void DirectMulticastStrategy::localMulticast(envelope *env, 
-                                             ComlibSectionHashObject *obj) {
-    int nIndices = obj->indices.size();
-    
-    //If the library is set to persistent. 
-    //The message is stored in the library. The applications should 
-    //use the message as a readonly and it exists till the next one 
-    //comes along
-    
-    if(obj->msg != NULL) {
-        CmiFree(obj->msg);
-       obj->msg = NULL;
-    } 
-    
-    if(nIndices > 0) {
-       void *msg = EnvToUsr(env);
-       void *msg1 = msg;
-        
-        msg1 = CkCopyMsg(&msg);
+void DirectMulticastStrategy::createObjectOnSrcPe(ComlibSectionHashObject *obj, int npes, ComlibMulticastIndexCount *pelist) {
+       ComlibPrintf("[%d] old createObjectOnSrcPe() npes=%d\n", CkMyPe(), npes);
        
-       if(isPersistent) {
-           CmiReference(UsrToEnv(msg1));
-           obj->msg = (void *)UsrToEnv(msg1);
+       obj->pelist = new int[npes];
+       obj->npes = npes;
+       for (int i=0; i<npes; ++i) {
+               obj->pelist[i] = pelist[i].pe;
        }
-       
-        ComlibArrayInfo::localMulticast(&(obj->indices), UsrToEnv(msg1));
-    }    
-}
-
-
-//Calls default multicast scheme to send the messages. It could 
-//also call a converse lower level strategy to do the muiticast.
-//For example pipelined multicast
-void DirectMulticastStrategy::remoteMulticast(envelope *env, 
-                                              ComlibSectionHashObject *obj) {
-    
-    int npes = obj->npes;
-    int *pelist = obj->pelist;
-    
-    if(npes == 0) {
-        CmiFree(env);
-        return;    
-    }
-    
-    //CmiSetHandler(env, handlerId);
-    CmiSetHandler(env, CkpvAccess(strategy_handlerid));
-
-    ((CmiMsgHeaderBasic *) env)->stratid = getInstance();
-
-    //Collect Multicast Statistics
-    RECORD_SENDM_STATS(getInstance(), env->getTotalsize(), pelist, npes);
-    
-    CkPackMessage(&env);
-    //Sending a remote multicast
-    CmiSyncListSendAndFree(npes, pelist, env->getTotalsize(), (char*)env);
-    //CmiSyncBroadcastAndFree(env->getTotalsize(), (char*)env);
-}
-
-void DirectMulticastStrategy::pup(PUP::er &p){
-
-    CharmStrategy::pup(p);
-    p | isPersistent; 
 }
 
-void DirectMulticastStrategy::beginProcessing(int numElements){
-    
-    //handlerId = CkRegisterHandler((CmiHandler)DMHandler);    
-    
-    CkArrayID dest;
-    int nidx;
-    CkArrayIndexMax *idx_list;
-
-    ainfo.getDestinationArray(dest, idx_list, nidx);
-    sinfo = ComlibSectionInfo(dest, myInstanceID);
-
-    ComlibLearner *learner = new ComlibLearner();
-    //setLearner(learner);
-}
 
-void DirectMulticastStrategy::handleMessage(void *msg){
-    envelope *env = (envelope *)msg;
-    RECORD_RECV_STATS(getInstance(), env->getTotalsize(), env->getSrcPe());
+void DirectMulticastStrategy::createObjectOnIntermediatePe(ComlibSectionHashObject *obj,
+                                                          int npes,
+                                                          ComlibMulticastIndexCount *counts,
+                                                          int srcpe) {
+       ComlibPrintf("[%d] old createObjectOnIntermediatePe()\n", CkMyPe());
 
-    //Section multicast base message
-    CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)EnvToUsr(env);
-    
-    int status = cbmsg->_cookie.sInfo.cInfo.status;
-    ComlibPrintf("[%d] In handleMulticastMessage %d\n", CkMyPe(), status);
-    
-    if(status == COMLIB_MULTICAST_NEW_SECTION)
-        handleNewMulticastMessage(env);
-    else {
-        //status == COMLIB_MULTICAST_OLD_SECTION, use the cached section id
-        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
-                                 cbmsg->_cookie.sInfo.cInfo.id);    
-        
-        ComlibSectionHashObject *obj;
-        obj = sec_ht.get(key);
-        
-        if(obj == NULL)
-            CkAbort("Destination indices is NULL\n");
-        
-        localMulticast(env, obj);
-        remoteMulticast(env, obj);
-    }
+    obj->pelist = 0;
+    obj->npes = 0;
 }
 
-#include <string>
-
-void DirectMulticastStrategy::handleNewMulticastMessage(envelope *env) {
-    
-    ComlibPrintf("%d : In handleNewMulticastMessage\n", CkMyPe());
 
-    CkUnpackMessage(&env);    
-
-    int localElems;
-    envelope *newenv;
-    CkArrayIndexMax *local_idx_list;    
-    
-    sinfo.unpack(env, localElems, local_idx_list, newenv);
-
-    ComlibMulticastMsg *cbmsg = (ComlibMulticastMsg *)EnvToUsr(env);
-    ComlibSectionHashKey key(cbmsg->_cookie.pe, 
-                             cbmsg->_cookie.sInfo.cInfo.id);
-    
-    ComlibSectionHashObject *old_obj = NULL;
-    
-    old_obj = sec_ht.get(key);
-    if(old_obj != NULL) {
-        delete old_obj;
-    }
-
-    /*
-    CkArrayIndexMax *idx_list_array = new CkArrayIndexMax[idx_list.size()];
-    for(int count = 0; count < idx_list.size(); count++)
-        idx_list_array[count] = idx_list[count];
-    */
-
-    ComlibSectionHashObject *new_obj = createObjectOnIntermediatePe(localElems, local_idx_list, cbmsg->nPes, cbmsg->indicesCount, cbmsg->_cookie.pe);
-
-    /*
-    char buf[100];
-    std::string tmp = "";
-    for (int i=0; i<idx_list.size(); i++) {
-      sprintf(buf, " (%d-%d-%d-%d)",((short*)&idx_list_array[i])[2],((short*)&idx_list_array[i])[3],((short*)&idx_list_array[i])[4],((short*)&idx_list_array[i])[5]);
-      tmp+=buf;
-    }
-    ComlibPrintf("[%d] LOCAL MULTICAST: %s\n",key.hash(),tmp.data());
-    tmp=""+new_obj->npes;
-    for (int i=0; i<new_obj->npes; i++) {
-      sprintf(buf, " %d",new_obj->pelist[i]);
-      tmp+=buf;
-    }
-    ComlibPrintf("[%d] REMOTE MULTICAST: %s\n",key.hash(),tmp.data());
-    */
-    //delete [] idx_list_array;
-    
-    sec_ht.put(key) = new_obj;
-
-    /*
-    CkPrintf("%u: Proc = %d (%d) forward:", key.hash(), CkMyPe(),cbmsg->nPes);
-    for (int i=0; i<new_obj->npes; ++i) CkPrintf(" %d",new_obj->pelist[i]);
-    CkPrintf(", deliver:");
-    for (int i=0; i<new_obj->indices.size(); ++i) CkPrintf(" %d",((int*)&new_obj->indices[i])[1]);
-    CkPrintf("\n");
-    */
-
-    remoteMulticast(env, new_obj);
-    localMulticast(newenv, new_obj); //local multicast always copies
-    CmiFree(newenv);                
-}
+/*@}*/
index 6d4d5c797a24b5803060b154cee36e0768a41f75..10051731825c04dd981765843f335f5dadac11c4 100644 (file)
@@ -1,35 +1,32 @@
+/**
+   @addtogroup ComlibCharmStrategy
+   @{
+   @file 
+   @brief Send a multicast by sending once directly to each processor owning destination elements.
+
+*/
+
 #ifndef DIRECT_MULTICAST_STRATEGY
 #define DIRECT_MULTICAST_STRATEGY
 
-#include "ComlibManager.h"
-#include "ComlibSectionInfo.h"
+#include "MulticastStrategy.h"
 
-void *DMHandler(void *msg);
-/**
- * Main class for multicast strategies. It defaults to sending a direct message
- * to all the processors involved in the multicast.
 
- * The definition of the section, as well as the location of all the elements in
- * the processors is determined by the sending processor. The other will obey to
- * it, even some elements have already migrated elsewhere.
- */
-class DirectMulticastStrategy: public CharmStrategy {
- protected:
-    //   int handlerId;    
-    ComlibSectionInfo sinfo;
+/** 
+  Send the multicast by sending once directly to each processor owning destination elements.
 
-    int isPersistent; 
-    
-    ///Array section support.
-    CkHashtableT<ComlibSectionHashKey, ComlibSectionHashObject *> sec_ht; 
-    
-    ///Add this section to the hash table locally.
-    void insertSectionID(CkSectionID *sid);
+  The definition of the section, as well as the location of all the elements in
+  the processors is determined by the sending processor. The other will obey to
+  it, even some elements have already migrated elsewhere.
 
+*/
+class DirectMulticastStrategy: public MulticastStrategy {
+ protected:
     ///Called when a new section multicast is called by the user locally.
     ///The strategy should then create a topology for it and return a hash
     ///object to store that topology.
-    virtual ComlibSectionHashObject *createObjectOnSrcPe(int nindices, CkArrayIndexMax *idx_list);
+  virtual void createObjectOnSrcPe(ComlibSectionHashObject *obj, int npes, ComlibMulticastIndexCount *pelist);
 
     /**   
      * Similar to createHashObjectOnSrcPe, but that this call is made on the
@@ -44,42 +41,19 @@ class DirectMulticastStrategy: public CharmStrategy {
      * @param srcpe processor which started the multicast
      * @return a hash object describing the section
      */
-    virtual ComlibSectionHashObject *createObjectOnIntermediatePe(int nindices, CkArrayIndexMax *idxlist, int npes, ComlibMulticastIndexCount *counts, int srcpe);
-        
-    ///Needed for getNewMulticastMessage, to specify if the list of processors need to be ordered
-    virtual int needSorting() { return 0; }
-
-    ///Called to multicast the message to local array elements.
-    void localMulticast(envelope *env, ComlibSectionHashObject *obj);
-    
-    ///Called to send to message out to the remote destinations.
-    ///This method can be overridden to call converse level strategies.
-    virtual void remoteMulticast(envelope *env, ComlibSectionHashObject *obj);
-
-    ///Process a new message by extracting the array elements from it and
-    ///creating a new hash object by calling createObjectOnIntermediatePe().
-    void handleNewMulticastMessage(envelope *env);
-
+    virtual void createObjectOnIntermediatePe(ComlibSectionHashObject *obj, int npes, ComlibMulticastIndexCount *counts, int srcpe);
+   
  public:
-
-    DirectMulticastStrategy(CkMigrateMessage *m): CharmStrategy(m){}
+    DirectMulticastStrategy(CkMigrateMessage *m): MulticastStrategy(m){}
                 
     ///Array constructor
-    DirectMulticastStrategy(CkArrayID aid, int isPersistent = 0);
-        
-    //Destuctor
-    ~DirectMulticastStrategy();
-        
-    virtual void insertMessage(CharmMessageHolder *msg);
-    virtual void doneInserting();
+    DirectMulticastStrategy(int isPersistent = 0): MulticastStrategy(isPersistent) {}
 
-    ///Called by the converse handler function
-    virtual void handleMessage(void *msg);    
+    PUPable_decl(DirectMulticastStrategy);
 
-    virtual void pup(PUP::er &p);    
-    virtual void beginProcessing(int nelements);
+    virtual void pup(PUP::er &p){ MulticastStrategy::pup(p);}   
     
-    PUPable_decl(DirectMulticastStrategy);
 };
 #endif
 
+/*@}*/
index a3c18578d7008872d85d997d54b43be5651f4eee..73946571e8a85e28c8aa288038e69bd9b1acd55a 100644 (file)
@@ -1,21 +1,25 @@
-#include "DummyStrategy.h"
+// #ifdef filippo
 
-DummyStrategy::DummyStrategy() : CharmStrategy(){
-}
+// #include "DummyStrategy.h"
 
-void DummyStrategy::insertMessage(CharmMessageHolder *cmsg){
-    ComlibPrintf("Sending Directly\n");
-    char *msg = cmsg->getCharmMessage();
-    CmiSyncSendAndFree(cmsg->dest_proc, UsrToEnv(msg)->getTotalsize(), 
-                       (char *)UsrToEnv(msg));
-    delete cmsg;
-}
+// DummyStrategy::DummyStrategy() : CharmStrategy(){
+// }
 
-void DummyStrategy::doneInserting(){
-}
+// void DummyStrategy::insertMessage(CharmMessageHolder *cmsg){
+//     ComlibPrintf("Sending Directly\n");
+//     char *msg = cmsg->getCharmMessage();
+//     CmiSyncSendAndFree(cmsg->dest_proc, UsrToEnv(msg)->getTotalsize(), 
+//                        (char *)UsrToEnv(msg));
+//     delete cmsg;
+// }
 
-void DummyStrategy::pup(PUP::er &p){
-   CharmStrategy::pup(p);
-}
+// void DummyStrategy::doneInserting(){
+// }
 
-//PUPable_def(DummyStrategy);
+// void DummyStrategy::pup(PUP::er &p){
+//    CharmStrategy::pup(p);
+// }
+
+// //PUPable_def(DummyStrategy);
+
+// #endif
index bdb834abfa168ccdd7c40fc85dbf0f37778aaa83..5b795e6cbda9c63431574d63ea17a684f7b183eb 100644 (file)
@@ -1,15 +1,19 @@
-#ifndef DUMMY_STRATEGY
-#define DUMMY_STRATEGY
-#include "ComlibManager.h"
-
-class DummyStrategy : public CharmStrategy {
- public:
-    DummyStrategy();
-    DummyStrategy(CkMigrateMessage *m): CharmStrategy(m){}
-    void insertMessage(CharmMessageHolder *msg);
-    void doneInserting();
-
-    virtual void pup(PUP::er &p);
-    PUPable_decl(DummyStrategy);
-};
-#endif
+/* #ifdef filippo */
+
+/* #ifndef DUMMY_STRATEGY */
+/* #define DUMMY_STRATEGY */
+/* #include "ComlibManager.h" */
+
+/* class DummyStrategy : public CharmStrategy { */
+/*  public: */
+/*     DummyStrategy(); */
+/*     DummyStrategy(CkMigrateMessage *m): CharmStrategy(m){} */
+/*     void insertMessage(CharmMessageHolder *msg); */
+/*     void doneInserting(); */
+
+/*     virtual void pup(PUP::er &p); */
+/*     PUPable_decl(DummyStrategy); */
+/* }; */
+/* #endif */
+
+/* #endif */
index 7f9c1f7ef5326a99cfade452eda866155a37306c..c4482d387ffcc0cd83a8c9ffd07f4d875017e217 100644 (file)
@@ -1,38 +1,29 @@
-
-/*********************************************************
-            The EachToManyMulticast Strategy optimizes all-to-all
-            communication. It combines messages and sends them along
-            virtual topologies 2d mesh, 3d mesh and hypercube.
-
-            For large messages send them directly.
-
-            This is the object level strategy. For processor level
-            optimizations routers are called.
-
-  - Sameer Kumar.
-
-**********************************************************/
-
+/**
+   @addtogroup ComlibCharmStrategy
+   @{
+   @file 
+*/
 
 #include "EachToManyMulticastStrategy.h"
 #include "string.h"
-#include "routerstrategy.h"
+//#include "routerstrategy.h"
 
 #include "AAPLearner.h"
 #include "AAMLearner.h"
 
 //EachToManyMulticastStrategy CODE
-CkpvExtern(int, RecvdummyHandle);
-CkpvExtern(CkGroupID, cmgrID);
+//CkpvExtern(int, RecvdummyHandle);
+//CkpvExtern(CkGroupID, cmgrID);
 
+/*
 void *itrDoneHandler(void *msg){
 
     EachToManyMulticastStrategy *nm_mgr;
-    
+
     DummyMsg *dmsg = (DummyMsg *)msg;
     comID id = dmsg->id;
     int instid = id.instanceID;
-    
+
     CmiFree(msg);
     ComlibPrintf("[%d] Iteration finished %d\n", CkMyPe(), instid);
 
@@ -40,81 +31,85 @@ void *itrDoneHandler(void *msg){
         CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()
         ->getStrategyTableEntry(instid);
     int nexpected = sentry->numElements;
-    
+
     if(nexpected == 0) {             
-        ComlibPrintf("[%d] Calling Dummy Done Inserting, %d, %d\n", 
-                     CkMyPe(), instid, nexpected);
+        ComlibPrintf("[%d] Calling Dummy Done Inserting, %d, %d\n", CkMyPe(), instid, nexpected);
         nm_mgr = (EachToManyMulticastStrategy *)sentry->strategy;    
         nm_mgr->doneInserting();
-       
+
        if (!nm_mgr->getOnFinish().isInvalid()) nm_mgr->getOnFinish().send(0);
-    
+
     }
 
     return NULL;
 }
-
+ */
+/*
 void *E2MHandler(void *msg){
     //CkPrintf("[%d]:In EachtoMany CallbackHandler\n", CkMyPe());
     EachToManyMulticastStrategy *nm_mgr;    
-    
+
     CmiMsgHeaderExt *conv_header = (CmiMsgHeaderExt *) msg;
     int instid = conv_header->stratid;
 
     envelope *env = (envelope *)msg;
-    
+
     nm_mgr = (EachToManyMulticastStrategy *) 
         CProxy_ComlibManager(CkpvAccess(cmgrID)).
         ckLocalBranch()->getStrategy(instid);
-    
+
     RECORD_RECV_STATS(instid, env->getTotalsize(), env->getSrcPe());
-    
+
     nm_mgr->localMulticast(msg);
     return NULL;
 }
+ */
 
 //Group Constructor
-EachToManyMulticastStrategy::EachToManyMulticastStrategy(int substrategy, 
-                                                         int n_srcpes, 
-                                                         int *src_pelist,
-                                                         int n_destpes, 
-                                                         int *dest_pelist) 
-    : routerID(substrategy), CharmStrategy() {
-    
-    setType(GROUP_STRATEGY);
-
-    CkGroupID gid;
-    gid.setZero();
-    ginfo.setSourceGroup(gid, src_pelist, n_srcpes);    
-    ginfo.setDestinationGroup(gid, dest_pelist, n_destpes);
-
-    //Written in this funny way to be symmetric with the array case.
-    ginfo.getDestinationGroup(gid, destpelist, ndestpes);
-    ginfo.getCombinedPeList(pelist, npes);
-
-    commonInit();
+EachToManyMulticastStrategy::EachToManyMulticastStrategy(int substrategy,
+               CkGroupID src,
+               CkGroupID dest,
+               int n_srcpes, 
+               int *src_pelist,
+               int n_destpes, 
+               int *dest_pelist) 
+: RouterStrategy(substrategy), CharmStrategy() {
+       ComlibPrintf("[%d] EachToManyMulticast group constructor\n",CkMyPe());
+
+       setType(GROUP_STRATEGY);
+
+       //CkGroupID gid;
+       //gid.setZero();
+       ginfo.setSourceGroup(src, src_pelist, n_srcpes);    
+       ginfo.setDestinationGroup(dest, dest_pelist, n_destpes);
+
+       //Written in this funny way to be symmetric with the array case.
+       //ginfo.getDestinationGroup(gid, destpelist, ndestpes);
+       //ginfo.getCombinedPeList(pelist, npes);
+
+       commonInit(ginfo.getCombinedCountList());
 }
 
 //Array Constructor
 EachToManyMulticastStrategy::EachToManyMulticastStrategy(int substrategy, 
-                                                         CkArrayID src, 
-                                                         CkArrayID dest, 
-                                                         int nsrc, 
-                                                         CkArrayIndexMax 
-                                                         *srcelements
-                                                         int ndest, 
-                                                         CkArrayIndexMax 
-                                                         *destelements)
-    :routerID(substrategy), CharmStrategy() {
-
-    setType(ARRAY_STRATEGY);
-    ainfo.setSourceArray(src, srcelements, nsrc);
-    ainfo.setDestinationArray(dest, destelements, ndest);
-
-    ainfo.getDestinationPeList(destpelist, ndestpes);
-    ainfo.getCombinedPeList(pelist, npes);
-    
-    /*
+               CkArrayID src, 
+               CkArrayID dest, 
+               int nsrc, 
+               CkArrayIndexMax *srcelements, 
+               int ndest
+               CkArrayIndexMax *destelements)
+: RouterStrategy(substrategy), CharmStrategy() {
+       ComlibPrintf("[%d] EachToManyMulticast array constructor. nsrc=%d ndest=%d\n",CkMyPe(), nsrc, ndest);
+
+       setType(ARRAY_STRATEGY);
+       ainfo.setSourceArray(src, srcelements, nsrc);
+       ainfo.setDestinationArray(dest, destelements, ndest);
+
+       int *count = ainfo.getCombinedCountList();
+       //ainfo.getSourcePeList(nsrcPe, srcPe);
+       //ainfo.getDestinationPeList(ndestPe, destPe);
+
+       /*
       char dump[1000];
       char sdump[100];
       sprintf(dump, "%d: Each To MANY PELIST :\n", CkMyPe());
@@ -123,39 +118,41 @@ EachToManyMulticastStrategy::EachToManyMulticastStrategy(int substrategy,
       strcat(dump, sdump);           
       }    
       ComlibPrintf("%s\n", dump);
-    */
+        */
 
-    commonInit();
+       commonInit(count);
 }
 
-extern char *router;
+extern char *routerName;
 //Common initialization for both group and array constructors
-void EachToManyMulticastStrategy::commonInit() {
-
-    setBracketed();
-    setForwardOnMigration(1);
-
-    if(CkMyPe() == 0 && router != NULL){
-        if(strcmp(router, "USE_MESH") == 0)
-            routerID = USE_MESH;
-        else if(strcmp(router, "USE_GRID") == 0)
-            routerID = USE_GRID;
-        else  if(strcmp(router, "USE_HYPERCUBE") == 0)
-            routerID = USE_HYPERCUBE;
-        else  if(strcmp(router, "USE_DIRECT") == 0)
-            routerID = USE_DIRECT;        
-        else  if(strcmp(router, "USE_PREFIX") == 0)
-            routerID = USE_PREFIX;        
-
-        //Just for the first step. After learning the learned
-        //strategies will be chosen
-        router = NULL;
-    }
-    
-    ComlibPrintf("Creating Strategy %d\n", routerID);
-
-    useLearner = 0;
-    rstrat = NULL;
+void EachToManyMulticastStrategy::commonInit(int *count) {
+
+       setBracketed();
+       //setForwardOnMigration(1);
+
+       if(CkMyPe() == 0 && routerName != NULL){
+               if(strcmp(routerName, "USE_MESH") == 0)
+                       routerIDsaved = USE_MESH;
+               else if(strcmp(routerName, "USE_GRID") == 0)
+                       routerIDsaved = USE_GRID;
+               else  if(strcmp(routerName, "USE_HYPERCUBE") == 0)
+                       routerIDsaved = USE_HYPERCUBE;
+               else  if(strcmp(routerName, "USE_DIRECT") == 0)
+                       routerIDsaved = USE_DIRECT;        
+               else  if(strcmp(routerName, "USE_PREFIX") == 0)
+                       routerIDsaved = USE_PREFIX;        
+
+               //Just for the first step. After learning the learned
+               //strategies will be chosen
+               //router = NULL;
+       }
+
+       //ComlibPrintf("Creating Strategy %d\n", routerID);
+
+       // finish the creation of the RouterStrategy superclass
+       bracketedUpdatePeKnowledge(count);
+       delete [] count;
+       //rstrat = NULL;
 }
 
 EachToManyMulticastStrategy::~EachToManyMulticastStrategy() {
@@ -164,154 +161,135 @@ EachToManyMulticastStrategy::~EachToManyMulticastStrategy() {
 
 void EachToManyMulticastStrategy::insertMessage(CharmMessageHolder *cmsg){
 
-    ComlibPrintf("[%d] EachToManyMulticast: insertMessage \n", 
-                 CkMyPe());   
-
-    envelope *env = UsrToEnv(cmsg->getCharmMessage());
-
-    if(cmsg->dest_proc == IS_BROADCAST) {
-        //All to all multicast
-        
-        cmsg->npes = ndestpes;
-        cmsg->pelist = destpelist;
-        
-        //Use Multicast Learner (Foobar will not work for combinations
-        //of personalized and multicast messages
-        
-        CmiSetHandler(env, handlerId);
-
-        //Collect Multicast Statistics
-        RECORD_SENDM_STATS(getInstance(), env->getTotalsize(), 
-                           cmsg->pelist, cmsg->npes);
-    }
-    else {
-        //All to all personalized
+  cmsg -> checkme();
+
+       ComlibPrintf("[%d] EachToManyMulticast: insertMessage\n", CkMyPe());
+
+       envelope *env = UsrToEnv(cmsg->getCharmMessage());
+
+       if(cmsg->dest_proc == IS_BROADCAST) {
+               //All to all multicast
 
-        //Collect Statistics
-        RECORD_SEND_STATS(getInstance(), env->getTotalsize(), 
-                          cmsg->dest_proc);
-    }        
+               // not necessary to set cmsg since the superclass will take care
+               //cmsg->npes = ndestPes;
+               //cmsg->pelist = destPes;
 
-    rstrat->insertMessage(cmsg);
+               //Use Multicast Learner (Foobar will not work for combinations
+               //of personalized and multicast messages
+
+               // this handler will ensure that handleMessage is called if direcly
+               // sent, or deliver is called if normal routing path is taken
+               CmiSetHandler(env, CkpvAccess(comlib_handler));
+
+               //Collect Multicast Statistics
+               RECORD_SENDM_STATS(getInstance(), env->getTotalsize(), 
+                               cmsg->pelist, cmsg->npes);
+       }
+       else {
+               //All to all personalized
+
+               //Collect Statistics
+               RECORD_SEND_STATS(getInstance(), env->getTotalsize(), 
+                               cmsg->dest_proc);
+       }
+
+       RouterStrategy::insertMessage(cmsg);
 }
 
+/*
 void EachToManyMulticastStrategy::doneInserting(){
 
     StrategyTableEntry *sentry = 
         CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()
         ->getStrategyTableEntry(getInstance());
     int nexpected = sentry->numElements;
-    
+
     if(routerID == USE_DIRECT && nexpected == 0)
         return;
-    
+
     if(MyPe < 0)
         return;
 
     //ComlibPrintf("%d: DoneInserting \n", CkMyPe());    
     rstrat->doneInserting();
 }
+ */
 
 void EachToManyMulticastStrategy::pup(PUP::er &p){
 
-    int count = 0;
-    ComlibPrintf("[%d] Each To many::pup %s\n", CkMyPe(), 
-                 ((!p.isUnpacking() == 0)?("UnPacking"):("Packing")));
-
-    CharmStrategy::pup(p);
-
-    p | routerID; 
-    p | npes; p | ndestpes;     
-    p | useLearner;
-
-    if(p.isUnpacking() && npes > 0) {
-        pelist = new int[npes];    
-    }
-
-    if(npes > 0)
-        p(pelist, npes);
+       ComlibPrintf("[%d] EachToManyMulticastStrategy::pup called for %s\n", CkMyPe(), 
+                       p.isPacking()?"packing":(p.isUnpacking()?"unpacking":"sizing"));
 
-    if(p.isUnpacking() && ndestpes > 0) {
-        destpelist = new int[ndestpes];    
-    }    
+       RouterStrategy::pup(p);
+       CharmStrategy::pup(p);
 
-    if(ndestpes > 0)
-        p(destpelist, ndestpes);
-
-    if(p.isUnpacking()){
-       handlerId = CkRegisterHandler((CmiHandler)E2MHandler);
-        int handler = CkRegisterHandler((CmiHandler)itrDoneHandler);        
-        
-        if(npes > 0) {
-            rstrat = new RouterStrategy(routerID, handler, npes, pelist);
-            setConverseStrategy(rstrat);
-            MyPe = rstrat->getProcMap()[CkMyPe()];
-        }
-        else MyPe = -1;
-    }
-    
-    ComlibPrintf("[%d] End of pup\n", CkMyPe());
 }
 
-void EachToManyMulticastStrategy::beginProcessing(int numElements){
-    
-    ComlibPrintf("[%d] Begin processing %d\n", CkMyPe(), numElements);
-    /*
-    char dump[1000];
-    char sdump[100];
-    sprintf(dump, "%d: Each To MANY PELIST :\n", CkMyPe());
-    for(int count = 0; count < npes; count ++){
-        sprintf(sdump, "%d, ", pelist[count]);
-        strcat(dump, sdump);           
-    }    
-    ComlibPrintf("%s\n", dump);
-    */
-
-    int expectedDeposits = 0;
-
-    rstrat->setInstance(getInstance());
-
-    if(ainfo.isSourceArray()) 
-        expectedDeposits = numElements;
-    
-    if(getType() == GROUP_STRATEGY) {
-        
-        CkGroupID gid;
-        int *srcpelist;
-        int nsrcpes;
-        
-        ginfo.getSourceGroup(gid, srcpelist, nsrcpes);
-        
-        for(int count = 0; count < nsrcpes; count ++)
-            if(srcpelist[count] == CkMyPe()){
-                expectedDeposits = 1;
-                break;
-            }
-        
-        StrategyTableEntry *sentry = 
-            CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()
-            ->getStrategyTableEntry(myInstanceID);
-        sentry->numElements = expectedDeposits;
-    }
-    
-    if(useLearner) {
-        if(!mflag) 
-            setLearner(new AAPLearner());    
-        else 
-            setLearner(new AAMLearner());                
-    }
-    
-    if(expectedDeposits > 0)
-        return;
-    
-    if(expectedDeposits == 0 && MyPe >= 0)
-        ConvComlibScheduleDoneInserting(myInstanceID);
+/* NOT NEEDED
+void EachToManyMulticastStrategy::finalizeCreation() {
+  ainfo.purge();
 }
-
+ */
+
+// void EachToManyMulticastStrategy::beginProcessing(int numElements){
+
+//     ComlibPrintf("[%d] Begin processing %d\n", CkMyPe(), numElements);
+//     /*
+//     char dump[1000];
+//     char sdump[100];
+//     sprintf(dump, "%d: Each To MANY PELIST :\n", CkMyPe());
+//     for(int count = 0; count < npes; count ++){
+//         sprintf(sdump, "%d, ", pelist[count]);
+//         strcat(dump, sdump);           
+//     }    
+//     ComlibPrintf("%s\n", dump);
+//     */
+
+//     int expectedDeposits = 0;
+
+//     rstrat->setInstance(getInstance());
+
+//     if(ainfo.isSourceArray()) 
+//         expectedDeposits = numElements;
+
+//     if(getType() == GROUP_STRATEGY) {
+
+//         CkGroupID gid;
+//         int *srcpelist;
+//         int nsrcpes;
+
+//         ginfo.getSourceGroup(gid, srcpelist, nsrcpes);
+
+//         for(int count = 0; count < nsrcpes; count ++)
+//             if(srcpelist[count] == CkMyPe()){
+//                 expectedDeposits = 1;
+//                 break;
+//             }
+
+//         StrategyTableEntry *sentry = 
+//             CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()
+//             ->getStrategyTableEntry(myInstanceID);
+//         sentry->numElements = expectedDeposits;
+//     }
+//     /*
+//     if(useLearner) 
+//         setLearner(new AAPLearner());    
+//     else 
+//         setLearner(new AAMLearner());                
+//     */
+
+//     if(expectedDeposits > 0)
+//         return;
+
+//     if(expectedDeposits == 0 && MyPe >= 0)
+//         ConvComlibScheduleDoneInserting(myInstanceID);
+// }
+
+/*
 void EachToManyMulticastStrategy::finalizeProcessing() {
     if(npes > 0)
         delete [] pelist;
-    
+
     if(ndestpes > 0)
         delete [] destpelist;
 
@@ -321,11 +299,47 @@ void EachToManyMulticastStrategy::finalizeProcessing() {
     if(useLearner && getLearner() != NULL)
         delete getLearner();
 }
+ */
+
+void EachToManyMulticastStrategy::deliver(char *msg, int size) {
+       ComlibPrintf("[%d] EachToManyMulticastStrategy::deliver for %s\n",
+                       CkMyPe(), isAllToAll()?"multicast":"personalized");
+       
+       envelope *env = (envelope *)msg;
+       RECORD_RECV_STATS(myHandle, env->getTotalsize(), env->getSrcPe());
+
+       if (isAllToAll()){
+               ComlibPrintf("Delivering via localMulticast()\n");
+               localMulticast(msg);
+       }
+       else { 
+               if (getType() == GROUP_STRATEGY) {
+                       ComlibPrintf("Delivering via personalized CkSendMsgBranchInline\n");
+                       CkUnpackMessage(&env);
+                       CkSendMsgBranchInline(env->getEpIdx(), EnvToUsr(env), CkMyPe(), env->getGroupNum());
+               }
+               else if (getType() == ARRAY_STRATEGY) {
+                       //        CkPrintf("[%d] Delivering via ComlibArrayInfo::deliver()\n", CkMyPe());
+                       //      ComlibArrayInfo::deliver(env);
+
+                       // call ainfo's localBroadcast(env);
+                       ComlibPrintf("[%d] Delivering via ComlibArrayInfo::localBroadcast()\n", CkMyPe());
+                       ainfo.localBroadcast(env);
+
+               }
+       }
+}
 
 void EachToManyMulticastStrategy::localMulticast(void *msg){
-    register envelope *env = (envelope *)msg;
-    CkUnpackMessage(&env);
-    
-    ainfo.localBroadcast(env);
+       register envelope *env = (envelope *)msg;
+       CkUnpackMessage(&env);
+       CkPrintf("localMulticast calls ainfo.localBroadcast()\n");
+       ainfo.localBroadcast(env);
+}
+
+void EachToManyMulticastStrategy::notifyDone() {
+       if (!getOnFinish().isInvalid()) getOnFinish().send(0);
+       RouterStrategy::notifyDone();
 }
 
+/*@}*/
index 924a828712efa5e647f482bf312de0958d5789c2..74910b2ee78ebaf750cb28a2e33af505c9bced0e 100644 (file)
@@ -1,53 +1,82 @@
+/**
+   @addtogroup ComlibCharmStrategy
+   @{
+   @file 
+   @brief Optimized all-to-all communication that can combine messages and send them along virtual topologies.
+*/
+
 #ifndef EACH_TO_MANY_MULTICAST_STRATEGY
 #define EACH_TO_MANY_MULTICAST_STRATEGY
 
 #include "ComlibManager.h"
 #include "routerstrategy.h"
 
-class EachToManyMulticastStrategy: public CharmStrategy {
- protected:
-    int routerID;      //Which topology
-    int npes, *pelist; //Domain of the topology
-    int MyPe;          //My id in that domain
+/**
+   The EachToManyMulticast Strategy optimizes all-to-all
+   communication. It combines messages and sends them along
+   virtual topologies 2d mesh, 3d mesh and hypercube using
+   the RouterStrategy as underlying strategy.
 
-    int ndestpes, *destpelist; //Destination processors
-    int handlerId;
-    
-    //Executes common code just after array and group constructors
-    virtual void commonInit();
+   For large messages send them directly.
+
+   This is the object level strategy. For processor level
+   optimizations the underlying RouterStrategy is called.
 
-    RouterStrategy *rstrat;
-    int useLearner;
+   @author Sameer Kumar, Filippo, and Isaac
+
+*/
+class EachToManyMulticastStrategy : public RouterStrategy, public CharmStrategy {
+ protected:
+    /// Executes common code just after array and group constructors
+    virtual void commonInit(int*);
 
  public:
-    //Group constructor
-    EachToManyMulticastStrategy(int strategyId, int nsrcpes=0, 
-                                int *srcpelist=0, 
-                                int ndestpes =0, int *destpelist =0);
-    
-    //Array constructor
+    /// Group constructor
+    /// If only the first three parameters are provided, the whole group will be used for the multicast(0 to CkNumPes)
+    /// TODO verify that the 0 parameter 
+    EachToManyMulticastStrategy(int strategyId, CkGroupID src, CkGroupID dest, 
+               int nsrcpes=0, int *srcpelist=0, 
+               int ndestpes =0, int *destpelist =0);
+
+    /// Array constructor
+    /// TODO: Fix this to allow for the same parameters as would be given to an array section creation(ranges of indices).
     EachToManyMulticastStrategy(int substrategy, CkArrayID src, 
                                 CkArrayID dest, int nsrc=0, 
                                 CkArrayIndexMax *srcelements=0, int ndest=0, 
                                 CkArrayIndexMax *destelements=0);
     
-    EachToManyMulticastStrategy(CkMigrateMessage *m) : CharmStrategy(m){};
+    EachToManyMulticastStrategy(CkMigrateMessage *m) : RouterStrategy(m), CharmStrategy(m) {
+      ComlibPrintf("[%d] EachToManyMulticast migration constructor\n",CkMyPe());
+    };
     
     ~EachToManyMulticastStrategy();
 
+    void insertMessage(MessageHolder *msg) {
+      ((CharmMessageHolder*)msg) -> checkme();
+      insertMessage((CharmMessageHolder*)msg);
+    }
+
     //Basic function, subclasses should not have to change it
     virtual void insertMessage(CharmMessageHolder *msg);
-    //More specielized function
-    virtual void doneInserting();
 
     virtual void pup(PUP::er &p);    
-    virtual void beginProcessing(int nelements);
-    virtual void finalizeProcessing();
     virtual void localMulticast(void *msg);
+
+    virtual void notifyDone();
+    virtual void deliver(char *, int);
+
+    /// this method can be called when the strategy is in DIRECT mode, so the
+    /// message will go the comlib_handler and then arrive here.
+    virtual void handleMessage(void *msg) {
+      envelope *env = (envelope*)msg;
+      
+      deliver((char*)msg, env->getTotalsize());
+    }
     
     PUPable_decl(EachToManyMulticastStrategy);
-    
-    inline void enableLearning() {useLearner = 1;}
+
 };
 #endif
 
+/*@}*/
index 99b5808e2c11d39e3d32a60341c65a073a5d79b5..56a532cfa9f6fe78bcc953d09731c2f3a08476bf 100644 (file)
-#include "KDirectMulticastStrategy.h"
-
-//Group Constructor
-KDirectMulticastStrategy::KDirectMulticastStrategy(int kf, 
-                                                   int ndest, int *pelist) 
-    : DirectMulticastStrategy(ndest, pelist), kfactor(kf) {
-    //FIXME: verify the list is sorted
-    commonKDirectInit();
-}
-
-//Array Constructor
-KDirectMulticastStrategy::KDirectMulticastStrategy(int kf, 
-                                                   CkArrayID dest_aid)
-    : DirectMulticastStrategy(dest_aid), kfactor(kf){
-    commonKDirectInit();    
-}
-
-void KDirectMulticastStrategy::commonKDirectInit(){
-    //sort list and create a reverse map
-}
-
-extern int _charmHandlerIdx;
-void KDirectMulticastStrategy::doneInserting(){
-    ComlibPrintf("%d: DoneInserting \n", CkMyPe());
+// #ifdef filippo
+
+// #include "KDirectMulticastStrategy.h"
+
+// //Group Constructor
+// KDirectMulticastStrategy::KDirectMulticastStrategy(int kf, 
+//                                                    int ndest, int *pelist) 
+//     : DirectMulticastStrategy(ndest, pelist), kfactor(kf) {
+//     //FIXME: verify the list is sorted
+//     commonKDirectInit();
+// }
+
+// //Array Constructor
+// KDirectMulticastStrategy::KDirectMulticastStrategy(int kf, 
+//                                                    CkArrayID dest_aid)
+//     : DirectMulticastStrategy(dest_aid), kfactor(kf){
+//     commonKDirectInit();    
+// }
+
+// void KDirectMulticastStrategy::commonKDirectInit(){
+//     //sort list and create a reverse map
+// }
+
+// extern int _charmHandlerIdx;
+// void KDirectMulticastStrategy::doneInserting(){
+//     ComlibPrintf("%d: DoneInserting \n", CkMyPe());
     
-    if(messageBuf->length() == 0) {
-        return;
-    }
+//     if(messageBuf->length() == 0) {
+//         return;
+//     }
     
-    while(!messageBuf->isEmpty()) {
-       CharmMessageHolder *cmsg = messageBuf->deq();
-        char *msg = cmsg->getCharmMessage();
-        register envelope* env = UsrToEnv(msg);
+//     while(!messageBuf->isEmpty()) {
+//     CharmMessageHolder *cmsg = messageBuf->deq();
+//         char *msg = cmsg->getCharmMessage();
+//         register envelope* env = UsrToEnv(msg);
 
-        ComlibPrintf("[%d] Calling KDirect %d %d %d\n", CkMyPe(),
-                     env->getTotalsize(), ndestpes, cmsg->dest_proc);
+//         ComlibPrintf("[%d] Calling KDirect %d %d %d\n", CkMyPe(),
+//                      env->getTotalsize(), ndestpes, cmsg->dest_proc);
                
-        if(cmsg->dest_proc == IS_MULTICAST) {      
-            CmiSetHandler(env, handlerId);
+//         if(cmsg->dest_proc == IS_MULTICAST) {      
+//             CmiSetHandler(env, handlerId);
             
-            int *cur_pelist = NULL;
-            int cur_npes = 0;
+//             int *cur_pelist = NULL;
+//             int cur_npes = 0;
             
-            if(cmsg->sec_id == NULL) {
-                cur_pelist = kdestpelist;
-                cur_npes = kfactor;
-            }
-            else {                
-                cur_npes = (kfactor <= cmsg->sid.npes)?kfactor : 
-                    cmsg->sid.npes;
-                cur_pelist = cmsg->sid.pe_list;
-            }
+//             if(cmsg->sec_id == NULL) {
+//                 cur_pelist = kdestpelist;
+//                 cur_npes = kfactor;
+//             }
+//             else {                
+//                 cur_npes = (kfactor <= cmsg->sid.npes)?kfactor : 
+//                     cmsg->sid.npes;
+//                 cur_pelist = cmsg->sid.pe_list;
+//             }
             
-            ComlibPrintf("[%d] Sending Message to %d\n", CkMyPe(), cur_npes);
-            CmiSyncListSendAndFree(cur_npes, cur_pelist, 
-                                   UsrToEnv(msg)->getTotalsize(), 
-                                   UsrToEnv(msg));
-        }
-        else {
-            CmiSyncSendAndFree(cmsg->dest_proc, 
-                               UsrToEnv(msg)->getTotalsize(), 
-                               (char *)UsrToEnv(msg));
-        }        
+//             ComlibPrintf("[%d] Sending Message to %d\n", CkMyPe(), cur_npes);
+//             CmiSyncListSendAndFree(cur_npes, cur_pelist, 
+//                                    UsrToEnv(msg)->getTotalsize(), 
+//                                    UsrToEnv(msg));
+//         }
+//         else {
+//             CmiSyncSendAndFree(cmsg->dest_proc, 
+//                                UsrToEnv(msg)->getTotalsize(), 
+//                                (char *)UsrToEnv(msg));
+//         }        
         
-        delete cmsg; 
-    }
-}
+//         delete cmsg; 
+//     }
+// }
 
-void KDirectMulticastStrategy::pup(PUP::er &p){
-    DirectMulticastStrategy::pup(p);
+// void KDirectMulticastStrategy::pup(PUP::er &p){
+//     DirectMulticastStrategy::pup(p);
 
-    p | kfactor;
-}
+//     p | kfactor;
+// }
 
-void KDirectMulticastStrategy::beginProcessing(int  nelements){
+// void KDirectMulticastStrategy::beginProcessing(int  nelements){
 
-    DirectMulticastStrategy::beginProcessing(nelements);
+//     DirectMulticastStrategy::beginProcessing(nelements);
 
-    kndestpelist = new int[kfactor]; 
+//     kndestpelist = new int[kfactor]; 
 
-    int next_pe = 0, count = 0;
-    //Assuming the destination pe list is sorted.
-    for(count = 0; count < ndestpes; count++)        
-        if(destpelist[count] > CkMyPe()) {
-            next_pe = count;
-            break;
-        }
+//     int next_pe = 0, count = 0;
+//     //Assuming the destination pe list is sorted.
+//     for(count = 0; count < ndestpes; count++)        
+//         if(destpelist[count] > CkMyPe()) {
+//             next_pe = count;
+//             break;
+//         }
 
-    int kpos = 0;
-    for(count = next_pe; count < next_pe + kfactor; count++){
-        int pe = destpelist[count % ndestpes];
-        kdestpelist[kpos ++] = pe;
-    }
-}
+//     int kpos = 0;
+//     for(count = next_pe; count < next_pe + kfactor; count++){
+//         int pe = destpelist[count % ndestpes];
+//         kdestpelist[kpos ++] = pe;
+//     }
+// }
 
-void KDirectMulticastStrategy::handleMulticastMessage(void *msg){
-    register envelope *env = (envelope *)msg;
+// void KDirectMulticastStrategy::handleMulticastMessage(void *msg){
+//     register envelope *env = (envelope *)msg;
     
-    CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)EnvToUsr(env);
-    int src_pe = cbmsg->_cookie.pe;
-    if(isDestinationGroup){               
-        CmiSetHandler(env, _charmHandlerIdx);
-        CmiSyncSend(CkMyPe(), env->getTotalsize(), (char *)env);
+//     CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)EnvToUsr(env);
+//     int src_pe = cbmsg->_cookie.pe;
+//     if(isDestinationGroup){               
+//         CmiSetHandler(env, _charmHandlerIdx);
+//         CmiSyncSend(CkMyPe(), env->getTotalsize(), (char *)env);
         
-        int nmsgs = getNumMessagesToSend(src_pe, CkMyPe, CkNumPes());
-        if(nmsgs > 0){            
-            CmiSetHandler(env, handlerId);            
-            CmiSyncListSendAndFree(nmsgs, kdestpelist, 
-                                   env->getTotalsize(), env);
-        }        
-        return;
-    }
-
-    int status = cbmsg->_cookie.sInfo.cInfo.status;
-    ComlibPrintf("[%d] In handle multicast message %d\n", CkMyPe(), status);
-
-    if(status == COMLIB_MULTICAST_ALL) {                        
-        int nmsgs = getNumMessagesToSend(src_pe. CkMyPe(), CkNumPes());
-        if(nmsgs > 0){ //Have to forward the messages           
-            void *msg = EnvToUsr(env);
-            void *newmsg = CkCopyMsg(&msg);
-            envelope *newenv = UsrToEnv(newmsg);        
-            CmiSyncListSendAndFree(nmsgs, kdestpelist, 
-                                   newenv->getTotalsize(), newenv);
-        }
-
-        //Multicast to all destination elements on current processor        
-        ComlibPrintf("[%d] Local multicast sending all %d\n", CkMyPe(), 
-                     localDestIndices.size());
+//         int nmsgs = getNumMessagesToSend(src_pe, CkMyPe, CkNumPes());
+//         if(nmsgs > 0){            
+//             CmiSetHandler(env, handlerId);            
+//             CmiSyncListSendAndFree(nmsgs, kdestpelist, 
+//                                    env->getTotalsize(), env);
+//         }        
+//         return;
+//     }
+
+//     int status = cbmsg->_cookie.sInfo.cInfo.status;
+//     ComlibPrintf("[%d] In handle multicast message %d\n", CkMyPe(), status);
+
+//     if(status == COMLIB_MULTICAST_ALL) {                        
+//         int nmsgs = getNumMessagesToSend(src_pe. CkMyPe(), CkNumPes());
+//         if(nmsgs > 0){ //Have to forward the messages           
+//             void *msg = EnvToUsr(env);
+//             void *newmsg = CkCopyMsg(&msg);
+//             envelope *newenv = UsrToEnv(newmsg);        
+//             CmiSyncListSendAndFree(nmsgs, kdestpelist, 
+//                                    newenv->getTotalsize(), newenv);
+//         }
+
+//         //Multicast to all destination elements on current processor        
+//         ComlibPrintf("[%d] Local multicast sending all %d\n", CkMyPe(), 
+//                      localDestIndices.size());
         
-        localMulticast(&localDestIndices, env);
-    }   
-    else if(status == COMLIB_MULTICAST_NEW_SECTION){        
-        CkUnpackMessage(&env);
-        ComlibPrintf("[%d] Received message for new section src=%d\n", 
-                     CkMyPe(), cbmsg->_cookie.pe);
-
-        ComlibMulticastMsg *ccmsg = (ComlibMulticastMsg *)cbmsg;
+//         localMulticast(&localDestIndices, env);
+//     }   
+//     else if(status == COMLIB_MULTICAST_NEW_SECTION){        
+//         CkUnpackMessage(&env);
+//         ComlibPrintf("[%d] Received message for new section src=%d\n", 
+//                      CkMyPe(), cbmsg->_cookie.pe);
+
+//         ComlibMulticastMsg *ccmsg = (ComlibMulticastMsg *)cbmsg;
         
-        KDirectHashObject *kobj = 
-            createHashObject(ccmsg->nIndices, ccmsg->indices);
+//         KDirectHashObject *kobj = 
+//             createHashObject(ccmsg->nIndices, ccmsg->indices);
         
-        envelope *usrenv = (envelope *) ccmsg->usrMsg;
+//         envelope *usrenv = (envelope *) ccmsg->usrMsg;
         
-        envelope *newenv = (envelope *)CmiAlloc(usrenv->getTotalsize());
-        memcpy(newenv, usrenv, usrenv->getTotalsize());
+//         envelope *newenv = (envelope *)CmiAlloc(usrenv->getTotalsize());
+//         memcpy(newenv, usrenv, usrenv->getTotalsize());
 
-        localMulticast(&kobj->indices, newenv);
+//         localMulticast(&kobj->indices, newenv);
 
-        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
-                                 cbmsg->_cookie.sInfo.cInfo.id);
+//         ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+//                                  cbmsg->_cookie.sInfo.cInfo.id);
 
-        KDirectHashObject *old_kobj = 
-            (KDirectHashObject*)sec_ht.get(key);
-        if(old_kobj != NULL)
-            delete old_kobj;
+//         KDirectHashObject *old_kobj = 
+//             (KDirectHashObject*)sec_ht.get(key);
+//         if(old_kobj != NULL)
+//             delete old_kobj;
         
-        sec_ht.put(key) = kobj;
-
-        if(kobj->npes > 0) {
-            ComlibPrintf("[%d] Forwarding Message of %d to %d pes\n", 
-                         CkMyPe(), cbmsg->_cookie.pe, kobj->npes);
-            CkPackMessage(&env);
-            CmiSyncListSendAndFree(kpbj->npes, kobj->pelist, 
-                                   env->getTotalsize(), env);
-        }
-        else
-            CmiFree(env);       
-    }
-    else {
-        //status == COMLIB_MULTICAST_OLD_SECTION, use the cached section id
-        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
-                                 cbmsg->_cookie.sInfo.cInfo.id);    
-        KDirectHashObject *kobj = (KDirectHashObject *)sec_ht.get(key);
+//         sec_ht.put(key) = kobj;
+
+//         if(kobj->npes > 0) {
+//             ComlibPrintf("[%d] Forwarding Message of %d to %d pes\n", 
+//                          CkMyPe(), cbmsg->_cookie.pe, kobj->npes);
+//             CkPackMessage(&env);
+//             CmiSyncListSendAndFree(kpbj->npes, kobj->pelist, 
+//                                    env->getTotalsize(), env);
+//         }
+//         else
+//             CmiFree(env);       
+//     }
+//     else {
+//         //status == COMLIB_MULTICAST_OLD_SECTION, use the cached section id
+//         ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+//                                  cbmsg->_cookie.sInfo.cInfo.id);    
+//         KDirectHashObject *kobj = (KDirectHashObject *)sec_ht.get(key);
         
-        if(kobj == NULL)
-            CkAbort("Destination indices is NULL\n");
+//         if(kobj == NULL)
+//             CkAbort("Destination indices is NULL\n");
         
-        if(kobj->npes > 0){
-            void *msg = EnvToUsr(env);
-            void *newmsg = CkCopyMsg(&msg);
-            envelope *newenv = UsrToEnv(newmsg);        
-            CmiSyncListSendAndFree(kpbj->npes, kobj->pelist, 
-                                   newenv->getTotalsize(), newenv);
-
-        }
+//         if(kobj->npes > 0){
+//             void *msg = EnvToUsr(env);
+//             void *newmsg = CkCopyMsg(&msg);
+//             envelope *newenv = UsrToEnv(newmsg);        
+//             CmiSyncListSendAndFree(kpbj->npes, kobj->pelist, 
+//                                    newenv->getTotalsize(), newenv);
+
+//         }
         
-        localMulticast(&kobj->indices, env);
-    }
-}
+//         localMulticast(&kobj->indices, env);
+//     }
+// }
 
-void KDirectMulticastStrategy::initSectionID(CkSectionID *sid){
+// void KDirectMulticastStrategy::initSectionID(CkSectionID *sid){
 
-    ComlibPrintf("KDirect Init section ID\n");
-    sid->pelist = NULL;
-    sid->npes = 0;
+//     ComlibPrintf("KDirect Init section ID\n");
+//     sid->pelist = NULL;
+//     sid->npes = 0;
 
-    int *pelist = new int[kfactor];
-    int npes;
-    getPeList(sid->_nElem,  sid->_elems, pelist, npes);
+//     int *pelist = new int[kfactor];
+//     int npes;
+//     getPeList(sid->_nElem,  sid->_elems, pelist, npes);
     
-    sid->destpelist = pelist;
-    sid->ndestpes = npes;    
-}
+//     sid->destpelist = pelist;
+//     sid->ndestpes = npes;    
+// }
 
-KDirectHashObject *KDirectMulticastStrategy::createHashObject(int nelements, CkArrayIndexMax *elements){
+// KDirectHashObject *KDirectMulticastStrategy::createHashObject(int nelements, CkArrayIndexMax *elements){
 
-    KDirectHashObject *kobj = new KDirectHashObject;
-    kobj->pelist = new int[kfactor];
-    getPeList(nelements,  elements, kobj->pelist, kobj->npes);
+//     KDirectHashObject *kobj = new KDirectHashObject;
+//     kobj->pelist = new int[kfactor];
+//     getPeList(nelements,  elements, kobj->pelist, kobj->npes);
 
-    return kobj;
-}
+//     return kobj;
+// }
 
 
-void KDirectMulticastStrategy::getPeList(int nelements, 
-                                         CkArrayIndexMax *elements, 
-                                         int *pelist, int &npes, 
-                                         int src_pe){
+// void KDirectMulticastStrategy::getPeList(int nelements, 
+//                                          CkArrayIndexMax *elements, 
+//                                          int *pelist, int &npes, 
+//                                          int src_pe){
     
-    npes = 0;
+//     npes = 0;
     
-    int *tmp_pelist = new int[CkNumPes()];
-    int num_pes;
+//     int *tmp_pelist = new int[CkNumPes()];
+//     int num_pes;
     
-    //make this a reusable function call later.
-    int count = 0, acount = 0;
-    for(acount = 0; acount < nelements; acount++){
-        int p = CkArrayID::CkLocalBranch(destArrayID)->
-            lastKnown(elements[acount]);
+//     //make this a reusable function call later.
+//     int count = 0, acount = 0;
+//     for(acount = 0; acount < nelements; acount++){
+//         int p = CkArrayID::CkLocalBranch(destArrayID)->
+//             lastKnown(elements[acount]);
         
-        for(count = 0; count < num_pes; count ++)
-            if(tmp_pelist[count] == p)
-                break;
+//         for(count = 0; count < num_pes; count ++)
+//             if(tmp_pelist[count] == p)
+//                 break;
         
-        if(count == num_pes) {
-            tmp_pelist[num_pes ++] = p;
-        }
-    }
+//         if(count == num_pes) {
+//             tmp_pelist[num_pes ++] = p;
+//         }
+//     }
 
-    if(num_pes == 0) {
-        delete [] tmp_pelist;
-        return;
-    }
+//     if(num_pes == 0) {
+//         delete [] tmp_pelist;
+//         return;
+//     }
 
-    qsort(tmp_pelist, num_pes, sizeof(int), intCompare);
+//     qsort(tmp_pelist, num_pes, sizeof(int), intCompare);
     
-    int pdiff = 0;
-    int my_pos = 0;
-    int src_pos = 0;
-
-    int count;
-    for(count = 0; count < num_pes; count ++) {
-        if(tmp_pelist[count] == CkMyPe()){
-            my_pos = count;
-        }
-
-        if(tmp_pelist[count] == src_pos){
-            src_pos = count;
-        }        
-    }            
-
-    int n_tosend = getNumMessagesToSend(src_pos, my_pos, num_pes);
-    for(count = 0; count < n_tosend; count ++) {
-        pelist[npes ++] = tmp_pelist[(src_pos + count)%num_pes];
-    }    
-
-    delete [] tmp_pelist;    
-}
-
-int KDirectMulticastStrategy::getNumMessagesToSend(int src_pe, int my_pe, 
-                                                   int num_pes){
+//     int pdiff = 0;
+//     int my_pos = 0;
+//     int src_pos = 0;
+
+//     int count;
+//     for(count = 0; count < num_pes; count ++) {
+//         if(tmp_pelist[count] == CkMyPe()){
+//             my_pos = count;
+//         }
+
+//         if(tmp_pelist[count] == src_pos){
+//             src_pos = count;
+//         }        
+//     }            
+
+//     int n_tosend = getNumMessagesToSend(src_pos, my_pos, num_pes);
+//     for(count = 0; count < n_tosend; count ++) {
+//         pelist[npes ++] = tmp_pelist[(src_pos + count)%num_pes];
+//     }    
+
+//     delete [] tmp_pelist;    
+// }
+
+// int KDirectMulticastStrategy::getNumMessagesToSend(int src_pe, int my_pe, 
+//                                                    int num_pes){
     
-    if(src_pe == my_pe) {
-        retutn 0;
-    }
+//     if(src_pe == my_pe) {
+//         retutn 0;
+//     }
 
-    int nToSend = 0;
+//     int nToSend = 0;
 
-    int pdiff = my_pe - src_pe;
+//     int pdiff = my_pe - src_pe;
     
-    if(pdiff < 0)
-        pdiff += num_pes;
+//     if(pdiff < 0)
+//         pdiff += num_pes;
     
-    if(pdiff % kfactor != 0)
-        return 0;
+//     if(pdiff % kfactor != 0)
+//         return 0;
     
-    return (num_pes - pdiff > kfactor)? kfactor : num_pes - pdiff;
-}
+//     return (num_pes - pdiff > kfactor)? kfactor : num_pes - pdiff;
+// }
+
+// #endif
index 1235e73d6979818a5f9b00c2eab112cf507fc3c6..c44b8742f884e9a0008713d3da349cc1d3a7bd5b 100644 (file)
@@ -1,53 +1,61 @@
-#ifndef KDIRECT_MULTICAST_STRATEGY
-#define KDIRECT_MULTICAST_STRATEGY
+/* #ifdef filippo */
 
-#include "DirectMulticastStrategy.h"
+/* /\*****************  DISCLAMER ********************* */
+/*  * This class is old and not compatible. Deprecated */
+/*  **************************************************\/ */
 
-class KDirectHashObject{
- public:
-    CkVec<CkArrayIndexMax> indices;
-    int npes;
-    int *pelist;
-};
+/* #ifndef KDIRECT_MULTICAST_STRATEGY */
+/* #define KDIRECT_MULTICAST_STRATEGY */
 
+/* #include "DirectMulticastStrategy.h" */
 
-class KDirectMulticastStrategy: public DirectMultcastStrategy {
- protected:
-    int kfactor;
+/* class KDirectHashObject{ */
+/*  public: */
+/*     CkVec<CkArrayIndexMax> indices; */
+/*     int npes; */
+/*     int *pelist; */
+/* }; */
 
-    //Initialize and cache information in a section id which can be
-    //used the next time the section is multicast to.
-    virtual void initSectionID(CkSectionID *sid);
+
+/* class KDirectMulticastStrategy: public DirectMultcastStrategy { */
+/*  protected: */
+/*     int kfactor; */
+
+/*     //Initialize and cache information in a section id which can be */
+/*     //used the next time the section is multicast to. */
+/*     virtual void initSectionID(CkSectionID *sid); */
     
-    //Common Initializer for group and array constructors
-    //Every substrategy should implement its own
-    void commonKDirectInit();
+/*     //Common Initializer for group and array constructors */
+/*     //Every substrategy should implement its own */
+/*     void commonKDirectInit(); */
     
-    //Create a new multicast message with the array section in it
-    ComlibMulticastMsg * getNewMulticastMessage(CharmMessageHolder *cmsg);
+/*     //Create a new multicast message with the array section in it */
+/*     ComlibMulticastMsg * getNewMulticastMessage(CharmMessageHolder *cmsg); */
 
- public:
+/*  public: */
     
-    //Group constructor
-    KDirectMulticastStrategy(int ndestpes = 0, int *destpelist = 0);    
+/*     //Group constructor */
+/*     KDirectMulticastStrategy(int ndestpes = 0, int *destpelist = 0);     */
 
-    //Array constructor
-    KDirectMulticastStrategy(CkArrayID aid);
+/*     //Array constructor */
+/*     KDirectMulticastStrategy(CkArrayID aid); */
 
-    KDirectMulticastStrategy(CkMigrateMessage *m): Strategy(m){}
+/*     KDirectMulticastStrategy(CkMigrateMessage *m): Strategy(m){} */
     
-    //    virtual void insertMessage(CharmMessageHolder *msg);
-    virtual void doneInserting();
+/*     //    virtual void insertMessage(CharmMessageHolder *msg); */
+/*     virtual void doneInserting(); */
 
-    //Called by the converse handler function
-    virtual void handleMulticastMessage(void *msg);
+/*     //Called by the converse handler function */
+/*     virtual void handleMulticastMessage(void *msg); */
     
-    //virtual void beginProcessing(int nelements);
+/*     //virtual void beginProcessing(int nelements); */
     
-    void setKFactor(int k){ kfactor = k; }
+/*     void setKFactor(int k){ kfactor = k; } */
     
-    virtual void pup(PUP::er &p);    
-    PUPable_decl(KDirectMulticastStrategy);
-};
+/*     virtual void pup(PUP::er &p);     */
+/*     PUPable_decl(KDirectMulticastStrategy); */
+/* }; */
+
+/* #endif */
 
-#endif
+/* #endif */
index 82ccfe95e9f1a506fff6fea647ebb6b3a7584097..a61e00fdfd253e5b9367905e724084053d28a80e 100644 (file)
-#include "MPIStrategy.h"
+// #ifdef filippo
 
-#if CHARM_MPI
-MPI_Comm groupComm;
-MPI_Group group, groupWorld;
-#endif
+// #include "MPIStrategy.h"
 
-MPIStrategy::MPIStrategy() {
-    messageBuf = NULL;
-    messageCount = 0;
-    npes = CkNumPes();
-    pelist = NULL;
-}
+// #if CHARM_MPI
+// MPI_Comm groupComm;
+// MPI_Group group, groupWorld;
+// #endif
 
-MPIStrategy::MPIStrategy(int npes, int *pelist) {
-    messageBuf = NULL;
-    messageCount = 0;
+// MPIStrategy::MPIStrategy() {
+//     messageBuf = NULL;
+//     messageCount = 0;
+//     npes = CkNumPes();
+//     pelist = NULL;
+// }
 
-    this->npes = npes;
-    this->pelist = pelist;
-}
+// MPIStrategy::MPIStrategy(int npes, int *pelist) {
+//     messageBuf = NULL;
+//     messageCount = 0;
 
-void MPIStrategy::insertMessage(CharmMessageHolder *cmsg){
-    cmsg->next = messageBuf;
-    messageBuf = cmsg;    
-}
+//     this->npes = npes;
+//     this->pelist = pelist;
+// }
 
-void MPIStrategy::doneInserting(){
-#if CHARM_MPI
-    ComlibPrintf("[%d] In MPI strategy\n", CkMyPe());
+// void MPIStrategy::insertMessage(CharmMessageHolder *cmsg){
+//     cmsg->next = messageBuf;
+//     messageBuf = cmsg;    
+// }
+
+// void MPIStrategy::doneInserting(){
+// #if CHARM_MPI
+//     ComlibPrintf("[%d] In MPI strategy\n", CkMyPe());
     
-    CharmMessageHolder *cmsg = messageBuf;
-    char *buf_ptr = mpi_sndbuf;
+//     CharmMessageHolder *cmsg = messageBuf;
+//     char *buf_ptr = mpi_sndbuf;
     
-    //if(npes == 0)
-    //  npes = CkNumPes();
+//     //if(npes == 0)
+//     //  npes = CkNumPes();
     
-    for(count = 0; count < npes; count ++) {
-        ((int *)buf_ptr)[0] = 0;
-        buf_ptr += MPI_MAX_MSG_SIZE;
-    }
+//     for(count = 0; count < npes; count ++) {
+//         ((int *)buf_ptr)[0] = 0;
+//         buf_ptr += MPI_MAX_MSG_SIZE;
+//     }
     
-    buf_ptr = mpi_sndbuf;
-    for(count = 0; count < messageCount; count ++) {
-        if(npes < CkNumPes()) {
-            ComlibPrintf("[%d] Copying data to %d and rank %d\n", 
-                         cmsg->dest_proc, procMap[cmsg->dest_proc]);
-            buf_ptr = mpi_sndbuf + MPI_MAX_MSG_SIZE * procMap[cmsg->dest_proc];  
-        }
-        else
-            buf_ptr = mpi_sndbuf + MPI_MAX_MSG_SIZE * cmsg->dest_proc; 
+//     buf_ptr = mpi_sndbuf;
+//     for(count = 0; count < messageCount; count ++) {
+//         if(npes < CkNumPes()) {
+//             ComlibPrintf("[%d] Copying data to %d and rank %d\n", 
+//                          cmsg->dest_proc, procMap[cmsg->dest_proc]);
+//             buf_ptr = mpi_sndbuf + MPI_MAX_MSG_SIZE * procMap[cmsg->dest_proc];  
+//         }
+//         else
+//             buf_ptr = mpi_sndbuf + MPI_MAX_MSG_SIZE * cmsg->dest_proc; 
         
-        char * msg = cmsg->getCharmMessage();
-        envelope * env = UsrToEnv(msg);
+//         char * msg = cmsg->getCharmMessage();
+//         envelope * env = UsrToEnv(msg);
         
-        ((int *)buf_ptr)[0] = env->getTotalsize();
+//         ((int *)buf_ptr)[0] = env->getTotalsize();
         
-        ComlibPrintf("[%d] Copying message\n", CkMyPe());
-        memcpy(buf_ptr + sizeof(int), (char *)env, env->getTotalsize());
+//         ComlibPrintf("[%d] Copying message\n", CkMyPe());
+//         memcpy(buf_ptr + sizeof(int), (char *)env, env->getTotalsize());
         
-        ComlibPrintf("[%d] Deleting message\n", CkMyPe());
-        CmiFree((char *) env);
-        CharmMessageHolder *prev = cmsg;
-        cmsg = cmsg->next;
-        delete prev;
-    }
+//         ComlibPrintf("[%d] Deleting message\n", CkMyPe());
+//         CmiFree((char *) env);
+//         CharmMessageHolder *prev = cmsg;
+//         cmsg = cmsg->next;
+//         delete prev;
+//     }
     
-    //ComlibPrintf("[%d] Calling Barrier\n", CkMyPe());
-    //PMPI_Barrier(groupComm);
+//     //ComlibPrintf("[%d] Calling Barrier\n", CkMyPe());
+//     //PMPI_Barrier(groupComm);
     
-    ComlibPrintf("[%d] Calling All to all\n", CkMyPe());
-    MPI_Alltoall(mpi_sndbuf, MPI_MAX_MSG_SIZE, MPI_CHAR, mpi_recvbuf, 
-                  MPI_MAX_MSG_SIZE, MPI_CHAR, groupComm);
+//     ComlibPrintf("[%d] Calling All to all\n", CkMyPe());
+//     MPI_Alltoall(mpi_sndbuf, MPI_MAX_MSG_SIZE, MPI_CHAR, mpi_recvbuf, 
+//                   MPI_MAX_MSG_SIZE, MPI_CHAR, groupComm);
     
-    ComlibPrintf("[%d] All to all finished\n", CkMyPe());
-    buf_ptr = mpi_recvbuf;
-    for(count = 0; count < npes; count ++) {
-        int recv_msg_size = ((int *)buf_ptr)[0];
-        char * recv_msg = buf_ptr + sizeof(int);
+//     ComlibPrintf("[%d] All to all finished\n", CkMyPe());
+//     buf_ptr = mpi_recvbuf;
+//     for(count = 0; count < npes; count ++) {
+//         int recv_msg_size = ((int *)buf_ptr)[0];
+//         char * recv_msg = buf_ptr + sizeof(int);
         
-        if((recv_msg_size > 0) && recv_msg_size < MPI_MAX_MSG_SIZE) {
-            ComlibPrintf("[%d] Receiving message of size %d\n", CkMyPe(), 
-                         recv_msg_size);
-            CmiSyncSend(CkMyPe(), recv_msg_size, recv_msg);
-        }
-        buf_ptr += MPI_MAX_MSG_SIZE;
-    }
-#endif
-}
+//         if((recv_msg_size > 0) && recv_msg_size < MPI_MAX_MSG_SIZE) {
+//             ComlibPrintf("[%d] Receiving message of size %d\n", CkMyPe(), 
+//                          recv_msg_size);
+//             CmiSyncSend(CkMyPe(), recv_msg_size, recv_msg);
+//         }
+//         buf_ptr += MPI_MAX_MSG_SIZE;
+//     }
+// #endif
+// }
 
-void MPIStrategy::pup(PUP::er &p) {
-    CharmStrategy::pup(p);
+// void MPIStrategy::pup(PUP::er &p) {
+//     CharmStrategy::pup(p);
 
-    p | messageCount;
-    p | npes; 
+//     p | messageCount;
+//     p | npes; 
        
-    if(p.isUnpacking())
-        pelist = new int[npes];
-    p(pelist , npes);
+//     if(p.isUnpacking())
+//         pelist = new int[npes];
+//     p(pelist , npes);
 
-    messageBuf = NULL;
+//     messageBuf = NULL;
     
-    if(p.isUnpacking()){
-#if CHARM_MPI
-        if(npes < CkNumPes()){
-            MPI_Comm_group(MPI_COMM_WORLD, &groupWorld);
-            MPI_Group_incl(groupWorld, npes, pelist, &group);
-            MPI_Comm_create(MPI_COMM_WORLD, group, &groupComm);
-        }
-        else groupComm = MPI_COMM_WORLD;
-#endif
-    }
-}
+//     if(p.isUnpacking()){
+// #if CHARM_MPI
+//         if(npes < CkNumPes()){
+//             MPI_Comm_group(MPI_COMM_WORLD, &groupWorld);
+//             MPI_Group_incl(groupWorld, npes, pelist, &group);
+//             MPI_Comm_create(MPI_COMM_WORLD, group, &groupComm);
+//         }
+//         else groupComm = MPI_COMM_WORLD;
+// #endif
+//     }
+// }
+
+// //PUPable_def(MPIStrategy);
 
-//PUPable_def(MPIStrategy);
+// #endif
index f00f37a00a4bacea43e5a2e97e512c6e387a8d3c..d0f15740e4ee4e1d9a7f5d242ed97739401a2743 100644 (file)
@@ -1,30 +1,40 @@
-#ifndef MPI_STRATEGY
-#define MPI_STRATEGY
-
-#include "ComlibManager.h"
-
-#if CHARM_MPI
-#include "mpi.h"
-#define MPI_MAX_MSG_SIZE 1000
-#define MPI_BUF_SIZE 2000000
-char mpi_sndbuf[MPI_BUF_SIZE];
-char mpi_recvbuf[MPI_BUF_SIZE];
-#endif
-
-class MPIStrategy : public CharmStrategy {
-    CharmMessageHolder *messageBuf;
-    int messageCount;
-    int npes, *pelist;
-
- public:
-    MPIStrategy();
-    MPIStrategy(CkMigrateMessage *m) {}
-    MPIStrategy(int npes, int *pelist);
-
-    virtual void insertMessage(CharmMessageHolder *msg);
-    virtual void doneInserting();
-
-    virtual void pup(PUP::er &p);
-    PUPable_decl(MPIStrategy);
-};
-#endif
+/* #ifdef filippo */
+
+/* /\*************  DISCLAMER ******************** */
+
+/*   Currently this strategy is not in a working state! */
+
+/* *********************************************\/ */
+
+/* #ifndef MPI_STRATEGY */
+/* #define MPI_STRATEGY */
+
+/* #include "ComlibManager.h" */
+
+/* #if CHARM_MPI */
+/* #include "mpi.h" */
+/* #define MPI_MAX_MSG_SIZE 1000 */
+/* #define MPI_BUF_SIZE 2000000 */
+/* char mpi_sndbuf[MPI_BUF_SIZE]; */
+/* char mpi_recvbuf[MPI_BUF_SIZE]; */
+/* #endif */
+
+/* class MPIStrategy : public CharmStrategy { */
+/*     CharmMessageHolder *messageBuf; */
+/*     int messageCount; */
+/*     int npes, *pelist; */
+
+/*  public: */
+/*     MPIStrategy(); */
+/*     MPIStrategy(CkMigrateMessage *m) {} */
+/*     MPIStrategy(int npes, int *pelist); */
+
+/*     virtual void insertMessage(CharmMessageHolder *msg); */
+/*     virtual void doneInserting(); */
+
+/*     virtual void pup(PUP::er &p); */
+/*     PUPable_decl(MPIStrategy); */
+/* }; */
+/* #endif */
+
+/* #endif */
index 6866c97cfdf4f836641d5a4a3b04ef0239fc7795..6a662913f269764ad5743b99cf4b0e0e063ff5e4 100644 (file)
-#include "MsgPacker.h"
+// #ifdef filippo
 
-CkpvExtern(int, RecvCombinedShortMsgHdlrIdx);
+// #include "MsgPacker.h"
 
-MsgPacker::MsgPacker(){
-    nShortMsgs = 0;
-    msgList = 0;    
-}
+// CkpvExtern(int, RecvCombinedShortMsgHdlrIdx);
 
-MsgPacker::MsgPacker(CkQ<CharmMessageHolder *> &msgq, int n_msgs){
+// MsgPacker::MsgPacker(){
+//     nShortMsgs = 0;
+//     msgList = 0;    
+// }
 
-    CkAssert(n_msgs < 65536);  //16 bit field for num messages
+// MsgPacker::MsgPacker(CkQ<CharmMessageHolder *> &msgq, int n_msgs){
 
-    nShortMsgs = n_msgs;
-    msgList = new short_envelope[n_msgs];    
+//     CkAssert(n_msgs < 65536);  //16 bit field for num messages
 
-    for(int count = 0; count < n_msgs; count ++){
-        CharmMessageHolder *cmsg = msgq.deq();
-        char *msg = cmsg->getCharmMessage();
-        envelope *env = (envelope *)UsrToEnv(msg);
-        CkPackMessage(&env);
+//     nShortMsgs = n_msgs;
+//     msgList = new short_envelope[n_msgs];    
 
-        if(count == 0) {
-            aid = env->getsetArrayMgr();
-            if(aid.isZero()) 
-                CkAbort("Array packing set and ArrayID is zero");
-        }        
+//     for(int count = 0; count < n_msgs; count ++){
+//         CharmMessageHolder *cmsg = msgq.deq();
+//         char *msg = cmsg->getCharmMessage();
+//         envelope *env = (envelope *)UsrToEnv(msg);
+//         CkPackMessage(&env);
+
+//         if(count == 0) {
+//             aid = env->getsetArrayMgr();
+//             if(aid.isZero()) 
+//                 CkAbort("Array packing set and ArrayID is zero");
+//         }        
         
-        msgList[count].epIdx = env->getsetArrayEp();
-        msgList[count].size = env->getTotalsize() - sizeof(envelope);
-        msgList[count].idx = env->getsetArrayIndex();
-        msgList[count].data = msg;
-
-        CkAssert(msgList[count].size < MAX_MESSAGE_SIZE);
-        delete cmsg;
-    }
-}
-
-//Takes a queue of envelopes as char* ptrs and not charm message holders
-//Used by mesh streaming strategy
-MsgPacker::MsgPacker(CkQ<char *> &msgq, int n_msgs){
+//         msgList[count].epIdx = env->getsetArrayEp();
+//         msgList[count].size = env->getTotalsize() - sizeof(envelope);
+//         msgList[count].idx = env->getsetArrayIndex();
+//         msgList[count].data = msg;
+
+//         CkAssert(msgList[count].size < MAX_MESSAGE_SIZE);
+//         delete cmsg;
+//     }
+// }
+
+// //Takes a queue of envelopes as char* ptrs and not charm message holders
+// //Used by mesh streaming strategy
+// MsgPacker::MsgPacker(CkQ<char *> &msgq, int n_msgs){
     
-    CkAssert(n_msgs < 65536);  //16 bit field for num messages
+//     CkAssert(n_msgs < 65536);  //16 bit field for num messages
     
-    nShortMsgs = n_msgs;
-    msgList = new short_envelope[n_msgs];    
+//     nShortMsgs = n_msgs;
+//     msgList = new short_envelope[n_msgs];    
     
-    for(int count = 0; count < n_msgs; count ++){
-        envelope *env = (envelope *)msgq.deq();
-        char *msg = (char *)EnvToUsr(env);
-        CkPackMessage(&env);
-
-        if(count == 0) {
-            aid = env->getsetArrayMgr();
-            if(aid.isZero()) 
-                CkAbort("Array packing set and ArrayID is zero");
-        }        
+//     for(int count = 0; count < n_msgs; count ++){
+//         envelope *env = (envelope *)msgq.deq();
+//         char *msg = (char *)EnvToUsr(env);
+//         CkPackMessage(&env);
+
+//         if(count == 0) {
+//             aid = env->getsetArrayMgr();
+//             if(aid.isZero()) 
+//                 CkAbort("Array packing set and ArrayID is zero");
+//         }        
         
-        msgList[count].epIdx = env->getsetArrayEp();
-        msgList[count].size = env->getTotalsize() - sizeof(envelope);
-        msgList[count].idx = env->getsetArrayIndex();
-        msgList[count].data = msg;
+//         msgList[count].epIdx = env->getsetArrayEp();
+//         msgList[count].size = env->getTotalsize() - sizeof(envelope);
+//         msgList[count].idx = env->getsetArrayIndex();
+//         msgList[count].data = msg;
         
-        CkAssert(msgList[count].size < MAX_MESSAGE_SIZE);
-    }
-}
-
-MsgPacker::~MsgPacker(){
-    if(nShortMsgs > 0 && msgList != NULL) {
-        for(int count = 0; count < nShortMsgs; count ++)
-            CkFreeMsg(msgList[count].data);        
+//         CkAssert(msgList[count].size < MAX_MESSAGE_SIZE);
+//     }
+// }
+
+// MsgPacker::~MsgPacker(){
+//     if(nShortMsgs > 0 && msgList != NULL) {
+//         for(int count = 0; count < nShortMsgs; count ++)
+//             CkFreeMsg(msgList[count].data);        
         
-        delete [] msgList;
-    }
-}
-
-void MsgPacker::getMessage(CombinedMessage *&cmb_msg, int &total_size){
-    int count;
-    PUP_cmiAllocSizer sp;
-
-    CombinedMessage cmb_hdr;
-    cmb_hdr.aid = aid;
-    cmb_hdr.srcPE = CkMyPe();
-    cmb_hdr.nmsgs = nShortMsgs;
-
-    sp | cmb_hdr;
-    for(count = 0; count < nShortMsgs; count ++)
-        sp | msgList[count];
+//         delete [] msgList;
+//     }
+// }
+
+// void MsgPacker::getMessage(CombinedMessage *&cmb_msg, int &total_size){
+//     int count;
+//     PUP_cmiAllocSizer sp;
+
+//     CombinedMessage cmb_hdr;
+//     cmb_hdr.aid = aid;
+//     cmb_hdr.srcPE = CkMyPe();
+//     cmb_hdr.nmsgs = nShortMsgs;
+
+//     sp | cmb_hdr;
+//     for(count = 0; count < nShortMsgs; count ++)
+//         sp | msgList[count];
     
-    total_size = sp.size();
-    ComlibPrintf("In MsgPacker with %d bytes and %d messages\n", total_size, 
-                 nShortMsgs);
+//     total_size = sp.size();
+//     ComlibPrintf("In MsgPacker with %d bytes and %d messages\n", total_size, 
+//                  nShortMsgs);
+
+//     cmb_msg = (CombinedMessage *)CmiAlloc(sp.size());
 
-    cmb_msg = (CombinedMessage *)CmiAlloc(sp.size());
+//     PUP_toCmiAllocMem mp(cmb_msg);
+//     mp | cmb_hdr;
 
-    PUP_toCmiAllocMem mp(cmb_msg);
-    mp | cmb_hdr;
+//     for(count = 0; count < nShortMsgs; count ++)
+//         mp | msgList[count];
 
-    for(count = 0; count < nShortMsgs; count ++)
-        mp | msgList[count];
+//     CmiSetHandler(cmb_msg, CkpvAccess(RecvCombinedShortMsgHdlrIdx));
+// }
 
-    CmiSetHandler(cmb_msg, CkpvAccess(RecvCombinedShortMsgHdlrIdx));
-}
+// #endif
index cf673b1d599171d3fc763f71325c2618d96355af..5ca24356c47e92d721f9624c60756c9ed14fa9f6 100644 (file)
@@ -1,6 +1,16 @@
 #ifndef MESSAGE_PACKER_H
 #define MESSAGE_PACKER_H
 
+/**
+   @addtogroup CharmComlib
+   *@{
+
+   @file
+   
+   @brief An envelope for packing multiple messages into a single message.
+*/
+
+
 #include "charm++.h"
 #include "envelope.h"
 #include "ComlibManager.h"
@@ -171,5 +181,6 @@ inline void MsgPacker::deliver(CombinedMessage *cmb_msg){
     CmiFree(cmb_msg);
 }
 
-
+/*@}*/
 #endif
+
index 4d4f2dbacef4d4310c89af423f5c1d3fadb01de9..3594a6358a21b19884c8169a15ab80cd1fcc9e6e 100644 (file)
+/**
+   @addtogroup ComlibCharmStrategy
+   @{
+   
+   @file 
+*/
 
 #include "MultiRingMulticast.h"
 
-//Array Constructor
-MultiRingMulticast::MultiRingMulticast(CkArrayID dest_aid, int flag)
-    : DirectMulticastStrategy(dest_aid, flag){
-}
-
-
-void MultiRingMulticast::pup(PUP::er &p){
-
-    DirectMulticastStrategy::pup(p);
-}
-
-void MultiRingMulticast::beginProcessing(int  nelements){
-
-    DirectMulticastStrategy::beginProcessing(nelements);
-}
-
-/*
-inline int getMyId(int *pelist, int npes, int mype) {
-    int myid = -1;
-
-    for(int count = 0; count < npes; count ++)
-        if(mype == pelist[count])
-            myid = count;
-
-    //if(myid == -1)
-    //  CkPrintf("Warning myid = -1\n");
-
-    return myid;
-}
-
-
-//Assumes a sorted input. Returns the next processor greater a given
-//current pe
-inline void getNextPe(int *pelist, int npes, int mype, int &nextpe) {
-    
-    nextpe = pelist[0];
-    
-    int count= 0;
-    for(count = 0; count < npes; count++)
-        if(pelist[count] > mype)
-            break;
-    
-    if(count < npes) 
-        nextpe = pelist[count];
-    
-    return;
-}
-
-
-inline int getMidPe(int *pelist, int npes, int src_pe) {
-    
-    int my_id = 0;
-    int mid_pe = -1;
-    
-    my_id = getMyId(pelist, npes, src_pe);    
-    
-    CkAssert(my_id >= 0 && my_id < npes);
-
-    if(my_id < npes/2)
-        mid_pe = pelist[npes/2 + my_id];        
-    else
-        mid_pe = pelist[my_id % (npes/2)];
-    
-    //if(mid_pe == -1)
-    //  CkPrintf("Warning midpe = -1\n");
-
-    return mid_pe;
-}
-*/
 
 //Unlike ring the source here sends two or more messages while all
 //elements along the ring only send one.
 
-ComlibSectionHashObject *MultiRingMulticast::createObjectOnSrcPe
-(int nelements, CkArrayIndexMax *elements){
+void MultiRingMulticastStrategy::createObjectOnSrcPe(ComlibSectionHashObject *obj, int npes, ComlibMulticastIndexCount *pelist) {
 
-    ComlibSectionHashObject *obj = new ComlibSectionHashObject();
+  obj->npes = 0;
+  obj->pelist = 0;
 
-    obj->npes = 0;
-    obj->pelist = 0;
+  if(npes == 0) return;
 
-    int *pelist;
-    int npes;
-    sinfo.getPeList(nelements, elements, npes, pelist);
-    
-    sinfo.getLocalIndices(nelements, elements, obj->indices);
-
-    if(npes == 0)
-        return obj;
-
-    if(npes < 4) {
-        // direct sending, take out ourself from the list!
-       for (int i=0; i<npes; ++i) {
-         if (pelist[i] == CkMyPe()) {
-           pelist[i] = pelist[--npes];
-           break;
-         }
-       }
-        obj->npes = npes;
-        obj->pelist = pelist;
-       //CkPrintf("MultiRingMulticast::createObjectOnSrcPe, less than 4 procs\n");
-
-        return obj;
-    }
-    
-    //CkPrintf("MultiRingMulticast::createObjectOnSrcPe, more than 3 procs\n");
-    //pelist[npes ++] = CkMyPe();
-    qsort(pelist, npes, sizeof(int), intCompare);
-
-    /*
-      char dump[2560];
-      sprintf(dump, "Section on %d : ", CkMyPe());
-      for(int count = 0; count < npes; count ++) {
-      sprintf(dump, "%s, %d", dump, pelist[count]);
+  if(npes < 4) {
+    // direct sending, take out ourself from the list!
+    obj->npes = npes;
+    for (int i=0; i<npes; ++i) if (pelist[i].pe == CkMyPe()) obj->npes --;
+    obj->pelist = new int[obj->npes];
+    for (int i=0, count=0; i<npes; ++i) {
+      if (pelist[i].pe != CkMyPe()) {
+       obj->pelist[count] = pelist[i].pe;
+       count++;
       }
-    
-      CkPrintf("%s\n\n", dump);
-    */
-    
-    int myid = -1; // getMyId(pelist, npes, CkMyPe());    
-    for (int i=0; i<npes; ++i) {
-      if (pelist[i] == CkMyPe()) {
-       myid = i;
-       break;
-      }
-    }
-
-    //CkAssert(myid >= 0 && myid < npes);
-
-    int breaking = npes/2; /* 0 : breaking-1    is the first ring
-                             breaking : npes-1 is the second ring
-                          */
-
-    int next_id = myid + 1;
-    // wrap nextpe around the ring
-    if(myid < breaking) {
-      if (next_id >= breaking) next_id = 0;
-    } else {
-      if (next_id >= npes) next_id = breaking;
     }
+    return;
+  }
     
-    int mid_id;
-    if (myid < breaking) {
-      mid_id = myid + breaking;
-      if (mid_id < breaking) mid_id = breaking;
-    } else {
-      mid_id = myid - breaking;
-      if (mid_id >= breaking) mid_id = 0;
-    }
-    //mid_pe = getMidPe(pelist, npes, CkMyPe());
-    
-    if(pelist[next_id] != CkMyPe()) {
-        obj->pelist = new int[2];
-        obj->npes = 2;
-        
-        obj->pelist[0] = pelist[next_id];
-        obj->pelist[1] = pelist[mid_id];
+  int myid = -1; // getMyId(pelist, npes, CkMyPe());    
+  for (int i=0; i<npes; ++i) {
+    if (pelist[i].pe == CkMyPe()) {
+      myid = i;
+      break;
     }
-    else {
-        CkAbort("Warning Should not be here !!!!!!!!!\n");
-        //obj->pelist = new int[1];
-        //obj->npes = 1;
+  }
+
+  int breaking = npes/2; /* 0 : breaking-1    is the first ring
+                           breaking : npes-1 is the second ring
+                        */
+
+  int next_id&nb