Adding communication library in src/ck-com and src/conv-com
authorSameer Kumar <skumar2@uiuc.edu>
Wed, 19 May 2004 23:31:05 +0000 (23:31 +0000)
committerSameer Kumar <skumar2@uiuc.edu>
Wed, 19 May 2004 23:31:05 +0000 (23:31 +0000)
53 files changed:
src/ck-com/BroadcastStrategy.C [new file with mode: 0644]
src/ck-com/BroadcastStrategy.h [new file with mode: 0644]
src/ck-com/ComlibManager.C [new file with mode: 0644]
src/ck-com/ComlibManager.ci [new file with mode: 0644]
src/ck-com/ComlibManager.h [new file with mode: 0644]
src/ck-com/DirectMulticastStrategy.C [new file with mode: 0644]
src/ck-com/DirectMulticastStrategy.h [new file with mode: 0644]
src/ck-com/DummyStrategy.C [new file with mode: 0644]
src/ck-com/DummyStrategy.h [new file with mode: 0644]
src/ck-com/EachToManyMulticastStrategy.C [new file with mode: 0644]
src/ck-com/EachToManyMulticastStrategy.h [new file with mode: 0644]
src/ck-com/KDirectMulticastStrategy.C [new file with mode: 0644]
src/ck-com/KDirectMulticastStrategy.h [new file with mode: 0644]
src/ck-com/MPIStrategy.C [new file with mode: 0644]
src/ck-com/MPIStrategy.h [new file with mode: 0644]
src/ck-com/MeshStreamingStrategy.C [new file with mode: 0644]
src/ck-com/MeshStreamingStrategy.h [new file with mode: 0644]
src/ck-com/MsgPacker.C [new file with mode: 0644]
src/ck-com/MsgPacker.h [new file with mode: 0644]
src/ck-com/NodeMulticast.C [new file with mode: 0644]
src/ck-com/NodeMulticast.h [new file with mode: 0644]
src/ck-com/PipeBroadcastStrategy.C [new file with mode: 0644]
src/ck-com/PipeBroadcastStrategy.h [new file with mode: 0644]
src/ck-com/PrioStreaming.C [new file with mode: 0644]
src/ck-com/PrioStreaming.h [new file with mode: 0644]
src/ck-com/RingMulticastStrategy.C [new file with mode: 0644]
src/ck-com/RingMulticastStrategy.h [new file with mode: 0644]
src/ck-com/StreamingStrategy.C [new file with mode: 0644]
src/ck-com/StreamingStrategy.h [new file with mode: 0644]
src/conv-com/3dgridrouter.C [new file with mode: 0644]
src/conv-com/3dgridrouter.h [new file with mode: 0644]
src/conv-com/comlib.h [new file with mode: 0644]
src/conv-com/convcomlibmanager.C [new file with mode: 0644]
src/conv-com/convcomlibmanager.h [new file with mode: 0644]
src/conv-com/convcomlibstrategy.C [new file with mode: 0644]
src/conv-com/convcomlibstrategy.h [new file with mode: 0644]
src/conv-com/de.C [new file with mode: 0644]
src/conv-com/de.h [new file with mode: 0644]
src/conv-com/dummystrategy.C [new file with mode: 0644]
src/conv-com/dummystrategy.h [new file with mode: 0644]
src/conv-com/graphrouter.C [new file with mode: 0644]
src/conv-com/graphrouter.h [new file with mode: 0644]
src/conv-com/gridrouter.C [new file with mode: 0644]
src/conv-com/gridrouter.h [new file with mode: 0644]
src/conv-com/hypercubetopology.C [new file with mode: 0644]
src/conv-com/hypercubetopology.h [new file with mode: 0644]
src/conv-com/petable.C [new file with mode: 0644]
src/conv-com/petable.h [new file with mode: 0644]
src/conv-com/router.h [new file with mode: 0644]
src/conv-com/routerstrategy.C [new file with mode: 0644]
src/conv-com/routerstrategy.h [new file with mode: 0644]
src/conv-com/treerouter.C [new file with mode: 0644]
src/conv-com/treerouter.h [new file with mode: 0644]

diff --git a/src/ck-com/BroadcastStrategy.C b/src/ck-com/BroadcastStrategy.C
new file mode 100644 (file)
index 0000000..63ea02f
--- /dev/null
@@ -0,0 +1,138 @@
+//Broadcast strategy for charm++ programs using the net version
+//This stategy will wonly work for groups.
+//This strategy implements a tree based broadcast
+//I will extent it for arrays later.
+//Developed by Sameer Kumar 04/10/04
+
+#include "BroadcastStrategy.h"
+
+CkpvExtern(CkGroupID, cmgrID);
+extern int sfactor;
+
+static void recv_bcast_handler(void *msg) {
+    int instid = CmiGetXHandler(msg);
+    BroadcastStrategy *bstrat = (BroadcastStrategy *)
+        CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()->getStrategy(instid);
+    
+    bstrat->handleMessage((char *)msg);    
+}
+
+//Constructor, 
+//Can read spanning factor from command line
+BroadcastStrategy::BroadcastStrategy(int topology) : 
+    CharmStrategy(), _topology(topology) {
+    spanning_factor = DEFAULT_BROADCAST_SPANNING_FACTOR;
+    if(sfactor > 0)
+        spanning_factor = sfactor;
+    
+}
+
+
+//Receives the message and sends it along the spanning tree.
+void BroadcastStrategy::insertMessage(CharmMessageHolder *cmsg){
+    CkPrintf("[%d] BRAODCASTING\n", CkMyPe());
+
+    char *msg = cmsg->getCharmMessage();
+    if(_topology == USE_HYPERCUBE) {
+        envelope *env = UsrToEnv(msg);
+        env->setSrcPe(0);    
+    }
+    handleMessage((char *)UsrToEnv(msg));
+    
+    delete cmsg;
+}
+
+//not implemented here because no bracketing is required for this strategy
+void BroadcastStrategy::doneInserting(){
+}
+
+
+//register the converse handler to recieve the broadcast message
+void BroadcastStrategy::beginProcessing(int nelements) {
+    handlerId = CkRegisterHandler((CmiHandler)recv_bcast_handler);
+}
+
+void BroadcastStrategy::handleMessage(char *msg) {
+    if(_topology == USE_TREE)
+        handleTree(msg);
+    else if(_topology == USE_HYPERCUBE) 
+        handleHypercube(msg);
+    else CkAbort("Unknown Topology");
+}
+
+void BroadcastStrategy::handleTree(char *msg){
+    
+    envelope *env = (envelope *)msg;
+    int startpe = env->getSrcPe();
+    int size = env->getTotalsize();
+    
+    CkAssert(startpe>=0 && startpe < CkNumPes());
+    
+    CmiSetHandler(msg, handlerId);
+    CmiSetXHandler(msg, getInstance());    
+    
+    //Sending along the spanning tree
+    //Gengbins tree building code stolen from the MPI machine layer    
+    int i;
+    for (i=1; i<=spanning_factor; i++) {
+        
+        int p = CkMyPe() - startpe;
+        if (p<0) 
+            p += CkNumPes();
+
+        p = spanning_factor*p + i;
+
+        if (p > CkNumPes() - 1) break;
+
+        p += startpe;
+        p = p % CkNumPes();
+
+        CkAssert(p>=0 && p < CkNumPes() && p != CkMyPe());
+
+        CmiSyncSend(p, size, msg);
+    }
+
+    CkSendMsgBranch(env->getEpIdx(), EnvToUsr(env), CkMyPe(), 
+                    env->getGroupNum());
+}
+
+
+void BroadcastStrategy::handleHypercube(char *msg){
+    envelope *env = (envelope *)msg;
+    int curcycle = env->getSrcPe();
+    int i;
+    int size = env->getTotalsize();
+    
+    double logp = CmiNumPes();
+    logp = log(logp)/log(2.0);
+    logp = ceil(logp);
+    
+    //CkPrintf("In hypercube %d, %d\n", (int)logp, curcycle); 
+    
+    /* assert(startpe>=0 && startpe<_Cmi_numpes); */
+    CmiSetHandler(msg, handlerId);
+    CmiSetXHandler(msg, getInstance());    
+
+    for (i = logp - curcycle - 1; i >= 0; i--) {
+        int p = CkMyPe() ^ (1 << i);
+
+        int newcycle = ++curcycle;
+        //CkPrintf("%d --> %d, %d\n", CkMyPe(), p, newcycle); 
+
+        env->setSrcPe(newcycle);
+        if(p < CmiNumPes()) {
+            CmiSyncSendFn(p, size, msg);
+        }
+    }
+
+    CkSendMsgBranch(env->getEpIdx(), EnvToUsr(env), CkMyPe(), 
+                    env->getGroupNum());
+}
+
+
+//Pack the group id and the entry point of the user message
+void BroadcastStrategy::pup(PUP::er &p){
+    Strategy::pup(p);    
+    p | spanning_factor;
+    p | _topology;
+}
diff --git a/src/ck-com/BroadcastStrategy.h b/src/ck-com/BroadcastStrategy.h
new file mode 100644 (file)
index 0000000..eeda41b
--- /dev/null
@@ -0,0 +1,35 @@
+#ifndef BRAODCAST_STRATEGY
+#define BRAODCAST_STRATEGY
+
+#define DEFAULT_BROADCAST_SPANNING_FACTOR 4
+
+#include "ComlibManager.h"
+
+//Broadcast strategy for charm++ programs using the net version
+//This stategy will wonly work for groups.
+//This strategy implements a tree based broadcast
+//I will extent it for arrays later.
+//Developed by Sameer Kumar 04/10/04
+
+class BroadcastStrategy : public CharmStrategy {
+
+    int handlerId;
+    int spanning_factor;
+    int _topology;
+
+    void handleTree(char *msg);
+    void handleHypercube(char *msg);
+
+ public:
+    BroadcastStrategy(int topology = USE_HYPERCUBE);
+    BroadcastStrategy(CkMigrateMessage *){}
+    void insertMessage(CharmMessageHolder *msg);
+    void doneInserting();
+
+    void handleMessage(char *msg);
+    void beginProcessing(int nelements);
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(BroadcastStrategy);
+};
+#endif
diff --git a/src/ck-com/ComlibManager.C b/src/ck-com/ComlibManager.C
new file mode 100644 (file)
index 0000000..81d7f1b
--- /dev/null
@@ -0,0 +1,800 @@
+#include "ComlibManager.h"
+
+#include "EachToManyMulticastStrategy.h"
+#include "DirectMulticastStrategy.h"
+#include "StreamingStrategy.h"
+#include "DummyStrategy.h"
+#include "MPIStrategy.h"
+#include "NodeMulticast.h"
+#include "MsgPacker.h"
+#include "RingMulticastStrategy.h"
+#include "PipeBroadcastStrategy.h"
+#include "BroadcastStrategy.h"
+#include "MeshStreamingStrategy.h"
+#include "PrioStreaming.h"
+
+CkpvExtern(int, RecvdummyHandle);
+
+CkpvDeclare(int, RecvmsgHandle);
+CkpvDeclare(int, RecvCombinedShortMsgHdlrIdx);
+CkpvDeclare(CkGroupID, cmgrID);
+CkpvExtern(ConvComlibManager *, conv_comm_ptr);
+
+//handler to receive array messages
+void recv_array_msg(void *msg){
+
+    if(msg == NULL)
+        return;
+    
+    ComlibPrintf("%d:In recv_msg\n", CkMyPe());
+
+    register envelope* env = (envelope *)msg;
+    env->setUsed(0);
+    env->getsetArrayHops()=1;
+    CkUnpackMessage(&env);
+
+    /*
+    CProxyElement_ArrayBase ap(env->getsetArrayMgr(), env->getsetArrayIndex());
+    ComlibPrintf("%d:Array Base created\n", CkMyPe());
+    ap.ckSend((CkArrayMessage *)EnvToUsr(env), env->getsetArrayEp());
+    */
+    
+    CkArray *a=(CkArray *)_localBranch(env->getsetArrayMgr());
+    a->deliver((CkArrayMessage *)EnvToUsr(env), CkDeliver_queue, CmiTrue);    
+
+    ComlibPrintf("%d:Out of recv_msg\n", CkMyPe());
+    return;
+}
+
+void recv_combined_array_msg(void *msg){
+    if(msg == NULL)
+        return;
+    
+    ComlibPrintf("%d:In recv_combined_array_msg\n", CkMyPe());
+
+    MsgPacker::deliver((CombinedMessage *)msg);
+}
+
+ComlibManager::ComlibManager(){
+    init();
+    ComlibPrintf("In comlibmanager constructor\n");
+}
+
+void ComlibManager::init(){
+    
+    PUPable_reg(CharmStrategy);
+    PUPable_reg(CharmMessageHolder);
+    
+    //comm_debug = 1;
+    
+    CkpvAccess(RecvmsgHandle) = CkRegisterHandler((CmiHandler)recv_array_msg);
+    CkpvAccess(RecvCombinedShortMsgHdlrIdx) = 
+        CkRegisterHandler((CmiHandler)recv_combined_array_msg);
+    
+    section_send_event = traceRegisterUserEvent("ArraySectionMulticast");
+    
+    npes = CkNumPes();
+    pelist = NULL;
+    nstrats = 0;
+
+    CkpvInitialize(CkGroupID, cmgrID);
+    CkpvAccess(cmgrID) = thisgroup;
+
+    dummyArrayIndex.nInts = 0;
+
+    curStratID = 0;
+    prevStratID = -1;
+    //prioEndIterationFlag = 1;
+
+    strategyTable = CpvAccess(conv_comm_ptr)->getStrategyTable();
+    
+    receivedTable = 0;
+    flushTable = 0;
+    totalMsgCount = 0;
+    totalBytes = 0;
+    nIterations = 0;
+    barrierReached = 0;
+    barrier2Reached = 0;
+
+    isRemote = 0;
+    remotePe = -1;
+
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+    cgproxy[0].barrier();
+}
+
+//First barrier makes sure that the communication library group 
+//has been created on all processors
+void ComlibManager::barrier(){
+  static int bcount = 0;
+  ComlibPrintf("In barrier %d\n", bcount);
+  if(CkMyPe() == 0) {
+    bcount ++;
+    if(bcount == CkNumPes()){
+      barrierReached = 1;
+      doneCreating();
+    }
+  }
+}
+
+//Has finished passing the strategy list to all the processors
+void ComlibManager::barrier2(){
+  static int bcount = 0;
+  if(CkMyPe() == 0) {
+    bcount ++;
+    ComlibPrintf("In barrier2 %d\n", bcount);
+    if(bcount == CkNumPes()) {
+        CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+        cgproxy.resumeFromBarrier2();
+    }
+  }
+}
+
+//Registers a set of strategies with the communication library
+ComlibInstanceHandle ComlibManager::createInstance() {
+  
+    ListOfStrategies.insertAtEnd(NULL);
+    nstrats++;
+    
+    ComlibInstanceHandle cinst(nstrats - 1, CkpvAccess(cmgrID));  
+    return cinst;
+}
+
+void ComlibManager::registerStrategy(int pos, CharmStrategy *strat) {
+    ListOfStrategies[pos] = strat;
+}
+
+//End of registering function, if barriers have been reached send them over
+void ComlibManager::doneCreating() {
+    if(!barrierReached)
+      return;    
+
+    ComlibPrintf("Sending Strategies %d, %d\n", nstrats, 
+                 ListOfStrategies.length());
+
+    if(nstrats == 0)
+        return;
+
+    StrategyWrapper sw;
+    sw.s_table = new Strategy* [nstrats];
+    sw.nstrats = nstrats;
+    
+    for (int count=0; count<nstrats; count++)
+        sw.s_table[count] = ListOfStrategies[count];
+
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+    cgproxy.receiveTable(sw);
+}
+
+//Called when the array/group element starts sending messages
+void ComlibManager::beginIteration(){
+    //right now does not do anything might need later
+    ComlibPrintf("[%d]:In Begin Iteration %d\n", CkMyPe(), (* strategyTable)[0].elementCount);
+    //prioEndIterationFlag = 0;
+}
+
+void ComlibManager::setInstance(int instID){
+
+    curStratID = instID;
+    ComlibPrintf("[%d]:In setInstance\n", CkMyPe(), (* strategyTable)[instID].elementCount);
+}
+
+//called when the array elements has finished sending messages
+void ComlibManager::endIteration(){
+    //    prioEndIterationFlag = 1;
+    prevStratID = -1;
+    
+    ComlibPrintf("[%d]:In End Iteration(%d) %d, %d\n", CkMyPe(), curStratID, 
+                 (* strategyTable)[curStratID].elementCount, (* strategyTable)[curStratID].numElements);
+
+    if(isRemote) {
+        isRemote = 0;
+        sendRemote();
+        return;
+    }
+
+    if(!receivedTable) {
+        (* strategyTable)[curStratID].nEndItr++;
+        return;
+    }        
+    
+    (* strategyTable)[curStratID].elementCount++;
+    int count = 0;
+    flushTable = 1;
+    
+    if((* strategyTable)[curStratID].elementCount == (* strategyTable)[curStratID].numElements) {
+        
+        ComlibPrintf("[%d]:In End Iteration %d\n", CkMyPe(), (* strategyTable)[curStratID].elementCount);
+        
+        nIterations ++;
+        
+        if(nIterations == LEARNING_PERIOD) {
+            //CkPrintf("Sending %d, %d\n", totalMsgCount, totalBytes);
+            CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+            cgproxy[0].learnPattern(totalMsgCount, totalBytes);
+        }
+        
+        if(barrier2Reached) {      
+           (* strategyTable)[curStratID].strategy->doneInserting();
+        }
+       else (* strategyTable)[curStratID].call_doneInserting = 1;
+       
+        (* strategyTable)[curStratID].elementCount = 0;
+    }
+    ComlibPrintf("After EndIteration\n");
+}
+
+//receive the list of strategies
+//Insert the strategies into the strategy table in converse comm lib.
+//CpvAccess(conv_comm_ptr) points to the converse commlib instance
+void ComlibManager::receiveTable(StrategyWrapper sw){
+    
+    ComlibPrintf("[%d] In receiveTable %d\n", CkMyPe(), sw.nstrats);
+
+    receivedTable = 1;
+    nstrats = sw.nstrats;
+
+    CkArrayID st_aid;
+    int st_nelements;
+    CkArrayIndexMax *st_elem;
+
+    int count = 0;
+    for(count = 0; count < nstrats; count ++) {
+        CharmStrategy *cur_strategy = (CharmStrategy *)sw.s_table[count];
+        
+        //set the instance to the current count
+        //currently all strategies are being copied to all processors
+        //later strategies will be selectively copied
+        cur_strategy->setInstance(count);  
+        CpvAccess(conv_comm_ptr)->insertStrategy(cur_strategy);
+        
+        ComlibPrintf("[%d] Inserting strategy \n", CkMyPe());       
+
+        if(cur_strategy->getType() == ARRAY_STRATEGY &&
+           cur_strategy->isBracketed()){ 
+
+            ComlibPrintf("Inserting Array Listener\n");
+
+            ComlibArrayInfo as = ((CharmStrategy *)cur_strategy)->ainfo;
+            as.getSourceArray(st_aid, st_elem, st_nelements);
+            
+            if(st_aid.isZero())
+                CkAbort("Array ID is zero");
+            
+            ComlibArrayListener *calistener = 
+                CkArrayID::CkLocalBranch(st_aid)->getComlibArrayListener();
+            
+            calistener->registerStrategy(&((* strategyTable)[count]));
+        }              
+  
+        if(cur_strategy->getType() == GROUP_STRATEGY){
+            (* strategyTable)[count].numElements = 1;
+        }
+        
+        cur_strategy->beginProcessing((* strategyTable)[count].numElements); 
+        
+        ComlibPrintf("[%d] endIteration from receiveTable %d, %d\n", 
+                     CkMyPe(), count,
+                     (* strategyTable)[count].nEndItr);
+                         
+        curStratID = count;
+        for(int itr = 0; itr < (* strategyTable)[count].nEndItr; itr++) 
+            endIteration();            
+    }           
+    
+    ComlibPrintf("receivedTable %d\n", nstrats);
+    
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+    cgproxy[0].barrier2();
+}
+
+void ComlibManager::resumeFromBarrier2(){
+    barrier2Reached = 1;
+    
+    ComlibPrintf("[%d] Barrier 2 reached\n", CkMyPe());
+
+    if(flushTable) {
+      for (int count = 0; count < nstrats; count ++) {
+          if (!(* strategyTable)[count].tmplist.isEmpty()) {
+              CharmMessageHolder *cptr;
+              while (!(* strategyTable)[count].tmplist.isEmpty())
+                  (* strategyTable)[count].strategy->insertMessage
+                      ((* strategyTable)[count].tmplist.deq());
+          }
+          
+          if ((* strategyTable)[count].call_doneInserting) {
+              ComlibPrintf("[%d] Calling done inserting \n", CkMyPe());
+              (* strategyTable)[count].strategy->doneInserting();
+          }
+      }
+    }
+    
+    ComlibPrintf("[%d] After Barrier2\n", CkMyPe());
+}
+
+extern int _charmHandlerIdx;
+
+void ComlibManager::ArraySend(CkDelegateData *pd,int ep, void *msg, 
+                              const CkArrayIndexMax &idx, CkArrayID a){
+    
+    ComlibPrintf("[%d] In Array Send\n", CkMyPe());
+    /*
+    if(curStratID != prevStratID && prioEndIterationFlag) {        
+        CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+        ComlibPrintf("[%d] Array Send calling prio end iteration\n", 
+                     CkMyPe());
+        PrioMsg *pmsg = new(8 * sizeof(int)) PrioMsg();
+        int mprio = -100;
+        *(int *)CkPriorityPtr(pmsg) = mprio;
+        pmsg->instID = curStratID;
+        CkSetQueueing(pmsg, CK_QUEUEING_BFIFO);
+        cgproxy[CkMyPe()].prioEndIteration(pmsg);
+        prioEndIterationFlag = 0;
+    }        
+    prevStratID = curStratID;            
+    */
+
+    CkArrayIndexMax myidx = idx;
+    int dest_proc = CkArrayID::CkLocalBranch(a)->lastKnown(myidx);
+    
+    //ComlibPrintf("Send Data %d %d %d %d\n", CkMyPe(), dest_proc, 
+    //  UsrToEnv(msg)->getTotalsize(), receivedTable);
+
+    register envelope * env = UsrToEnv(msg);
+    
+    env->getsetArrayMgr()=a;
+    env->getsetArraySrcPe()=CkMyPe();
+    env->getsetArrayEp()=ep;
+    env->getsetArrayHops()=0;
+    env->getsetArrayIndex()=idx;
+    env->setUsed(0);
+    
+    CkPackMessage(&env);
+    CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
+    
+    if(isRemote) {
+        CharmMessageHolder *cmsg = new 
+            CharmMessageHolder((char *)msg, dest_proc);
+
+        remoteQ.enq(cmsg);
+        return;
+    }
+
+    if(dest_proc == CkMyPe()){
+        CProxyElement_ArrayBase ap(a,idx);
+        ap.ckSend((CkArrayMessage *)msg, ep);
+        return;
+    }
+
+    totalMsgCount ++;
+    totalBytes += UsrToEnv(msg)->getTotalsize();
+
+    CharmMessageHolder *cmsg = new 
+        CharmMessageHolder((char *)msg, dest_proc);
+    //get rid of the new.
+
+    ComlibPrintf("Before Insert\n");
+
+    if (receivedTable)
+      (* strategyTable)[curStratID].strategy->insertMessage(cmsg);
+    else {
+        flushTable = 1;
+        (* strategyTable)[curStratID].tmplist.enq(cmsg);
+    }
+
+    //CmiPrintf("After Insert\n");
+}
+
+
+#include "qd.h"
+//CpvExtern(QdState*, _qd);
+
+void ComlibManager::GroupSend(CkDelegateData *pd,int ep, void *msg, int onPE, CkGroupID gid){
+    
+    int dest_proc = onPE;
+    /*
+    if(curStratID != prevStratID && prioEndIterationFlag) {        
+        CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+        ComlibPrintf("[%d] Array Send calling prio end iteration\n", 
+                     CkMyPe());
+        PrioMsg *pmsg = new(8 * sizeof(int)) PrioMsg;
+        *(int *)CkPriorityPtr(pmsg) = -0x7FFFFFFF;
+        CkSetQueueing(pmsg, CK_QUEUEING_IFIFO);
+        cgproxy[CkMyPe()].prioEndIteration(pmsg);
+        prioEndIterationFlag = 0;
+    }        
+    prevStratID = curStratID;            
+    */
+
+    ComlibPrintf("Send Data %d %d %d\n", CkMyPe(), dest_proc, 
+                 UsrToEnv(msg)->getTotalsize());
+
+    register envelope * env = UsrToEnv(msg);
+    if(dest_proc == CkMyPe()){
+        _SET_USED(env, 0);
+        CkSendMsgBranch(ep, msg, dest_proc, gid);
+        return;
+    }
+    
+    CpvAccess(_qd)->create(1);
+
+    env->setMsgtype(ForBocMsg);
+    env->setEpIdx(ep);
+    env->setGroupNum(gid);
+    env->setSrcPe(CkMyPe());
+    env->setUsed(0);
+
+    CkPackMessage(&env);
+    CmiSetHandler(env, _charmHandlerIdx);
+
+    CharmMessageHolder *cmsg = new CharmMessageHolder((char *)msg, dest_proc); 
+    //get rid of the new.
+    
+    if(receivedTable)
+        (* strategyTable)[curStratID].strategy->insertMessage(cmsg);
+    else {
+        flushTable = 1;
+        (* strategyTable)[curStratID].tmplist.enq(cmsg);
+    }
+}
+
+void ComlibManager::ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a){
+    ComlibPrintf("[%d] Array Broadcast \n", CkMyPe());
+
+    register envelope * env = UsrToEnv(m);
+    env->getsetArrayMgr()=a;
+    env->getsetArraySrcPe()=CkMyPe();
+    env->getsetArrayEp()=ep;
+    env->getsetArrayHops()=0;
+    env->getsetArrayIndex()= dummyArrayIndex;
+    
+    CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
+
+    CkSectionInfo minfo;
+    minfo.type = COMLIB_MULTICAST_MESSAGE;
+    minfo.sInfo.cInfo.instId = curStratID;
+    minfo.sInfo.cInfo.status = COMLIB_MULTICAST_ALL;  
+    minfo.sInfo.cInfo.id = 0; 
+    minfo.pe = CkMyPe();
+    ((CkMcastBaseMsg *)m)->_cookie = minfo;       
+
+    CharmMessageHolder *cmsg = new 
+        CharmMessageHolder((char *)m, IS_MULTICAST);
+    cmsg->npes = 0;
+    cmsg->pelist = NULL;
+    cmsg->sec_id = NULL;
+
+    multicast(cmsg);
+}
+
+void ComlibManager::ArraySectionSend(CkDelegateData *pd,int ep, void *m, CkArrayID a, 
+                                     CkSectionID &s) {
+
+#ifndef CMK_OPTIMIZE
+    traceUserEvent(section_send_event);
+#endif
+
+    ComlibPrintf("[%d] Array Section Send \n", CkMyPe());
+
+    register envelope * env = UsrToEnv(m);
+    env->getsetArrayMgr()=a;
+    env->getsetArraySrcPe()=CkMyPe();
+    env->getsetArrayEp()=ep;
+    env->getsetArrayHops()=0;
+    env->getsetArrayIndex()= dummyArrayIndex;
+    
+    CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
+    
+    env->setUsed(0);    
+    CkPackMessage(&env);
+    
+    totalMsgCount ++;
+    totalBytes += env->getTotalsize();
+
+    //Provide a dummy dest proc as it does not matter for mulitcast 
+    CharmMessageHolder *cmsg = new CharmMessageHolder((char *)m,IS_MULTICAST);
+    cmsg->npes = 0;
+    cmsg->sec_id = &s;
+
+    CkSectionInfo minfo;
+    minfo.type = COMLIB_MULTICAST_MESSAGE;
+    minfo.sInfo.cInfo.instId = curStratID;
+    minfo.sInfo.cInfo.status = COMLIB_MULTICAST_ALL;  
+    minfo.sInfo.cInfo.id = 0; 
+    minfo.pe = CkMyPe();
+    ((CkMcastBaseMsg *)m)->_cookie = minfo;    
+    //    s.npes = 0;
+    //s.pelist = NULL;
+
+    multicast(cmsg);
+}
+
+void ComlibManager::GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g) {
+    register envelope * env = UsrToEnv(m);
+
+    CpvAccess(_qd)->create(1);
+
+    env->setMsgtype(ForBocMsg);
+    env->setEpIdx(ep);
+    env->setGroupNum(g);
+    env->setSrcPe(CkMyPe());
+    env->setUsed(0);
+
+    CkPackMessage(&env);
+    CmiSetHandler(env, _charmHandlerIdx);
+    
+    //Provide a dummy dest proc as it does not matter for mulitcast 
+    CharmMessageHolder *cmsg = new CharmMessageHolder((char *)m,IS_MULTICAST);
+    
+    cmsg->npes = 0;
+    cmsg->pelist = NULL;
+
+    multicast(cmsg);
+}
+
+void ComlibManager::multicast(CharmMessageHolder *cmsg) {
+
+    register envelope * env = UsrToEnv(cmsg->getCharmMessage());    
+    ComlibPrintf("[%d]: In multicast\n", CkMyPe());
+
+    env->setUsed(0);    
+    CkPackMessage(&env);
+
+    //Will be used to detect multicast message for learning
+    totalMsgCount ++;
+    totalBytes += env->getTotalsize();
+    
+    if (receivedTable)
+       (* strategyTable)[curStratID].strategy->insertMessage(cmsg);
+    else {
+        flushTable = 1;
+       ComlibPrintf("Enqueuing message in tmplist at %d\n", curStratID);
+        (* strategyTable)[curStratID].tmplist.enq(cmsg);
+    }
+
+    ComlibPrintf("After multicast\n");
+}
+
+/*
+void ComlibManager::multicast(void *msg, int npes, int *pelist) {
+    register envelope * env = UsrToEnv(msg);
+    
+    ComlibPrintf("[%d]: In multicast\n", CkMyPe());
+
+    env->setUsed(0);    
+    CkPackMessage(&env);
+    CmiSetHandler(env, CkpvAccess(RecvmsgHandle));
+    
+    totalMsgCount ++;
+    totalBytes += env->getTotalsize();
+
+    CharmMessageHolder *cmsg = new 
+    CharmMessageHolder((char *)msg,IS_MULTICAST);
+    cmsg->npes = npes;
+    cmsg->pelist = pelist;
+    //Provide a dummy dest proc as it does not matter for mulitcast 
+    //get rid of the new.
+    
+    if (receivedTable)
+       (* strategyTable)[curStratID].strategy->insertMessage(cmsg);
+    else {
+        flushTable = 1;
+       ComlibPrintf("Enqueuing message in tmplist\n");
+        (* strategyTable)[curStratID].tmplist.enq(cmsg);
+    }
+
+    ComlibPrintf("After multicast\n");
+}
+*/
+
+
+void ComlibManager::learnPattern(int total_msg_count, int total_bytes) {
+    static int nrecvd = 0;
+    static double avg_message_count = 0;
+    static double avg_message_bytes = 0;
+
+    avg_message_count += ((double) total_msg_count) / LEARNING_PERIOD;
+    avg_message_bytes += ((double) total_bytes) /  LEARNING_PERIOD;
+
+    nrecvd ++;
+    
+    if(nrecvd == CkNumPes()) {
+        //Number of messages and bytes a processor sends in each iteration
+        avg_message_count /= CkNumPes();
+        avg_message_bytes /= CkNumPes();
+        
+        //CkPrintf("STATS = %5.3lf, %5.3lf", avg_message_count,
+        //avg_message_bytes);
+
+        //Learning, ignoring contention for now! 
+        double cost_dir, cost_mesh, cost_grid, cost_hyp;
+       double p=(double)CkNumPes();
+        cost_dir = ALPHA * avg_message_count + BETA * avg_message_bytes;
+        cost_mesh = ALPHA * 2 * sqrt(p) + BETA * avg_message_bytes * 2;
+        cost_grid = ALPHA * 3 * pow(p,1.0/3.0) + BETA * avg_message_bytes * 3;
+        cost_hyp =  (log(p)/log(2.0))*(ALPHA  + BETA * avg_message_bytes/2.0);
+        
+        // Find the one with the minimum cost!
+        int min_strat = USE_MESH; 
+        double min_cost = cost_mesh;
+        if(min_cost > cost_hyp)
+            min_strat = USE_HYPERCUBE;
+        if(min_cost > cost_grid)
+            min_strat = USE_GRID;
+
+        if(min_cost > cost_dir)
+            min_strat = USE_DIRECT;
+
+        switchStrategy(min_strat);        
+    }
+}
+
+void ComlibManager::switchStrategy(int strat){
+    //CkPrintf("Switching to %d\n", strat);
+}
+
+void ComlibManager::setRemote(int remote_pe){
+
+    ComlibPrintf("Setting remote flag on\n");
+
+    remotePe = remote_pe;
+    isRemote = 1;
+}
+
+
+void ComlibManager::receiveRemoteSend(CkQ<CharmMessageHolder *> &rq, 
+                                      int strat_id) {
+    setInstance(strat_id);
+    
+    int nmsgs = rq.length();
+
+    for(int count = 0; count < nmsgs; count++) {
+        char *msg = rq.deq()->getCharmMessage();
+        envelope *env = UsrToEnv(msg);
+        
+        ArraySend(NULL, env->getsetArrayEp(), msg, env->getsetArrayIndex(), 
+                  env->getsetArrayMgr());
+    }
+
+    endIteration();
+}
+
+void ComlibManager::sendRemote(){
+    
+    int nmsgs = remoteQ.length();
+
+    if(nmsgs == 0)
+        return;
+
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); 
+    cgproxy[remotePe].receiveRemoteSend(remoteQ, curStratID);
+    
+    for(int count = 0; count < nmsgs; count++) {
+        CharmMessageHolder *cmsg = remoteQ.deq();
+        CkFreeMsg(cmsg->getCharmMessage());
+        delete cmsg;
+    }
+}
+
+
+/*
+void ComlibManager::prioEndIteration(PrioMsg *pmsg){
+    CkPrintf("[%d] In Prio End Iteration\n", CkMyPe());
+    setInstance(pmsg->instID);
+    endIteration();
+    delete pmsg;
+}
+*/
+
+void ComlibDelegateProxy(CProxy *proxy){
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+    proxy->ckDelegate(cgproxy.ckLocalBranch());
+}
+
+ComlibInstanceHandle CkCreateComlibInstance(){
+    return CkGetComlibInstance();
+}
+
+ComlibInstanceHandle CkGetComlibInstance() {
+    if(CkMyPe() != 0)
+        CkAbort("Comlib Instance can only be created on Processor 0");
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));    
+    return (cgproxy.ckLocalBranch())->createInstance();
+}
+
+ComlibInstanceHandle CkGetComlibInstance(int id) {
+    ComlibInstanceHandle cinst(id, CkpvAccess(cmgrID));
+    return cinst;
+}
+
+void ComlibDoneCreating(){
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));
+    (cgproxy.ckLocalBranch())->doneCreating();
+}
+
+char *router;
+int sfactor=0;
+
+class ComlibManagerMain {
+public:
+    ComlibManagerMain(CkArgMsg *msg) {
+        
+        if(CkMyPe() == 0 && msg !=  NULL)
+            CmiGetArgString(msg->argv, "+strategy", &router);         
+
+        if(CkMyPe() == 0 && msg !=  NULL)
+            CmiGetArgInt(msg->argv, "+spanning_factor", &sfactor);
+        
+        CProxy_ComlibManager::ckNew();
+    }
+};
+
+//Called by user code
+ComlibInstanceHandle::ComlibInstanceHandle(){
+    _instid = -1;
+    _dmid.setZero();
+    _srcPe = -1;
+}
+
+//Called by user code
+ComlibInstanceHandle::ComlibInstanceHandle(const ComlibInstanceHandle &h){
+    _instid = h._instid;
+    _dmid = h._dmid;
+
+    ComlibPrintf("In Copy Constructor\n");
+
+    //We DO NOT copy the source processor
+    //Source PE is initialized here
+    _srcPe = CkMyPe();
+}
+
+void ComlibInstanceHandle::init(){
+    CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));    
+    *this = (cgproxy.ckLocalBranch())->createInstance();
+}
+
+//Called by the communication library
+ComlibInstanceHandle::ComlibInstanceHandle(int instid, CkGroupID dmid){
+    _instid = instid;
+    _dmid   = dmid;
+    _srcPe  = -1;
+}
+
+void ComlibInstanceHandle::beginIteration() { 
+    CProxy_ComlibManager cgproxy(_dmid);
+
+    ComlibPrintf("Instance Handle beginIteration %d, %d\n", CkMyPe(), _srcPe);
+
+    if(_srcPe != CkMyPe()) {
+        (cgproxy.ckLocalBranch())->setRemote(_srcPe);
+    }
+
+    (cgproxy.ckLocalBranch())->setInstance(_instid);
+    (cgproxy.ckLocalBranch())->beginIteration();   
+}
+
+void ComlibInstanceHandle::endIteration() {
+    CProxy_ComlibManager cgproxy(_dmid);
+    (cgproxy.ckLocalBranch())->endIteration();
+}
+
+void ComlibInstanceHandle::setStrategy(CharmStrategy *s) {
+    CProxy_ComlibManager cgproxy(_dmid);
+    (cgproxy.ckLocalBranch())->registerStrategy(_instid, s);
+}
+
+CkGroupID ComlibInstanceHandle::getComlibManagerID() {return _dmid;}    
+
+void ComlibInitSectionID(CkSectionID &sid){
+
+    sid._cookie.type = COMLIB_MULTICAST_MESSAGE;
+    sid._cookie.pe = CkMyPe();
+
+    sid._cookie.sInfo.cInfo.id = 0;    
+    sid.npes = 0;
+    sid.pelist = NULL;
+}
+
+#include "comlib.def.h"
+
+
diff --git a/src/ck-com/ComlibManager.ci b/src/ck-com/ComlibManager.ci
new file mode 100644 (file)
index 0000000..561117c
--- /dev/null
@@ -0,0 +1,50 @@
+module comlib {
+  initproc void initComlibManager();
+
+  message ComlibDummyMsg;
+  //  message PrioMsg;
+  message ComlibMulticastMsg {
+       CkArrayIndexMax indices[];
+       char usrMsg[];
+  };
+
+  group [migratable] ComlibManager {
+    entry ComlibManager();
+
+    entry void barrier(void);
+    entry void barrier2(void);
+    entry void resumeFromBarrier2(void);
+
+    //entry void beginIteration();
+    //entry void endIteration();
+    //entry void prioEndIteration(PrioMsg *);
+
+    entry void receiveTable(StrategyWrapper sw);
+    entry void receiveRemoteSend(CkQ<CharmMessageHolder *> &remoteQ, int id);
+
+    entry void learnPattern(int, int);
+  }
+
+  mainchare ComlibManagerMain {
+    entry ComlibManagerMain(); 
+  }
+
+  //PUPable CharmStrategy;
+  //PUPable MessageHolder;
+  //PUPable CharmMessageHolder;
+
+  PUPable DummyStrategy;
+  PUPable NodeMulticast;
+  PUPable StreamingStrategy;
+  PUPable MPIStrategy;
+  PUPable DirectMulticastStrategy;
+  PUPable EachToManyMulticastStrategy; 
+  PUPable RingMulticastStrategy;       
+  PUPable PipeBroadcastStrategy;
+  PUPable MeshStreamingStrategy;
+  PUPable PrioStreaming;
+  PUPable BroadcastStrategy;
+
+  //Strategy pupable defined in the array manager code, along with the
+  //array listener code
+};
diff --git a/src/ck-com/ComlibManager.h b/src/ck-com/ComlibManager.h
new file mode 100644 (file)
index 0000000..a6607e3
--- /dev/null
@@ -0,0 +1,201 @@
+#ifndef COMMLIBMANAGER_H
+#define COMMLIBMANAGER_H
+
+#include "charm++.h"
+#include "cksection.h"
+#include "envelope.h"
+#include "comlib.h"
+#include <math.h>
+
+#include "charm++.h"
+#include "convcomlibmanager.h"
+
+#define USE_TREE 1            //Organizes the all to all as a tree
+#define USE_MESH 2            //Virtual topology is a mesh here
+#define USE_HYPERCUBE 3       //Virtual topology is a hypercube
+#define USE_DIRECT 4          //A dummy strategy that directly forwards 
+                              //messages without any processing.
+#define USE_GRID 5            //Virtual topology is a 3d grid
+#define USE_LINEAR 6          //Virtual topology is a linear array
+
+#define CHARM_MPI 0 
+#define MAX_NSTRAT 1024
+#define LEARNING_PERIOD 1000 //Number of iterations after which the
+                             //learning framework will discover 
+                             //the appropriate strategy, not completely 
+                             //implemented
+#define IS_MULTICAST -1
+
+#define ALPHA 5E-6
+#define BETA 3.33E-9
+
+PUPbytes(comID);
+
+#include "comlib.decl.h"
+
+//Dummy message to be sent incase there are no messages to send. 
+//Used by only the EachToMany strategy!
+class ComlibDummyMsg: public CMessage_ComlibDummyMsg {
+    int dummy;
+};
+
+/*
+//Priority message to call end iteration
+class PrioMsg: public CMessage_PrioMsg {
+ public:
+    int instID;
+};
+*/
+
+class ComlibMulticastMsg : public CkMcastBaseMsg, 
+               public CMessage_ComlibMulticastMsg {
+    
+  public:
+    int nIndices;
+    char *usrMsg;        
+    CkArrayIndexMax *indices;
+};
+
+class ComlibManager;
+
+extern CkGroupID cmgrID;
+
+//An Instance of the communication library.
+class ComlibInstanceHandle {
+ private:    
+    
+    int _instid;
+    CkGroupID _dmid;
+    int _srcPe;
+    
+ public:
+    ComlibInstanceHandle();
+    ComlibInstanceHandle(const ComlibInstanceHandle &h);
+    ComlibInstanceHandle(int instid, CkGroupID dmid);    
+    
+    void init();
+    void beginIteration();
+    void endIteration();
+    
+    CkGroupID getComlibManagerID();
+    void setStrategy(CharmStrategy *);
+
+    int getSourcePe() {return _srcPe;}
+
+    friend class ComlibManager;
+};
+
+PUPbytes(ComlibInstanceHandle);
+
+class ComlibManager: public CkDelegateMgr {
+    friend class ComlibInstanceHandle;
+
+    int section_send_event;
+
+    int remotePe;
+    CmiBool isRemote;
+
+    int npes;
+    int *pelist;
+
+    CkArrayIndexMax dummyArrayIndex;
+
+    //For compatibility and easier use!
+    int strategyID; //Identifier of the strategy
+
+    //Pointer to the converse comm lib strategy table
+    StrategyTable *strategyTable;
+
+    CkVec<CharmStrategy *> ListOfStrategies; //temporary list of strategies
+    
+    CkQ<CharmMessageHolder *> remoteQ;  //list of remote messages
+                                        //after the object has
+                                        //migrated
+
+    //The number of strategies created by the user
+    int nstrats; 
+    
+    int curStratID, prevStratID;      
+    //Number of strategies created by the user.
+
+    //flags
+    int receivedTable, flushTable, barrierReached, barrier2Reached;
+    int totalMsgCount, totalBytes, nIterations;
+
+    ComlibArrayListener *alistener;
+    int prioEndIterationFlag;
+
+    void init(); //initialization function
+
+    //charm_message for multicast for a section of that group
+    void multicast(CharmMessageHolder *cmsg); //charm message holder here
+    //void multicast(void *charm_msg, int npes, int *pelist);
+
+    //The following funtions can be accessed only from ComlibInstanceHandle
+    void beginIteration();     //Notify begining of a bracket with
+                               //strategy identifier
+
+    void endIteration();       //Notify end, endIteration must be
+                               //called if a beginIteration is
+                               //called. Otherwise end of the entry
+                               //method is assumed to be the end of
+                               //the bracket.
+    
+    void setInstance(int id); 
+    //void prioEndIteration(PrioMsg *pmsg);
+    void registerStrategy(int pos, CharmStrategy *s);
+
+ public:
+    ComlibManager();  //Recommended constructor
+
+    ComlibManager(CkMigrateMessage *m){ }
+    int useDefCtor(void){ return 1; } //Use default constructor should
+    //be pupped and store all the strategies.
+    
+    void barrier(void);
+    void barrier2(void);
+    void resumeFromBarrier2(void);
+    void receiveTable(StrategyWrapper sw); //Receive table of strategies.
+
+    void ArraySend(CkDelegateData *pd,int ep, void *msg, const CkArrayIndexMax &idx, 
+                   CkArrayID a);
+
+    void receiveRemoteSend(CkQ<CharmMessageHolder*> &rq, int id);
+    void sendRemote();
+
+    void GroupSend(CkDelegateData *pd,int ep, void *msg, int onpe, CkGroupID gid);
+    
+    virtual void ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a);
+    virtual void GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g);
+    virtual void ArraySectionSend(CkDelegateData *pd,int ep,void *m,CkArrayID a,CkSectionID &s);
+
+    CharmStrategy *getStrategy(int instid)
+        {return (CharmStrategy *)(* strategyTable)[instid].strategy;}
+
+    StrategyTableEntry *getStrategyTableEntry(int instid)
+        {return &((*strategyTable)[instid]);}
+
+    //To create a new strategy, returns handle to the strategy table;
+    ComlibInstanceHandle createInstance();  
+    void doneCreating();             //Done creating instances
+
+    //Learning functions
+    void learnPattern(int totalMessageCount, int totalBytes);
+    void switchStrategy(int strat);
+
+    void setRemote(int remotePe);
+};
+
+void ComlibDelegateProxy(CProxy *proxy);
+
+ComlibInstanceHandle CkCreateComlibInstance();
+ComlibInstanceHandle CkGetComlibInstance();
+ComlibInstanceHandle CkGetComlibInstance(int id);
+
+//Only Called when the strategies are not being created in main::main
+void ComlibDoneCreating(); 
+
+void ComlibInitSectionID(CkSectionID &sid);
+void ComlibDeleteSection();
+
+#endif
diff --git a/src/ck-com/DirectMulticastStrategy.C b/src/ck-com/DirectMulticastStrategy.C
new file mode 100644 (file)
index 0000000..695cb0c
--- /dev/null
@@ -0,0 +1,385 @@
+#include "DirectMulticastStrategy.h"
+
+int intCompare(void *a, void *b){
+    int a1 = *(int *) a;
+    int b1 = *(int *) b;
+
+    if(a1 < b1)
+        return -1;
+    
+    if(a1 == b1)
+        return 0;
+
+    if(a1 > b1)
+        return 1;
+
+    return 0;
+}    
+
+//ComlibSectionHashKey CODE
+int ComlibSectionHashKey::staticCompare(const void *k1,const void *k2,size_t ){
+    return ((const ComlibSectionHashKey *)k1)->
+                compare(*(const ComlibSectionHashKey *)k2);
+}
+
+CkHashCode ComlibSectionHashKey::staticHash(const void *v,size_t){
+    return ((const ComlibSectionHashKey *)v)->hash();
+}
+
+CkpvExtern(CkGroupID, cmgrID);
+
+void *DMHandler(void *msg){
+    ComlibPrintf("[%d]:In CallbackHandler\n", CkMyPe());
+    DirectMulticastStrategy *nm_mgr;    
+    
+    CkMcastBaseMsg *bmsg = (CkMcastBaseMsg *)EnvToUsr((envelope *)msg);
+    int instid = bmsg->_cookie.sInfo.cInfo.instId;
+    
+    nm_mgr = (DirectMulticastStrategy *) 
+        CProxy_ComlibManager(CkpvAccess(cmgrID)).
+        ckLocalBranch()->getStrategy(instid);
+    
+    nm_mgr->handleMulticastMessage(msg);
+    return NULL;
+}
+
+//Group Constructor
+DirectMulticastStrategy::DirectMulticastStrategy(int ndest, int *pelist)
+    : CharmStrategy() {
+    isDestinationArray = 0;
+    isDestinationGroup = 1;
+
+    setType(GROUP_STRATEGY);
+    
+    ndestpes = ndest;
+    destpelist = pelist;
+
+    commonInit();
+}
+
+DirectMulticastStrategy::DirectMulticastStrategy(CkArrayID aid)
+    : destArrayID(aid), CharmStrategy() {
+
+    isDestinationArray = 1;
+    isDestinationGroup = 0;
+
+    setType(ARRAY_STRATEGY);
+
+    ndestpes = 0;
+    destpelist = 0;
+
+    commonInit();
+}
+
+void DirectMulticastStrategy::commonInit(){
+    if(ndestpes == 0) {
+        ndestpes = CkNumPes();
+        destpelist = new int[CkNumPes()];
+        for(int count = 0; count < CkNumPes(); count ++)
+            destpelist[count] = count;        
+    }
+}
+
+void DirectMulticastStrategy::insertMessage(CharmMessageHolder *cmsg){
+    if(messageBuf == NULL) {
+       CkPrintf("ERROR MESSAGE BUF IS NULL\n");
+       return;
+    }
+
+    ComlibPrintf("[%d] Comlib Direct Multicast: insertMessage \n", 
+                 CkMyPe());   
+   
+    if(cmsg->dest_proc == IS_MULTICAST && cmsg->sec_id != NULL) {        
+        int cur_sec_id = cmsg->sec_id->_cookie.sInfo.cInfo.id;
+
+        if(cur_sec_id > 0) {        
+            //Old section id, send the id with the message
+            CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)cmsg->getCharmMessage();
+            cbmsg->_cookie.sInfo.cInfo.id = cur_sec_id;
+            cbmsg->_cookie.sInfo.cInfo.status = COMLIB_MULTICAST_OLD_SECTION;
+        }
+        else {
+            CkSectionID *sid = cmsg->sec_id;
+
+            //New sec id, so send it along with the message
+            void *newmsg = (void *)getNewMulticastMessage(cmsg);
+            CkFreeMsg(cmsg->getCharmMessage());
+            delete cmsg;
+            
+            initSectionID(sid);
+
+            cmsg = new CharmMessageHolder((char *)newmsg, IS_MULTICAST); 
+            cmsg->sec_id = sid;
+        }        
+    }
+    
+    messageBuf->enq(cmsg);
+    if(!isBracketed())
+        doneInserting();
+}
+
+void DirectMulticastStrategy::doneInserting(){
+    ComlibPrintf("%d: DoneInserting \n", CkMyPe());
+    
+    if(messageBuf->length() == 0) {
+        return;
+    }
+
+    while(!messageBuf->isEmpty()) {
+       CharmMessageHolder *cmsg = messageBuf->deq();
+        char *msg = cmsg->getCharmMessage();
+               
+        if(cmsg->dest_proc == IS_MULTICAST) {      
+            if(isDestinationArray)
+                CmiSetHandler(UsrToEnv(msg), handlerId);
+            
+            int *cur_map = destpelist;
+            int cur_npes = ndestpes;
+            if(cmsg->sec_id != NULL && cmsg->sec_id->pelist != NULL) {
+                cur_map = cmsg->sec_id->pelist;
+                cur_npes = cmsg->sec_id->npes;
+            }
+            
+            ComlibPrintf("[%d] Calling Direct Multicast %d %d %d\n", CkMyPe(),
+                         UsrToEnv(msg)->getTotalsize(), cur_npes, 
+                         cmsg->dest_proc);
+            /*
+            for(int i=0; i < cur_npes; i++)
+                CkPrintf("[%d] Sending to %d %d\n", CkMyPe(), 
+                         cur_map[i], cur_npes);
+            */
+
+            CmiSyncListSendAndFree(cur_npes, cur_map, 
+                                   UsrToEnv(msg)->getTotalsize(), 
+                                   (char*)(UsrToEnv(msg)));            
+        }
+        else {
+            //CkPrintf("SHOULD NOT BE HERE\n");
+            CmiSyncSendAndFree(cmsg->dest_proc, UsrToEnv(msg)->getTotalsize(), 
+                               (char *)UsrToEnv(msg));
+        }        
+        
+       delete cmsg; 
+    }
+}
+
+void DirectMulticastStrategy::pup(PUP::er &p){
+
+    CharmStrategy::pup(p);
+
+    p | ndestpes;
+    p | destArrayID;
+    p | isDestinationArray;
+    p | isDestinationGroup;           
+
+    if(p.isUnpacking())
+        destpelist = new int[ndestpes];
+    p(destpelist, ndestpes);        
+}
+
+void DirectMulticastStrategy::beginProcessing(int numElements){
+    
+    messageBuf = new CkQ<CharmMessageHolder *>;    
+    handlerId = CkRegisterHandler((CmiHandler)DMHandler);    
+    
+    if(isDestinationArray) {
+        CkArray *dest_array = CkArrayID::CkLocalBranch(destArrayID);
+        dest_array->getComlibArrayListener()->getLocalIndices
+            (localDestIndices);
+    }
+
+    MaxSectionID = 1;
+}
+
+void DirectMulticastStrategy::handleMulticastMessage(void *msg){
+    register envelope *env = (envelope *)msg;
+    
+    CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)EnvToUsr(env);
+
+    int status = cbmsg->_cookie.sInfo.cInfo.status;
+    ComlibPrintf("[%d] In local multicast %d\n", CkMyPe(), status);
+    
+    CkVec<CkArrayIndexMax> *dest_indices; 
+    if(status == COMLIB_MULTICAST_ALL) {        
+        //Multicast to all destination elements on current processor        
+        ComlibPrintf("[%d] Local multicast sending all %d\n", CkMyPe(), 
+                     localDestIndices.size());
+
+        localMulticast(&localDestIndices, env);
+    }   
+    else if(status == COMLIB_MULTICAST_NEW_SECTION){        
+        CkUnpackMessage(&env);
+        dest_indices = new CkVec<CkArrayIndexMax>;
+
+        ComlibPrintf("[%d] Received message for new section %d %d\n", 
+                     CkMyPe(), cbmsg->_cookie.pe, 
+                     cbmsg->_cookie.sInfo.cInfo.id);
+
+        ComlibMulticastMsg *ccmsg = (ComlibMulticastMsg *)cbmsg;
+        for(int count = 0; count < ccmsg->nIndices; count++){
+            CkArrayIndexMax idx = ccmsg->indices[count];
+            //idx.print();
+            int dest_proc =CkArrayID::CkLocalBranch(destArrayID)
+                ->lastKnown(idx);
+            
+            if(dest_proc == CkMyPe())
+                dest_indices->insertAtEnd(idx);                        
+        }            
+        
+        envelope *usrenv = (envelope *) ccmsg->usrMsg;
+        envelope *newenv = (envelope *)CmiAlloc(usrenv->getTotalsize());
+        memcpy(newenv, ccmsg->usrMsg, usrenv->getTotalsize());
+        localMulticast(dest_indices, newenv);
+
+        CkVec<CkArrayIndexMax> *old_dest_indices;
+        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                                 cbmsg->_cookie.sInfo.cInfo.id);
+
+        old_dest_indices = (CkVec<CkArrayIndexMax> *)sec_ht.get(key);
+        if(old_dest_indices != NULL)
+            delete old_dest_indices;
+        
+        sec_ht.put(key) = dest_indices;
+        CmiFree(env);                
+    }
+    else {
+        //status == COMLIB_MULTICAST_OLD_SECTION, use the cached section id
+        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                                 cbmsg->_cookie.sInfo.cInfo.id);    
+        dest_indices = (CkVec<CkArrayIndexMax> *)sec_ht.get(key);
+        
+        if(dest_indices == NULL)
+            CkAbort("Destination indices is NULL\n");
+        
+        localMulticast(dest_indices, env);
+    }
+}
+
+#include "register.h"
+void DirectMulticastStrategy::localMulticast(CkVec<CkArrayIndexMax>*vec, 
+                                                 envelope *env){
+    
+    //Multicast the messages to all elements in vec
+    int nelements = vec->size();
+    if(nelements == 0) {
+        CmiFree(env);
+        return;
+    }
+    
+    void *msg = EnvToUsr(env);    
+    int ep = env->getsetArrayEp();
+    CkUnpackMessage(&env);
+
+    env->getsetArrayMgr() = destArrayID;
+    env->setPacked(0); 
+    env->getsetArrayHops()=1;  
+    env->setUsed(0);
+
+    for(int count = 0; count < nelements; count ++){        
+        CkArrayIndexMax idx = (*vec)[count];
+        
+        ComlibPrintf("[%d] Sending multicast message to ", CkMyPe());        
+        if(comm_debug) idx.print();     
+        /*
+        CProxyElement_ArrayBase ap(destArrayID, idx);
+        ArrayElement *elem = ap.ckLocal();
+        if(elem != NULL) {
+            CkDeliverMessageReadonly(ep, msg, elem);        
+        }
+        else { //Element migrated away?
+            void *newmsg = CkCopyMsg(&msg);
+            ap.ckSend((CkArrayMessage *)newmsg, ep);            
+        }        
+        */        
+
+        env->getsetArrayIndex() = idx;
+
+        CkArray *a=(CkArray *)_localBranch(destArrayID);
+        if(_entryTable[ep]->noKeep) 
+            a->deliver((CkArrayMessage *)msg, CkDeliver_inline, CmiFalse);        
+        else {
+            void *newmsg = CkCopyMsg(&msg);
+            a->deliver((CkArrayMessage *)newmsg, CkDeliver_queue, CmiTrue);             
+        }
+        
+    }
+    
+    CmiFree(env);
+}
+
+void DirectMulticastStrategy::initSectionID(CkSectionID *sid){
+    
+    if(sid->npes > 0) 
+        return;
+    
+    //CkPrintf("NDESTPES = %d\n", ndestpes);
+
+    sid->pelist = new int[ndestpes];
+    sid->npes = 0;
+    
+    int count = 0, acount = 0;
+    for(acount = 0; acount < sid->_nElems; acount++){
+        int p = CkArrayID::CkLocalBranch(destArrayID)->
+            lastKnown(sid->_elems[acount]);
+        
+        for(count = 0; count < sid->npes; count ++)
+            if(sid->pelist[count] == p)
+                break;
+        
+        if(count == sid->npes) {
+            sid->pelist[sid->npes ++] = p;
+        }
+    } 
+    
+}
+
+
+ComlibMulticastMsg * DirectMulticastStrategy::getNewMulticastMessage
+(CharmMessageHolder *cmsg){
+    
+    if(cmsg->sec_id == NULL || cmsg->sec_id->_nElems == 0)
+        return NULL;
+
+    void *m = cmsg->getCharmMessage();
+    envelope *env = UsrToEnv(m);
+    
+    //if(cmsg->sec_id->_cookie.sInfo.cInfo.id == 0) {  //New Section ID;
+    CkPackMessage(&env);
+    int sizes[2];
+    sizes[0] = cmsg->sec_id->_nElems;
+    sizes[1] = env->getTotalsize();                
+    
+    cmsg->sec_id->_cookie.sInfo.cInfo.id = MaxSectionID ++;
+    
+    ComlibPrintf("Creating new comlib multicast message %d, %d\n", sizes[0], sizes[1]);
+    
+    ComlibMulticastMsg *msg = new(sizes, 0) ComlibMulticastMsg;
+    msg->nIndices = cmsg->sec_id->_nElems;
+    msg->_cookie.sInfo.cInfo.instId = myInstanceID;
+    msg->_cookie.sInfo.cInfo.id = MaxSectionID - 1;
+    msg->_cookie.sInfo.cInfo.status = COMLIB_MULTICAST_NEW_SECTION;
+    msg->_cookie.type = COMLIB_MULTICAST_MESSAGE;
+    msg->_cookie.pe = CkMyPe();
+    
+    memcpy(msg->indices, cmsg->sec_id->_elems, 
+           sizes[0] * sizeof(CkArrayIndexMax));
+    memcpy(msg->usrMsg, env, sizes[1] * sizeof(char));         
+    envelope *newenv = UsrToEnv(msg);
+    
+    newenv->getsetArrayMgr() = env->getsetArrayMgr();
+    newenv->getsetArraySrcPe() = env->getsetArraySrcPe();
+    newenv->getsetArrayEp() = env->getsetArrayEp();
+    newenv->getsetArrayHops() = env->getsetArrayHops();
+    newenv->getsetArrayIndex() = env->getsetArrayIndex();
+    // for trace projections
+    newenv->setEvent(env->getEvent());
+    newenv->setSrcPe(env->getSrcPe());
+    
+    CkPackMessage(&newenv);        
+    return (ComlibMulticastMsg *)EnvToUsr(newenv);
+    //}   
+
+    return NULL;
+}
+
diff --git a/src/ck-com/DirectMulticastStrategy.h b/src/ck-com/DirectMulticastStrategy.h
new file mode 100644 (file)
index 0000000..7754ab9
--- /dev/null
@@ -0,0 +1,96 @@
+#ifndef DIRECT_MULTICAST_STRATEGY
+#define DIRECT_MULTICAST_STRATEGY
+
+#include "ComlibManager.h"
+
+void *DMHandler(void *msg);
+
+class ComlibSectionHashKey{
+ public:
+
+    int srcPe;
+    int id;
+    ComlibSectionHashKey(int _pe, int _id):srcPe(_pe), id(_id){};
+
+    //These routines allow ComlibSectionHashKey to be used in
+    //  a CkHashtableT
+    CkHashCode hash(void) const;
+    static CkHashCode staticHash(const void *a,size_t);
+    int compare(const ComlibSectionHashKey &ind) const;
+    static int staticCompare(const void *a,const void *b,size_t);
+};
+
+inline CkHashCode ComlibSectionHashKey::hash(void) const
+{
+    register int _id = id;
+    register int _pe = srcPe;
+    
+    register CkHashCode ret = (_id << 16) + _pe;
+    return ret;
+}
+
+inline int ComlibSectionHashKey::compare(const ComlibSectionHashKey &k2) const
+{
+    if(id == k2.id && srcPe == k2.srcPe)
+        return 1;
+    
+    return 0;
+}
+
+/*For calls to qsort*/
+int intCompare(void *a, void *b);
+
+class DirectMulticastStrategy: public CharmStrategy {
+ protected:
+    CkQ <CharmMessageHolder*> *messageBuf;
+
+    int ndestpes, *destpelist; //Destination processors
+    int handlerId;
+    int MaxSectionID;
+
+    int isDestinationArray, isDestinationGroup;
+
+    //Array support
+    CkArrayID destArrayID;
+    CkVec<CkArrayIndexMax> localDestIndices;
+    //Array section support
+    CkHashtableT<ComlibSectionHashKey, void *> sec_ht; 
+    
+    //Initialize and cache information in a section id which can be
+    //used the next time the section is multicast to.
+    virtual void initSectionID(CkSectionID *sid);
+    
+    //Common Initializer for group and array constructors
+    //Every substrategy should implement its own
+    void commonInit();
+    
+    //Called to multicast an array message locally
+    void localMulticast(CkVec<CkArrayIndexMax> *, envelope *env);
+
+    //Create a new multicast message with the array section in it
+    ComlibMulticastMsg * getNewMulticastMessage(CharmMessageHolder *cmsg);
+
+ public:
+    
+    //Group constructor
+    DirectMulticastStrategy(int ndestpes = 0, int *destpelist = 0);    
+
+    //Array constructor
+    DirectMulticastStrategy(CkArrayID aid);
+
+    DirectMulticastStrategy(CkMigrateMessage *m): CharmStrategy(m){}
+    
+    virtual void insertMessage(CharmMessageHolder *msg);
+    virtual void doneInserting();
+
+    //Called by the converse handler function
+    virtual void handleMulticastMessage(void *msg);
+    
+    virtual void pup(PUP::er &p);    
+    virtual void beginProcessing(int nelements);
+    
+    PUPable_decl(DirectMulticastStrategy);
+
+};
+#endif
+
diff --git a/src/ck-com/DummyStrategy.C b/src/ck-com/DummyStrategy.C
new file mode 100644 (file)
index 0000000..2255202
--- /dev/null
@@ -0,0 +1,19 @@
+#include "DummyStrategy.h"
+
+DummyStrategy::DummyStrategy() : CharmStrategy(){
+}
+
+void DummyStrategy::insertMessage(CharmMessageHolder *cmsg){
+    ComlibPrintf("Sending Directly\n");
+    char *msg = cmsg->getCharmMessage();
+    CmiSyncSendAndFree(cmsg->dest_proc, UsrToEnv(msg)->getTotalsize(), 
+                       (char *)UsrToEnv(msg));
+    delete cmsg;
+}
+
+void DummyStrategy::doneInserting(){
+}
+
+void DummyStrategy::pup(PUP::er &p){}
+
+//PUPable_def(DummyStrategy);
diff --git a/src/ck-com/DummyStrategy.h b/src/ck-com/DummyStrategy.h
new file mode 100644 (file)
index 0000000..bdb834a
--- /dev/null
@@ -0,0 +1,15 @@
+#ifndef DUMMY_STRATEGY
+#define DUMMY_STRATEGY
+#include "ComlibManager.h"
+
+class DummyStrategy : public CharmStrategy {
+ public:
+    DummyStrategy();
+    DummyStrategy(CkMigrateMessage *m): CharmStrategy(m){}
+    void insertMessage(CharmMessageHolder *msg);
+    void doneInserting();
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(DummyStrategy);
+};
+#endif
diff --git a/src/ck-com/EachToManyMulticastStrategy.C b/src/ck-com/EachToManyMulticastStrategy.C
new file mode 100644 (file)
index 0000000..c1a66a6
--- /dev/null
@@ -0,0 +1,407 @@
+
+#include "EachToManyMulticastStrategy.h"
+#include "string.h"
+#include "routerstrategy.h"
+
+//EachToManyMulticastStrategy CODE
+CkpvExtern(int, RecvdummyHandle);
+CkpvExtern(CkGroupID, cmgrID);
+
+void *itrDoneHandler(void *msg){
+
+    EachToManyMulticastStrategy *nm_mgr;
+    
+    DummyMsg *dmsg = (DummyMsg *)msg;
+    comID id = dmsg->id;
+    int instid = id.instanceID;
+
+    CmiFree(msg);
+    ComlibPrintf("[%d] Iteration finished %d\n", CkMyPe(), instid);
+
+    StrategyTableEntry *sentry = 
+        CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()
+        ->getStrategyTableEntry(instid);
+    int nexpected = sentry->numElements;
+    
+    if(nexpected == 0) {             
+        CkPrintf("[%d] Calling Dummy Done Inserting\n", CkMyPe());
+        nm_mgr = (EachToManyMulticastStrategy *)sentry->strategy;    
+        nm_mgr->doneInserting();
+    }
+    
+    return NULL;
+}
+
+void *E2MHandler(void *msg){
+    //CkPrintf("[%d]:In EachtoMany CallbackHandler\n", CkMyPe());
+    EachToManyMulticastStrategy *nm_mgr;    
+    
+    CkMcastBaseMsg *bmsg = (CkMcastBaseMsg *)EnvToUsr((envelope *)msg);
+    int instid = bmsg->_cookie.sInfo.cInfo.instId;
+    
+    nm_mgr = (EachToManyMulticastStrategy *) 
+        CProxy_ComlibManager(CkpvAccess(cmgrID)).
+        ckLocalBranch()->getStrategy(instid);
+    
+    nm_mgr->localMulticast(msg);
+    return NULL;
+}
+
+//Group Constructor
+EachToManyMulticastStrategy::EachToManyMulticastStrategy(int substrategy, 
+                                                         int n_srcpes, 
+                                                         int *src_pelist,
+                                                         int n_destpes, 
+                                                         int *dest_pelist) 
+    : routerID(substrategy), CharmStrategy() {
+    
+    setType(GROUP_STRATEGY);
+
+    int count = 0;
+
+    if(n_srcpes == 0) {
+        n_srcpes = CkNumPes();
+        src_pelist = new int[n_srcpes];
+        for(count =0; count < n_srcpes; count ++)
+            src_pelist[count] = count;
+    }
+    
+    CkGroupID gid;
+    gid.setZero();
+    ginfo.setSourceGroup(gid, src_pelist, n_srcpes);    
+
+    if(n_destpes == 0) {
+        ndestpes = CkNumPes();
+        destpelist = new int[ndestpes];
+        for(count =0; count < ndestpes; count ++)
+            destpelist[count] = count;
+    }
+    else {
+        ndestpes = n_destpes;
+        destpelist = dest_pelist;
+    }
+
+    if(n_srcpes == 0){
+        pelist = src_pelist;
+        npes = n_srcpes;
+
+        commonInit();
+        return;
+    }
+
+    if(n_destpes == 0) {
+        pelist = destpelist;
+        npes = ndestpes;
+        
+        commonInit();
+        return;
+    }
+    
+    //source and destination lists are both subsets
+    pelist = new int[CkNumPes()];
+    npes = n_srcpes;
+    memcpy(pelist, src_pelist, n_srcpes * sizeof(int));
+    
+    for(int dcount = 0; dcount < ndestpes; dcount++) {
+        int p = destpelist[dcount];
+        
+        for(count = 0; count < npes; count ++)
+            if(pelist[count] == p)
+                break;
+        
+        if(count == npes)
+            pelist[npes++] = p;
+    }    
+
+    commonInit();
+}
+
+//Array Constructor
+EachToManyMulticastStrategy::EachToManyMulticastStrategy(int substrategy, 
+                                                         CkArrayID src, 
+                                                         CkArrayID dest, 
+                                                         int nsrc, 
+                                                         CkArrayIndexMax 
+                                                         *srcelements, 
+                                                         int ndest, 
+                                                         CkArrayIndexMax 
+                                                         *destelements)
+    :routerID(substrategy), CharmStrategy() {
+
+    setType(ARRAY_STRATEGY);
+    ainfo.setSourceArray(src, srcelements, nsrc);
+    ainfo.setDestinationArray(dest, destelements, ndest);
+
+    ainfo.getDestinationPeList(destpelist, ndestpes);
+    ainfo.getCombinedPeList(pelist, npes);
+    
+    //    for(int count = 0; count < npes; count ++){
+    //CkPrintf("%d, ", pelist[count]);
+    //}    
+    //CkPrintf("\n");
+
+    commonInit();
+}
+
+extern char *router;
+//Common initialization for both group and array constructors
+void EachToManyMulticastStrategy::commonInit() {
+
+    setBracketed();
+
+    if(CkMyPe() == 0 && router != NULL){
+        if(strcmp(router, "USE_MESH") == 0)
+            routerID = USE_MESH;
+        else if(strcmp(router, "USE_GRID") == 0)
+            routerID = USE_GRID;
+        else  if(strcmp(router, "USE_HYPERCUBE") == 0)
+            routerID = USE_HYPERCUBE;
+        else  if(strcmp(router, "USE_DIRECT") == 0)
+            routerID = USE_DIRECT;        
+    }
+    
+    ComlibPrintf("Creating Strategy %d\n", routerID);
+
+    rstrat = NULL;
+}
+
+
+void EachToManyMulticastStrategy::insertMessage(CharmMessageHolder *cmsg){
+
+    ComlibPrintf("[%d] EachToManyMulticast: insertMessage \n", 
+                 CkMyPe());   
+
+    if(cmsg->dest_proc == IS_MULTICAST && cmsg->sec_id != NULL) {        
+        int cur_sec_id = cmsg->sec_id->_cookie.sInfo.cInfo.id;
+
+        if(cur_sec_id > 0) {        
+            //Old section id, send the id with the message
+            CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)cmsg->getCharmMessage();
+            cbmsg->_cookie.sInfo.cInfo.id = cur_sec_id;
+            cbmsg->_cookie.sInfo.cInfo.status = COMLIB_MULTICAST_OLD_SECTION;
+        }
+        else {
+            //New sec id, so send it along with the message
+            void *newmsg = (void *)getNewMulticastMessage(cmsg);
+            CkFreeMsg(cmsg->getCharmMessage());
+            CkSectionID *sid = cmsg->sec_id;
+            delete cmsg;
+            
+            cmsg = new CharmMessageHolder((char *)newmsg, IS_MULTICAST); 
+            cmsg->sec_id = sid;
+            initSectionID(cmsg->sec_id);
+        }        
+
+        if(cmsg->sec_id != NULL && cmsg->sec_id->pelist != NULL) {
+            cmsg->pelist = cmsg->sec_id->pelist;
+            cmsg->npes = cmsg->sec_id->npes;
+        }
+        
+        CmiSetHandler(UsrToEnv(cmsg->getCharmMessage()), handlerId);
+    }
+    
+    rstrat->insertMessage(cmsg);
+}
+
+void EachToManyMulticastStrategy::doneInserting(){
+    ComlibPrintf("%d: DoneInserting \n", CkMyPe());
+    
+    rstrat->doneInserting();
+}
+
+void EachToManyMulticastStrategy::pup(PUP::er &p){
+
+    int count = 0;
+    ComlibPrintf("[%d] Each To many::pup %s\n", CkMyPe(), 
+                 ((p.isPacking()==0)?("UnPacking"):("Packing")));
+
+    CharmStrategy::pup(p);
+
+    p | routerID; 
+    p | npes; p | ndestpes;     
+    
+    if(p.isUnpacking()) {
+        pelist = new int[npes];    
+    }
+    p(pelist, npes);
+
+    if(p.isUnpacking()) {
+        destpelist = new int[ndestpes];    
+    }    
+
+    p(destpelist, ndestpes);
+
+    if(p.isUnpacking()){
+       handlerId = CkRegisterHandler((CmiHandler)E2MHandler);
+        int handler = CkRegisterHandler((CmiHandler)itrDoneHandler);
+        
+        rstrat = new RouterStrategy(routerID, handler, npes, pelist);
+        setConverseStrategy(rstrat);
+        MyPe = rstrat->getProcMap()[CkMyPe()];
+    }
+    
+    ComlibPrintf("[%d] End of pup\n", CkMyPe());
+}
+
+void EachToManyMulticastStrategy::beginProcessing(int numElements){
+
+    int expectedDeposits = 0;
+    MaxSectionID = 0;
+
+    if(ainfo.isSourceArray()) 
+        expectedDeposits = numElements;
+
+    if(getType() == GROUP_STRATEGY) {
+        
+        CkGroupID gid;
+        int *srcpelist;
+        int nsrcpes;
+        
+        ginfo.getSourceGroup(gid, srcpelist, nsrcpes);
+        
+        for(int count = 0; count < nsrcpes; count ++)
+            if(srcpelist[count] == CkMyPe()){
+                expectedDeposits = 1;
+                break;
+            }
+        
+        StrategyTableEntry *sentry = 
+            CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()
+            ->getStrategyTableEntry(myInstanceID);
+        sentry->numElements = expectedDeposits;
+    }
+    
+    if(expectedDeposits > 0)
+        return;
+    
+    if(expectedDeposits == 0 && MyPe >= 0)
+        doneInserting();
+}
+
+void EachToManyMulticastStrategy::localMulticast(void *msg){
+    register envelope *env = (envelope *)msg;
+    CkUnpackMessage(&env);
+    
+    CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)EnvToUsr(env);
+
+    int status = cbmsg->_cookie.sInfo.cInfo.status;
+    ComlibPrintf("[%d] In local multicast %d\n", CkMyPe(), status);
+        
+    if(status == COMLIB_MULTICAST_ALL) {        
+        ainfo.localMulticast(env);
+        return;
+    }   
+
+    CkVec<CkArrayIndexMax> *dest_indices;    
+    if(status == COMLIB_MULTICAST_NEW_SECTION){        
+
+        dest_indices = new CkVec<CkArrayIndexMax>;
+
+        //CkPrintf("[%d] Received message for new section\n", CkMyPe());
+
+        CkArrayID destArrayID;
+        int nDestElements;
+        CkArrayIndexMax *destelements;
+        ainfo.getSourceArray(destArrayID, destelements, nDestElements);
+
+        ComlibMulticastMsg *ccmsg = (ComlibMulticastMsg *)cbmsg;
+        for(int count = 0; count < ccmsg->nIndices; count++){
+            CkArrayIndexMax idx = ccmsg->indices[count];
+            //idx.print();
+            int dest_proc =CkArrayID::CkLocalBranch(destArrayID)
+                ->lastKnown(idx);
+            
+            if(dest_proc == CkMyPe())
+                dest_indices->insertAtEnd(idx);                        
+        }            
+
+        envelope *usrenv = (envelope *) ccmsg->usrMsg;
+        envelope *newenv = (envelope *)CmiAlloc(usrenv->getTotalsize());
+        memcpy(newenv, ccmsg->usrMsg, usrenv->getTotalsize());
+
+        ainfo.localMulticast(dest_indices, newenv);
+
+        CkVec<CkArrayIndexMax> *old_dest_indices;
+        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                                 cbmsg->_cookie.sInfo.cInfo.id);
+
+        old_dest_indices = (CkVec<CkArrayIndexMax> *)sec_ht.get(key);
+        if(old_dest_indices != NULL)
+            delete old_dest_indices;
+        
+        sec_ht.put(key) = dest_indices;
+        CmiFree(env);
+        return;       
+    }
+
+    //status == COMLIB_MULTICAST_OLD_SECTION, use the cached section id
+    ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                             cbmsg->_cookie.sInfo.cInfo.id);    
+    dest_indices = (CkVec<CkArrayIndexMax> *)sec_ht.get(key);
+
+    if(dest_indices == NULL)
+        CkAbort("Destination indices is NULL\n");
+
+    ainfo.localMulticast(dest_indices, env);
+}
+
+ComlibMulticastMsg * EachToManyMulticastStrategy::getNewMulticastMessage
+(CharmMessageHolder *cmsg){
+    
+    if(cmsg->sec_id == NULL || cmsg->sec_id->_nElems == 0)
+        return NULL;
+
+    void *m = cmsg->getCharmMessage();
+    envelope *env = UsrToEnv(m);
+    
+    if(cmsg->sec_id->_cookie.sInfo.cInfo.id == 0) {  //New Section ID;
+        CkPackMessage(&env);
+        int sizes[2];
+        sizes[0] = cmsg->sec_id->_nElems;
+        sizes[1] = env->getTotalsize();                
+
+        cmsg->sec_id->_cookie.sInfo.cInfo.id = MaxSectionID ++;
+
+        ComlibPrintf("Creating new comlib multicast message %d, %d\n", sizes[0], sizes[1]);
+
+        ComlibMulticastMsg *msg = new(sizes, 0) ComlibMulticastMsg;
+        msg->nIndices = cmsg->sec_id->_nElems;
+        msg->_cookie.sInfo.cInfo.instId = myInstanceID;
+        msg->_cookie.type = COMLIB_MULTICAST_MESSAGE;
+        msg->_cookie.sInfo.cInfo.id = MaxSectionID - 1;
+        msg->_cookie.sInfo.cInfo.status = COMLIB_MULTICAST_NEW_SECTION;
+        msg->_cookie.pe = CkMyPe();
+
+        memcpy(msg->indices, cmsg->sec_id->_elems, 
+               sizes[0] * sizeof(CkArrayIndexMax));
+        memcpy(msg->usrMsg, env, sizes[1] * sizeof(char));         
+        envelope *newenv = UsrToEnv(msg);
+        
+        newenv->getsetArrayMgr() = env->getsetArrayMgr();
+        newenv->getsetArraySrcPe() = env->getsetArraySrcPe();
+        newenv->getsetArrayEp() = env->getsetArrayEp();
+        newenv->getsetArrayHops() = env->getsetArrayHops();
+        newenv->getsetArrayIndex() = env->getsetArrayIndex();
+       // for trace projections
+        newenv->setEvent(env->getEvent());
+        newenv->setSrcPe(env->getSrcPe());
+
+        CkPackMessage(&newenv);        
+        return (ComlibMulticastMsg *)EnvToUsr(newenv);
+    }   
+
+    return NULL;
+}
+
+
+void EachToManyMulticastStrategy::initSectionID(CkSectionID *sid){
+
+    ainfo.initSectionID(sid);    
+
+    //Convert real processor numbers to virtual processors in the all
+    //to all multicast group
+    for(int count = 0; count < sid->npes; count ++) {
+        sid->pelist[count] = rstrat->getProcMap()[sid->pelist[count]];        
+        if(sid->pelist[count] == -1) CkAbort("Invalid Section\n");
+    }
+}
diff --git a/src/ck-com/EachToManyMulticastStrategy.h b/src/ck-com/EachToManyMulticastStrategy.h
new file mode 100644 (file)
index 0000000..a6ada90
--- /dev/null
@@ -0,0 +1,70 @@
+#ifndef EACH_TO_MANY_MULTICAST_STRATEGY
+#define EACH_TO_MANY_MULTICAST_STRATEGY
+
+#include "ComlibManager.h"
+#include "DirectMulticastStrategy.h"
+#include "routerstrategy.h"
+
+class EachToManyMulticastStrategy: public CharmStrategy {
+ protected:
+    //CkQ <CharmMessageHolder*> *messageBuf;
+    int routerID;      //Which topology
+    //comID comid;
+    
+    int npes, *pelist; //Domain of the topology
+    int MyPe;          //My id in that domain
+
+    int ndestpes, *destpelist, *destMap; //Destination processors
+
+    int handlerId;
+    
+    //CkVec<CkArrayIndexMax> localDestIndices;
+    
+    //Dynamically set by the application
+    CkHashtableT<ComlibSectionHashKey, void *> sec_ht;
+
+    //void localMulticast(CkVec<CkArrayIndexMax> *vec, envelope *env);
+    
+    ComlibMulticastMsg *getPackedMulticastMessage(CharmMessageHolder *cmsg);
+    //void setReverseMap();    
+    
+    //Executes common code just after array and group constructors
+    virtual void commonInit();
+    virtual void initSectionID(CkSectionID *sid);
+
+    int MaxSectionID;
+
+    RouterStrategy *rstrat;
+
+ public:
+    //Group constructor
+    EachToManyMulticastStrategy(int strategyId, int nsrcpes=0, 
+                                int *srcpelist=0, 
+                                int ndestpes =0, int *destpelist =0);
+    
+    //Array constructor
+    EachToManyMulticastStrategy(int substrategy, CkArrayID src, 
+                                CkArrayID dest, int nsrc=0, 
+                                CkArrayIndexMax *srcelements=0, int ndest=0, 
+                                CkArrayIndexMax *destelements=0);
+    
+    EachToManyMulticastStrategy(CkMigrateMessage *m) : CharmStrategy(m){}
+
+    //Basic function, subclasses should not have to change it
+    virtual void insertMessage(CharmMessageHolder *msg);
+    //More specielized function
+    virtual void doneInserting();
+
+    virtual void pup(PUP::er &p);    
+    virtual void beginProcessing(int nelements);
+    virtual void localMulticast(void *msg);
+    
+    //virtual void setDestArray(CkArrayID dest) {destArrayID=dest;}
+    
+    ComlibMulticastMsg *getNewMulticastMessage(CharmMessageHolder *m);
+
+    PUPable_decl(EachToManyMulticastStrategy);
+
+};
+#endif
+
diff --git a/src/ck-com/KDirectMulticastStrategy.C b/src/ck-com/KDirectMulticastStrategy.C
new file mode 100644 (file)
index 0000000..99b5808
--- /dev/null
@@ -0,0 +1,289 @@
+#include "KDirectMulticastStrategy.h"
+
+//Group Constructor
+KDirectMulticastStrategy::KDirectMulticastStrategy(int kf, 
+                                                   int ndest, int *pelist) 
+    : DirectMulticastStrategy(ndest, pelist), kfactor(kf) {
+    //FIXME: verify the list is sorted
+    commonKDirectInit();
+}
+
+//Array Constructor
+KDirectMulticastStrategy::KDirectMulticastStrategy(int kf, 
+                                                   CkArrayID dest_aid)
+    : DirectMulticastStrategy(dest_aid), kfactor(kf){
+    commonKDirectInit();    
+}
+
+void KDirectMulticastStrategy::commonKDirectInit(){
+    //sort list and create a reverse map
+}
+
+extern int _charmHandlerIdx;
+void KDirectMulticastStrategy::doneInserting(){
+    ComlibPrintf("%d: DoneInserting \n", CkMyPe());
+    
+    if(messageBuf->length() == 0) {
+        return;
+    }
+    
+    while(!messageBuf->isEmpty()) {
+       CharmMessageHolder *cmsg = messageBuf->deq();
+        char *msg = cmsg->getCharmMessage();
+        register envelope* env = UsrToEnv(msg);
+
+        ComlibPrintf("[%d] Calling KDirect %d %d %d\n", CkMyPe(),
+                     env->getTotalsize(), ndestpes, cmsg->dest_proc);
+               
+        if(cmsg->dest_proc == IS_MULTICAST) {      
+            CmiSetHandler(env, handlerId);
+            
+            int *cur_pelist = NULL;
+            int cur_npes = 0;
+            
+            if(cmsg->sec_id == NULL) {
+                cur_pelist = kdestpelist;
+                cur_npes = kfactor;
+            }
+            else {                
+                cur_npes = (kfactor <= cmsg->sid.npes)?kfactor : 
+                    cmsg->sid.npes;
+                cur_pelist = cmsg->sid.pe_list;
+            }
+            
+            ComlibPrintf("[%d] Sending Message to %d\n", CkMyPe(), cur_npes);
+            CmiSyncListSendAndFree(cur_npes, cur_pelist, 
+                                   UsrToEnv(msg)->getTotalsize(), 
+                                   UsrToEnv(msg));
+        }
+        else {
+            CmiSyncSendAndFree(cmsg->dest_proc, 
+                               UsrToEnv(msg)->getTotalsize(), 
+                               (char *)UsrToEnv(msg));
+        }        
+        
+        delete cmsg; 
+    }
+}
+
+void KDirectMulticastStrategy::pup(PUP::er &p){
+    DirectMulticastStrategy::pup(p);
+
+    p | kfactor;
+}
+
+void KDirectMulticastStrategy::beginProcessing(int  nelements){
+
+    DirectMulticastStrategy::beginProcessing(nelements);
+
+    kndestpelist = new int[kfactor]; 
+
+    int next_pe = 0, count = 0;
+    //Assuming the destination pe list is sorted.
+    for(count = 0; count < ndestpes; count++)        
+        if(destpelist[count] > CkMyPe()) {
+            next_pe = count;
+            break;
+        }
+
+    int kpos = 0;
+    for(count = next_pe; count < next_pe + kfactor; count++){
+        int pe = destpelist[count % ndestpes];
+        kdestpelist[kpos ++] = pe;
+    }
+}
+
+void KDirectMulticastStrategy::handleMulticastMessage(void *msg){
+    register envelope *env = (envelope *)msg;
+    
+    CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)EnvToUsr(env);
+    int src_pe = cbmsg->_cookie.pe;
+    if(isDestinationGroup){               
+        CmiSetHandler(env, _charmHandlerIdx);
+        CmiSyncSend(CkMyPe(), env->getTotalsize(), (char *)env);
+        
+        int nmsgs = getNumMessagesToSend(src_pe, CkMyPe, CkNumPes());
+        if(nmsgs > 0){            
+            CmiSetHandler(env, handlerId);            
+            CmiSyncListSendAndFree(nmsgs, kdestpelist, 
+                                   env->getTotalsize(), env);
+        }        
+        return;
+    }
+
+    int status = cbmsg->_cookie.sInfo.cInfo.status;
+    ComlibPrintf("[%d] In handle multicast message %d\n", CkMyPe(), status);
+
+    if(status == COMLIB_MULTICAST_ALL) {                        
+        int nmsgs = getNumMessagesToSend(src_pe. CkMyPe(), CkNumPes());
+        if(nmsgs > 0){ //Have to forward the messages           
+            void *msg = EnvToUsr(env);
+            void *newmsg = CkCopyMsg(&msg);
+            envelope *newenv = UsrToEnv(newmsg);        
+            CmiSyncListSendAndFree(nmsgs, kdestpelist, 
+                                   newenv->getTotalsize(), newenv);
+        }
+
+        //Multicast to all destination elements on current processor        
+        ComlibPrintf("[%d] Local multicast sending all %d\n", CkMyPe(), 
+                     localDestIndices.size());
+        
+        localMulticast(&localDestIndices, env);
+    }   
+    else if(status == COMLIB_MULTICAST_NEW_SECTION){        
+        CkUnpackMessage(&env);
+        ComlibPrintf("[%d] Received message for new section src=%d\n", 
+                     CkMyPe(), cbmsg->_cookie.pe);
+
+        ComlibMulticastMsg *ccmsg = (ComlibMulticastMsg *)cbmsg;
+        
+        KDirectHashObject *kobj = 
+            createHashObject(ccmsg->nIndices, ccmsg->indices);
+        
+        envelope *usrenv = (envelope *) ccmsg->usrMsg;
+        
+        envelope *newenv = (envelope *)CmiAlloc(usrenv->getTotalsize());
+        memcpy(newenv, usrenv, usrenv->getTotalsize());
+
+        localMulticast(&kobj->indices, newenv);
+
+        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                                 cbmsg->_cookie.sInfo.cInfo.id);
+
+        KDirectHashObject *old_kobj = 
+            (KDirectHashObject*)sec_ht.get(key);
+        if(old_kobj != NULL)
+            delete old_kobj;
+        
+        sec_ht.put(key) = kobj;
+
+        if(kobj->npes > 0) {
+            ComlibPrintf("[%d] Forwarding Message of %d to %d pes\n", 
+                         CkMyPe(), cbmsg->_cookie.pe, kobj->npes);
+            CkPackMessage(&env);
+            CmiSyncListSendAndFree(kpbj->npes, kobj->pelist, 
+                                   env->getTotalsize(), env);
+        }
+        else
+            CmiFree(env);       
+    }
+    else {
+        //status == COMLIB_MULTICAST_OLD_SECTION, use the cached section id
+        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                                 cbmsg->_cookie.sInfo.cInfo.id);    
+        KDirectHashObject *kobj = (KDirectHashObject *)sec_ht.get(key);
+        
+        if(kobj == NULL)
+            CkAbort("Destination indices is NULL\n");
+        
+        if(kobj->npes > 0){
+            void *msg = EnvToUsr(env);
+            void *newmsg = CkCopyMsg(&msg);
+            envelope *newenv = UsrToEnv(newmsg);        
+            CmiSyncListSendAndFree(kpbj->npes, kobj->pelist, 
+                                   newenv->getTotalsize(), newenv);
+
+        }
+        
+        localMulticast(&kobj->indices, env);
+    }
+}
+
+void KDirectMulticastStrategy::initSectionID(CkSectionID *sid){
+
+    ComlibPrintf("KDirect Init section ID\n");
+    sid->pelist = NULL;
+    sid->npes = 0;
+
+    int *pelist = new int[kfactor];
+    int npes;
+    getPeList(sid->_nElem,  sid->_elems, pelist, npes);
+    
+    sid->destpelist = pelist;
+    sid->ndestpes = npes;    
+}
+
+KDirectHashObject *KDirectMulticastStrategy::createHashObject(int nelements, CkArrayIndexMax *elements){
+
+    KDirectHashObject *kobj = new KDirectHashObject;
+    kobj->pelist = new int[kfactor];
+    getPeList(nelements,  elements, kobj->pelist, kobj->npes);
+
+    return kobj;
+}
+
+
+void KDirectMulticastStrategy::getPeList(int nelements, 
+                                         CkArrayIndexMax *elements, 
+                                         int *pelist, int &npes, 
+                                         int src_pe){
+    
+    npes = 0;
+    
+    int *tmp_pelist = new int[CkNumPes()];
+    int num_pes;
+    
+    //make this a reusable function call later.
+    int count = 0, acount = 0;
+    for(acount = 0; acount < nelements; acount++){
+        int p = CkArrayID::CkLocalBranch(destArrayID)->
+            lastKnown(elements[acount]);
+        
+        for(count = 0; count < num_pes; count ++)
+            if(tmp_pelist[count] == p)
+                break;
+        
+        if(count == num_pes) {
+            tmp_pelist[num_pes ++] = p;
+        }
+    }
+
+    if(num_pes == 0) {
+        delete [] tmp_pelist;
+        return;
+    }
+
+    qsort(tmp_pelist, num_pes, sizeof(int), intCompare);
+    
+    int pdiff = 0;
+    int my_pos = 0;
+    int src_pos = 0;
+
+    int count;
+    for(count = 0; count < num_pes; count ++) {
+        if(tmp_pelist[count] == CkMyPe()){
+            my_pos = count;
+        }
+
+        if(tmp_pelist[count] == src_pos){
+            src_pos = count;
+        }        
+    }            
+
+    int n_tosend = getNumMessagesToSend(src_pos, my_pos, num_pes);
+    for(count = 0; count < n_tosend; count ++) {
+        pelist[npes ++] = tmp_pelist[(src_pos + count)%num_pes];
+    }    
+
+    delete [] tmp_pelist;    
+}
+
+int KDirectMulticastStrategy::getNumMessagesToSend(int src_pe, int my_pe, 
+                                                   int num_pes){
+    
+    if(src_pe == my_pe) {
+        retutn 0;
+    }
+
+    int nToSend = 0;
+
+    int pdiff = my_pe - src_pe;
+    
+    if(pdiff < 0)
+        pdiff += num_pes;
+    
+    if(pdiff % kfactor != 0)
+        return 0;
+    
+    return (num_pes - pdiff > kfactor)? kfactor : num_pes - pdiff;
+}
diff --git a/src/ck-com/KDirectMulticastStrategy.h b/src/ck-com/KDirectMulticastStrategy.h
new file mode 100644 (file)
index 0000000..1235e73
--- /dev/null
@@ -0,0 +1,53 @@
+#ifndef KDIRECT_MULTICAST_STRATEGY
+#define KDIRECT_MULTICAST_STRATEGY
+
+#include "DirectMulticastStrategy.h"
+
+class KDirectHashObject{
+ public:
+    CkVec<CkArrayIndexMax> indices;
+    int npes;
+    int *pelist;
+};
+
+
+class KDirectMulticastStrategy: public DirectMultcastStrategy {
+ protected:
+    int kfactor;
+
+    //Initialize and cache information in a section id which can be
+    //used the next time the section is multicast to.
+    virtual void initSectionID(CkSectionID *sid);
+    
+    //Common Initializer for group and array constructors
+    //Every substrategy should implement its own
+    void commonKDirectInit();
+    
+    //Create a new multicast message with the array section in it
+    ComlibMulticastMsg * getNewMulticastMessage(CharmMessageHolder *cmsg);
+
+ public:
+    
+    //Group constructor
+    KDirectMulticastStrategy(int ndestpes = 0, int *destpelist = 0);    
+
+    //Array constructor
+    KDirectMulticastStrategy(CkArrayID aid);
+
+    KDirectMulticastStrategy(CkMigrateMessage *m): Strategy(m){}
+    
+    //    virtual void insertMessage(CharmMessageHolder *msg);
+    virtual void doneInserting();
+
+    //Called by the converse handler function
+    virtual void handleMulticastMessage(void *msg);
+    
+    //virtual void beginProcessing(int nelements);
+    
+    void setKFactor(int k){ kfactor = k; }
+    
+    virtual void pup(PUP::er &p);    
+    PUPable_decl(KDirectMulticastStrategy);
+};
+
+#endif
diff --git a/src/ck-com/MPIStrategy.C b/src/ck-com/MPIStrategy.C
new file mode 100644 (file)
index 0000000..2f1e266
--- /dev/null
@@ -0,0 +1,113 @@
+#include "MPIStrategy.h"
+
+#if CHARM_MPI
+MPI_Comm groupComm;
+MPI_Group group, groupWorld;
+#endif
+
+MPIStrategy::MPIStrategy() {
+    messageBuf = NULL;
+    messageCount = 0;
+    npes = CkNumPes();
+    pelist = NULL;
+}
+
+MPIStrategy::MPIStrategy(int npes, int *pelist) {
+    messageBuf = NULL;
+    messageCount = 0;
+
+    this->npes = npes;
+    this->pelist = pelist;
+}
+
+void MPIStrategy::insertMessage(CharmMessageHolder *cmsg){
+    cmsg->next = messageBuf;
+    messageBuf = cmsg;    
+}
+
+void MPIStrategy::doneInserting(){
+#if CHARM_MPI
+    ComlibPrintf("[%d] In MPI strategy\n", CkMyPe());
+    
+    CharmMessageHolder *cmsg = messageBuf;
+    char *buf_ptr = mpi_sndbuf;
+    
+    //if(npes == 0)
+    //  npes = CkNumPes();
+    
+    for(count = 0; count < npes; count ++) {
+        ((int *)buf_ptr)[0] = 0;
+        buf_ptr += MPI_MAX_MSG_SIZE;
+    }
+    
+    buf_ptr = mpi_sndbuf;
+    for(count = 0; count < messageCount; count ++) {
+        if(npes < CkNumPes()) {
+            ComlibPrintf("[%d] Copying data to %d and rank %d\n", 
+                         cmsg->dest_proc, procMap[cmsg->dest_proc]);
+            buf_ptr = mpi_sndbuf + MPI_MAX_MSG_SIZE * procMap[cmsg->dest_proc];  
+        }
+        else
+            buf_ptr = mpi_sndbuf + MPI_MAX_MSG_SIZE * cmsg->dest_proc; 
+        
+        char * msg = cmsg->getCharmMessage();
+        envelope * env = UsrToEnv(msg);
+        
+        ((int *)buf_ptr)[0] = env->getTotalsize();
+        
+        ComlibPrintf("[%d] Copying message\n", CkMyPe());
+        memcpy(buf_ptr + sizeof(int), (char *)env, env->getTotalsize());
+        
+        ComlibPrintf("[%d] Deleting message\n", CkMyPe());
+        CmiFree((char *) env);
+        CharmMessageHolder *prev = cmsg;
+        cmsg = cmsg->next;
+        delete prev;
+    }
+    
+    //ComlibPrintf("[%d] Calling Barrier\n", CkMyPe());
+    //PMPI_Barrier(groupComm);
+    
+    ComlibPrintf("[%d] Calling All to all\n", CkMyPe());
+    MPI_Alltoall(mpi_sndbuf, MPI_MAX_MSG_SIZE, MPI_CHAR, mpi_recvbuf, 
+                  MPI_MAX_MSG_SIZE, MPI_CHAR, groupComm);
+    
+    ComlibPrintf("[%d] All to all finished\n", CkMyPe());
+    buf_ptr = mpi_recvbuf;
+    for(count = 0; count < npes; count ++) {
+        int recv_msg_size = ((int *)buf_ptr)[0];
+        char * recv_msg = buf_ptr + sizeof(int);
+        
+        if((recv_msg_size > 0) && recv_msg_size < MPI_MAX_MSG_SIZE) {
+            ComlibPrintf("[%d] Receiving message of size %d\n", CkMyPe(), 
+                         recv_msg_size);
+            CmiSyncSend(CkMyPe(), recv_msg_size, recv_msg);
+        }
+        buf_ptr += MPI_MAX_MSG_SIZE;
+    }
+#endif
+}
+
+void MPIStrategy::pup(PUP::er &p) {
+    p | messageCount;
+    p | npes; 
+       
+    if(p.isUnpacking())
+        pelist = new int[npes];
+    p(pelist , npes);
+
+    messageBuf = NULL;
+    
+    if(p.isUnpacking()){
+#if CHARM_MPI
+        if(npes < CkNumPes()){
+            MPI_Comm_group(MPI_COMM_WORLD, &groupWorld);
+            MPI_Group_incl(groupWorld, npes, pelist, &group);
+            MPI_Comm_create(MPI_COMM_WORLD, group, &groupComm);
+        }
+        else groupComm = MPI_COMM_WORLD;
+#endif
+    }
+}
+
+//PUPable_def(MPIStrategy);
diff --git a/src/ck-com/MPIStrategy.h b/src/ck-com/MPIStrategy.h
new file mode 100644 (file)
index 0000000..f00f37a
--- /dev/null
@@ -0,0 +1,30 @@
+#ifndef MPI_STRATEGY
+#define MPI_STRATEGY
+
+#include "ComlibManager.h"
+
+#if CHARM_MPI
+#include "mpi.h"
+#define MPI_MAX_MSG_SIZE 1000
+#define MPI_BUF_SIZE 2000000
+char mpi_sndbuf[MPI_BUF_SIZE];
+char mpi_recvbuf[MPI_BUF_SIZE];
+#endif
+
+class MPIStrategy : public CharmStrategy {
+    CharmMessageHolder *messageBuf;
+    int messageCount;
+    int npes, *pelist;
+
+ public:
+    MPIStrategy();
+    MPIStrategy(CkMigrateMessage *m) {}
+    MPIStrategy(int npes, int *pelist);
+
+    virtual void insertMessage(CharmMessageHolder *msg);
+    virtual void doneInserting();
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(MPIStrategy);
+};
+#endif
diff --git a/src/ck-com/MeshStreamingStrategy.C b/src/ck-com/MeshStreamingStrategy.C
new file mode 100644 (file)
index 0000000..f9cb46d
--- /dev/null
@@ -0,0 +1,666 @@
+/**************************************************************************
+** Greg Koenig (koenig@uiuc.edu)
+**
+** This is MeshStreamingStrategy, a strategy in the Charm++ communications
+** library.  In this strategy, processes are organized into a mesh as
+** depicted in the following diagram:
+**
+**  1    2    3    4
+**  5    6    7    8
+**  9   10   11   12
+** 13   14   15   16
+**
+** If, for example, PE 6 sends a message to PE 4 and a message to PE 12,
+** both messages will be stored in a column bucket on PE 6 for destination
+** column 3.  After DEFAULT_FLUSH_PERIOD milliseconds elapse or
+** DEFAULT_MAX_BUCKET_SIZE messages accumulate, all messages in the bucket
+** for column 3 are flushed by bundling them together and sending them at
+** once to PE 8.  When they arrive on PE 8 and are delivered to its
+** column_handler(), PE 8 breaks the messages in the bundle apart and
+** stores messages destined for individual rows in separate row buckets.
+** In the case of our example, the message destined for PE 4 would be
+** stored in the row bucket for row 0 and the message destined for PE 12
+** would be stored in the row bucket for row 2.  Again, after
+** DEFAULT_FLUSH_PERIOD milliseconds elapse or DEFAULT_MAX_BUCKET_SIZE
+** messages accumulate, all messages in the row buckets are flushed by
+** bundling them together and sending them at once to their respective
+** destination PEs by calling CmiMultipleSend().
+**
+** The advantage of bundling messages together is that to send to N
+** PEs in a computation, sqrt(N) actual messages need to be sent on the
+** network.  This trades computational overhead of bundling messages
+** for communication overhead; if the processors are fast relative to
+** the network, we win.
+**
+** To understand the bundling/unbundling operations, knowledge of Converse
+** and Charm++ message header formats is required.  I have attempted to
+** provide documentation within this code to describe what is going on.
+*/
+
+#include "MeshStreamingStrategy.h"
+
+// These macros are taken directly from convcore.c.
+#define SIZEFIELD(m) (((CmiChunkHeader *)(m))[-1].size)
+#define REFFIELD(m) (((CmiChunkHeader *)(m))[-1].ref)
+#define BLKSTART(m) (((CmiChunkHeader *)(m))-1)
+
+// These externs are defined inside ComlibManager.C.
+CkpvExtern(CkGroupID, cmgrID);
+CkpvExtern(int, RecvmsgHandle);
+
+
+/**************************************************************************
+** This handler is invoked automatically when the processor goes idle.
+**
+** The idle handler automatically re-registers itself, so there is no need
+** to re-register it from here.
+**
+** If nothing else is going on anyway, we might as well flush the buffers
+** now instead of waiting for the flush period.
+*/
+void idle_flush_handler (void *ptr, double curT)
+{
+  ComlibPrintf ("[%d] idle_flush_handler() invoked.\n", CkMyPe());
+
+  MeshStreamingStrategy *classptr = (MeshStreamingStrategy *) ptr;
+  classptr->FlushBuffers ();
+}
+
+
+
+/**************************************************************************
+** This handler is invoked automatically after a timeout occurs.
+**
+** The periodic handler does not automatically re-register itself, so it
+** calls RegisterPeriodicFlush() to do so after it finishes flushing
+** buffers.
+*/
+void periodic_flush_handler (void *ptr, double curT)
+{
+  ComlibPrintf ("[%d] periodic_flush_handler() invoked.\n", CkMyPe());
+
+  MeshStreamingStrategy *classptr = (MeshStreamingStrategy *) ptr;
+  classptr->FlushBuffers ();
+  classptr->RegisterPeriodicFlush ();
+}
+
+
+
+/**************************************************************************
+** This handler is invoked automatically when a packed message for a column
+** is received.
+**
+** The layout of the message received is shown in the diagram below.
+**
+**             \  /
+** +------------||---------------------------------------------------+
+** |Conv| I | # || dest || size | ref || Converse | Charm++  || user | ...
+** |hdr | D |   ||  PE  ||      | cnt ||  header  | envelope || data | ...
+** +------------||---------------------------------------------------+
+**             /  \
+**
+** The function first retrieves the strategy ID and the number of messages
+** in the packed message and then uses the strategy ID to obtain a pointer
+** to the MeshStreamingStrategy class.  It also obtains the row length by
+** calling GetRowLength().
+**
+** The function then iterates through the messages in the packed message.
+** For each message within, it allocates space by calling CmiAlloc() and
+** then copies the message from the packed buffer into the new message
+** buffer.  It is also able to obtain the destination PE for the message
+** because this information is included in the packed message data for
+** each packed message.  If the destination PE is the current PE, the
+** message is delivered immediately via a call to CmiSyncSendAndFree().
+** This routine calls CmiFree() on the message, which is appropriate
+** since it was allocated with CmiAlloc().  Otherwise, the message is
+** inserted into the row bucket for the necessary row by calling
+** InsertIntoRowBucket().  When messages are delivered from the row
+** bucket, they are freed by CmiFree().
+*/
+void column_handler (char *msg)
+{
+  int dest_pe;
+  int dest_row;
+  int msgsize;
+  int my_pe;
+  int num_msgs;
+  int row_length;
+  int strategy_id;
+  char *msgptr;
+  char *newmsg;
+  MeshStreamingStrategy *classptr;
+
+
+  ComlibPrintf ("[%d] column_handler() invoked.\n", CkMyPe());
+
+  my_pe = CkMyPe ();
+
+  strategy_id = ((int *) (msg + CmiMsgHeaderSizeBytes))[0];
+  num_msgs = ((int *) (msg + CmiMsgHeaderSizeBytes))[1];
+
+  classptr = (MeshStreamingStrategy *)
+             CProxy_ComlibManager (CkpvAccess (cmgrID)).
+             ckLocalBranch()->getStrategy (strategy_id);
+
+  row_length = classptr->GetRowLength ();
+
+  msgptr = (char *) (msg + CmiMsgHeaderSizeBytes + 2 * sizeof(int));
+  for (int i = 0; i < num_msgs; i++) {
+    dest_pe = ((int *) msgptr)[0];
+    msgsize = ((int *) msgptr)[1];
+
+    newmsg = (char *) CmiAlloc (msgsize);
+
+    memcpy (newmsg, (msgptr + 3 * sizeof(int)), msgsize);
+
+    if (dest_pe == my_pe) {
+      CmiSyncSendAndFree (my_pe, msgsize, newmsg);
+    } else {
+      dest_row = dest_pe / row_length;
+      classptr->InsertIntoRowBucket (dest_row, newmsg);
+    }
+
+    msgptr += msgsize + 3 * sizeof(int);
+  }
+
+  CmiFree (msg);
+}
+
+
+
+/**************************************************************************
+** This is the MeshStreamingStrategy constructor.
+**
+** The period and bucket_size have default values specified in the .h file.
+**
+** The constructor is invoked when the client code instantiates this
+** strategy.  The constructor executes on a SINGLE PROCESS in the
+** computation, so it cannot do things like determine an individual
+** process's position within the mesh.
+**
+** After the constructor is invoked, the communications library creates
+** instances that get pup'ed and shipped to each processor in the
+** computation.  To that end, the process that instantiates this strategy
+** (most likely PE 0) will then use pup to pack copies of the strategy
+** and then ship them off to other processes.  They will be un-pup'ed
+** there.  Finally, beginProcessing() will be called on EACH instance on
+** its target processor.
+*/
+MeshStreamingStrategy::MeshStreamingStrategy (int period, int bucket_size) 
+    : CharmStrategy() 
+{
+  ComlibPrintf ("[%d] MeshStreamingStrategy::MeshStreamingStrategy() invoked.\n", CkMyPe());
+
+  num_pe = CkNumPes ();
+
+  num_columns = (int) (ceil (sqrt ((double) num_pe)));
+  num_rows = num_columns;
+  row_length = num_columns;
+
+  flush_period = period;
+  max_bucket_size = bucket_size;
+
+  column_bucket = new CkQ<char *>[num_columns];
+  column_bytes = new int[num_columns];
+  row_bucket = new CkQ<char *>[num_rows];
+}
+
+
+
+/**************************************************************************
+** This method is called when the communications library sends a message
+** from one PE to another PE.  This could be due to a direct message being
+** sent, or due to a method invocation with marshalled parameters.
+**
+** The method begins by getting the destination PE from the
+** CharmMessageHolder that is passed in (and from this, computing the
+** destination column) and getting a pointer to the User data for the
+** message (and from this, computing the Envelope pointer and the Block
+** pointer).  The following diagram shows the layout of the message.
+**
+**     +--------------------------------------------------------------+
+**     | size | refcount || Converse | Charm++  ||        user        |
+**     |      |          ||  header  | envelope ||        data        |
+**     +--------------------------------------------------------------+
+**     ^                  ^                      ^
+**     |                  |                      |
+**     blk (Block)        env (Envelope)         usr (User)
+**
+** Calling cmsg->getCharmMessage() on a CharmMessageHolder object returns
+** a pointer to the user data.  From this, a pointer to the envelope can
+** be obtained by calling UsrToEnv().  All Converse messages are allocated
+** by CmiAlloc() which prepends two ints to all memory regions to hold a
+** size field and a refcount field.  BLKSTART() is a macro that gets the
+** start of a block from the envelope pointer.
+**
+** If the destination PE is our current PE, we just deliver the message
+** immediately.
+**
+** Otherwise, if the destination PE is in the same column as our PE, we
+** allocate a new region of memory with CmiAlloc() and copy from the
+** envelope pointer into the new region, and then deposit this new message
+** into the appropriate row bucket for our column.  (The row buckets are
+** queues of pointers to memory regions exactly like the diagram above.
+** All entries in the row bucket are allocated with CmiAlloc() and must
+** be deallocated with CmiFree()!)
+**
+** Otherwise, the destination PE must be in a different column from our
+** PE.  We allocate a new region of memory with "new" that looks like
+** the diagram below.
+**
+** +----------------------------------------------------------------------+
+** | dest || size | refcount || Converse | Charm++  ||        user        |
+** |  PE  ||      |          ||  header  | envelope ||        data        |
+** +----------------------------------------------------------------------+
+** ^       ^                  ^                      ^
+** |       |                  |                      |
+** newmsg  blk (Block)        env (Envelope)         usr (User)
+**
+** We then deposit this new message into the appropriate column bucket.
+** (The column buckets are queues of pointers that are allocated with
+** "new" and must be deallocated with "delete"!)
+*/
+void MeshStreamingStrategy::insertMessage (CharmMessageHolder *cmsg)
+{
+  int dest_pe;
+  int dest_row;
+  int dest_col;
+  int env_size;
+  int misc_size;
+  int total_size;
+  char *usr;
+  char *env;
+  char *blk;
+  char *newmsg;
+
+
+  ComlibPrintf ("[%d] MeshStreamingStrategy::insertMessage() invoked.\n", CkMyPe());
+
+  dest_pe = cmsg->dest_proc;
+  dest_col = dest_pe % num_columns;
+  usr = cmsg->getCharmMessage ();
+  env = (char *) UsrToEnv (usr);
+  blk = (char *) BLKSTART (env);
+  env_size = SIZEFIELD (env);
+  misc_size = (env - blk);
+  total_size = sizeof (int) + misc_size + env_size;
+
+  if (dest_pe == my_pe) {
+    CmiSyncSend (my_pe, env_size, env);
+  } else if (dest_col == my_column) {
+    newmsg = (char *) CmiAlloc (env_size);
+    memcpy (newmsg, env, env_size);
+
+    dest_row = dest_pe / row_length;
+
+    InsertIntoRowBucket (dest_row, newmsg);
+  } else {
+    newmsg = new char[total_size];
+    ((int *) newmsg)[0] = dest_pe;
+    memcpy ( (void *) &(((int *) newmsg)[1]), blk, misc_size + env_size);
+
+    column_bucket[dest_col].enq (newmsg);
+    column_bytes[dest_col] += total_size;
+
+    if (column_bucket[dest_col].length() > max_bucket_size) {
+      FlushColumn (dest_col);
+    }
+  }
+
+  delete cmsg;
+}
+
+
+
+/**************************************************************************
+** This method is not used for streaming strategies.
+*/
+void MeshStreamingStrategy::doneInserting ()
+{
+  ComlibPrintf ("[%d] MeshStreamingStrategy::doneInserting() invoked.\n", CkMyPe());
+
+  // Empty for this strategy.
+}
+
+
+
+/**************************************************************************
+** This method is invoked prior to any processing taking place in the
+** class.  Various initializations take place here that cannot take place
+** in the class constructor due to the communications library itself not
+** being totally initialized.
+**
+** See MeshStreamingStrategy::MeshStreamingStrategy() for more details.
+*/
+void MeshStreamingStrategy::beginProcessing (int ignored)
+{
+  ComlibPrintf ("[%d] MeshStreamingStrategy::beginProcessing() invoked.\n", CkMyPe());
+
+  strategy_id = myInstanceID;
+
+  my_pe = CkMyPe ();
+
+  my_column = my_pe % num_columns;
+  my_row = my_pe / row_length;
+
+  column_bucket = new CkQ<char *>[num_columns];
+  column_bytes = new int[num_columns];
+  for (int i = 0; i < num_columns; i++) {
+    column_bytes[i] = 0;
+  }
+
+  row_bucket = new CkQ<char *>[num_rows];
+
+  column_handler_id = CkRegisterHandler ((CmiHandler) column_handler);
+
+  CcdCallOnConditionKeepOnPE(CcdPROCESSOR_BEGIN_IDLE, idle_flush_handler,
+                         (void *) this, CkMyPe());
+  RegisterPeriodicFlush ();
+}
+
+
+
+/**************************************************************************
+** This method exists so periodic_flush_handler() can re-register itself to
+** be invoked periodically to flush buffers.
+*/
+void MeshStreamingStrategy::RegisterPeriodicFlush (void)
+{
+  ComlibPrintf ("[%d] MeshStreamingStrategy::RegisterPeriodicFlush() invoked.\n", CkMyPe());
+
+  CcdCallFnAfterOnPE(periodic_flush_handler, (void *) this, flush_period, CkMyPe());
+}
+
+
+
+/**************************************************************************
+** This method is used to flush a specified column bucket, either as the
+** result of the column bucket reaching its maximum capacity, as a result
+** of the periodic flush handler being invoked, or as a result of the
+** processor going idle.
+**
+** The method first finds the destination PE for the column.  This is the
+** PE in the target column that is within the same row as the current PE.
+**
+** If there are actually messages in the bucket, then space is allocated
+** to hold the new message which will pack all of the messages in the
+** column bucket together.  The layout of this message is shown below:
+**
+**             \  /
+** +------------||---------------------------------------------------+
+** |Conv| I | # || dest || size | ref || Converse | Charm++  || user | ...
+** |hdr | D |   ||  PE  ||      | cnt ||  header  | envelope || data | ...
+** +------------||---------------------------------------------------+
+**             /  \
+**
+** Since the buffer represents a Converse message, it must begin with a
+** Converse header.  After the header is an int representing the Commlib
+** strategy ID for this strategy.  This is needed only so that the
+** column_handler() can get a pointer to the MeshStreamingStrategy class
+** later.  Next, comes an int containing the number of messages within
+** the packed message.  Finally, the messages are removed from the column
+** bucket and appended one after another into the buffer.
+**
+** After packing, a handler is set on the message to cause it to invoke
+** column_handler() on the destination PE and the message is finally
+** sent with CmiSyncSendAndFree().
+**
+** The buffer that is allocated in this message is used as a Converse
+** message, so it is allocated with CmiAlloc() so the send routine can
+** properly free it with CmiFree().  Therefore it has two ints for size
+** and ref count at the beginning of the buffer.  These are not shown in
+** the diagram above since they are basically irrelevant to this software.
+*/
+void MeshStreamingStrategy::FlushColumn (int column)
+{
+  int dest_pe;
+  int num_msgs;
+  int newmsgsize;
+  int msgsize;
+  char *newmsg;
+  char *newmsgptr;
+  char *msgptr;
+
+
+  ComlibPrintf ("[%d] MeshStreamingStrategy::FlushColumn() invoked.\n", CkMyPe());
+
+  assert (column < num_columns);
+
+  dest_pe = column + (my_row * row_length);
+  if (dest_pe >= num_pe) {
+    // This means that there is a hole in the mesh.
+    dest_pe = column + ((my_row % (num_rows - 1) - 1) * row_length);
+  }
+
+  num_msgs = column_bucket[column].length ();
+
+  if (num_msgs > 0) {
+    newmsgsize = CmiMsgHeaderSizeBytes + 2 * sizeof (int) +
+                 column_bytes[column];
+    newmsg = (char *) CmiAlloc (newmsgsize);
+
+    ((int *) (newmsg + CmiMsgHeaderSizeBytes))[0] = strategy_id;
+    ((int *) (newmsg + CmiMsgHeaderSizeBytes))[1] = num_msgs;
+
+    newmsgptr = (char *) (newmsg + CmiMsgHeaderSizeBytes + 2 * sizeof (int));
+    for (int i = 0; i < num_msgs; i++) {
+      msgptr = column_bucket[column].deq ();
+      msgsize = ((int *) msgptr)[1] + (3 * sizeof (int));
+      memcpy (newmsgptr, msgptr, msgsize);
+
+      newmsgptr += msgsize;
+
+      delete [] msgptr;
+    }
+
+    column_bytes[column] = 0;
+
+    CmiSetHandler (newmsg, column_handler_id);
+
+    CmiSyncSendAndFree (dest_pe, newmsgsize, newmsg);
+  }
+}
+
+
+
+/**************************************************************************
+** This method is used to flush a specified row bucket, either as the
+** result of the row bucket reaching its maximum capacity, as a result
+** of the periodic flush handler being invoked, or as a result of the
+** processor going idle.
+**
+** The method first finds the destination PE for the row.  The method then
+** iterates through the messages in the row bucket and constructs an array
+** for sizes[] of the message sizes and an array for msgComps[] of
+** pointers to the messages in the row bucket.  The method also sets the
+** handler for each message to be "RecvmsgHandle" which is the handler
+** for multi-message sends.  Finally, the method calls CmiMultiSend() to
+** send all messages to the destination PE in one go.
+**
+** After the row bucket is emptied, the method calls CmiFree() to
+** deallocate space for the individual messages.  Since each message was
+** allocated via CmiAlloc() this is appropriate.
+**
+** Each message in the row bucket has the layout shown in the diagram
+** below.
+**
+**     +--------------------------------------------------------------+
+**     | size | refcount || Converse | Charm++  ||        user        |
+**     |      |          ||  header  | envelope ||        data        |
+**     +--------------------------------------------------------------+
+**                        ^
+**                        |
+**                        msg
+**
+*/
+void MeshStreamingStrategy::FlushRow (int row)
+{
+  int dest_pe;
+  int num_msgs;
+  int *sizes;
+  char *msg;
+  char **msgComps;
+  int i;
+
+  ComlibPrintf ("[%d] MeshStreamingStrategy::FlushRow() invoked.\n", CkMyPe());
+
+  assert (row < num_rows);
+
+  dest_pe = my_column + (row * row_length);
+
+  num_msgs = row_bucket[row].length ();
+  if (num_msgs > 0) {
+    sizes = new int[num_msgs];
+    msgComps = new char *[num_msgs];
+
+    for (i = 0; i < num_msgs; i++) {
+      msg = row_bucket[row].deq ();
+      CmiSetHandler (msg, CkpvAccess(RecvmsgHandle));
+      sizes[i] = SIZEFIELD (msg);
+      msgComps[i] = msg;
+    }
+
+    CmiMultipleSend (dest_pe, num_msgs, sizes, msgComps);
+
+    for (i = 0; i < num_msgs; i++) {
+      CmiFree (msgComps[i]);
+    }
+
+    delete [] sizes;
+    delete [] msgComps;
+  }
+}
+
+
+
+/**************************************************************************
+** This method exists so various handlers can easily trigger all column
+** buckets and row buckets to flush.
+*/
+void MeshStreamingStrategy::FlushBuffers (void)
+{
+  ComlibPrintf ("[%d] MeshStreamingStrategy::PeriodicFlush() invoked.\n", CkMyPe());
+
+  for (int column = 0; column < num_columns; column++) {
+    FlushColumn (column);
+  }
+
+  for (int row = 0; row < num_rows; row++) {
+    FlushRow (row);
+  }
+}
+
+
+
+/**************************************************************************
+** This method exists primarily so column_handler() can insert messages
+** into a specified row bucket.
+*/
+void MeshStreamingStrategy::InsertIntoRowBucket (int row, char *msg)
+{
+  ComlibPrintf ("[%d] MeshStreamingStrategy::InsertIntoRowBucket() invoked.\n", CkMyPe());
+
+  assert (row < num_rows);
+
+  row_bucket[row].enq (msg);
+  if (row_bucket[row].length() > max_bucket_size) {
+    FlushRow (row);
+  }
+}
+
+
+
+/**************************************************************************
+** This method exists only so column_handler() can get the length of a row
+** in the mesh.  Since it is outside of the MeshStreamingStrategy class, it
+** does not have direct access to the class variables.
+*/
+int MeshStreamingStrategy::GetRowLength (void)
+{
+  ComlibPrintf ("[%d] MeshStreamingStrategy::GetRowLength() invoked.\n", CkMyPe());
+
+  return (row_length);
+}
+
+
+
+/**************************************************************************
+** This is a very complicated pack/unpack method.
+**
+** This method must handle the column_bucket[] and row_bucket[] data
+** structures.  These are arrays of queues of (char *).  To pack these,
+** we must iterate through the data structures and pack the sizes of
+** each message (char *) pointed to by each queue entry.
+*/
+void MeshStreamingStrategy::pup (PUP::er &p)
+{
+  int i;
+  ComlibPrintf ("[%d] MeshStreamingStrategy::pup() invoked.\n", CkMyPe());
+
+  // Call the superclass method -- easy.
+  Strategy::pup (p);
+
+  // Pup the instance variables -- easy.
+  p | num_pe;
+  p | num_columns;
+  p | num_rows;
+  p | row_length;
+
+  p | my_pe;
+  p | my_column;
+  p | my_row;
+
+  p | max_bucket_size;
+  p | flush_period;
+  p | strategy_id;
+  p | column_handler_id;
+
+  // Handle the column_bucket[] data structure.
+  // For each element in column_bucket[], pup the length of the queue
+  // at that element followed by the contents of that queue.  For each
+  // queue, pup the size of the message pointed to by the (char *)
+  // entry, followed by the memory for the (char *) entry.
+  if (p.isUnpacking ()) {
+    column_bucket = new CkQ<char *>[num_columns];
+  }
+
+  for (i = 0; i < num_columns; i++) {
+    int length = column_bucket[i].length ();
+
+    p | length;
+
+    for (int j = 0; j < length; j++) {
+      char *msg = column_bucket[i].deq ();
+      int size = sizeof (int) + ((int *) msg)[1];
+      p | size;
+      p(msg, size);
+    }
+  }
+
+  // Handle the column_bytes[] data structure.
+  // This is a straightforward packing of an int array.
+  if (p.isUnpacking ()) {
+    column_bytes = new int[num_columns];
+  }
+
+  p(column_bytes, num_columns);
+
+  // Handle the row_bucket[] data structure.
+  // This works exactly like the column_bucket[] above.
+  if (p.isUnpacking ()) {
+    row_bucket = new CkQ<char *>[num_rows];
+  }
+
+  for (i = 0; i < num_rows; i++) {
+    int length = row_bucket[i].length ();
+
+    p | length;
+
+    for (int j = 0; j < length; j++) {
+      char *msg = row_bucket[i].deq ();
+      int size = ((int *) msg)[0];
+      p | size;
+      p(msg, size);
+    }
+  }
+}
diff --git a/src/ck-com/MeshStreamingStrategy.h b/src/ck-com/MeshStreamingStrategy.h
new file mode 100644 (file)
index 0000000..f5cfae1
--- /dev/null
@@ -0,0 +1,59 @@
+/**************************************************************************
+** Greg Koenig (koenig@uiuc.edu)
+**
+** This is MeshStreamingStrategy, a strategy in the Charm++ communications
+** library.  See MeshStreamingStrategy.C for detailed comments.
+*/
+
+#ifndef MESH_STREAMING_STRATEGY
+#define MESH_STREAMING_STRATEGY
+
+#include <assert.h>
+#include <math.h>
+
+#include "ComlibManager.h"
+
+#define DEFAULT_FLUSH_PERIOD 10        // milliseconds
+#define DEFAULT_MAX_BUCKET_SIZE 1000   // number of messages
+
+class MeshStreamingStrategy : public CharmStrategy
+{
+  public:
+    MeshStreamingStrategy (int period=DEFAULT_FLUSH_PERIOD,
+                          int bucket_size=DEFAULT_MAX_BUCKET_SIZE);
+    MeshStreamingStrategy (CkMigrateMessage *m) : CharmStrategy(m){ }
+
+    void insertMessage (CharmMessageHolder *msg);
+    void doneInserting ();
+    void beginProcessing (int ignored);
+    void RegisterPeriodicFlush (void);
+    void FlushColumn (int column);
+    void FlushRow (int row);
+    void FlushBuffers (void);
+    void InsertIntoRowBucket (int row, char *msg);
+    int GetRowLength (void);
+    virtual void pup (PUP::er &p);
+    PUPable_decl (MeshStreamingStrategy);
+
+  private:
+    int num_pe;
+    int num_columns;
+    int num_rows;
+    int row_length;
+
+    int my_pe;
+    int my_column;
+    int my_row;
+
+    int flush_period;
+    int max_bucket_size;
+
+    int strategy_id;
+
+    int column_handler_id;
+
+    CkQ<char *> *column_bucket;
+    int *column_bytes;
+    CkQ<char *> *row_bucket;
+};
+#endif
diff --git a/src/ck-com/MsgPacker.C b/src/ck-com/MsgPacker.C
new file mode 100644 (file)
index 0000000..8d938a4
--- /dev/null
@@ -0,0 +1,169 @@
+#include "ComlibManager.h"
+#include "MsgPacker.h"
+#include "register.h"
+
+CkpvExtern(int, RecvCombinedShortMsgHdlrIdx);
+
+void short_envelope::pup(PUP::er &p){
+    p | idx;
+    p | epIdx;
+    p | size;
+    
+    if(p.isUnpacking()) 
+        data = new char[size];    
+    p(data, size);
+}
+
+short_envelope::short_envelope(){
+    epIdx = 0;
+    data = NULL;
+}
+
+short_envelope::~short_envelope(){
+    /*
+      if(data) 
+      delete [] data;        
+      data = NULL;
+    */
+}
+
+
+MsgPacker::MsgPacker(){
+    nShortMsgs = 0;
+    msgList = 0;    
+}
+
+MsgPacker::MsgPacker(CkQ<CharmMessageHolder *> &msgq, int n_msgs){
+
+    CkAssert(n_msgs < 65536);  //16 bit field for num messages
+
+    nShortMsgs = n_msgs;
+    msgList = new short_envelope[n_msgs];    
+
+    for(int count = 0; count < n_msgs; count ++){
+        CharmMessageHolder *cmsg = msgq.deq();
+        envelope *env = (envelope *)UsrToEnv(cmsg->getCharmMessage());
+        CkPackMessage(&env);
+
+        if(count == 0) {
+            aid = env->getsetArrayMgr();
+            if(aid.isZero()) CkAbort("Array packing set and ArrayID is zero");
+        }        
+        
+        msgList[count].epIdx = env->getsetArrayEp();
+        msgList[count].size = env->getTotalsize() - sizeof(envelope);
+        msgList[count].idx = env->getsetArrayIndex();
+        msgList[count].data = cmsg->getCharmMessage();
+
+        if(msgList[count].size > MAX_MESSAGE_SIZE)
+            CkAbort("Can't send messges larger than 64KB\n");
+
+        delete cmsg;
+    }
+}
+
+MsgPacker::~MsgPacker(){
+    if(nShortMsgs > 0 && msgList != NULL) {
+        for(int count = 0; count < nShortMsgs; count ++)
+            CkFreeMsg(msgList[count].data);        
+        
+        delete [] msgList;
+    }
+}
+
+void MsgPacker::getMessage(CombinedMessage *&cmb_msg, int &total_size){
+    int count;
+    PUP::sizer sp;
+    for(count = 0; count < nShortMsgs; count ++)
+        sp | msgList[count];
+
+    int size = sp.size();  
+    total_size = ALIGN8(sizeof(CombinedMessage)) + size;
+    
+    //CkPrintf("In MsgPacker with %d bytes and %d messages\n", total_size, 
+    //           nShortMsgs);
+
+    cmb_msg = (CombinedMessage *)CmiAlloc(total_size);
+
+    PUP::toMem mp((char *)cmb_msg + ALIGN8(sizeof(CombinedMessage)));
+    for(count = 0; count < nShortMsgs; count ++)
+        mp | msgList[count];
+
+    cmb_msg->aid = aid;
+    cmb_msg->srcPE = CkMyPe();
+    cmb_msg->nmsgs = nShortMsgs;
+
+    CmiSetHandler(cmb_msg, CkpvAccess(RecvCombinedShortMsgHdlrIdx));
+}
+
+
+void MsgPacker::deliver(CombinedMessage *cmb_msg){
+    int nmsgs = cmb_msg->nmsgs;
+
+    ComlibPrintf("In MsgPacker::deliver\n");
+
+    char *from_addr = (char *)cmb_msg + ALIGN8(sizeof(CombinedMessage));
+    PUP::fromMem fp(from_addr);
+    CkArrayID aid = cmb_msg->aid;
+
+    int src_pe = cmb_msg->srcPE;
+
+    for(int count = 0; count < nmsgs; count ++){
+        short_envelope senv;
+        fp | senv;
+        
+        int ep = senv.epIdx;
+        CkArrayIndexMax idx = senv.idx;
+        int size = senv.size;
+
+        CProxyElement_ArrayBase ap(aid, idx);
+        ArrayElement *a_elem = ap.ckLocal();
+        CkArray *a=(CkArray *)_localBranch(aid);
+
+        int msgIdx = _entryTable[ep]->msgIdx;
+        if(_entryTable[ep]->noKeep && a_elem != NULL) {
+            //Unpack the message
+            senv.data = (char *)_msgTable[msgIdx]->unpack(senv.data); 
+            CkDeliverMessageReadonly(ep, senv.data, a_elem);            
+            delete[] senv.data;
+        }
+        else {
+            //envelope *env = (envelope *)CmiAlloc(sizeof(envelope) + size);
+            envelope *env = _allocEnv(ForArrayEltMsg, sizeof(envelope) + size);
+
+            void *data = EnvToUsr(env);
+            memcpy(data, senv.data, size);
+            
+            //Unpack the message
+            data = (char *)_msgTable[msgIdx]->unpack(data); 
+            
+            env->getsetArrayMgr() = aid;
+            env->getsetArrayIndex() = idx;
+            env->getsetArrayEp() = ep;
+            env->setPacked(0); 
+            env->getsetArraySrcPe()=src_pe;  
+            env->getsetArrayHops()=1;  
+            env->setQueueing(CK_QUEUEING_FIFO);            
+            env->setUsed(0);
+            env->setMsgIdx(msgIdx);
+
+            env->setTotalsize(sizeof(envelope) + size);
+
+            //if(a_elem)
+            //  CkDeliverMessageFree(ep, data, a_elem);                     
+            //else
+            //ap.ckSend((CkArrayMessage *)data, ep);
+            
+            a->deliver((CkArrayMessage *)data, CkDeliver_queue, CmiTrue);
+
+            delete[] senv.data;
+        }        
+    }      
+        
+    CmiFree(cmb_msg);
+}
+
+
+
+
+
diff --git a/src/ck-com/MsgPacker.h b/src/ck-com/MsgPacker.h
new file mode 100644 (file)
index 0000000..ebe74a4
--- /dev/null
@@ -0,0 +1,46 @@
+#ifndef MESSAGE_PACKER_H
+#define MESSAGE_PACKER_H
+
+#include "charm++.h"
+#include "envelope.h"
+
+#define MAX_MESSAGE_SIZE 65535
+
+class short_envelope {
+ public:
+    UShort epIdx;
+    UShort size;  //Can only send messages up to 64KB :)    
+    CkArrayIndexMax idx;
+    char *data;
+
+    short_envelope();
+    ~short_envelope();
+    short_envelope(CkMigrateMessage *){}
+    
+    void pup(PUP::er &p);
+};
+PUPmarshall(short_envelope);
+
+struct CombinedMessage{
+
+    char header[CmiReservedHeaderSize];
+    CkArrayID aid;
+    int srcPE;
+    int nmsgs;
+};
+
+class MsgPacker {        
+    CkArrayID aid;
+    short_envelope * msgList;
+    int nShortMsgs;   
+
+ public:
+    MsgPacker();
+    ~MsgPacker();    
+    MsgPacker(CkQ<CharmMessageHolder*> &cmsg_list, int n_msgs);
+    void getMessage(CombinedMessage *&msg, int &size);
+
+    static void deliver(CombinedMessage *cmb_msg);
+};
+
+#endif
diff --git a/src/ck-com/NodeMulticast.C b/src/ck-com/NodeMulticast.C
new file mode 100644 (file)
index 0000000..2694207
--- /dev/null
@@ -0,0 +1,338 @@
+#include "NodeMulticast.h"
+#include "converse.h"
+
+#define MAX_BUF_SIZE 165000
+#define MAX_SENDS_PER_BATCH 16
+#define MULTICAST_DELAY 5
+
+static NodeMulticast *nm_mgr;
+
+static void call_doneInserting(void *ptr,double curWallTime){
+    NodeMulticast *mgr = (NodeMulticast *)ptr;
+    mgr->doneInserting();
+}
+
+static void* NodeMulticastHandler(void *msg){
+    ComlibPrintf("In Node MulticastHandler\n");
+    nm_mgr->recvHandler(msg);
+    return NULL;
+}
+
+static void* NodeMulticastCallbackHandler(void *msg){
+    ComlibPrintf("[%d]:In Node MulticastCallbackHandler\n", CkMyPe());
+    register envelope *env = (envelope *)msg;
+    CkUnpackMessage(&env);
+    //nm_mgr->getCallback().send(EnvToUsr(env));
+
+    //nm_mgr->getHandler()(env);
+    return NULL;
+}
+
+//Handles multicast by sending only one message to a nodes and making 
+//them multicast locally
+void NodeMulticast::setDestinationArray(CkArrayID a, int nelem, 
+                                       CkArrayIndexMax **idx, int ep){
+
+    mode = ARRAY_MODE;
+    messageBuf = NULL;
+    pes_per_node = 4;
+    if(getenv("RMS_NODES") != NULL)
+        pes_per_node = CkNumPes()/atoi(getenv("RMS_NODES"));
+
+    mAid = a;
+    nelements = nelem;
+    entryPoint = ep;
+  
+    numNodes = CkNumPes()/pes_per_node;
+    numCurDestPes = CkNumPes();
+    myRank = 0;
+    nodeMap = new int[numNodes];
+  
+    ComlibPrintf("In SetDestinationArray %d, %d, %d, %d\n", numNodes, 
+                 pes_per_node, nelements, ep);
+  
+    indexVec = new CkVec<CkArrayIndexMax> [CkNumPes()];
+    
+    for(int count = 0; count < nelements; count++) {
+        ComlibPrintf("Before lastKnown %d\n", count);
+        int dest_proc = CkArrayID::CkLocalBranch(a)->lastKnown(*idx[count]);
+        ComlibPrintf("After lastKnown %d\n", dest_proc);
+        nodeMap[dest_proc/pes_per_node] = 1;
+        
+        indexVec[dest_proc].insertAtEnd(*idx[count]);
+    }    
+
+    ComlibPrintf("After SetDestinationArray\n");
+}
+/*
+void NodeMulticast::setPeList(int npes, int *pelist, ComlibMulticastHandler handler){
+    mode = PROCESSOR_MODE;
+    messageBuf = NULL;
+    pes_per_node = 4;
+    //if(getenv("RMS_NODES") != NULL)
+    //pes_per_node = CkNumPes()/atoi(getenv("RMS_NODES"));
+
+    //cb = callback;
+    this->handler = (long)handler;
+  
+    numNodes = CkNumPes()/pes_per_node;
+    numCurDestPes = npes;
+    
+    myRank = 0;
+    nodeMap = new int[numNodes];
+  
+    this->npes = npes;
+    this->pelist = new int[npes];
+    memcpy(this->pelist, pelist, npes * sizeof(int));
+
+    ComlibPrintf("In setPeList %d, %d, %d\n", numNodes, 
+                 pes_per_node, npes);
+    
+    for(int count = 0; count < npes; count++)
+        nodeMap[pelist[count]/pes_per_node] = 1;        
+    
+    ComlibPrintf("After setPeList\n");
+}
+*/
+
+void NodeMulticast::recvHandler(void *msg) {
+    register envelope* env = (envelope *)msg;
+    void *charm_msg = (void *)EnvToUsr(env);
+
+    env->setUsed(0);
+    ComlibPrintf("In receive Handler\n");
+    if(mode == ARRAY_MODE) {
+        env->getsetArrayMgr()=mAid;
+       env->getsetArrayEp()=entryPoint;
+       env->getsetArrayHops()=0;       
+       CkUnpackMessage(&env);
+
+       for(int count = 0; count < pes_per_node; count ++){
+           int dest_pe = (CkMyPe()/pes_per_node) * pes_per_node + count;
+           int size = indexVec[dest_pe].size();
+           
+           ComlibPrintf("[%d], %d elements to send to %d of size %d\n", CkMyPe(), size, dest_pe, env->getTotalsize());
+           
+           CkArrayIndexMax * idx_arr = indexVec[dest_pe].getVec();
+           for(int itr = 0; itr < size; itr ++) {
+               void *newcharmmsg = CkCopyMsg(&charm_msg); 
+               envelope* newenv = UsrToEnv(newcharmmsg);
+               CProxyElement_ArrayBase ap(mAid, idx_arr[itr]);         
+               newenv->getsetArrayIndex()=idx_arr[itr];
+               ap.ckSend((CkArrayMessage *)newcharmmsg, entryPoint);
+           }
+       }
+    }
+    else {
+      CkUnpackMessage(&env);
+      for(int count = 0; count < pes_per_node; count++) 
+       if(validRank[count]){
+            void *newcharmmsg;
+            envelope* newenv;
+         
+            if(count <  pes_per_node - 1) {
+                newcharmmsg = CkCopyMsg(&charm_msg); 
+                newenv = UsrToEnv(newcharmmsg);
+            }
+            else {
+                newcharmmsg = charm_msg;
+                newenv = UsrToEnv(newcharmmsg);
+            }
+
+            CmiSetHandler(newenv, NodeMulticastCallbackHandlerId);
+            ComlibPrintf("[%d] In receive Handler (proc mode), sending message to %d at handler %d\n", 
+                         CkMyPe(), (CkMyPe()/pes_per_node) * pes_per_node 
+                         + count, NodeMulticastCallbackHandlerId);
+            
+            CkPackMessage(&newenv);
+            CmiSyncSendAndFree((CkMyPe()/pes_per_node) *pes_per_node + count, 
+                               newenv->getTotalsize(), (char *)newenv);
+       }
+    }
+    ComlibPrintf("[%d] CmiFree (Code) (%x)\n", CkMyPe(), 
+                 (long) msg - 2*sizeof(int));
+    //CmiFree(msg);
+}
+
+void NodeMulticast::insertMessage(CharmMessageHolder *cmsg){
+
+    ComlibPrintf("In insertMessage \n");
+    envelope *env = UsrToEnv(cmsg->getCharmMessage());
+
+    CmiSetHandler(env, NodeMulticastHandlerId);
+    messageBuf->enq(cmsg);
+}
+
+void NodeMulticast::doneInserting(){
+    CharmMessageHolder *cmsg;
+    char *msg;
+    register envelope *env;
+    
+    ComlibPrintf("NodeMulticast :: doneInserting\n");
+    
+    if(messageBuf->length() > 1) {
+        //CkPrintf("NodeMulticast :: doneInserting length > 1\n");
+        /*
+        char **msgComps;
+        int *sizes, msg_count;
+    
+        msgComps = new char*[messageBuf->length()];
+        sizes = new int[messageBuf->length()];
+        msg_count = 0;
+        while (!messageBuf->isEmpty()) {
+            cmsg = messageBuf->deq();
+            msg = cmsg->getCharmMessage();
+            env = UsrToEnv(msg);
+            sizes[msg_count] = env->getTotalsize();
+            msgComps[msg_count] = (char *)env;
+            msg_count++;
+            
+            delete cmsg;
+        }
+        
+        for(int count = 0; count < numNodes; count++)
+            if(nodeMap[count])
+                CmiMultipleSend(count * pes_per_node + myRank, msg_count, 
+                                sizes, msgComps);
+        
+        delete [] msgComps;
+        delete [] sizes;
+        */
+    }
+    else if (messageBuf->length() == 1){
+        static int prevCount = 0;
+        int count = 0;
+        ComlibPrintf("Sending Node Multicast\n");
+        cmsg = messageBuf->deq();
+        msg = cmsg->getCharmMessage();
+        env = UsrToEnv(msg);
+       
+       if(mode == ARRAY_MODE)
+           env->getsetArraySrcPe()=CkMyPe();
+       CkPackMessage(&env);
+
+        CmiSetHandler(env, NodeMulticastHandlerId);
+        ComlibPrintf("After set handler\n");
+
+        //CmiPrintf("cursedtpes = %d, %d\n", cmsg->npes, numCurDestPes);
+        
+        if((mode != ARRAY_MODE) && cmsg->npes < numCurDestPes) {
+            numCurDestPes = cmsg->npes;
+            for(count = 0; count < numNodes; count++) 
+                nodeMap[count] = 0;        
+            
+            for(count = 0; count < cmsg->npes; count++) 
+                nodeMap[(cmsg->pelist[count])/pes_per_node] = 1;        
+        }
+        
+        for(count = prevCount; count < numNodes; count++) {
+           //int dest_node = count;
+           int dest_node = (count + (CkMyPe()/pes_per_node))%numNodes;
+           if(nodeMap[dest_node]) {
+                void *newcharmmsg;
+                envelope* newenv;
+                
+                if(count < numNodes - 1) {
+                    newcharmmsg = CkCopyMsg((void **)&msg); 
+                    newenv = UsrToEnv(newcharmmsg);
+                }
+                else {
+                    newcharmmsg = msg;
+                    newenv = UsrToEnv(newcharmmsg);
+                }
+               
+               ComlibPrintf("[%d]In cmisyncsend to %d\n", CkMyPe(), 
+                            dest_node * pes_per_node + myRank);
+#if CMK_PERSISTENT_COMM
+               if(env->getTotalsize() < MAX_BUF_SIZE)
+                    CmiUsePersistentHandle(&persistentHandlerArray[dest_node],1);
+#endif
+               CkPackMessage(&newenv);
+               CmiSyncSendAndFree(dest_node * pes_per_node + myRank, 
+                                  newenv->getTotalsize(), (char *)newenv);
+#if CMK_PERSISTENT_COMM
+               if(env->getTotalsize() < MAX_BUF_SIZE)
+                    CmiUsePersistentHandle(NULL, 0);
+#endif          
+            }
+            prevCount ++;
+            if((prevCount % MAX_SENDS_PER_BATCH == 0) &&
+               (prevCount != numNodes)) {
+                CcdCallFnAfterOnPE((CcdVoidFn)call_doneInserting, (void *)this, 
+                               MULTICAST_DELAY, CkMyPe());
+                return;
+            }
+            prevCount = 0;
+       }
+
+        ComlibPrintf("[%d] CmiFree (Code) (%x)\n", CkMyPe(), (char *)env - 2*sizeof(int));
+        //CmiFree(env);
+        delete cmsg;
+    }
+}
+
+void NodeMulticast::pup(PUP::er &p){
+    
+    CharmStrategy::pup(p);
+
+    p | pes_per_node;
+    p | numNodes;
+    p | nelements;
+    p | entryPoint;
+    p | npes;
+    p | mode;
+    p | numCurDestPes;
+    p | mAid;
+    
+    if(p.isUnpacking()) {
+        nodeMap = new int[numNodes];
+       
+       if(mode == ARRAY_MODE) {
+           typedef CkVec<CkArrayIndexMax> CkVecArrayIndex;
+           CkVecArrayIndex *vec = new CkVecArrayIndex[CkNumPes()];
+           indexVec = vec;
+       }
+
+       if(mode == PROCESSOR_MODE)
+           pelist = new int[npes];
+    }
+
+    p | cb;
+    p | handler;
+    p(nodeMap, numNodes);
+
+    if(mode == PROCESSOR_MODE)
+      p(pelist, npes);
+
+    if(mode == ARRAY_MODE)
+      for(int count = 0; count < CkNumPes(); count++)
+        p | indexVec[count];
+    
+    if(p.isUnpacking()) {
+        messageBuf = new CkQ <CharmMessageHolder *>;
+        myRank = CkMyPe() % pes_per_node;
+        
+        NodeMulticastHandlerId = CkRegisterHandler((CmiHandler)NodeMulticastHandler);
+       NodeMulticastCallbackHandlerId = CkRegisterHandler
+           ((CmiHandler)NodeMulticastCallbackHandler);
+       
+        nm_mgr = this;
+
+       //validRank[0] =  validRank[1] = validRank[2] = validRank[3] = 0;
+        memset(validRank, 0, MAX_PES_PER_NODE * sizeof(int));
+       for(int count = 0; count < npes; count ++){
+           if(CkMyPe()/pes_per_node == pelist[count] / pes_per_node)
+               validRank[pelist[count] % pes_per_node] = 1;
+       }
+
+#if CMK_PERSISTENT_COMM
+       persistentHandlerArray = new PersistentHandle[numNodes];
+       for(int count = 0; count < numNodes; count ++)
+            //if(nodeMap[count])
+            persistentHandlerArray[count] = CmiCreatePersistent
+                (count * pes_per_node + myRank, MAX_BUF_SIZE);
+#endif
+    }
+}
+
+//PUPable_def(NodeMulticast);
diff --git a/src/ck-com/NodeMulticast.h b/src/ck-com/NodeMulticast.h
new file mode 100644 (file)
index 0000000..2944eb4
--- /dev/null
@@ -0,0 +1,50 @@
+#ifndef NODE_MULTICAST
+#define NODE_MULTICAST
+#include "ComlibManager.h"
+
+#if CMK_PERSISTENT_COMM
+#include "persistent.h"
+#endif
+
+#define MAX_PES_PER_NODE 16
+#define PROCESSOR_MODE 0
+#define ARRAY_MODE 1
+
+class NodeMulticast : public CharmStrategy {
+    CkQ <CharmMessageHolder*> *messageBuf;
+    int pes_per_node, *nodeMap, numNodes, myRank, numCurDestPes;
+    int mode; //Array destinations or processor destinations
+
+    CkArrayID mAid;
+    CkVec<CkArrayIndexMax> *indexVec;
+    int NodeMulticastHandlerId, entryPoint, nelements;
+    
+    int npes, *pelist, NodeMulticastCallbackHandlerId;
+    int validRank[MAX_PES_PER_NODE];
+    CkCallback cb;
+    long handler;
+
+#if CMK_PERSISTENT_COMM
+    PersistentHandle *persistentHandlerArray;
+#endif
+
+ public:
+    NodeMulticast(){}
+    void setDestinationArray(CkArrayID a, int nelem, 
+                            CkArrayIndexMax **idx, int ep);
+
+    //void setPeList(int npes, int *pelist, CkCallback callback);
+    //void setPeList(int npes, int *pelist, ComlibMulticastHandler handler);
+    
+    NodeMulticast(CkMigrateMessage *m) : CharmStrategy(m){}
+    void recvHandler(void *msg);
+    void insertMessage(CharmMessageHolder *msg);
+    void doneInserting();
+
+    CkCallback getCallback() { return cb;}
+    //ComlibMulticastHandler getHandler() { return (ComlibMulticastHandler)handler;}
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(NodeMulticast);
+};
+#endif
diff --git a/src/ck-com/PipeBroadcastStrategy.C b/src/ck-com/PipeBroadcastStrategy.C
new file mode 100644 (file)
index 0000000..943a953
--- /dev/null
@@ -0,0 +1,311 @@
+#include "PipeBroadcastStrategy.h"
+
+//PipeBcastHashKey CODE
+int PipeBcastHashKey::staticCompare(const void *k1,const void *k2,size_t ){
+    return ((const PipeBcastHashKey *)k1)->
+                compare(*(const PipeBcastHashKey *)k2);
+}
+
+CkHashCode PipeBcastHashKey::staticHash(const void *v,size_t){
+    return ((const PipeBcastHashKey *)v)->hash();
+}
+
+CkpvExtern(CkGroupID, cmgrID);
+
+void propagate_handler(void *message) {
+  // call the appropriate function PipeBroadcastStrategy::propagate
+  //int instid = ((envelope *)message)->getEpIdx();
+  //int instid = ((CkMcastBaseMsg*)(EnvToUsr((envelope*)message)))->_cookie.sInfo.cInfo.instId;
+  int instid = CmiGetXHandler(message);
+  PipeBroadcastStrategy *myStrategy = (PipeBroadcastStrategy *)CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()->getStrategy(instid);
+  myStrategy->propagate((envelope *)message, false);
+}
+
+void propagate_handler_frag(void *message) {
+  int instid = CmiGetXHandler(message);
+  PipeBroadcastStrategy *myStrategy = (PipeBroadcastStrategy *)CProxy_ComlibManager(CkpvAccess(cmgrID)).ckLocalBranch()->getStrategy(instid);
+  myStrategy->propagate((envelope *)message, true);
+}
+
+
+void PipeBroadcastStrategy::propagate(envelope *env, int isFragmented){
+  // find destination processors and send
+  int destination, tmp, k;
+  int num_pes, *dest_pes;
+  PipeBcastInfo *info = (PipeBcastInfo*)(((char*)env)+CmiReservedHeaderSize);
+  int srcPeNumber = isFragmented ? info->srcPe : env->getSrcPe();
+  int totalSendingSize = isFragmented ? info->chunkSize+CmiReservedHeaderSize+sizeof(PipeBcastInfo) : env->getTotalsize();
+
+  switch (topology) {
+  case USE_LINEAR:
+    if (srcPeNumber == (CkMyPe()+1)%CkNumPes()) break;
+    destination = (CkMyPe()+1) % CkNumPes();
+    ComlibPrintf("[%d] Pipebroadcast sending to %d\n",CkMyPe(), destination);
+    CmiSyncSend(destination, totalSendingSize, (char *)env);
+    break;
+  case USE_HYPERCUBE:
+    tmp = srcPeNumber ^ CkMyPe();
+    k = int(log((double)CkNumPes()) * log_of_2_inv + 2);
+    if (tmp) {
+      do {--k;} while (!(tmp>>k));
+    }
+    ComlibPrintf("[%d] tmp=%d, k=%d\n",CkMyPe(),tmp,k);
+    // now 'k' is the last dimension in the hypercube used for exchange
+    if (isFragmented) info->srcPe = CkMyPe();
+    else env->setSrcPe(CkMyPe());  // where the message is coming from
+    dest_pes = (int *)malloc(k*sizeof(int));
+    --k;  // next dimension in the cube to be used
+    num_pes = HypercubeGetBcastDestinations(k, dest_pes);
+
+    /*
+    for ( ; k>=0; --k) {
+      // add the processor destination at level k if it exist
+      dest_pes[num_pes] = CkMyPe() ^ (1<<k);
+      if (dest_pes[num_pes] >= CkNumPes()) {
+       dest_pes[num_pes] &= (-1)<<k;
+       if (CkNumPes()>dest_pes[num_pes]) dest_pes[num_pes] += (CkMyPe() - (CkMyPe() & ((-1)<<k))) % (CkNumPes() - dest_pes[num_pes]);
+      }
+      if (dest_pes[num_pes] < CkNumPes()) {
+       ComlibPrintf("[%d] PipeBroadcast sending to %d\n",CkMyPe(), dest_pes[num_pes]);
+       ++num_pes;
+      }
+    }
+    */
+
+    //CmiSyncListSend(num_pes, dest_pes, env->getTotalsize(), (char *)env);
+    for (k=0; k<num_pes; ++k) CmiSyncSend(dest_pes[k], totalSendingSize, (char *)env);
+    free(dest_pes);
+    break;
+
+    // for other strategies
+
+  default:
+    // should NEVER reach here!
+    CmiPrintf("Error, topology %d not known\n",topology);
+    CkExit();
+  }
+
+  // deliver messages to local objects (i.e. send it to ComlibManager)
+  deliverer(env, isFragmented);
+  //CmiSetHandler(env, CmiGetXHandler(env));
+  //CmiSyncSendAndFree(CkMyPe(), env->getTotalsize(), (char *)env);
+
+}
+
+void PipeBroadcastStrategy::deliverer(envelope *env_frag, int isFragmented) {
+  envelope *env;
+  int isFinished=0;
+  ComlibPrintf("isArray = %d\n", (getType() == ARRAY_STRATEGY));
+
+  // check if the message is fragmented
+  if (isFragmented) {
+    // store the fragment in the hash table until completed
+    ComlibPrintf("[%d] deliverer: received fragmented message, storing\n",CkMyPe());
+    PipeBcastInfo *info = (PipeBcastInfo*)(((char*)env_frag)+CmiReservedHeaderSize);
+
+    PipeBcastHashKey key (info->bcastPe, info->seqNumber);
+    PipeBcastHashObj *position = fragments.get(key);
+
+    char *incomingMsg;
+    if (position) {
+      // the message already exist, add to it
+      ComlibPrintf("[%d] adding to an existing message for id %d/%d (%d remaining)\n",CkMyPe(),info->bcastPe,info->seqNumber,position->remaining-1);
+      incomingMsg = position->message;
+      memcpy (incomingMsg+CmiReservedHeaderSize+((pipeSize-CmiReservedHeaderSize-sizeof(PipeBcastInfo))*info->chunkNumber), ((char*)env_frag)+CmiReservedHeaderSize+sizeof(PipeBcastInfo), info->chunkSize);
+
+      if (--position->remaining == 0) {  // message completely received
+       isFinished = 1;
+       env = (envelope*)incomingMsg;
+       // delete from the hash table
+       fragments.remove(key);
+      }
+
+    } else {
+      // the message doesn't exist, create it
+      ComlibPrintf("[%d] creating new message of size %d for id %d/%d; chunk=%d chunkSize=%d\n",CkMyPe(),info->messageSize,info->bcastPe,info->seqNumber,info->chunkNumber,info->chunkSize);
+      incomingMsg = (char*)CmiAlloc(info->messageSize);
+      memcpy (incomingMsg, env_frag, CmiReservedHeaderSize);
+      memcpy (incomingMsg+CmiReservedHeaderSize+((pipeSize-CmiReservedHeaderSize-sizeof(PipeBcastInfo))*info->chunkNumber), ((char*)env_frag)+CmiReservedHeaderSize+sizeof(PipeBcastInfo), info->chunkSize);
+      int remaining = (int)ceil((double)info->messageSize/(pipeSize-CmiReservedHeaderSize-sizeof(PipeBcastInfo)))-1;
+      if (remaining) {  // more than one chunk (it was not forced to be splitted)
+       PipeBcastHashObj *object = new PipeBcastHashObj(info->messageSize, remaining, incomingMsg);
+       fragments.put(key) = object;
+      } else {  // only one chunk, it was forces to be splitted
+       isFinished = 1;
+       env = (envelope*)incomingMsg;
+       // nothing to delete from fragments since nothing has been added
+      }
+    }
+    CmiFree(env_frag);
+
+  } else {  // message not fragmented
+    ComlibPrintf("[%d] deliverer: received message in single chunk\n",CkMyPe());
+    isFinished = 1;
+    env = env_frag;
+  }
+
+  if (isFinished) {
+    if (getType() == ARRAY_STRATEGY) {
+      CkArray *dest_array = CkArrayID::CkLocalBranch(aid);
+      localDest = new CkVec<CkArrayIndexMax>;
+      dest_array->getComlibArrayListener()->getLocalIndices(*localDest);
+      void *msg = EnvToUsr(env);
+      CkArrayIndexMax idx;
+      ArrayElement *elem;
+      int ep = env->getsetArrayEp();
+      CkUnpackMessage(&env);
+
+      ComlibPrintf("[%d] deliverer: delivering a finished message\n",CkMyPe());
+      for (int count = 0; count < localDest->size(); ++count) {
+       idx = (*localDest)[count];
+       ComlibPrintf("[%d] Sending message to ",CkMyPe());
+       if (comm_debug) idx.print();
+
+       CProxyElement_ArrayBase ap(aid, idx);
+       elem = ap.ckLocal();
+       CkDeliverMessageReadonly (ep, msg, elem);
+      }
+      delete localDest;
+      // the envelope env should be deleted only if the message is delivered
+      CmiFree(env);
+    }
+
+    if (getType() == GROUP_STRATEGY) {
+      // deliver the message to the predifined group "gid"
+      CkSendMsgBranchInline(env->getEpIdx(), EnvToUsr(env), CkMyPe(), gid);
+    }
+  }
+}
+
+PipeBroadcastStrategy::PipeBroadcastStrategy()
+  :topology(USE_HYPERCUBE), pipeSize(DEFAULT_PIPE), CharmStrategy() {
+
+    //isArray = 0;
+    commonInit();
+}
+
+PipeBroadcastStrategy::PipeBroadcastStrategy(int _topology)
+  :topology(_topology), pipeSize(DEFAULT_PIPE), CharmStrategy() {
+    //isArray = 0;
+  commonInit();
+}
+
+PipeBroadcastStrategy::PipeBroadcastStrategy(int _topology, int _pipeSize)
+  :topology(_topology), pipeSize(_pipeSize), CharmStrategy() {
+    //isArray = 0;
+  commonInit();
+}
+
+PipeBroadcastStrategy::PipeBroadcastStrategy(int _topology, CkArrayID _aid)
+  :topology(_topology), pipeSize(DEFAULT_PIPE), CharmStrategy() {
+    //isArray = 1;
+  setType(ARRAY_STRATEGY);
+  aid = _aid;
+  CmiPrintf("init: %d %d\n",topology, pipeSize);
+  commonInit();
+}
+
+PipeBroadcastStrategy::PipeBroadcastStrategy(int _topology, CkArrayID _aid, int _pipeSize)
+  :topology(_topology), pipeSize(_pipeSize), CharmStrategy() {
+    setType(ARRAY_STRATEGY);
+  aid = _aid;
+  commonInit();
+}
+
+PipeBroadcastStrategy::PipeBroadcastStrategy(CkGroupID _gid, int _topology, int _pipeSize)
+  :topology(_topology), pipeSize(_pipeSize), CharmStrategy() {
+    setType(GROUP_STRATEGY);
+  gid = _gid;
+  commonInit();
+}
+
+void PipeBroadcastStrategy::commonInit(){
+  log_of_2_inv = 1/log((double)2);
+  seqNumber = 0;
+}
+
+void PipeBroadcastStrategy::insertMessage(CharmMessageHolder *cmsg){
+  ComlibPrintf("[%d] Pipelined Broadcast with strategy %d\n",CkMyPe(),topology);
+  messageBuf->enq(cmsg);
+  doneInserting();
+}
+
+// routine for interfacing with converse.
+// Require only the converse reserved header if forceSplit is true
+void PipeBroadcastStrategy::conversePipeBcast(envelope *env, int totalSize, int forceSplit) {
+  // set the instance ID to be used by the receiver using the XHandler variable
+  CmiSetXHandler(env, myInstanceID);
+
+  if (totalSize > pipeSize || forceSplit) {
+    ++seqNumber;
+    // message doesn't fit into the pipe: split it into chunks and propagate them individually
+    ComlibPrintf("[%d] Propagating message in multiple chunks\n",CkMyPe());
+
+    char *sendingMsg;
+    char *nextChunk = ((char*)env)+CmiReservedHeaderSize;
+    int remaining = totalSize-CmiReservedHeaderSize;
+    int reducedPipe = pipeSize-CmiReservedHeaderSize-sizeof(PipeBcastInfo);
+    ComlibPrintf("reducedPipe = %d, CmiReservedHeaderSize = %d, sizeof(PipeBcastInfo) = %d\n",reducedPipe,CmiReservedHeaderSize,sizeof(PipeBcastInfo));
+    ComlibPrintf("sending %d chunks of size %d, total=%d\n",(int)ceil(((double)totalSize-CmiReservedHeaderSize)/reducedPipe),reducedPipe,remaining);
+    for (int i=0; i<(int)ceil(((double)totalSize-CmiReservedHeaderSize)/reducedPipe); ++i) {
+      sendingMsg = (char*)CmiAlloc(pipeSize);
+      CmiSetHandler(env, propagateHandle_frag);
+      memcpy (sendingMsg, env, CmiReservedHeaderSize);
+      PipeBcastInfo *info = (PipeBcastInfo*)(sendingMsg+CmiReservedHeaderSize);
+      info->srcPe = CkMyPe();
+      info->bcastPe = CkMyPe();
+      info->seqNumber = seqNumber;
+      info->chunkNumber = i;
+      info->chunkSize = reducedPipe<remaining ? reducedPipe : remaining;
+      info->messageSize = totalSize;
+      memcpy (sendingMsg+CmiReservedHeaderSize+sizeof(PipeBcastInfo), nextChunk, reducedPipe);
+
+      remaining -= reducedPipe;
+      nextChunk += reducedPipe;
+
+      propagate((envelope*)sendingMsg, true);
+    }
+
+  } else {
+    // the message fit into the pipe, so send it in a single chunk
+    ComlibPrintf("[%d] Propagating message in one single chunk\n",CkMyPe());
+    CmiSetHandler(env, propagateHandle);
+    env->setSrcPe(CkMyPe());
+    //env->setEpIdx(myInstanceID);
+    propagate(env, false);
+  }
+}
+
+void PipeBroadcastStrategy::doneInserting(){
+  ComlibPrintf("[%d] DoneInserting\n",CkMyPe());
+  while (!messageBuf->isEmpty()) {
+    CharmMessageHolder *cmsg = messageBuf->deq();
+    // modify the Handler to deliver the message to the propagator
+    envelope *env = UsrToEnv(cmsg->getCharmMessage());
+
+    conversePipeBcast(env, env->getTotalsize(), false);
+  }
+}
+
+void PipeBroadcastStrategy::pup(PUP::er &p){
+  ComlibPrintf("[%d] PipeBroadcast pupping %s\n",CkMyPe(), (p.isPacking()==0)?(p.isUnpacking()?"UnPacking":"sizer"):("Packing"));
+  CharmStrategy::pup(p);
+
+  p | aid;
+  p | gid;
+  p | pipeSize;
+  p | topology;
+  p | seqNumber;
+
+  if (p.isUnpacking()) {
+    log_of_2_inv = 1/log((double)2);
+    messageBuf = new CkQ<CharmMessageHolder *>;
+    propagateHandle = CkRegisterHandler((CmiHandler)propagate_handler);
+    propagateHandle_frag = CkRegisterHandler((CmiHandler)propagate_handler_frag);
+  }
+  //p|(*messageBuf);
+  //p|fragments;
+
+}
+
+//PUPable_def(PipeBroadcastStrategy);
diff --git a/src/ck-com/PipeBroadcastStrategy.h b/src/ck-com/PipeBroadcastStrategy.h
new file mode 100644 (file)
index 0000000..af1d0d2
--- /dev/null
@@ -0,0 +1,94 @@
+#ifndef PIPE_BROADCAST_STRATEGY
+#define PIPE_BROADCAST_STRATEGY
+#include "ComlibManager.h"
+
+#define DEFAULT_PIPE   8196
+
+struct PipeBcastInfo {
+  short bcastPe;     // pe who is doing the broadcast, used for the hash key
+  short seqNumber;
+  int chunkSize;   // it is the size of the data part of the message (without the converse header)
+  int chunkNumber;
+  int messageSize;   // the entire message size, all included
+  short srcPe;       // pe from whom I'm receiving the message
+};
+
+class PipeBcastHashKey{
+ public:
+
+    int srcPe;
+    int seq;
+    PipeBcastHashKey(int _pe, int _seq):srcPe(_pe), seq(_seq){};
+
+    //These routines allow PipeBcastHashKey to be used in
+    //  a CkHashtableT
+    CkHashCode hash(void) const;
+    static CkHashCode staticHash(const void *a,size_t);
+    int compare(const PipeBcastHashKey &ind) const;
+    static int staticCompare(const void *a,const void *b,size_t);
+};
+
+// sequential numbers must be below 2^16, so the number of processors must
+inline CkHashCode PipeBcastHashKey::hash(void) const
+{
+    register int _seq = seq;
+    register int _pe = srcPe;
+    
+    register CkHashCode ret = (_seq << 16) + _pe;
+    return ret;
+}
+
+inline int PipeBcastHashKey::compare(const PipeBcastHashKey &k2) const
+{
+    if(seq == k2.seq && srcPe == k2.srcPe)
+        return 1;
+    
+    return 0;
+}
+
+class PipeBcastHashObj{
+ public:
+  char *message;
+  int dimension;
+  int remaining;
+  PipeBcastHashObj (int dim, int rem, char *msg) :dimension(dim),remaining(rem),message(msg) {};
+
+};
+
+class PipeBroadcastStrategy : public CharmStrategy {
+ protected:
+
+  int pipeSize; // this is the size of the splitted messages, including the converse header
+  int topology;
+  double log_of_2_inv;
+  int seqNumber;
+  CkQ <CharmMessageHolder*> *messageBuf;
+  CkHashtableT<PipeBcastHashKey, PipeBcastHashObj *> fragments;
+  int propagateHandle;
+  int propagateHandle_frag;
+  CkVec<CkArrayIndexMax> *localDest;
+
+  CkArrayID aid;
+  CkGroupID gid;
+
+  void commonInit();
+  void deliverer(envelope *env, int isFrag);
+
+ public:
+  PipeBroadcastStrategy();
+  PipeBroadcastStrategy(int _topology);
+  PipeBroadcastStrategy(int _topology, int _pipeSize);
+  PipeBroadcastStrategy(int _topology, CkArrayID _aid);
+  PipeBroadcastStrategy(int _topology, CkArrayID _aid, int _pipeSize);
+  PipeBroadcastStrategy(CkGroupID _gid, int _topology=USE_HYPERCUBE, int _pipeSize=DEFAULT_PIPE);
+  PipeBroadcastStrategy(CkMigrateMessage *){}
+  void insertMessage(CharmMessageHolder *msg);
+  void doneInserting();
+  void conversePipeBcast(envelope *env, int size, int forceSplit);
+
+  void propagate(envelope *env, int isFrag);
+
+  virtual void pup(PUP::er &p);
+  PUPable_decl(PipeBroadcastStrategy);
+};
+#endif
diff --git a/src/ck-com/PrioStreaming.C b/src/ck-com/PrioStreaming.C
new file mode 100644 (file)
index 0000000..c32d950
--- /dev/null
@@ -0,0 +1,36 @@
+#include "PrioStreaming.h"
+#include "MsgPacker.h"
+
+PrioStreaming::PrioStreaming(int periodMs,int bufferMax_, int prio)
+    : StreamingStrategy(periodMs, bufferMax_), basePriority(prio)
+{
+}
+
+void PrioStreaming::insertMessage(CharmMessageHolder *cmsg) {
+
+    ComlibPrintf("Prio Straming: InsertMessage %d, %d\n",  
+                 PERIOD, bufferMax);
+
+    int pe=cmsg->dest_proc;
+    streamingMsgBuf[pe].enq(cmsg);
+    streamingMsgCount[pe]++;
+    if (streamingMsgCount[pe] > bufferMax) {
+        flushPE(pe);
+        return;
+    }
+    
+    char* msg = cmsg->getCharmMessage();
+    envelope *env = UsrToEnv(msg);
+    int msg_prio = *(int*)env->getPrioPtr();
+    
+    if(msg_prio <= basePriority)
+        flushPE(pe);
+}
+
+void PrioStreaming::pup(PUP::er &p){
+
+    StreamingStrategy::pup(p);
+
+    p | basePriority;
+
+}
diff --git a/src/ck-com/PrioStreaming.h b/src/ck-com/PrioStreaming.h
new file mode 100644 (file)
index 0000000..3f310cc
--- /dev/null
@@ -0,0 +1,35 @@
+#ifndef PRIO_STREAMING
+#define PRIO_STREAMING
+#include "ComlibManager.h"
+#include "StreamingStrategy.h"
+
+
+class PrioStreaming : public StreamingStrategy {
+ protected:
+    int basePriority;
+    
+ public:
+    /**
+     Create a priority based streaming strategy, suitable for passing
+     to ComlibManager.  
+     These are the criteria for flushing all pending messages: 
+
+     - it's been at least period (in ms) since the last flush, or 
+
+     - the processor just went idle.  Thses criteria flush a single 
+     PE's pending messages: 
+
+     - more than bufferMax messages to buffered for one PE.
+
+     - Current message is a high priority message
+    */
+
+    PrioStreaming(int periodMs=10, int bufferMax=1000, int prio=0);
+    PrioStreaming(CkMigrateMessage *){}
+    
+    virtual void insertMessage(CharmMessageHolder *msg);
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(PrioStreaming);
+};
+#endif
diff --git a/src/ck-com/RingMulticastStrategy.C b/src/ck-com/RingMulticastStrategy.C
new file mode 100644 (file)
index 0000000..b73165d
--- /dev/null
@@ -0,0 +1,269 @@
+#include "RingMulticastStrategy.h"
+
+//Group Constructor
+RingMulticastStrategy::RingMulticastStrategy(int ndest, int *pelist) 
+    : DirectMulticastStrategy(ndest, pelist) {
+    commonRingInit();
+}
+
+//Array Constructor
+RingMulticastStrategy::RingMulticastStrategy(CkArrayID dest_aid)
+    : DirectMulticastStrategy(dest_aid){
+    commonRingInit();    
+}
+
+void RingMulticastStrategy::commonRingInit(){
+    //Sort destpelist
+}
+
+extern int _charmHandlerIdx;
+void RingMulticastStrategy::doneInserting(){
+    ComlibPrintf("%d: DoneInserting \n", CkMyPe());
+    
+    if(messageBuf->length() == 0) {
+        return;
+    }
+
+    while(!messageBuf->isEmpty()) {
+       CharmMessageHolder *cmsg = messageBuf->deq();
+        char *msg = cmsg->getCharmMessage();
+        register envelope* env = UsrToEnv(msg);
+
+        ComlibPrintf("[%d] Calling Ring %d %d %d\n", CkMyPe(),
+                     env->getTotalsize(), ndestpes, cmsg->dest_proc);
+               
+        if(cmsg->dest_proc == IS_MULTICAST) {      
+            CmiSetHandler(env, handlerId);
+            
+            int dest_pe = -1;
+            RingMulticastHashObject *robj;
+
+            if(cmsg->sec_id == NULL)
+                dest_pe = nextPE;
+            else {
+                robj = getHashObject(CkMyPe(), 
+                                     cmsg->sec_id->_cookie.sInfo.cInfo.id);
+                                
+                ComlibPrintf("Gotten has obect %d\n",  robj);
+
+                CkAssert(robj != NULL);
+
+                dest_pe = robj->nextPE;
+            }
+            
+            ComlibPrintf("[%d] Sending Message to %d\n", CkMyPe(), dest_pe);
+
+            if(dest_pe != -1)
+                CmiSyncSend(dest_pe, env->getTotalsize(), (char *)env); 
+            
+            if(isDestinationArray) {
+                /*
+                if(robj != NULL)
+                    localMulticast(&robj->indices, env);
+                else
+                    localMulticast(&localDestIndices, env);
+                */
+                CmiSyncSendAndFree(CkMyPe(), env->getTotalsize(), (char *)env);
+            }
+            else {
+                CmiSetHandler(env, _charmHandlerIdx);
+                CmiSyncSendAndFree(CkMyPe(), env->getTotalsize(), (char *)env);
+            }
+        }
+        else {
+            CmiSyncSendAndFree(cmsg->dest_proc, UsrToEnv(msg)->getTotalsize(), 
+                               (char *)UsrToEnv(msg));
+        }        
+        
+       delete cmsg; 
+    }
+}
+
+void RingMulticastStrategy::pup(PUP::er &p){
+
+    DirectMulticastStrategy::pup(p);
+}
+
+void RingMulticastStrategy::beginProcessing(int  nelements){
+
+    DirectMulticastStrategy::beginProcessing(nelements);
+
+    nextPE = -1;
+    if(ndestpes == 1)
+        return;
+
+    for(int count = 0; count < ndestpes; count++)
+        if(destpelist[count] > CkMyPe()) {
+            nextPE = destpelist[count];
+            break;
+        }
+    if(nextPE == -1)
+        nextPE = destpelist[0];
+}
+
+void RingMulticastStrategy::handleMulticastMessage(void *msg){
+    register envelope *env = (envelope *)msg;
+       
+    CkMcastBaseMsg *cbmsg = (CkMcastBaseMsg *)EnvToUsr(env);
+    int src_pe = cbmsg->_cookie.pe;
+    if(isDestinationGroup){               
+
+        if(!isEndOfRing(nextPE, src_pe)) {
+            ComlibPrintf("[%d] Forwarding Message to %d\n", CkMyPe(), nextPE);
+            CmiSyncSend(nextPE, env->getTotalsize(), (char *)env);        
+        }
+        CmiSetHandler(env, _charmHandlerIdx);
+        CmiSyncSendAndFree(CkMyPe(), env->getTotalsize(), (char *)env);
+        
+        return;
+    }
+
+    int status = cbmsg->_cookie.sInfo.cInfo.status;
+    ComlibPrintf("[%d] In handle multicast message %d\n", CkMyPe(), status);
+
+    if(status == COMLIB_MULTICAST_ALL) {                        
+        if(src_pe != CkMyPe() && !isEndOfRing(nextPE, src_pe)) {
+            ComlibPrintf("[%d] Forwarding Message to %d\n", CkMyPe(), nextPE);
+            CmiSyncSend(nextPE, env->getTotalsize(), (char *)env); 
+        }
+
+        //Multicast to all destination elements on current processor        
+        ComlibPrintf("[%d] Local multicast sending all %d\n", CkMyPe(), 
+                     localDestIndices.size());
+        
+        localMulticast(&localDestIndices, env);
+    }   
+    else if(status == COMLIB_MULTICAST_NEW_SECTION){        
+        CkUnpackMessage(&env);
+        ComlibPrintf("[%d] Received message for new section src=%d\n", 
+                     CkMyPe(), cbmsg->_cookie.pe);
+
+        ComlibMulticastMsg *ccmsg = (ComlibMulticastMsg *)cbmsg;
+        
+        RingMulticastHashObject *robj = 
+            createHashObject(ccmsg->nIndices, ccmsg->indices);
+        
+        envelope *usrenv = (envelope *) ccmsg->usrMsg;
+        
+        envelope *newenv = (envelope *)CmiAlloc(usrenv->getTotalsize());
+        memcpy(newenv, usrenv, usrenv->getTotalsize());
+
+        localMulticast(&robj->indices, newenv);
+
+        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                                 cbmsg->_cookie.sInfo.cInfo.id);
+
+        RingMulticastHashObject *old_robj = 
+            (RingMulticastHashObject*)sec_ht.get(key);
+        if(old_robj != NULL)
+            delete old_robj;
+        
+        sec_ht.put(key) = robj;
+
+        if(src_pe != CkMyPe() && !isEndOfRing(robj->nextPE, src_pe)) {
+            ComlibPrintf("[%d] Forwarding Message of %d to %d\n", CkMyPe(), 
+                         cbmsg->_cookie.pe, robj->nextPE);
+            CkPackMessage(&env);
+            CmiSyncSendAndFree(robj->nextPE, env->getTotalsize(), 
+                               (char *)env);
+        }
+        else
+            CmiFree(env);       
+    }
+    else {
+        //status == COMLIB_MULTICAST_OLD_SECTION, use the cached section id
+        ComlibSectionHashKey key(cbmsg->_cookie.pe, 
+                                 cbmsg->_cookie.sInfo.cInfo.id);    
+        RingMulticastHashObject *robj = (RingMulticastHashObject *)sec_ht.
+            get(key);
+        
+        if(robj == NULL)
+            CkAbort("Destination indices is NULL\n");
+        
+        if(src_pe != CkMyPe() && !isEndOfRing(robj->nextPE, src_pe)) {
+            CmiSyncSend(robj->nextPE, env->getTotalsize(), (char *)env);
+            ComlibPrintf("[%d] Forwarding Message to %d\n", CkMyPe(), 
+                         robj->nextPE);
+        }
+        
+        localMulticast(&robj->indices, env);
+    }
+}
+
+void RingMulticastStrategy::initSectionID(CkSectionID *sid){
+
+    CkPrintf("Ring Init section ID\n");
+    sid->pelist = NULL;
+    sid->npes = 0;
+
+    RingMulticastHashObject *robj = 
+        createHashObject(sid->_nElems, sid->_elems);
+    
+    ComlibSectionHashKey key(CkMyPe(), sid->_cookie.sInfo.cInfo.id);
+    sec_ht.put(key) = robj;
+}
+
+RingMulticastHashObject *RingMulticastStrategy::createHashObject
+(int nelements, CkArrayIndexMax *elements){
+
+    RingMulticastHashObject *robj = new RingMulticastHashObject;
+
+    int next_pe = CkNumPes();
+    int acount = 0;
+    int min_dest = CkNumPes();
+    for(acount = 0; acount < nelements; acount++){
+        //elements[acount].print();
+        int p = CkArrayID::CkLocalBranch(destArrayID)->
+            lastKnown(elements[acount]);
+        
+        if(p < min_dest)
+            min_dest = p;
+        
+        if(p > CkMyPe() && next_pe > p) 
+            next_pe = p;       
+
+        if (p == CkMyPe())
+            robj->indices.insertAtEnd(elements[acount]);
+    }
+    
+    //Recycle the destination pelist and start from the begining
+    if(next_pe == CkNumPes() && min_dest != CkMyPe())        
+        next_pe = min_dest;
+    
+    if(next_pe == CkNumPes())
+        next_pe = -1;
+
+    robj->nextPE = next_pe;
+
+    return robj;
+}
+
+
+RingMulticastHashObject *RingMulticastStrategy::getHashObject(int pe, int id){
+    
+    ComlibSectionHashKey key(pe, id);
+    RingMulticastHashObject *robj = (RingMulticastHashObject *)sec_ht.get(key);
+    return robj;
+}
+
+int RingMulticastStrategy::isEndOfRing(int next_pe, int src_pe){
+
+    if(next_pe < 0)
+        return 1;
+
+    ComlibPrintf("[%d] isEndofring %d, %d\n", CkMyPe(), next_pe, src_pe);
+
+    if(next_pe > CkMyPe()){
+        if(src_pe <= next_pe && src_pe > CkMyPe())
+            return 1;
+
+        return 0;
+    }
+    
+    //next_pe < CkMyPe()
+
+    if(src_pe > CkMyPe() || src_pe <= next_pe)
+        return 1;
+    
+    return 0;
+}
diff --git a/src/ck-com/RingMulticastStrategy.h b/src/ck-com/RingMulticastStrategy.h
new file mode 100644 (file)
index 0000000..d4a60cd
--- /dev/null
@@ -0,0 +1,43 @@
+#ifndef RING_MULTICAST_STRATEGY
+#define RING_MULTICAST_STRATEGY
+
+#include "DirectMulticastStrategy.h"
+
+class RingMulticastHashObject{
+ public:
+    CkVec<CkArrayIndexMax> indices;
+    int nextPE;
+};
+
+
+class RingMulticastStrategy: public DirectMulticastStrategy {
+    
+    int nextPE;
+    
+    void commonRingInit();
+    int isEndOfRing(int next_pe, int src_pe);
+    RingMulticastHashObject *getHashObject(int pe, int id);
+    RingMulticastHashObject *createHashObject(int nelements, 
+                                              CkArrayIndexMax *elements);
+    void initSectionID(CkSectionID *sid);
+
+ public:
+    
+    //Group constructor
+    RingMulticastStrategy(int ndestpes, int *destpelist);
+    
+    //Array constructor
+    RingMulticastStrategy(CkArrayID dest_id);    
+    RingMulticastStrategy(CkMigrateMessage *m) {}
+    
+    //void insertMessage(CharmMessageHolder *msg);
+    void doneInserting();
+    void handleMulticastMessage(void *msg);
+    
+    void pup(PUP::er &p);    
+    void beginProcessing(int nelements);
+    
+    PUPable_decl(RingMulticastStrategy);
+};
+#endif
+
diff --git a/src/ck-com/StreamingStrategy.C b/src/ck-com/StreamingStrategy.C
new file mode 100644 (file)
index 0000000..2495495
--- /dev/null
@@ -0,0 +1,154 @@
+#include "StreamingStrategy.h"
+#include "MsgPacker.h"
+
+StreamingStrategy::StreamingStrategy(int periodMs,int bufferMax_)
+    : PERIOD(periodMs), bufferMax(bufferMax_), CharmStrategy()
+{
+    streamingMsgBuf=NULL;
+    streamingMsgCount=NULL;
+    shortMsgPackingFlag = CmiFalse;
+    idleFlush = CmiTrue;
+}
+
+void StreamingStrategy::insertMessage(CharmMessageHolder *cmsg) {
+
+    int pe=cmsg->dest_proc;
+    char *msg = cmsg->getCharmMessage();
+    envelope *env = UsrToEnv(msg);
+    int size = env->getTotalsize();
+
+    if(size > MAX_STREAMING_MESSAGE_SIZE) {//AVOID COPYING
+      ComlibPrintf("StreamingStrategy::insertMessage: direct send\n");
+      CmiSyncSendAndFree(pe, size, (char *)env);
+      delete cmsg;
+      return;
+    }
+
+    ComlibPrintf("StreamingStrategy::insertMessage: buffering t=%d, n=%d, s=%d\n",  
+                PERIOD, bufferMax, size);
+    
+    streamingMsgBuf[pe].enq(cmsg);
+    streamingMsgCount[pe]++;
+    if (streamingMsgCount[pe] > bufferMax) flushPE(pe);
+}
+
+void StreamingStrategy::doneInserting() {
+  ComlibPrintf("[%d] In Streaming strategy::doneInserting\n", CkMyPe());
+  //Do nothing
+}
+
+/// Send off all accumulated messages for this PE:
+void StreamingStrategy::flushPE(int pe) {
+  if(streamingMsgCount[pe] == 0)
+    return; //Nothing to do.
+  
+  CharmMessageHolder *cmsg, *toBeDeleted = NULL;
+  if(shortMsgPackingFlag){
+    MsgPacker mpack(streamingMsgBuf[pe], streamingMsgCount[pe]);
+    CombinedMessage *msg; 
+    int size;
+    mpack.getMessage(msg, size);
+    ComlibPrintf("[%d] StreamingStrategy::flushPE: packed %d short messages to %d\n", 
+                CkMyPe(), streamingMsgCount[pe], pe); 
+    CmiSyncSendAndFree(pe, size, (char *)msg);
+    streamingMsgCount[pe] = 0;
+  }
+  else {
+    // Build a CmiMultipleSend list of messages to be sent off:
+    int msg_count=streamingMsgCount[pe], msg_pe=0;
+    if(msg_count == 1) {
+      cmsg = streamingMsgBuf[pe].deq();
+      char *msg = cmsg->getCharmMessage();
+      envelope *env = UsrToEnv(msg);
+      int size = env->getTotalsize();
+      CmiSyncSendAndFree(pe, size, (char *)env);
+      ComlibPrintf("[%d] StreamingStrategy::flushPE: one message to %d\n", 
+                  CkMyPe(), pe);            
+      delete cmsg;
+      streamingMsgCount[pe] = 0;
+      return;
+    }
+    char **msgComps = new char*[msg_count];
+    int *sizes = new int[msg_count];
+    ComlibPrintf("[%d] StreamingStrategy::flushPE: %d messages to %d\n", 
+                CkMyPe(), msg_count, pe);            
+    while (!streamingMsgBuf[pe].isEmpty()) {
+      cmsg = streamingMsgBuf[pe].deq();
+      char *msg = cmsg->getCharmMessage();
+      envelope *env = UsrToEnv(msg);
+      sizes[msg_pe] = env->getTotalsize();
+      msgComps[msg_pe] = (char *)env;
+      msg_pe++;
+      
+      // Link cmsg into the toBeDeleted list:
+      cmsg->next = toBeDeleted;
+      toBeDeleted = cmsg;            
+    }
+    
+    if (msg_count!=msg_pe) 
+      CkAbort("streamingMsgCount doesn't match streamingMsgBuf!\n");
+
+    ComlibPrintf("--> Sending %d Messages to PE %d\n", msg_count, pe);
+
+    CmiMultipleSend(pe, msg_count, sizes, msgComps);
+    delete [] msgComps;
+    delete [] sizes;
+    streamingMsgCount[pe] = 0;
+    
+    // Traverse the tobeDeleted list:
+    cmsg = toBeDeleted;
+    while (toBeDeleted) {
+        toBeDeleted = (CharmMessageHolder *)toBeDeleted->next;
+        CkFreeMsg(cmsg->getCharmMessage());
+        delete cmsg;
+        cmsg = toBeDeleted;            
+    }     
+  }
+}
+
+void StreamingStrategy::periodicFlush() {
+  for (int pe=0; pe<CkNumPes(); pe++) flushPE(pe);
+}
+
+/// This routine is called via CcdCallFnAfter to flush all messages:
+static void call_delayFlush(void *arg,double curWallTime) {
+  StreamingStrategy *s=(StreamingStrategy *)arg;
+  s->periodicFlush();
+  s->registerFlush(); //Set ourselves up to be called again
+}
+
+void StreamingStrategy::registerFlush(void) {
+  // CkPrintf("[%d] Will call function again every %d ms\n",CkMyPe(),PERIOD);
+  CcdCallFnAfterOnPE((CcdVoidFn)call_delayFlush, (void *)this, PERIOD, CkMyPe());
+}
+
+/// This routine is called via CcdCallOnCondition to flush all messages:
+static void call_idleFlush(void *arg,double curWallTime) {
+  StreamingStrategy *s=(StreamingStrategy *)arg;
+  s->periodicFlush();
+}
+
+// When we're finally ready to go, register for timeout and idle flush.
+void StreamingStrategy::beginProcessing(int ignored) {
+  registerFlush();
+  if(idleFlush)
+    CcdCallOnConditionKeepOnPE(CcdPROCESSOR_BEGIN_IDLE,
+                              (CcdVoidFn)call_idleFlush, 
+                              (void *)this, CkMyPe());
+}
+
+void StreamingStrategy::pup(PUP::er &p){
+  p | PERIOD;
+  p | bufferMax;
+  p | shortMsgPackingFlag;
+  p | idleFlush;
+
+  if(p.isUnpacking()) {
+    streamingMsgBuf = new CkQ<CharmMessageHolder *>[CkNumPes()];
+    streamingMsgCount = new int[CkNumPes()];
+    for(int count = 0; count < CkNumPes(); count ++)
+      streamingMsgCount[count] = 0;
+  }
+}
+
+//PUPable_def(StreamingStrategy);
diff --git a/src/ck-com/StreamingStrategy.h b/src/ck-com/StreamingStrategy.h
new file mode 100644 (file)
index 0000000..8650457
--- /dev/null
@@ -0,0 +1,49 @@
+#ifndef STREAMING_STRATEGY
+#define STREAMING_STRATEGY
+#include "ComlibManager.h"
+
+#define MAX_STREAMING_MESSAGE_SIZE 2048*2
+
+
+class StreamingStrategy : public CharmStrategy {
+ protected:
+    CkQ<CharmMessageHolder *> *streamingMsgBuf;
+    int *streamingMsgCount;
+    int PERIOD, bufferMax;
+    CmiBool shortMsgPackingFlag, idleFlush;
+
+    /// Flush all messages destined for this processor:
+    void flushPE(int destPE);
+    
+ public:
+    /**
+     Create a streaming strategy, suitable for passing to ComlibManager.
+     These are the criteria for flushing all pending messages:
+       - it's been at least period (in ms) since the last flush, or
+       - the processor just went idle.
+     Thses criteria flush a single PE's pending messages:
+       - more than bufferMax messages to buffered for one PE.
+    */
+    StreamingStrategy(int periodMs=10,int bufferMax=1000);
+    StreamingStrategy(CkMigrateMessage *m) : CharmStrategy(m) {}
+    
+    virtual void insertMessage(CharmMessageHolder *msg);
+    virtual void doneInserting();
+    
+    virtual void beginProcessing(int ignored);
+
+    virtual void pup(PUP::er &p);
+    virtual void enableShortArrayMessagePacking()
+        {shortMsgPackingFlag=CmiTrue;} //Should be used only for array
+                                       //messages
+
+    virtual void disableIdleFlush() { idleFlush = CmiFalse;}
+
+    /// Register self to be flushed again after a delay.
+    void registerFlush(void);
+    /// Flush all pending messages:
+    void periodicFlush();
+
+    PUPable_decl(StreamingStrategy);
+};
+#endif
diff --git a/src/conv-com/3dgridrouter.C b/src/conv-com/3dgridrouter.C
new file mode 100644 (file)
index 0000000..d520b1a
--- /dev/null
@@ -0,0 +1,377 @@
+/************************************************************
+ * File : D3Gridrouter.C
+ *
+ * Author : Sameer Kumar
+ *
+ * Grid (3d grid) based router
+ ***********************************************************/
+#include <math.h>
+#include "3dgridrouter.h"
+//#define NULL 0
+
+#define gmap(pe) {if (gpes) pe=gpes[pe];}
+
+/**The only communication op used. Modify this to use
+ ** vector send */
+#define GRIDSENDFN(kid, u1, u2, knpe, kpelist, khndl, knextpe)  \
+       {int len;\
+       char *newmsg;\
+        newmsg=PeGrid->ExtractAndPack(kid, u1, knpe, kpelist, &len);\
+       if (newmsg) {\
+         CmiSetHandler(newmsg, khndl);\
+          CmiSyncSendAndFree(knextpe, len, (char *)newmsg);\
+        }\
+       else {\
+         SendDummyMsg(kid, knextpe, u2);\
+       }\
+}
+
+#define ROWLEN COLLEN
+#define NPLANES (int)ceil(((double)NumPes) / (ROWLEN * COLLEN))
+
+#define RowLen(pe) ColLen3D(pe)
+#define PELISTSIZE ((ROWLEN-1)/sizeof(int)+1)
+
+double cubeRoot(double d) {
+  return pow(d,1.0/3.0);
+}
+
+
+inline int ColLen3D(int npes)
+{
+    int len= (int)cubeRoot((double)npes);
+    //    ComlibPrintf("%d:collen len = %d\n", CkMyPe(), len);
+    if (npes > (len * len * len)) len++;
+    return(len);
+}
+
+inline int Expect1(int gpe, int gnpes)
+{
+    int i, len=ColLen3D(gnpes);
+    int pe = gpe % (len * len);
+    
+    int npes = len * len;
+    if((gnpes - 1)/(len * len) == gpe / (len * len))
+        npes = ((gnpes - 1) % (len*len)) + 1;
+    
+    for (i=len-1;i>=0;i--) {
+        int myrow=pe/len;
+        int toprep=i*len;
+        int offset=pe-myrow*len;
+        if ((toprep+offset) <= (npes-1)) return(i+1);
+    }
+    return 0;
+    //return(len);
+}
+
+inline int Expect2(int gpe, int gnpes) {
+    int len=RowLen(gnpes);
+    int myplane = gpe / (len * len);
+    int lastplane = (gnpes - 1)/(len * len);
+    
+    if(myplane < lastplane)
+        return len;
+
+    int pe = gpe % (len * len);
+    int myrow = pe / len;
+    int lastrow = ((gnpes - 1) % (len * len)) / len;
+    
+    if (myrow < lastrow)
+        return len;
+    
+    int ret = ((gnpes - 1) % (len * len)) - myrow * len + 1;
+    if(ret < 0)
+        ret = 0;
+    return ret;
+}
+    
+
+inline int LPMsgExpect(int gpe, int gnpes)
+{
+    int i;
+    int row = RowLen(gnpes);
+    int col = ColLen3D(gnpes);
+    int len = (int)ceil(((double)gnpes) / (row * col));
+
+    for (i=len-1;i>=0;i--) {
+        int myrow = (gpe%(row * col))/col;
+        int toprep=i*(row * col);
+        
+        if ((toprep + (gpe % (row * col))) <= (gnpes-1)) return(i+1);
+    }
+    return(len);
+}
+
+/****************************************************
+ * Preallocated memory=P ints + MAXNUMMSGS msgstructs
+ *****************************************************/
+D3GridRouter::D3GridRouter(int n, int me)
+{
+    ComlibPrintf("PE=%d me=%d NUMPES=%d\n", CkMyPe(), me, n);
+    
+    NumPes=n;
+    MyPe=me;
+    gpes=NULL;
+
+    COLLEN=ColLen3D(NumPes);
+
+    recvExpected[0] = 0;
+    recvExpected[1] = 0;
+    routerStage = 0;
+    
+    int myrow = (MyPe % (ROWLEN * COLLEN)) / COLLEN;
+    int myrep = myrow * COLLEN + MyPe - (MyPe % (ROWLEN * COLLEN));
+    int numunmappedpes=myrep+ROWLEN-NumPes;
+    int nummappedpes=ROWLEN;
+    
+    if (numunmappedpes >0) {
+       nummappedpes=NumPes-myrep;
+       int i=NumPes+MyPe-myrep;
+       while (i<myrep+ROWLEN) {
+            recvExpected[0] += Expect1(i, NumPes);
+            i+=nummappedpes;
+       }
+    }
+    
+    if((NumPes % (COLLEN * ROWLEN) != 0) && ((NumPes - 1)/(ROWLEN*COLLEN) - MyPe/(ROWLEN*COLLEN) == 1)){
+        if(myrep + ROWLEN * COLLEN >= NumPes) 
+            recvExpected[0] += Expect1(MyPe + ROWLEN*COLLEN, NumPes);
+        
+        if(MyPe + ROWLEN * COLLEN >= NumPes) 
+            recvExpected[1] += Expect2(MyPe + ROWLEN*COLLEN, NumPes);
+        ComlibPrintf("%d: here\n");
+    }
+
+    recvExpected[0] += Expect1(MyPe, NumPes);
+    recvExpected[1] += Expect2(MyPe, NumPes);
+
+    LPMsgExpected = LPMsgExpect(MyPe, NumPes);
+    //ComlibPrintf("%d LPMsgExpected=%d\n", MyPe, LPMsgExpected);
+    
+    PeGrid = new PeTable(/*CkNumPes()*/NumPes);
+    
+    oneplane = new int[NPLANES * ROWLEN];
+    zline = new int[NPLANES];
+    
+    InitVars();
+    ComlibPrintf("%d:%d:COLLEN=%d, ROWLEN=%d, recvexpected=%d,%d\n", CkMyPe(), MyPe, COLLEN, ROWLEN, recvExpected[0], recvExpected[1]);
+}
+
+D3GridRouter::~D3GridRouter()
+{
+    delete PeGrid;
+    delete[] zline;
+    delete[] oneplane;
+}
+
+void D3GridRouter :: InitVars()
+{
+    recvCount[0]=0;
+    recvCount[1]=0;
+    
+    LPMsgCount=0;
+}
+
+void D3GridRouter::NumDeposits(comID, int num)
+{
+}
+
+void D3GridRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
+{
+    int npe=NumPes;
+    int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
+    for (int i=0;i<npe;i++) destpes[i]=i;
+    EachToManyMulticast(id, size, msg, npe, destpes, more);
+}
+
+void D3GridRouter::EachToManyMulticast(comID id, int size, void *msg, int numpes, int *destpes, int more)
+{
+    int i;
+    
+    //Buffer the message
+    if (size) {
+       PeGrid->InsertMsgs(numpes, destpes, size, msg);
+    }
+    
+    if (more) return;
+
+    routerStage = 0;
+    ComlibPrintf("All messages received %d %d\n", CkMyPe(), COLLEN);
+    
+    //Send the messages
+    int firstproc = MyPe - (MyPe % (ROWLEN * COLLEN));
+    for (i=0;i<COLLEN;i++) {
+        
+        ComlibPrintf("ROWLEN = %d, COLLEN =%d first proc = %d\n", ROWLEN, COLLEN, firstproc);
+        
+        int MYROW = (MyPe % (ROWLEN * COLLEN))/ROWLEN;
+        int nextrowrep = firstproc + i*ROWLEN;
+        
+        int nextpe = (MyPe % (ROWLEN * COLLEN)) % ROWLEN + nextrowrep;
+        int nummappedpes=NumPes-nextrowrep;
+        
+        if (nummappedpes <= 0) { // looks for nextpe in the previous plane
+            nextpe -= ROWLEN * COLLEN;
+            if(nextpe < 0)
+                continue;
+        }
+
+        if (nextpe >= NumPes) {
+            int mm=(nextpe-NumPes) % nummappedpes;
+            nextpe=nextrowrep+mm;
+        }
+        
+        int nplanes = (int)ceil(((double)NumPes) / (ROWLEN * COLLEN));
+        int idx = 0;
+        
+        ComlibPrintf("%d->%d:(", MyPe, nextpe);
+        for (int j=0;j< ROWLEN;j++) 
+            for(int k = 0; k < nplanes; k++) {
+                int dest = i * ROWLEN + j + ROWLEN * COLLEN * k;
+                if(dest < NumPes) {
+                    oneplane[idx++] = dest;
+                    ComlibPrintf("%d,", oneplane[idx-1]);
+                }
+                else break;
+            }
+        ComlibPrintf(")\n");
+        
+        if (nextpe == MyPe) {
+            ComlibPrintf("%d calling recv directly\n", MyPe);
+            recvCount[0]++;
+            RecvManyMsg(id, NULL);
+            continue;
+        }
+        
+        ComlibPrintf("nummappedpes = %d, NumPes = %d, nextrowrep = %d, nextpe = %d, mype = %d\n", nummappedpes, NumPes, nextrowrep,  nextpe, MyPe);
+        
+        gmap(nextpe);
+        ComlibPrintf("sending to column %d and dest %d in %d\n", i, nextpe, CkMyPe());
+        GRIDSENDFN(id, 0, 0, idx, oneplane, CkpvAccess(RecvHandle), nextpe); 
+    }
+}
+
+void D3GridRouter::RecvManyMsg(comID id, char *msg)
+{
+    ComlibPrintf("%d recvcount=%d,%d recvexpected = %d,%d\n", MyPe, recvCount[0],  recvCount[1], recvExpected[0],recvExpected[1]);
+    int stage = 0;
+    if (msg) {
+        stage = PeGrid->UnpackAndInsert(msg);
+        recvCount[stage]++;
+    }
+    
+    if ((recvCount[0] == recvExpected[0]) && (routerStage == 0)){
+        routerStage = 1;
+        int myrow = (MyPe % (ROWLEN * COLLEN)) / COLLEN;
+        int myrep = myrow*ROWLEN + MyPe - (MyPe % (ROWLEN * COLLEN));
+        for (int i=0;i<ROWLEN;i++) {
+            int nextpe = myrep + i;
+            
+            //if (nextpe >= NumPes || nextpe==MyPe) continue;
+            
+            if(nextpe == MyPe) {
+                recvCount[1]++;
+                RecvManyMsg(id, NULL);
+                continue;
+            }
+            
+            if(nextpe >= NumPes) {
+                nextpe -= ROWLEN * COLLEN;
+                if(nextpe < 0)
+                    continue;
+            }
+
+            int *pelist = zline;
+            int nplanes = (int)ceil(((double)NumPes) / (ROWLEN * COLLEN));
+            int k = 0;
+            
+            ComlibPrintf("recv:myrow = %d, nplanes = %d\n", myrow, nplanes);
+            ComlibPrintf("recv:%d->%d:", MyPe, nextpe);
+            for(k = 0; k < nplanes; k++) {
+                int dest = myrow * ROWLEN + i + ROWLEN * COLLEN * k;
+                ComlibPrintf("%d,", dest);
+                if(dest >= NumPes)
+                    break;
+                zline[k] = dest;
+            }
+            ComlibPrintf(")\n");
+
+            ComlibPrintf("Before gmap %d\n", nextpe);
+            
+            gmap(nextpe);
+            
+            ComlibPrintf("After gmap %d\n", nextpe);
+            
+            ComlibPrintf("%d:sending recv message %d %d\n", MyPe, nextpe, myrep);
+            GRIDSENDFN(id, 1, 1, k, pelist, CkpvAccess(RecvHandle), nextpe);
+        }
+    }
+    
+    if((recvCount[1] == recvExpected[1]) && (routerStage == 1)){
+        routerStage = 2;
+        int nplanes = (int)ceil(((double)NumPes) / (ROWLEN * COLLEN));
+        for (int k=0; k < nplanes; k++) {
+            int nextpe = (MyPe % (ROWLEN * COLLEN)) + k * ROWLEN * COLLEN;
+
+            if (nextpe >= NumPes || nextpe==MyPe) continue;
+            
+            int gnextpe = nextpe;
+            int *pelist = &gnextpe;
+            
+            ComlibPrintf("Before gmap %d\n", nextpe);
+            
+            gmap(nextpe);
+            
+            ComlibPrintf("After gmap %d\n", nextpe);
+            
+            ComlibPrintf("%d:sending proc message %d %d\n", MyPe, nextpe, nplanes);
+            GRIDSENDFN(id, 2, 2, 1, pelist, CkpvAccess(ProcHandle), nextpe);
+        }
+        LocalProcMsg(id);
+    }
+}
+
+void D3GridRouter::DummyEP(comID id, int stage)
+{
+    if (stage == 2) {
+        ComlibPrintf("%d dummy calling lp\n", MyPe);
+       LocalProcMsg(id);
+    }
+    else {
+        recvCount[stage]++;
+       ComlibPrintf("%d dummy calling recv\n", MyPe);
+       RecvManyMsg(id, NULL);
+    }
+}
+
+void D3GridRouter:: ProcManyMsg(comID id, char *m)
+{
+    PeGrid->UnpackAndInsert(m);
+    ComlibPrintf("%d proc calling lp\n", MyPe);
+    LocalProcMsg(id);
+}
+
+void D3GridRouter:: LocalProcMsg(comID id)
+{
+    ComlibPrintf("%d local procmsg called\n", MyPe);
+    
+    LPMsgCount++;
+    PeGrid->ExtractAndDeliverLocalMsgs(MyPe);
+    
+    if (LPMsgCount==LPMsgExpected) {
+       PeGrid->Purge();
+       InitVars();
+        routerStage = 0;
+        ComlibPrintf("%d:Round Done\n", CkMyPe());
+       Done(id);
+    }
+}
+
+void D3GridRouter :: SetMap(int *pes)
+{
+    gpes=pes;
+    
+    //ComlibPrintf("%d:GPES[1] = %d\n", CkMyPe(), gpes[1]);
+    //gpes[1] = 4;
+}
+
diff --git a/src/conv-com/3dgridrouter.h b/src/conv-com/3dgridrouter.h
new file mode 100644 (file)
index 0000000..f4d6ad5
--- /dev/null
@@ -0,0 +1,41 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#ifndef _D3GRIDROUTER_H
+#define _D3GRIDROUTER_H
+
+#include <math.h>
+#include <converse.h>
+#include "comlib.h"
+#include "petable.h"
+
+//3DGrid based router
+class D3GridRouter : public Router
+{
+  private:
+       PeTable *PeGrid;
+       int *oneplane, *zline, *gpes;
+       int MyPe, NumPes, COLLEN, nPlanes;
+       int LPMsgCount, LPMsgExpected;
+       int recvExpected[2], recvCount[2];
+        int routerStage;
+       void InitVars();
+       void LocalProcMsg(comID id);
+  public:
+       D3GridRouter(int, int);
+       ~D3GridRouter();
+       void NumDeposits(comID, int);
+       void EachToAllMulticast(comID , int , void *, int);
+       void EachToManyMulticast(comID , int , void *, int, int *, int);
+       void RecvManyMsg(comID, char *);
+       void ProcManyMsg(comID, char *);
+       void DummyEP(comID id, int);
+       void ProcMsg(int, msgstruct **) {;}
+       void SetMap(int *);
+};
+
+#endif
diff --git a/src/conv-com/comlib.h b/src/conv-com/comlib.h
new file mode 100644 (file)
index 0000000..550c781
--- /dev/null
@@ -0,0 +1,83 @@
+
+#ifndef COMLIB_H
+#define COMLIB_H
+
+#include <converse.h>
+#include <stdlib.h>
+#include "charm++.h"
+
+#if CMK_BLUEGENE_CHARM
+#define CmiReservedHeaderSize   CmiBlueGeneMsgHeaderSizeBytes
+#else
+#define CmiReservedHeaderSize   CmiExtHeaderSizeBytes
+#endif
+
+extern int comm_debug;
+#if CMK_OPTIMIZE
+inline void ComlibPrintf(...) {}
+#else
+#define ComlibPrintf if(comm_debug) CmiPrintf
+#endif
+
+enum{BCAST=0,TREE, GRID, HCUBE};  
+
+#define USE_TREE 1            //Organizes the all to all as a tree
+#define USE_MESH 2            //Virtual topology is a mesh here
+#define USE_HYPERCUBE 3       //Virtual topology is a hypercube
+#define USE_DIRECT 4          //A dummy strategy that directly forwards 
+                              //messages without any processing.
+#define USE_GRID 5            //Virtual topology is a 3d grid
+#define USE_LINEAR 6          //Virtual topology is a linear array
+
+#define IS_MULTICAST -1
+
+#define MAXNUMMSGS 1000
+//#define MAXNUMSTRATEGY 10
+#define MSGSIZETHRESHOLD 5000000
+#define MAXBUFSIZE 65536
+#define PERSISTENT_BUFSIZE 131072
+
+typedef struct {
+    //int srcpe;
+    //short ImplType;
+    //short ImplIndex;
+    //int callbackHandler;
+    //short SwitchVal;
+    //int NumMembers;
+    //CmiGroup grp;
+
+    int refno;
+    int instanceID;  
+    char isAllToAll;
+} comID;
+
+
+typedef struct {
+  int msgsize;
+  void *msg;
+} msgstruct ;
+
+typedef struct { 
+    char core[CmiReservedHeaderSize];
+    comID id;
+    int magic;
+    int refno;
+} DummyMsg ;
+
+//The handler to invoke the RecvManyMsg method of router
+CkpvExtern(int, RecvHandle);
+//The handler to invoke the ProcManyMsg method of router
+CkpvExtern(int, ProcHandle);
+//The handler to invoke the DoneEP method of router
+CkpvExtern(int, DummyHandle);
+
+//Dummy msg handle.
+//Just deletes and ignores the message
+CkpvExtern(int, RecvdummyHandle);
+
+
+#include "router.h"
+
+#endif
+       
+
diff --git a/src/conv-com/convcomlibmanager.C b/src/conv-com/convcomlibmanager.C
new file mode 100644 (file)
index 0000000..ce47bcd
--- /dev/null
@@ -0,0 +1,77 @@
+/* Converse ComlibManager 
+   Enables communication library strategies to be called from converse code.
+   Also called by the Charm Comlibmanager.
+   
+   Stores a strategy table. Strategies can be inserted and accessed
+   from this table.
+
+   Sameer Kumar 28/03/04
+*/
+
+#include "convcomlibmanager.h"
+#include "routerstrategy.h"
+
+int comm_debug;
+
+CkpvDeclare(ConvComlibManager *, conv_comm_ptr);
+CkpvDeclare(int, RecvdummyHandle);
+
+ConvComlibManager::ConvComlibManager(): strategyTable(10){
+    nstrats = 0;
+}
+
+void ConvComlibManager::insertStrategy(Strategy *s) {
+    StrategyTableEntry &st = strategyTable[nstrats];
+    st.strategy = s;
+
+    s->setInstance(nstrats);
+    nstrats ++;
+}
+
+
+void ConvComlibManager::insertStrategy(Strategy *s, int loc) {
+
+    //For now allow insertion of any location    
+    StrategyTableEntry &st = strategyTable[loc];
+
+    st.strategy = s;
+}
+
+//handler for dummy messages
+void recv_dummy(void *msg){
+    ComlibPrintf("Received Dummy %d\n", CmiMyPe());    
+    CmiFree(msg);
+}
+
+
+//An initialization routine which does prelimnary initialization of the 
+//Converse commlib manager. Currently also initialized krishnans code
+void initComlibManager(){ 
+    ConvComlibManager *conv_com = new ConvComlibManager();
+    
+    CkpvInitialize(ConvComlibManager *, conv_comm_ptr);
+    CkpvAccess(conv_comm_ptr) = conv_com;
+
+    //comm_debug = 1;
+    ComlibPrintf("Init Call\n");
+
+    CkpvInitialize(int, RecvmsgHandle);
+    CkpvInitialize(int, RecvCombinedShortMsgHdlrIdx);
+    
+    CkpvInitialize(int, RecvdummyHandle);
+    CkpvAccess(RecvdummyHandle) = CkRegisterHandler((CmiHandler)recv_dummy);
+
+    PUPable_reg(Strategy);
+    PUPable_reg(RouterStrategy);
+    PUPable_reg(MessageHolder);
+}
+Strategy *ConvComlibGetStrategy(int loc) {
+    //Calling converse strategy lets Charm++ strategies one strategy
+    //table entry but multiple layers of strategies (Charm on top of Converse).
+    return (CkpvAccess(conv_comm_ptr))->getStrategy(loc)->getConverseStrategy();
+}
+
+void ConvComlibRegisterStrategy(Strategy *s) {
+    (CkpvAccess(conv_comm_ptr))->insertStrategy(s);
+}
diff --git a/src/conv-com/convcomlibmanager.h b/src/conv-com/convcomlibmanager.h
new file mode 100644 (file)
index 0000000..d79a76b
--- /dev/null
@@ -0,0 +1,37 @@
+/* Converse ComlibManager 
+   Enables communication library strategies to be called from converse code.
+   Reused by the Charm Comlibmanager.
+   
+   Stores a strategy table. Strategies can be inserted and accessed
+   from this table.
+
+   Sameer Kumar 28/03/04.
+*/
+
+#ifndef CONVCOMLIBMANAGER
+#define CONVCOMLIBMANAGER
+
+#include <converse.h>
+#include "comlib.h"
+#include <convcomlibstrategy.h>
+
+class ConvComlibManager {
+
+    StrategyTable strategyTable;
+    int nstrats;
+
+ public:
+
+    ConvComlibManager();
+    void insertStrategy(Strategy *s);
+    void insertStrategy(Strategy *s, int loc);
+    Strategy * getStrategy(int loc) {return strategyTable[loc].strategy;}
+    StrategyTable *getStrategyTable() {return &strategyTable;}
+};
+
+
+void initComlibManager();
+Strategy *ConvComlibGetStrategy(int loc);
+void ConvComlibRegisterStrategy(Strategy *s);
+
+#endif
diff --git a/src/conv-com/convcomlibstrategy.C b/src/conv-com/convcomlibstrategy.C
new file mode 100644 (file)
index 0000000..636c2dd
--- /dev/null
@@ -0,0 +1,102 @@
+
+#include <converse.h>
+#include "convcomlibstrategy.h"
+
+//Class that defines the entry methods that a strategy must define.
+//To write a new strategy inherit from this class and define the
+//virtual methods.  Every strategy can also define its own constructor
+//and have any number of arguments. Also call the parent class methods
+//in those methods. The strategies defined here can only be used in
+//converse. For Charm++ strategies please see ck-core/ComlibStraetgy.h
+
+Strategy::Strategy() : PUP::able() {
+    myInstanceID = 0;
+    type = CONVERSE_STRATEGY;
+    converseStrategy = this;
+};
+
+//Each strategy must define his own Pup interface.
+void Strategy::pup(PUP::er &p){ 
+
+    PUP::able::pup(p);
+
+    p | isStrategyBracketed;
+    p | type;
+}
+
+
+//Message holder functions. Message holder is a wrapper around a
+//message. Has other useful data like destination processor list for a
+//multicast etc.
+
+MessageHolder::MessageHolder(char * msg, int proc, int sz) : PUP::able() {
+    data = msg;
+    dest_proc = proc;
+    size = sz;
+    
+    isDummy = 0;
+
+    npes = 0;
+    pelist = 0;
+}
+
+MessageHolder::~MessageHolder() {
+    /*
+      if(pelist != NULL && npes > 0)
+      delete[] pelist;
+    */
+}
+
+char * MessageHolder::getMessage(){
+    return data;
+}
+
+void MessageHolder::pup(PUP::er &p) {
+    PUP::able::pup(p);
+
+    p | dest_proc;
+    p | isDummy;
+    p | size;
+    p | npes;    
+
+    if(p.isUnpacking()) {
+        data = (char *)CmiAlloc(size);
+        
+        if(npes >0)
+            pelist = new int[npes];
+    }
+
+    p(data, size);
+    if(npes > 0)
+        p(pelist, npes);    
+    else
+        pelist = 0;
+}
+
+void StrategyWrapper::pup (PUP::er &p) {
+
+    //CkPrintf("In PUP of StrategyWrapper\n");
+
+    p | nstrats;
+    if(p.isUnpacking())
+       s_table = new Strategy * [nstrats];
+    
+    for(int count = 0; count < nstrats; count ++)
+        p | s_table[count];
+}
+
+
+StrategyTableEntry::StrategyTableEntry() {
+    strategy = NULL;
+    
+    numElements = 0;   //used by the array listener, 
+                       //could also be used for other objects
+    elementCount = 0;  //Count of how many elements have deposited
+                       //their data
+    nEndItr = 0;       //#elements that called end iteration
+    call_doneInserting = 0;
+}
+
+
+PUPable_def(Strategy);
+PUPable_def(MessageHolder);
diff --git a/src/conv-com/convcomlibstrategy.h b/src/conv-com/convcomlibstrategy.h
new file mode 100644 (file)
index 0000000..6c76eb6
--- /dev/null
@@ -0,0 +1,134 @@
+#ifndef CONVCOMMLIBSTRATEGY_H
+#define CONVCOMMLIBSTRATEGY_H
+
+#include "converse.h"
+#include "pup.h"
+#include "cklists.h"
+
+//An abstract data structure that holds a converse message and which
+//can be buffered by the communication library Message holder is a
+//wrapper around a message. Has other useful data like destination
+//processor list for a multicast etc.
+
+class MessageHolder : public PUP::able {
+ public:
+    int dest_proc;
+    char *data;
+    int size;
+    MessageHolder *next; // also used for the refield at the receiver
+    int isDummy;
+    
+    //For multicast, the user can pass the pelist and list of Pes he
+    //wants to send the data to.
+    int npes;
+    int *pelist;
+    
+    MessageHolder() : PUP::able() 
+        {dest_proc = size = isDummy = 0; data = NULL;}    
+
+    MessageHolder(CkMigrateMessage *m) : PUP::able(m) {}
+
+    MessageHolder(char * msg, int dest_proc, int size);
+    ~MessageHolder();
+
+    char * getMessage();
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(MessageHolder);
+};
+
+#define CONVERSE_STRATEGY 0     //The strategy works for converse programs
+#define NODEGROUP_STRATEGY 1    //Node group level optimizations 
+#define GROUP_STRATEGY 2        //Charm Processor level optimizations
+#define ARRAY_STRATEGY 3        //Array level optimizations
+
+//Class that defines the entry methods that a Converse level strategy
+//must define. To write a new strategy inherit from this class and
+//define the virtual methods.  Every strategy can also define its own
+//constructor and have any number of arguments. Also call the parent
+//class methods in the virtual methods.
+
+class Strategy : public PUP::able{
+ protected:
+    int type;
+    int isStrategyBracketed;
+    int myInstanceID;
+
+    //Charm strategies for modularity may have converse strategies in
+    //them.  For the code to work in both Charm and converse, this
+    //variable can be used.    
+    Strategy *converseStrategy;
+
+ public:
+    Strategy();
+    Strategy(CkMigrateMessage *m) : PUP::able(m) {
+        converseStrategy = this;
+    }
+
+    void setBracketed(){isStrategyBracketed = 1;}
+    int isBracketed(){return isStrategyBracketed;}
+
+    //Called for each message
+    virtual void insertMessage(MessageHolder *msg) {}
+    
+    //Called after all chares and groups have finished depositing their 
+    //messages on that processor.
+    virtual void doneInserting() {}
+
+    void setInstance(int instid){myInstanceID = instid;}
+    int getInstance(){return myInstanceID;}
+    int getType() {return type;}
+    void setType(int t) {type = t;}
+
+    void setConverseStrategy(Strategy *s){
+        converseStrategy = s;
+    }
+
+    Strategy * getConverseStrategy() {
+        return converseStrategy;
+    }
+
+    //Each strategy must define his own Pup interface.
+    virtual void pup(PUP::er &p);
+    PUPable_decl(Strategy);
+};
+
+//Enables a list of strategies to be stored in a message through the
+//pupable framework
+class StrategyWrapper  {
+ public:
+    Strategy **s_table;
+    int nstrats;
+
+    void pup(PUP::er &p);
+};
+PUPmarshall(StrategyWrapper);
+
+//Table of strategies. Each entry in the table points to a strategy.
+//Strategies can change during the execution of the program but the
+//StrategyTableEntry stores some persistent information for the
+//strategy. The communication library on receiving a message, calls
+//the strategy in this table given by the strategy id in the message.
+
+struct StrategyTableEntry {
+    Strategy *strategy;
+    //A buffer for all strategy messages
+    CkQ<MessageHolder*> tmplist;
+    
+    int numElements;   //used by the array listener, 
+                       //could also be used for other objects
+    int elementCount;  //Count of how many elements have deposited
+                       //their data
+
+    //Used during a fence barrier at the begining or during the
+    //learning phases. Learning is only available for Charm++
+    //programs.
+    int nEndItr;       //#elements that called end iteration
+    int call_doneInserting; //All elements deposited their data
+
+    StrategyTableEntry();
+};
+
+typedef CkVec<StrategyTableEntry> StrategyTable;
+
+#endif
diff --git a/src/conv-com/de.C b/src/conv-com/de.C
new file mode 100644 (file)
index 0000000..5699a02
--- /dev/null
@@ -0,0 +1,399 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+/*************************************************
+ * File : de.C
+ *
+ * Author : Krishnan V.
+ *
+ * Dimensional Exchange (Hypercube) Router 
+ *  
+ * Modified to send last k stages directly by Sameer Kumar 9/07/03
+ *
+ ************************************************/
+#include "de.h"
+
+#define gmap(pe) (gpes ? gpes[pe] : pe)
+
+/**The only communication op used. Modify this to use
+ ** vector send */
+
+#define HCUBESENDFN(kid, u1, u2, knpe, kpelist, khndl, knextpe, pehcube)  \
+       {int len;\
+       char *newmsg;\
+       newmsg=pehcube->ExtractAndPack(kid, u1, knpe, kpelist, &len);\
+       if (newmsg) {\
+         CmiSetHandler(newmsg, khndl);\
+         CmiSyncSendAndFree(knextpe, len, newmsg);\
+       }\
+       else {\
+         SendDummyMsg(kid, knextpe, u2);\
+       }\
+}
+
+inline int maxdim(int n)
+{
+  int maxpes=1, dim=0;
+
+  while (maxpes< n) {
+       maxpes *=2;
+       dim++;
+  }
+  if (maxpes==n) return(dim);
+  else return(dim-1);
+}
+
+inline int neighbor(int pe, int dim)
+{
+  return(pe ^ (1<<dim));
+}
+
+inline int adjust(int dim, int pe)
+{
+  int mymax=1<<dim;
+  if (pe >= mymax) return(neighbor(pe, dim));
+  else return(pe);
+}
+
+inline int setIC(int dim, int pe, int N)
+{
+  int mymax= 1<< dim;
+  int initCounter=1, myneighb;
+  if (mymax < N) {
+       myneighb= neighbor(pe, dim);
+       if (myneighb < N && myneighb >= mymax) {
+              initCounter=0;
+       }
+  }
+  if (pe >= mymax) initCounter = -1;
+  return(initCounter);
+}
+
+/*********************************************************************
+ * Total preallocated memory=(P+Dim+Dim*P)ints + MAXNUMMSGS msgstruct
+ **********************************************************************/
+DimexRouter::DimexRouter(int n, int me, int ndirect)
+{
+  int i;
+  //last ndirect steps will be sent directly
+  numDirectSteps = ndirect;
+  //2 raised to the power of ndirect
+  two_pow_ndirect = 1;
+  for(int count = 0; count < ndirect; count ++)
+      two_pow_ndirect *= 2;
+
+  //Initialize the no: of pes and my Pe number
+  NumPes=n;
+  MyPe=me;
+  gpes=NULL;
+
+  //Initialize Dimension and no: of stages
+  Dim=maxdim(NumPes);
+
+  PeHcube=new PeTable(NumPes);
+  PeHcube1 = new PeTable(NumPes);
+
+  InitVars();
+
+  //Create the message array, buffer and the next stage table
+  buffer=new int[Dim+1];
+  next=(int **)CmiAlloc(sizeof(int *)*Dim);
+  for (i=0;i<Dim;i++) {
+       next[i]=new int[NumPes];
+       buffer[i]=0;
+       for (int j=0;j<NumPes;j++) next[i][j]=-1;
+  }
+  buffer[Dim]=0;
+
+  //Create and initialize the indexes to the above table
+  penum=new int[NumPes];
+  int *dp=new int[NumPes];
+  for (i=0;i<NumPes;i++) {
+       penum[i]=0;
+       dp[i]=i;
+  }
+
+  CreateStageTable(NumPes, dp);
+  delete(dp);
+
+  //CmiPrintf("%d DE constructor done dim=%d, mymax=%d IC=%d\n", CkMyPe(), Dim, 1<<Dim, InitCounter);
+
+  if(numDirectSteps > Dim - 1)
+      numDirectSteps = Dim - 1;
+}
+DimexRouter :: ~DimexRouter()
+{
+  int i;
+  delete PeHcube;
+  delete PeHcube1;
+  delete buffer;
+  for (i=0;i<Dim;i++) {
+       delete next[i];
+  }
+  delete next;
+}
+
+void DimexRouter :: SetMap(int *pes)
+{
+  gpes=pes;
+}
+
+void DimexRouter :: InitVars()
+{
+  stage=Dim-1;
+  InitCounter=setIC(Dim, MyPe, NumPes);
+  procMsgCount = 0;
+}
+
+void DimexRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
+{
+  int npe=NumPes;
+  int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
+  for (int i=0;i<npe;i++) destpes[i]=i;
+  EachToManyMulticast(id, size, msg, npe, destpes, more);
+  CmiFree(destpes);
+}
+
+void DimexRouter::NumDeposits(comID, int num)
+{
+  //CmiPrintf("Deposit=%d\n", num);
+}
+
+void DimexRouter::EachToManyMulticast(comID id, int size, void *msg, int numpes, int *destpes, int more)
+{
+
+    SetID(id);
+
+    //Create the message
+    if (msg && size) {
+       PeHcube->InsertMsgs(numpes, destpes, size, msg);
+    }
+    
+    if (more >0) return;
+    
+    if (InitCounter <0) {
+        ComlibPrintf("%d Sending to the lower hypercube\n", MyPe);
+       int nextpe=neighbor(MyPe, Dim);
+       int * pelist=(int *)CmiAlloc(NumPes*sizeof(int));
+       for (int i=0;i<NumPes;i++) {
+            pelist[i]=i;
+       }
+        ComlibPrintf("Before Gmap %d\n", nextpe);
+       nextpe=gmap(nextpe);
+       HCUBESENDFN(id, Dim, Dim, NumPes, pelist, CkpvAccess(RecvHandle), nextpe, PeHcube);
+       CmiFree(pelist);
+       return;
+    }
+    
+    //Done: no more stages.
+    if (stage <0) {
+       //CmiPrintf("calling lp in multicast call %d\n", stage);
+       LocalProcMsg(id);
+       return;
+    }
+    
+    InitCounter++;
+    RecvManyMsg(id,NULL);
+}
+
+//Send the messages for the next stage to the next dimension neighbor
+//If only numDirectStage's are left send messages directly using prefix send
+void DimexRouter::RecvManyMsg(comID id, char *msg)
+{
+  //CmiPrintf("%d recv called\n", MyPe);
+    int msgstage;
+    if (msg) {
+        msgstage=PeHcube->UnpackAndInsert(msg);
+        //CmiPrintf("%d recvd msg for stage=%d\n", MyPe, msgstage);
+        if (msgstage == Dim) InitCounter++;
+        else buffer[msgstage]=1;
+    }
+  
+    
+    //Check the buffers 
+    while ((InitCounter==2) || (stage >=numDirectSteps && buffer[stage+1])) {
+       InitCounter=setIC(Dim, MyPe, NumPes);
+       if (InitCounter != 2) { 
+            buffer[stage+1]=0;
+       }
+
+       //Send the data to the neighbor in this stage
+       int nextpe=neighbor(MyPe, stage);
+        
+        ComlibPrintf("Before Gmap %d\n", nextpe);
+        nextpe=gmap(nextpe);
+        ComlibPrintf("%d Sending to %d\n", MyPe, nextpe);
+       HCUBESENDFN(id, stage, stage, penum[stage], next[stage], CkpvAccess(RecvHandle), nextpe, PeHcube);
+
+       //Go to the next stage
+       stage--; 
+    }        
+
+    if (stage < numDirectSteps && buffer[numDirectSteps]) {
+                
+        InitCounter=setIC(Dim, MyPe, NumPes);
+        
+        //I am a processor in the smaller hypercube and there are some
+        //processors to send directly
+        if(InitCounter >= 0 && numDirectSteps > 0) {
+            //Sending through prefix send to save on copying overhead   
+            //of the hypercube algorithm            
+            
+            int *pelist = (int *)CmiAlloc(two_pow_ndirect * sizeof(int));
+            for(int count = 0; count < two_pow_ndirect; count ++){
+                int nextpe = count ^ MyPe;
+                gmap(nextpe);
+                
+                ComlibPrintf("%d Sending to %d\n", MyPe, nextpe);
+                pelist[count] = nextpe;
+            }
+            
+            int len;
+            char *newmsg;
+            newmsg=PeHcube->ExtractAndPackAll(id, stage, &len);
+            if (newmsg) {
+                CmiSetHandler(newmsg, CkpvAccess(ProcHandle));
+                CmiSyncListSendAndFree(two_pow_ndirect, pelist, len, newmsg);
+            }
+            
+            stage -= numDirectSteps;
+
+            //if(procMsgCount == two_pow_ndirect)
+            //  LocalProcMsg();
+        }
+        else if(numDirectSteps == 0) {
+            LocalProcMsg(id);
+            ComlibPrintf("Calling local proc msg %d\n", 
+                         buffer[numDirectSteps]);
+        }
+        
+       buffer[numDirectSteps]=0;
+    }
+}
+
+void DimexRouter :: ProcManyMsg(comID id, char *m)
+{
+
+    InitCounter=setIC(Dim, MyPe, NumPes);
+    if(id.isAllToAll) {
+        int pe_list[2];
+        int npes = 2;
+
+        if(InitCounter > 0)
+            npes = 1;
+        
+        pe_list[0] = MyPe;
+        pe_list[1] = neighbor(MyPe, Dim);
+        
+        PeHcube1->UnpackAndInsertAll(m, npes, pe_list);
+    }
+    else
+        PeHcube->UnpackAndInsert(m);
+    
+    procMsgCount ++;
+
+    if(InitCounter >= 0){
+        if((procMsgCount == two_pow_ndirect) && stage < 0) {
+            ComlibPrintf("%d Calling lp %d %d\n", CkMyPe(), 
+                         procMsgCount, stage);
+            LocalProcMsg(id);
+        }
+    }
+    else
+        //CmiPrintf("calling lp in procmsg call\n");
+        LocalProcMsg(id);
+}
+
+void DimexRouter:: LocalProcMsg(comID id)
+{
+    //CmiPrintf("%d local procmsg called\n", CkMyPe());
+
+    int mynext=neighbor(MyPe, Dim);
+    int mymax=1<<Dim;
+    
+    if (mynext >=mymax && mynext < NumPes) {
+        ComlibPrintf("Before Gmap %d\n", mynext);
+        mynext=gmap(mynext);
+        int *pelist=&mynext;
+        ComlibPrintf("%d Sending to %d\n", MyPe, mynext);        
+        
+        if(id.isAllToAll){
+            HCUBESENDFN(id, Dim, -1, 1, pelist, CkpvAccess(ProcHandle), mynext, PeHcube1);
+        }
+        else {
+            HCUBESENDFN(id, Dim, -1, 1, pelist, CkpvAccess(ProcHandle), mynext, PeHcube);
+        }
+    }
+  
+    if(id.isAllToAll)
+        PeHcube1->ExtractAndDeliverLocalMsgs(MyPe);
+    else
+        PeHcube->ExtractAndDeliverLocalMsgs(MyPe);
+
+    PeHcube->Purge();
+    PeHcube1->Purge();
+    InitVars();
+    Done(id);
+}
+
+void DimexRouter::DummyEP(comID id, int msgstage)
+{
+  if (msgstage >= 0) {
+       buffer[msgstage]=1;
+       RecvManyMsg(id, NULL);
+  }
+  else {
+       //CmiPrintf("%d Dummy calling lp\n", MyPe);
+       LocalProcMsg(id);
+  }
+}
+
+void DimexRouter::CreateStageTable(int numpes, int *destpes)
+{
+  int *dir=new int[numpes];
+  int nextdim, j, i;
+  for (i=0;i<numpes;i++) {
+       dir[i]=MyPe ^ adjust(Dim, destpes[i]);
+  }
+  for (nextdim=Dim-1; nextdim>=0; nextdim--) {
+    int mask=1<<nextdim;
+    for (i=0;i<numpes;i++) {
+       if (dir[i] & mask) {
+               dir[i]=0;
+               for (j=0;(j<penum[nextdim]) && (destpes[i]!=next[nextdim][j]);j++);
+               if (destpes[i]==next[nextdim][j]) { 
+                       //CmiPrintf("EQUAL %d\n", destpes[i]);
+                       continue;
+               }
+               next[nextdim][penum[nextdim]]=destpes[i];
+               penum[nextdim]+=1;
+               //CmiPrintf("%d next[%d][%d]=%d\n",MyPe, nextdim, penum[nextdim],destpes[i]);
+        }
+    }
+  }
+  delete dir;
+  return;
+}
+
+Router * newhcubeobject(int n, int me)
+{
+    Router *obj=new DimexRouter(n, me);
+    return(obj);
+}
+
+//MOVE this CODE else where, this method has been depricated!!!
+void DimexRouter::SetID(comID id) { 
+
+    if(id.isAllToAll) {
+        numDirectSteps = 2;
+        two_pow_ndirect = 1;
+          for(int count = 0; count < numDirectSteps; count ++)
+              two_pow_ndirect *= 2;
+    }
+}
diff --git a/src/conv-com/de.h b/src/conv-com/de.h
new file mode 100644 (file)
index 0000000..b37c1f1
--- /dev/null
@@ -0,0 +1,44 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#ifndef _DE_H
+#define _DE_H
+#include <converse.h>
+#include "comlib.h"
+#include "petable.h"
+
+//Dimensional Exchange (Hypercube) based router
+class DimexRouter : public Router
+{
+  private:
+       PeTable *PeHcube, *PeHcube1;
+       int *buffer;
+       int* msgnum, InitCounter;
+       int *penum,*gpes;
+       int **next;
+       int Dim, stage, MyPe, NumPes, numDirectSteps, two_pow_ndirect;
+        int procMsgCount;
+       void InitVars();
+       void CreateStageTable(int, int *);
+       void LocalProcMsg(comID id);
+  public:
+        
+       DimexRouter(int, int, int ndirect = 0);
+       ~DimexRouter();
+       void NumDeposits(comID, int);
+       void EachToAllMulticast(comID , int , void *, int);
+       void EachToManyMulticast(comID , int , void *, int, int *, int);
+       void ProcMsg(int, msgstruct **) {;}
+       void RecvManyMsg(comID, char *);
+       void ProcManyMsg(comID, char *);
+       void DummyEP(comID id, int);
+       void SetMap(int *);
+
+        //FIX this, some initialization done here
+       void SetID(comID id);
+};
+#endif
diff --git a/src/conv-com/dummystrategy.C b/src/conv-com/dummystrategy.C
new file mode 100644 (file)
index 0000000..58e5145
--- /dev/null
@@ -0,0 +1,18 @@
+
+#include "RouterStrategy.h"
+
+RouterStrategy::RouterStrategy() : Strategy(){
+}
+
+void RouterStrategy::insertMessage(MessageHolder *cmsg){
+    ComlibPrintf("Sending Directly\n");
+    char *msg = cmsg->getMessage();
+    CmiSyncSendAndFree(cmsg->dest_proc, cmsg->size, msg);
+    delete cmsg;
+}
+
+void RouterStrategy::doneInserting(){
+}
+
+void RouterStrategy::pup(PUP::er &p){}
+
diff --git a/src/conv-com/dummystrategy.h b/src/conv-com/dummystrategy.h
new file mode 100644 (file)
index 0000000..1e87909
--- /dev/null
@@ -0,0 +1,23 @@
+
+/* Class that calls Krishnan's routers from the new Comlib.
+   Developed to be called from Converse
+   Sameer Kumar 05/14/04
+*/
+   
+
+#ifndef ROUTER_STRATEGY
+#define ROUTER_STRATEGY
+#include "convcomlibmanager.h"
+
+class RouterStrategy : public Strategy {
+ public:
+    RouterStrategy();
+    RouterStrategy(CkMigrateMessage *m): Strategy(m){}
+
+    void insertMessage(MessageHolder *msg);
+    void doneInserting();
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(RouterStrategy);
+};
+#endif
diff --git a/src/conv-com/graphrouter.C b/src/conv-com/graphrouter.C
new file mode 100644 (file)
index 0000000..c0977eb
--- /dev/null
@@ -0,0 +1,251 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#include "graphrouter.h"
+#include "hypercubetopology.h"
+
+#define gmap(pe) {if (gpes) pe=gpes[pe];}
+
+GraphRouter::GraphRouter(int n, int me){
+    init(n, me, new HypercubeTopology(n, me));
+}
+
+void GraphRouter::init(int n, int me, TopologyDescriptor *tp)
+{  
+    NumPes=n;
+    MyPe=me;
+    gpes=NULL;
+    this->tp = tp;
+    
+    PeGraph = new PeTable(NumPes);
+    pesToSend = new int[NumPes];
+    nstages = tp->getNumStages() + 1;
+    currentIteration = 0;
+    
+    stageComplete = new int[nstages];
+    recvExpected = new int[nstages];
+    recvCount = new int[nstages];
+
+    memset(stageComplete, 0, nstages * sizeof(int));
+    memset(recvCount, 0, nstages * sizeof(int));
+    
+    for(int count = 1; count < nstages; count++)
+        recvExpected[count] = tp->getNumMessagesExpected(count);
+    
+    curStage = 0;
+    ComlibPrintf("me=%d NUMPES=%d nstages=%d\n", MyPe, n, nstages);
+}
+
+GraphRouter::~GraphRouter()
+{
+    delete PeGraph;
+    delete pesToSend;
+    delete tp;
+    delete [] stageComplete;
+    delete [] recvExpected;
+    delete [] recvCount;
+    delete [] neighborPeList;
+}
+
+void GraphRouter::NumDeposits(comID, int num)
+{
+}
+
+void GraphRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
+{
+    int npe=NumPes;
+    int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
+    for (int i=0;i<npe;i++) destpes[i]=i;
+    EachToManyMulticast(id, size, msg, npe, destpes, more);
+}
+
+void GraphRouter::sendMessages(comID id, int cur_stage){
+    int nsteps = tp->getNumSteps(cur_stage);
+    int nextpe = 0, npestosend = 0;
+    
+    for(int stepcount = 0; stepcount < nsteps; stepcount ++){
+        tp->getPesToSend(stepcount, cur_stage, npestosend, pesToSend, nextpe);
+        
+        gmap(nextpe);
+        ComlibPrintf("%d:sending to %d for %d pes in stage %d\n", MyPe, nextpe, npestosend, cur_stage);
+
+        int len;
+       char *newmsg;
+        newmsg=PeGraph->ExtractAndPack(id, cur_stage + 1, npestosend, 
+                                       pesToSend, &len);
+        
+#if CMK_PERSISTENT_COMM
+        if(len < PERSISTENT_BUFSIZE)
+            if(currentIteration % 2)
+                CmiUsePersistentHandle(&handlerArrayOdd[cur_stage], 1);
+            else
+                CmiUsePersistentHandle(&handlerArrayEven[cur_stage], 1);
+#endif          
+        
+       if (newmsg) {
+            if(cur_stage < nstages - 2)
+                CmiSetHandler(newmsg, CkpvAccess(RecvHandle));
+            else
+                CmiSetHandler(newmsg, CkpvAccess(ProcHandle));
+
+            CmiSyncSendAndFree(nextpe, len, newmsg);
+        }
+       else {
+            SendDummyMsg(id, nextpe, cur_stage + 1);
+       }
+        
+#if CMK_PERSISTENT_COMM
+        if(len < PERSISTENT_BUFSIZE)
+            CmiUsePersistentHandle(NULL, 0);
+#endif          
+    }
+}
+
+void GraphRouter::EachToManyMulticast(comID id, int size, void *msg, 
+                                      int numpes, int *destpes, int more)
+{
+    PeGraph->InsertMsgs(numpes, destpes, size, msg);
+    if (more) return;
+
+    ComlibPrintf("All messages received %d\n", MyPe);
+    sendMessages(id, 0);
+
+    curStage = 1;
+
+    int stage_itr;
+    for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
+        if(stageComplete[stage_itr]){
+            sendMessages(id, stage_itr);
+            stageComplete[stage_itr] = 0;
+        }
+        else break;
+    }
+    curStage = stage_itr;
+    if(curStage == nstages - 1)
+        ProcManyMsg(id, NULL);
+    else 
+        PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
+}
+
+void GraphRouter::RecvManyMsg(comID id, char *msg)
+{
+    int stage = 0;
+    stage = PeGraph->UnpackAndInsert(msg);
+    
+    recvCount[stage] ++;
+    if (recvCount[stage] == recvExpected[stage]) {
+        ComlibPrintf("%d recvcount=%d recvexpected = %d stage=%d\n", MyPe, recvCount[stage], recvExpected[stage], stage);
+        
+        recvCount[stage] = 0;
+        stageComplete[stage] = 1;
+    }
+    
+    int stage_itr;
+    for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
+        if(stageComplete[stage_itr]){
+            sendMessages(id, stage_itr);
+            stageComplete[stage_itr] = 0;
+        }
+        else break;
+    }
+    curStage = stage_itr;
+    if(curStage == nstages - 1)
+        ProcManyMsg(id, NULL);
+    else 
+        PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
+}
+
+void GraphRouter::DummyEP(comID id, int stage)
+{
+    if(stage < nstages - 1) {
+        recvCount[stage] ++;
+        if (recvCount[stage] == recvExpected[stage]) {
+            ComlibPrintf("%d DUMMY recvcount=%d recvexpected = %d\n", MyPe, recvCount[stage], recvExpected[stage]);
+            recvCount[stage] = 0;
+            stageComplete[stage] = 1;
+        }
+
+        int stage_itr;
+        for(stage_itr = curStage; stage_itr < nstages - 1; stage_itr ++){
+            if(stageComplete[stage_itr]){
+                sendMessages(id, stage_itr);
+                stageComplete[stage] = 0;
+            }
+            else break;
+        }
+        curStage = stage_itr;
+        if(curStage == nstages - 1)
+            ProcManyMsg(id, NULL);
+        else 
+            PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
+    }
+    else 
+        ProcManyMsg(id, NULL);
+}
+
+void GraphRouter:: ProcManyMsg(comID id, char *m)
+{
+    int stage = nstages - 1;
+    if(m) {
+        PeGraph->UnpackAndInsert(m);
+        recvCount[stage] ++;
+    }
+
+    if(recvCount[stage] == recvExpected[stage]) {
+        ComlibPrintf("%d proc many msg %d\n", MyPe, stage);
+        stageComplete[stage] = 1;
+    }
+    else 
+        return;
+    
+    if(curStage != nstages -1)
+        return;
+
+    currentIteration ++;
+    recvCount[stage] = 0;
+    PeGraph->ExtractAndDeliverLocalMsgs(MyPe);
+    
+    PeGraph->Purge();
+    curStage = 0;
+    Done(id);
+}
+
+Router * newgraphobject(int n, int me)
+{
+    ComlibPrintf("In create graph router \n");
+    Router *obj = new GraphRouter(n, me);
+    return(obj);
+}
+
+void GraphRouter :: SetMap(int *pes)
+{
+    gpes=pes;
+
+#if CMK_PERSISTENT_COMM
+    numNeighbors=0;
+    neighborPeList = new int[NumPes];
+
+    tp->getNeighbors(numNeighbors, neighborPeList);
+    handlerArrayOdd = new PersistentHandle[numNeighbors];
+    handlerArrayEven = new PersistentHandle[numNeighbors];
+
+    //Persistent handlers for all the neighbors
+    int pcount = 0;
+    for (pcount = 0; pcount < numNeighbors; pcount++) {
+        int dest = neighborPeList[pcount];
+        gmap(dest);
+        ComlibPrintf("%d:Creating Persistent Buffer of size %d at %d\n", MyPe,
+                     PERSISTENT_BUFSIZE, dest);
+        handlerArrayOdd[pcount] = CmiCreatePersistent(dest, 
+                                                      PERSISTENT_BUFSIZE);
+        ComlibPrintf("%d:Creating Even Persistent Buffer of size %d at %d\n",
+                     MyPe, PERSISTENT_BUFSIZE, dest);
+        handlerArrayEven[pcount] = CmiCreatePersistent(dest, 
+                                                       PERSISTENT_BUFSIZE);
+    }
+#endif
+}
diff --git a/src/conv-com/graphrouter.h b/src/conv-com/graphrouter.h
new file mode 100644 (file)
index 0000000..2f48366
--- /dev/null
@@ -0,0 +1,68 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#ifndef _GRAPHROUTER_H
+#define _GRAPHROUTER_H
+
+#include <math.h>
+#include <converse.h>
+#include "comlib.h"
+#include "petable.h"
+
+#include "persistent.h"
+
+class TopologyDescriptor {
+ public:
+    //TopologyDescriptor(int npes, int mype)=0;
+    //Entry methods which will define the specific graph.
+    virtual void getNeighbors(int &npes, int *pelist) = 0;
+    virtual int getNumStages() = 0;
+    virtual int getNumSteps(int stage) = 0;
+    virtual void getPesToSend(int step, int stage, int &npesToSend, 
+                              int *pelist, int &nextpe) = 0;
+    virtual int getNumMessagesExpected(int stage) = 0;
+};
+
+//A generalized virtual topology based router. To make it a specific
+//router a new topology class needs to br created and passed to this.
+class GraphRouter : public Router
+{
+    PeTable *PeGraph;
+    TopologyDescriptor *tp;
+    
+    int *pesToSend, *gpes;
+    int numNeighbors, *neighborPeList, nstages;
+    
+    int MyPe, NumPes, currentIteration;
+    int *recvExpected, *recvCount;
+    int curStage, *stageComplete;
+
+    void sendMessages(comID id, int stage);
+    void init(int numPes, int myPe, TopologyDescriptor *tp);
+    
+#if CMK_PERSISTENT_COMM
+    PersistentHandle *handlerArrayOdd, *handlerArrayEven;
+#endif          
+    
+ public:
+    GraphRouter(int numPes, int myPe);
+    //GraphRouter(int numPes, int myPe, int topid);
+    ~GraphRouter();
+  
+    //Router enrty methods overridden here.
+
+    void NumDeposits(comID, int);
+    void EachToAllMulticast(comID , int , void *, int);
+    void EachToManyMulticast(comID , int , void *, int, int *, int);
+    void RecvManyMsg(comID, char *);
+    void ProcManyMsg(comID, char *);
+    void DummyEP(comID id, int);
+    
+    void SetMap(int *);
+};
+
+#endif
diff --git a/src/conv-com/gridrouter.C b/src/conv-com/gridrouter.C
new file mode 100644 (file)
index 0000000..19d0ac5
--- /dev/null
@@ -0,0 +1,319 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+/************************************************************
+ * File : gridrouter.C
+ *
+ * Author : Krishnan V.
+ *
+ * Grid (mesh) based router
+ ***********************************************************/
+#include "gridrouter.h"
+
+#define gmap(pe) {if (gpes) pe=gpes[pe];}
+
+/**The only communication op used. Modify this to use
+ ** vector send */
+#define GRIDSENDFN(kid, u1, u2, knpe, kpelist, khndl, knextpe)  \
+       {int len;\
+       char *newmsg;\
+        newmsg=PeMesh->ExtractAndPack(kid, u1, knpe, kpelist, &len);\
+       if (newmsg) {\
+         CmiSetHandler(newmsg, khndl);\
+          CmiSyncSendAndFree(knextpe, len, newmsg);\
+        }\
+       else {\
+         SendDummyMsg(kid, knextpe, u2);\
+       }\
+}
+
+/****************************************************
+ * Preallocated memory=P ints + MAXNUMMSGS msgstructs
+ *****************************************************/
+GridRouter::GridRouter(int n, int me)
+{
+  //CmiPrintf("PE=%d me=%d NUMPES=%d\n", MyPe, me, n);
+  
+  NumPes=n;
+  MyPe=me;
+  gpes=NULL;
+  COLLEN=ColLen(NumPes);
+  LPMsgExpected = Expect(MyPe, NumPes);
+  recvExpected = 0;
+
+  int myrow=MyPe/COLLEN;
+  int mycol=MyPe%COLLEN;  
+  int lastrow = (NumPes - 1)/COLLEN;
+  
+  if(myrow < lastrow) 
+      recvExpected = ROWLEN;
+  else
+      recvExpected = (NumPes - 1)%ROWLEN + 1;
+
+  if(lastrow * COLLEN + mycol > NumPes - 1) {
+      //We have a hole in the lastrow
+      if(lastrow * COLLEN + myrow <= NumPes - 1) 
+          //We have a processor which wants to send data to that hole
+          recvExpected ++;
+      
+      if((myrow == 0) && (NumPes == ROWLEN*(COLLEN-1) - 1))
+          //Special case with one hole only
+         recvExpected ++;      
+  }
+  
+  ComlibPrintf("%d LPMsgExpected=%d\n", MyPe, LPMsgExpected);
+
+  PeMesh = new PeTable(NumPes);
+  PeMesh1 = new PeTable(NumPes);
+  PeMesh2 = new PeTable(NumPes);
+
+  onerow=(int *)CmiAlloc(ROWLEN*sizeof(int));
+
+  rowVector = (int *)CmiAlloc(ROWLEN*sizeof(int));
+  colVector = (int *)CmiAlloc(COLLEN*sizeof(int));
+
+  int myrep=myrow*COLLEN;
+  int count = 0;
+  int pos = 0;
+
+  for(count = myrow; count < ROWLEN+myrow; count ++){
+      int nextpe= myrep + count%ROWLEN;
+      
+      if (nextpe >= NumPes) {
+          int new_row = mycol % (myrow+1);
+          
+          if(new_row >= myrow)
+              new_row = 0;
+          
+          nextpe = COLLEN * new_row + count;
+      }
+      
+      if(nextpe == MyPe)
+          continue;
+
+      rowVector[pos ++] = nextpe;
+  }
+  rvecSize = pos;
+
+  pos = 0;
+  for(count = mycol; count < COLLEN+mycol; count ++){
+      int nextrowrep = (count % COLLEN) *COLLEN;
+      int nextpe = nextrowrep+mycol;
+      
+      if(nextpe < NumPes && nextpe != MyPe)
+          colVector[pos ++] = nextpe;
+  }
+  
+  cvecSize = pos;
+
+  growVector = new int[rvecSize];
+  gcolVector = new int[cvecSize];
+
+  for(count = 0; count < rvecSize; count ++)
+      growVector[count] = rowVector[count];
+  
+  for(count = 0; count < cvecSize; count ++)
+      gcolVector[count] = colVector[count];
+  
+
+  InitVars();
+  ComlibPrintf("%d:COLLEN=%d, ROWLEN=%d, recvexpected=%d\n", MyPe, COLLEN, ROWLEN, recvExpected);
+}
+
+GridRouter::~GridRouter()
+{
+  delete PeMesh;
+  delete PeMesh1;
+  delete PeMesh2;
+    
+  CmiFree(onerow);
+}
+
+void GridRouter :: InitVars()
+{
+  recvCount=0;
+  LPMsgCount=0;
+}
+
+void GridRouter::NumDeposits(comID, int num)
+{
+}
+
+void GridRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
+{
+  int npe=NumPes;
+  int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
+  for (int i=0;i<npe;i++) destpes[i]=i;
+  EachToManyMulticast(id, size, msg, npe, destpes, more);
+}
+
+extern void CmiReference(void *blk);
+
+void GridRouter::EachToManyMulticast(comID id, int size, void *msg, int numpes, int *destpes, int more)
+{
+  int i=0;
+  
+  if(id.isAllToAll)
+      PeMesh->InsertMsgs(1, &MyPe, size, msg);
+  else
+      PeMesh->InsertMsgs(numpes, destpes, size, msg);
+  
+  if (more) return;
+
+  ComlibPrintf("All messages received %d %d %d\n", MyPe, COLLEN,id.isAllToAll);
+
+  char *a2amsg = NULL;
+  int a2a_len;
+  if(id.isAllToAll) {
+      ComlibPrintf("ALL to ALL flag set\n");
+
+      a2amsg = PeMesh->ExtractAndPackAll(id, 0, &a2a_len);
+      CmiSetHandler(a2amsg, CkpvAccess(RecvHandle));
+      CmiReference(a2amsg);
+      CmiSyncListSendAndFree(rvecSize, growVector, a2a_len, a2amsg);      
+      RecvManyMsg(id, a2amsg);
+      return;
+  }
+
+  //Send the messages
+  int MYROW=MyPe/COLLEN;
+  int MYCOL = MyPe%COLLEN;
+  int myrep=MYROW*COLLEN;
+  
+  for (int colcount = 0; colcount < rvecSize; ++colcount) {
+      int nextpe = rowVector[colcount];
+      i = nextpe % COLLEN;
+      
+      int length = (NumPes - 1)/COLLEN + 1;
+      if((length - 1)* COLLEN + i >= NumPes)
+          length --;
+      
+      for (int j = 0; j < length; j++) {
+          onerow[j]=j * COLLEN + i;
+      }
+      
+      ComlibPrintf("%d: before gmap sending to %d of column %d\n",
+                   MyPe, nextpe, i);
+      gmap(nextpe);
+      ComlibPrintf("%d:sending to %d of column %d\n", MyPe, nextpe, i);
+      
+      GRIDSENDFN(id, 0, 0, length, onerow, CkpvAccess(RecvHandle), nextpe); 
+  }
+  RecvManyMsg(id, NULL);
+}
+
+void GridRouter::RecvManyMsg(comID id, char *msg)
+{
+  if (msg) {  
+      if(id.isAllToAll)
+          PeMesh1->UnpackAndInsertAll(msg, 1, &MyPe);
+      else
+          PeMesh->UnpackAndInsert(msg);
+  }
+
+  recvCount++;
+  if (recvCount == recvExpected) {
+      ComlibPrintf("%d recvcount=%d recvexpected = %d\n", MyPe, recvCount, recvExpected);
+      
+      int myrow=MyPe/COLLEN;
+      int mycol=MyPe%COLLEN;
+      
+      char *a2amsg;
+      int a2a_len;
+      if(id.isAllToAll) {
+          a2amsg = PeMesh1->ExtractAndPackAll(id, 1, &a2a_len);
+          CmiSetHandler(a2amsg, CkpvAccess(ProcHandle));
+          CmiReference(a2amsg);
+          CmiSyncListSendAndFree(cvecSize, gcolVector, a2a_len, a2amsg);   
+          ProcManyMsg(id, a2amsg);
+          return;
+      }
+
+      for (int rowcount=0; rowcount < cvecSize; rowcount++) {
+          int nextpe = colVector[rowcount];
+                    
+          int gnextpe = nextpe;
+          int *pelist=&gnextpe;
+          
+          ComlibPrintf("Before gmap %d\n", nextpe);
+          
+          gmap(nextpe);
+
+          ComlibPrintf("After gmap %d\n", nextpe);
+          
+          ComlibPrintf("%d:sending message to %d of row %d\n", MyPe, nextpe, 
+                       rowcount);
+          
+          GRIDSENDFN(id, 0, 1, 1, pelist, CkpvAccess(ProcHandle), nextpe);
+      }
+      
+      LocalProcMsg(id);
+  }
+}
+
+void GridRouter::DummyEP(comID id, int magic)
+{
+  if (magic == 1) {
+       //ComlibPrintf("%d dummy calling lp\n", MyPe);
+       LocalProcMsg(id);
+  }
+  else {
+       //ComlibPrintf("%d dummy calling recv\n", MyPe);
+       RecvManyMsg(id, NULL);
+  }
+}
+
+void GridRouter:: ProcManyMsg(comID id, char *m)
+{
+    if(id.isAllToAll)
+        PeMesh2->UnpackAndInsertAll(m, 1, &MyPe);
+    else
+        PeMesh->UnpackAndInsert(m);
+    //ComlibPrintf("%d proc calling lp\n");
+    
+    LocalProcMsg(id);
+}
+
+void GridRouter:: LocalProcMsg(comID id)
+{
+    LPMsgCount++;
+    PeMesh->ExtractAndDeliverLocalMsgs(MyPe);
+    PeMesh2->ExtractAndDeliverLocalMsgs(MyPe);
+    
+    ComlibPrintf("%d local procmsg called\n", MyPe);
+    if (LPMsgCount==LPMsgExpected) {
+        PeMesh->Purge();
+        PeMesh2->Purge();
+        
+        InitVars();
+        Done(id);
+    }
+}
+
+Router * newgridobject(int n, int me)
+{
+  Router *obj=new GridRouter(n, me);
+  return(obj);
+}
+
+void GridRouter :: SetMap(int *pes)
+{
+    
+  gpes=pes;
+  
+  if(!gpes)
+      return;
+
+  int count = 0;
+
+  for(count = 0; count < rvecSize; count ++)
+      growVector[count] = gpes[rowVector[count]];
+  
+  for(count = 0; count < cvecSize; count ++)
+      gcolVector[count] = gpes[colVector[count]];
+}
+
diff --git a/src/conv-com/gridrouter.h b/src/conv-com/gridrouter.h
new file mode 100644 (file)
index 0000000..c1287a5
--- /dev/null
@@ -0,0 +1,74 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#ifndef _GRIDROUTER_H
+#define _GRIDROUTER_H
+
+#include <math.h>
+#include <converse.h>
+#include "comlib.h"
+#include "petable.h"
+
+#define ROWLEN COLLEN
+#define RowLen(pe) ColLen(pe)
+#define PELISTSIZE ((ROWLEN-1)/sizeof(int)+1)
+
+inline int ColLen(int npes)
+{
+  int len= (int)sqrt((double)npes);
+  if (npes > (len*len)) len++;
+  return(len);
+}
+
+inline int Expect(int pe, int npes)
+{
+  int i, len=ColLen(npes);
+  for (i=len-1;i>=0;i--) {
+       int myrow=pe/len;
+       int toprep=i*len;
+       int offset=pe-myrow*len;
+       if ((toprep+offset) <= (npes-1)) return(i+1);
+  }
+  return(len);
+}
+
+#include "persistent.h"
+
+//Grid based router
+class GridRouter : public Router
+{
+  private:
+       PeTable *PeMesh, *PeMesh1, *PeMesh2;
+        int *onerow, *gpes, 
+            *rowVector, *colVector, 
+            *growVector, *gcolVector;
+
+        int rvecSize, cvecSize;
+       int MyPe, NumPes, COLLEN;
+       int LPMsgCount, LPMsgExpected;
+       int recvExpected, recvCount;
+       void InitVars();
+       void LocalProcMsg(comID id);
+#if CMK_PERSISTENT_COMM
+       PersistentHandle *rowHandleArray, *columnHandleArray;
+        PersistentHandle *rowHandleArrayEven, *columnHandleArrayEven;
+#endif          
+
+  public:
+       GridRouter(int, int);
+       ~GridRouter();
+       void NumDeposits(comID, int);
+       void EachToAllMulticast(comID , int , void *, int);
+       void EachToManyMulticast(comID , int , void *, int, int *, int);
+       void RecvManyMsg(comID, char *);
+       void ProcManyMsg(comID, char *);
+       void DummyEP(comID id, int);
+       void ProcMsg(int, msgstruct **) {;}
+       void SetMap(int *);
+};
+
+#endif
diff --git a/src/conv-com/hypercubetopology.C b/src/conv-com/hypercubetopology.C
new file mode 100644 (file)
index 0000000..b25c6bb
--- /dev/null
@@ -0,0 +1,111 @@
+
+#include "hypercubetopology.h"
+
+inline int neighbor(int pe, int dim)
+{
+    return(pe ^ (1<<dim));
+}
+
+inline int maxdim(int n)
+{
+    int maxpes=1, dim=0;
+
+    while (maxpes< n) {
+        maxpes *=2;
+        dim++;
+    }
+    if (maxpes==n) return(dim);
+    else return(dim-1);
+}
+
+inline int adjust(int dim, int pe)
+{
+    int mymax=1<<dim;
+    if (pe >= mymax) return(neighbor(pe, dim));
+    else return(pe);
+}
+
+HypercubeTopology::HypercubeTopology(int npes, int mype) {
+    
+    int i = 0;
+    NumPes = npes;
+    MyPe = mype;
+    Dim = maxdim(npes);
+
+    next = new int *[Dim];
+    for (i=0;i<Dim;i++) {
+        next[i]=new int[NumPes];
+        for (int j=0;j<NumPes;j++) next[i][j]=-1;
+    }
+    
+    //Create and initialize the indexes to the above table
+    penum=new int[NumPes];
+    int *dp=new int[NumPes];
+    for (i=0;i<NumPes;i++) {
+        penum[i]=0;
+        dp[i]=i;
+    }
+    
+    CreateStageTable(NumPes, dp);
+    delete(dp);
+}
+
+void HypercubeTopology::getNeighbors(int &np, int *pelist){
+    np = Dim;
+
+    for(int count = 0; count < Dim; count ++)
+        pelist[count] = MyPe ^ (1 << (Dim - count - 1));
+}
+
+int HypercubeTopology::getNumStages(){
+    return Dim;
+}
+
+int HypercubeTopology::getNumSteps(int stage) {
+    return 1;
+}
+
+void HypercubeTopology::getPesToSend(int step, int stage, int &np, 
+                                     int *pelist, int &nextpe) {
+    if(step > 0) {
+        np = 0;
+        return;
+    }
+
+    np = penum[Dim - stage - 1];
+    memcpy(pelist, next[Dim - stage - 1], np *sizeof(int));
+        
+    nextpe = neighbor(MyPe, Dim - stage - 1);
+}
+
+int HypercubeTopology::getNumMessagesExpected(int stage) {
+    return 1;
+}
+
+void HypercubeTopology::CreateStageTable(int numpes, int *destpes)
+{
+    int *dir=new int[numpes];
+    int nextdim, j, i;
+    for (i=0;i<numpes;i++) {
+        dir[i]=MyPe ^ adjust(Dim, destpes[i]);
+    }
+    
+    for (nextdim=Dim-1; nextdim>=0; nextdim--) {
+        int mask=1<<nextdim;
+        for (i=0;i<numpes;i++) {
+            if (dir[i] & mask) {
+                dir[i]=0;
+                for (j=0;(j<penum[nextdim]) && (destpes[i]!=next[nextdim][j]);j++);
+                if (destpes[i]==next[nextdim][j]) { 
+                    //CmiPrintf("EQUAL %d\n", destpes[i]);
+                    continue;
+                }
+                next[nextdim][penum[nextdim]]=destpes[i];
+                penum[nextdim]+=1;
+                //CmiPrintf("%d next[%d][%d]=%d\n",MyPe, nextdim, penum[nextdim],destpes[i]);
+            }
+        }
+    }
+    delete dir;
+    return;
+}
diff --git a/src/conv-com/hypercubetopology.h b/src/conv-com/hypercubetopology.h
new file mode 100644 (file)
index 0000000..171e65f
--- /dev/null
@@ -0,0 +1,24 @@
+#ifndef __HYPERCUBE_TOPOLOGY_H
+#define __HYPERCUBE_TOPOLOGY_H
+
+#include "graphrouter.h"
+#include "string.h"
+
+class HypercubeTopology: public TopologyDescriptor {
+    int NumPes, MyPe, Dim;
+    int *penum, **next;
+
+    void CreateStageTable(int, int *);
+    
+ public:
+    HypercubeTopology(int npes, int mype);
+    //Entry methods which will define the specific graph.
+    void getNeighbors(int &npes, int *pelist);
+    int getNumStages();
+    int getNumSteps(int stage);
+    void getPesToSend(int step, int stage, int &npesToSend, 
+                              int *pelist, int &nextpe);
+    int getNumMessagesExpected(int stage);
+};
+
+#endif
diff --git a/src/conv-com/petable.C b/src/conv-com/petable.C
new file mode 100644 (file)
index 0000000..8b3aee3
--- /dev/null
@@ -0,0 +1,684 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+/*********************************************
+ * File : petable.C
+ *
+ * Author: Krishnan V
+ *
+ * The message buffer
+ *********************************************/
+#include <string.h>
+#include <stdlib.h>
+#include <converse.h>
+#include "comlib.h"
+#include "petable.h"
+#include "converse.h"
+
+#define BIGBUFFERSIZE 65536
+#define PTPREALLOC    100
+
+struct AllToAllHdr{
+    char dummy[CmiReservedHeaderSize];
+    int refno;
+    comID id;
+    int ufield;
+    int nmsgs;
+};
+
+#define ALIGN8(x)       (int)((~7)&((x)+7))
+
+/* Reduce the no. of mallocs by allocating from
+ * a free list */
+#define PTALLOC(ktmp) {\
+  if (PTFreeList) {\
+       ktmp=PTFreeList;\
+       PTFreeList=ktmp->next;\
+  }\
+  else {\
+       ktmp=(PTinfo *)CmiAlloc(sizeof(PTinfo));\
+       }\
+}
+
+#define PTFREE(ktmp) {\
+  ktmp->next=PTFreeList;\
+  PTFreeList=ktmp;\
+}
+
+#define REALLOC(ktmp, ksize) {\
+   PTinfo **junkptr=(PTinfo **)CmiAlloc(2*ksize*sizeof(void *));\
+   for (int ki=0; ki<ksize;ki++) junkptr[ki]=ktmp[ki];\
+   CmiFree(ktmp);\
+   ktmp=junkptr;\
+}
+
+/**************************************************************
+ * Preallocated memory=P*MSGQLEN ptr + 2P ints + 1000 ptrs
+ **************************************************************/
+PeTable :: PeTable(int n)
+{
+  NumPes=n;
+  magic=0;
+  PeList = (PTinfo ***)CmiAlloc(sizeof(PTinfo *)*NumPes);
+  //  ComlibPrintf("Pelist[%d][%d]\n", NumPes, MSGQLEN);
+  msgnum=new int[NumPes];
+  MaxSize=new int[NumPes];
+  for (int i=0;i<NumPes;i++) {
+       msgnum[i]=0;
+       MaxSize[i]=MSGQLEN;
+       PeList[i]=(PTinfo **)CmiAlloc(sizeof(PTinfo *)*MSGQLEN);
+       for (int j=0;j<MSGQLEN;j++) PeList[i][j]=0;
+  }
+
+  //ptrlist=(PTinfo **)CmiAlloc(1000*sizeof(PTinfo *));
+  //  FreeList= new GList;
+  //CombBuffer=(char *)CmiAlloc(BIGBUFFERSIZE);
+
+  PTFreeList=NULL;
+}
+
+PeTable :: ~PeTable()
+{
+  int i;
+  for (i=0;i<NumPes;i++) CmiFree(PeList[i]);
+  CmiFree(PeList);
+  delete msgnum;
+  delete MaxSize;
+  GarbageCollect();
+  //CmiFree(ptrlist);
+  PTinfo *tmp;
+  while (PTFreeList) {
+       tmp=PTFreeList;
+       PTFreeList=tmp->next;
+       CmiFree(tmp);
+  }
+ // delete FreeList;
+
+}
+
+void PeTable:: Purge()
+{
+  for (int i=0; i<NumPes;i++) {
+       if (msgnum[i]) {
+            // ComlibPrintf("%d Warning: %d Undelivered Messages for %d\n", CkMyPe(), msgnum[i], i);
+         //msgnum[i]=0;
+       }
+  }
+  GarbageCollect();
+  //  ComlibPrintf("combcount = %d\n", combcount);
+  //combcount = 0;
+}
+
+void PeTable :: InsertMsgs(int npe, int *pelist, int size, void *msg)
+{
+  PTinfo *tmp;
+  PTALLOC(tmp);
+  tmp->refCount=0;
+  tmp->magic=0;
+  tmp->offset=0;
+  tmp->freelistindex=-1;
+  tmp->msgsize=size;
+  tmp->msg=msg;
+
+  for (int j=0;j<npe;j++) {
+    tmp->refCount++;
+    int index=pelist[j];
+    
+    ComlibPrintf("[%d] Inserting %d %d %d\n", CkMyPe(), msgnum[index], index, size);
+    
+    if (msgnum[index] >= MaxSize[index]) {
+        REALLOC(PeList[index], MaxSize[index]);
+        MaxSize[index] *= 2;
+    }
+    PeList[index][msgnum[index]]=tmp;
+    msgnum[index]++;
+  }
+}
+
+void PeTable :: InsertMsgs(int npe, int *pelist, int nmsgs, void **msglist)
+{
+  msgstruct **m=(msgstruct **)msglist;
+  for (int i=0;i<nmsgs;i++)
+      InsertMsgs(npe, pelist, m[i]->msgsize, m[i]->msg);
+}
+
+void PeTable :: ExtractAndDeliverLocalMsgs(int index)
+{
+  int j;
+  msgstruct m;
+
+  ComlibPrintf("%d:Delivering %d local messages\n", CkMyPe(), msgnum[index]);
+  for (j=msgnum[index]-1;(j>=0);j--) {
+
+       m.msgsize=PeList[index][j]->msgsize;
+       m.msg=PeList[index][j]->msg;
+
+       if (--(PeList[index][j]->refCount) <=0) {
+            CmiSyncSendAndFree(CkMyPe()/*index*/, m.msgsize, (char*)m.msg);
+            PTFREE(PeList[index][j]);
+       }
+       else {
+            CmiSyncSend(CkMyPe()/*index*/, m.msgsize, (char*)m.msg);
+        }
+       PeList[index][j]=NULL;
+  }
+  msgnum[index]=j+1;
+
+  return;
+}
+
+int PeTable :: TotalMsgSize(int npe, int *pelist, int *nm, int *nd)
+{
+    register int totsize=0;
+    magic++;
+    *nm=0;
+    *nd=0;        
+
+    for (int i=0;i<npe;i++) {
+        
+        int index = pelist[i];
+        
+        *nm += msgnum[index];
+
+        ComlibPrintf("%d: NUM MSGS %d, %d\n", CmiMyPe(), index, 
+                     msgnum[index]);
+
+        for (int j=0;j<msgnum[index];j++) {
+            if (PeList[index][j]->magic != magic) {
+
+                int tmp_size = PeList[index][j]->msgsize;
+                if(tmp_size % 8 != 0)
+                    tmp_size += 8 - tmp_size % 8;
+                
+                totsize += tmp_size;                
+                totsize += sizeof(int)+sizeof(int);
+                
+                PeList[index][j]->magic=magic;
+                (*nd)++;
+            }
+        }
+    }
+    return(totsize);
+}
+
+
+#undef PACK
+#undef PACKMSG
+#define PACKINT(data) {((int*)t)[0] = data; t+=sizeof(int);}
+#define PACK(type,data) {junk=(char *)&(data); memcpy(t, junk, sizeof(type)); t+=sizeof(type);}
+#define PACKMSG(data, size) { memcpy(p+msg_offset, (data), size); msg_offset += size; }
+
+/*Used for all to all multicast operations.  Assumes that each message
+  is destined to all the processors, to speed up all to all
+  substantially --Sameer 09/03/03 
+  
+  Protocol:
+  |ref|comid|ufield|nmsgs|size|ref|msg1|size2|ref2|msg2|....
+*/
+
+char * PeTable ::ExtractAndPackAll(comID id, int ufield, int *length)
+{
+    int nmsgs = 0, i, j;
+    int index = 0;
+
+    ComlibPrintf("[%d] In Extract And Pack All\n", CkMyPe());
+
+    //Increment magic to detect duplicate messages
+    magic++;
+
+    register int total_msg_size = 0;
+
+    //first compute size
+    for (i=0;i<NumPes;i++) {
+        index = i;
+        for (j=msgnum[index]-1; (j>=0); j--) {
+            if (PeList[index][j]->magic != magic) {                
+                total_msg_size += ALIGN8(PeList[index][j]->msgsize);
+                total_msg_size += 2 * sizeof(int);
+                PeList[index][j]->magic=magic;
+
+                nmsgs ++;
+            }            
+        }
+    }
+    
+    total_msg_size += ALIGN8(sizeof(AllToAllHdr));
+
+    ComlibPrintf("[%d] Message Size %d, nmsgs %d **%d**\n", CkMyPe(), total_msg_size, nmsgs, sizeof(AllToAllHdr));
+    
+    //poiter to the combined message, UGLY NAME
+    char *p = (char *) CmiAlloc(total_msg_size * sizeof(char));    
+
+    ComlibPrintf("After cmialloc\n");
+
+    //buffer to copy stuff into
+    char *t = p; 
+    char *junk = NULL;
+    
+    int dummy = 0;
+    
+    int refno = 0;
+
+    AllToAllHdr ahdr;
+    ahdr.refno = refno;
+    ahdr.id = id;
+    ahdr.ufield = ufield;
+    ahdr.nmsgs = nmsgs;
+
+    /*
+      PACKINT(refno);    
+      PACK(comID, id);
+      
+      PACKINT(ufield);
+      PACKINT(nmsgs);
+      //    PACKINT(dummy); //Aligning to 8 bytes
+    */
+
+    PACK(AllToAllHdr, ahdr);   
+
+    int msg_offset = ALIGN8(sizeof(AllToAllHdr));
+    
+    //Increment magic again for creating the message
+    magic++;
+    for (i=0;i<NumPes;i++) {
+        index=i;
+        int ref = 0;
+        int size;
+
+        for (j=msgnum[index]-1; (j>=0); j--) {
+            //Check if it is a duplicate
+            if (PeList[index][j]->magic != magic) {                
+                size = PeList[index][j]->msgsize;
+                PACKMSG(&size, sizeof(int));
+                PACKMSG(&ref, sizeof(int));
+                PeList[index][j]->magic=magic;
+                PACKMSG(PeList[index][j]->msg, size);
+                if(msg_offset % 8 != 0)
+                    msg_offset += 8 - msg_offset%8;
+            }
+
+            //Free it when all the processors have gotten rid of it
+            if (--(PeList[index][j]->refCount) <=0) {
+                ComlibPrintf("before cmifree \n");
+                CmiFree(PeList[index][j]->msg);   
+                ComlibPrintf("after cmifree \n");
+
+                PTFREE(PeList[index][j]);
+            }
+            //Assign the current processors message pointer to NULL
+            PeList[index][j] = NULL;
+        }
+        msgnum[index] = 0;
+    }
+    
+    *length = total_msg_size;
+    return p;
+}
+
+char * PeTable ::ExtractAndPack(comID id, int ufield, int npe, 
+                                int *pelist, int *length)
+{
+    char *junk;
+    int mask=~7;
+    int nummsgs, offset, actual_msgsize=0, num_distinctmsgs;
+    
+    ComlibPrintf("In ExtractAndPack %d\n", npe); 
+    
+    int tot_msgsize=TotalMsgSize(npe, pelist, &nummsgs, &num_distinctmsgs);
+    if (tot_msgsize ==0) {
+       *length=0;
+        
+        ComlibPrintf("Returning NULL\n");
+
+       return(NULL);
+    }
+    
+    //int ave_msgsize=(tot_msgsize>MSGSIZETHRESHOLD) ? 
+    //  tot_msgsize/(num_distinctmsgs):tot_msgsize;
+    
+    int msg_offset = CmiReservedHeaderSize + sizeof(comID) 
+        + (npe + 4 + nummsgs) * sizeof(int);  
+
+    if(msg_offset % 8 != 0)
+        msg_offset += 8 - msg_offset % 8;
+    
+    int headersize=msg_offset;
+    
+    *length = tot_msgsize;
+    *length += msg_offset;
+    char *p;
+    p=(char *)CmiAlloc(*length);
+    
+    int l1 = *length;
+    
+    char *t = p + CmiReservedHeaderSize;
+    int i, j;
+    if (!p) CmiAbort("Big time problem\n");
+    magic++;
+
+    int refno = id.refno;    
+
+    PACKINT(refno);
+    PACK(comID, id);
+    PACKINT(ufield);
+    PACKINT(npe);
+    
+    int lesspe=0;
+    int npacked = 0;
+    for (i=0;i<npe;i++) {
+        int index=pelist[i];
+        if (msgnum[index]<=0) {
+            lesspe++;
+            continue;
+        }
+        
+        //ComlibPrintf("%d Packing pelist[%d]\n", CkMyPe(), i);
+        register int newval=-1*pelist[i];
+        PACKINT(newval); 
+        for (j=0;j<msgnum[index];j++) {
+            if (PeList[index][j]->magic != magic) {
+               int tmpms=actual_msgsize+PeList[index][j]->msgsize;
+               if (tmpms >= MSGSIZETHRESHOLD 
+                    /*|| (PeList[index][j]->msgsize>=ave_msgsize)*/ ) {
+                    
+                    CmiPrintf("%d sending directly\n", CkMyPe());
+                    if (--(PeList[index][j]->refCount) <=0) {
+                        CmiSyncSendAndFree(index, PeList[index][j]->msgsize, 
+                                           (char*)PeList[index][j]->msg);
+                       //ComlibPrintf("%d Freeing msg\n", CkMyPe());
+                        PTFREE(PeList[index][j]);
+                       }
+                    else
+                        CmiSyncSend(index, PeList[index][j]->msgsize, 
+                                    (char*)PeList[index][j]->msg);
+                    PeList[index][j]=NULL;
+                    continue;
+               }
+               
+               npacked ++;
+                
+               offset=msg_offset;
+               PeList[index][j]->magic=magic;
+               PeList[index][j]->offset=msg_offset;
+               PTinfo *tempmsg=PeList[index][j];
+               PACKMSG((&(tempmsg->msgsize)), sizeof(int));
+               actual_msgsize += tempmsg->msgsize;
+               int nullptr=-1;
+               PACKMSG(&nullptr, sizeof(int));
+
+               PACKMSG(tempmsg->msg, tempmsg->msgsize);
+                if(msg_offset % 8 != 0)
+                    msg_offset += 8 - msg_offset%8;
+
+                if(actual_msgsize % 8 != 0)                    
+                    actual_msgsize += 8 - actual_msgsize % 8;
+                
+               actual_msgsize += 2*sizeof(int);
+            }
+            else {
+               offset=(PeList[index][j]->offset);
+            }
+            
+            //ComlibPrintf("%d Packing msg_offset=%d\n", CkMyPe(), offset);
+            PACKINT(offset); 
+            if (--(PeList[index][j]->refCount) <=0) {
+                CmiFree(PeList[index][j]->msg);
+                
+                PTFREE(PeList[index][j]);
+            }
+            PeList[index][j]=NULL;
+        }
+        msgnum[index]=0;
+    }
+    offset=-1;
+    PACKINT(offset);
+    
+    if (lesspe) {
+        t=p+CmiReservedHeaderSize+2*sizeof(int) + sizeof(comID);
+       npe=npe-lesspe;
+       PACK(int, npe);
+    }
+    if (!actual_msgsize) {
+        if (l1 >= MAXBUFSIZE) 
+            CmiFree(p);
+        *length=0;
+        return(NULL);
+    }
+    
+    *length=actual_msgsize+headersize;
+    return(p);
+} 
+
+#undef UNPACK
+#define UNPACK(type,data) {junk=(char *)&(data); memcpy(junk, t, sizeof(type));t+=sizeof(type);}
+#undef UNPACKMSG
+#define UNPACKMSG(dest,src, size) { memcpy(dest, src, size); offset += size;}
+
+int PeTable :: UnpackAndInsert(void *in)
+{
+  char *junk;
+  char *t =(char *)in + CmiReservedHeaderSize;
+  int i, ufield, npe, pe, tot_msgsize, ptrlistindex=0;
+  comID id;
+  int refno = 0;
+
+  UNPACK(int, refno);
+  
+  //ComlibPrintf("%d UnPacking id\n", CkMyPe());
+  UNPACK(comID, id);
+  UNPACK(int, ufield);
+  UNPACK(int, npe);
+  
+  register int offset;
+  for (i=0;i<npe;i++) {
+       UNPACK(int, pe);
+       pe *= -1;
+
+       UNPACK(int, offset);
+       while (offset > 0) {
+            int tempmsgsize;
+            UNPACKMSG(&(tempmsgsize), (char *)in+offset, sizeof(int));
+            int ptr;
+            UNPACKMSG(&ptr, (char *)in+offset, sizeof(int));
+
+            if (ptr >=0 )  {
+                if (msgnum[pe] >= MaxSize[pe]) {
+                    REALLOC(PeList[pe], MaxSize[pe]);
+                    MaxSize[pe] *= 2;
+                }
+                PeList[pe][msgnum[pe]]=ptrvec[ptr];
+                (ptrvec[ptr])->refCount++;
+                msgnum[pe]++;
+
+                UNPACK(int, offset);
+                continue;
+            }
+            PTinfo *temp;
+            PTALLOC(temp);
+            temp->msgsize=tempmsgsize;
+            temp->refCount=1;
+            temp->magic=0;
+            temp->offset=0;
+
+            ptrvec.insert(ptrlistindex, temp);
+            memcpy((char *)in+offset-sizeof(int), &ptrlistindex, sizeof(int));
+
+            ptrlistindex++;
+            temp->msg=(void *)((char *)in+offset);
+            if (msgnum[pe] >= MaxSize[pe]) {
+
+                REALLOC(PeList[pe], MaxSize[pe]);
+                MaxSize[pe] *= 2;
+            }
+            PeList[pe][msgnum[pe]]=temp;
+            msgnum[pe]++;
+            UNPACK(int, offset);
+       }
+       t -=sizeof(int);
+  }
+  *(int *)((char *)in -sizeof(int))=ptrlistindex; 
+  
+  if (ptrlistindex==0)
+      CmiFree(in);
+  
+  for (i=0;i<ptrlistindex;i++) {
+      char * actualmsg=(char *)(ptrvec[i]->msg);
+      int *rc=(int *)(actualmsg-sizeof(int));
+      *rc=(int)((char *)in-actualmsg);
+      //ComlibPrintf("I am inserting %d\n", *rc);
+  }
+  
+  return(ufield);
+}
+
+/* Unpack and insert an all to all message, the router provides the
+   list of processors to insert into.
+   Same protocol as mentioned earlier.
+*/
+
+int PeTable :: UnpackAndInsertAll(void *in, int npes, int *pelist){
+  char *junk;
+  char *t =(char *)in /*+CmiReservedHeaderSize*/;
+  int i,  
+      ufield,   // user field or ths stage of the iteration 
+      nmsgs,    // number of messages in combo message
+      refno,    // reference number
+      dummy;    // alignment dummy
+  
+  comID id;
+
+  /*
+    UNPACK(int, refno);      
+    UNPACK(comID, id);
+    
+    UNPACK(int, ufield);
+    UNPACK(int, nmsgs);
+    //UNPACK(int, dummy);
+    int header_size = sizeof(comID) + CmiReservedHeaderSize + 3 *sizeof(int);
+    if(header_size % 8 != 0)
+    t+= 8 - header_size % 8;
+  */
+
+  AllToAllHdr ahdr;
+  UNPACK(AllToAllHdr, ahdr);
+
+  if(sizeof(AllToAllHdr) % 8 != 0)
+      t += 8 - sizeof(AllToAllHdr) % 8;
+
+  refno = ahdr.refno;
+  id = ahdr.id;
+  nmsgs = ahdr.nmsgs;
+  ufield = ahdr.ufield;
+
+  ComlibPrintf("[%d] unpack and insert all %d, %d\n", CkMyPe(), ufield, nmsgs);
+  
+  //Inserting a memory foot print may, change later
+  CmiChunkHeader *chdr= (CmiChunkHeader*)((char*)in - sizeof(CmiChunkHeader));
+
+  for(int count = 0; count < nmsgs; count++){
+      int *ref = 0;
+      int size = 0;
+      char *msg = 0;
+
+      UNPACK(int, size);
+      ref = (int *)t;
+      t += sizeof(int);
+
+      *ref = (int)((char *)(&chdr->ref) - (char *)ref);
+      chdr->ref ++;
+
+      ComlibPrintf("ref = %d, global_ref = %d\n", *ref, chdr->ref);
+
+      msg = t;
+      t += ALIGN8(size);
+      
+      InsertMsgs(npes, pelist, size, msg);
+  }  
+
+  CmiFree(in);
+  return ufield;
+}
+
+void PeTable :: GarbageCollect()
+{
+}
+
+GList :: GList()
+{
+  InList=(InNode *)CmiAlloc(10*sizeof(InNode));
+  InListIndex=0;
+}
+
+GList :: ~GList()
+{
+  CmiFree(InList);
+}
+
+int GList :: AddWholeMsg(void *ptr)
+{
+  InList[InListIndex].flag=0;
+  InList[InListIndex++].ptr=ptr;
+  return(InListIndex-1);
+}
+
+void GList :: setRefcount(int indx, int ref)
+{
+  //ComlibPrintf("setting InList[%d]=%d\n", indx, ref);
+  InList[indx].refCount=ref;
+}
+
+void GList :: DeleteWholeMsg(int indx)
+{
+  //ComlibPrintf("DeleteWholeMsg indx=%d\n", indx);
+  InList[indx].refCount--;
+  if ((InList[indx].refCount <=0) && (InList[indx].flag==0)) {
+       //ComlibPrintf("Deleting msgwhole\n");
+       CmiFree(InList[indx].ptr);
+  }
+}
+void GList :: DeleteWholeMsg(int indx, int flag)
+{
+  InList[indx].refCount--;
+  InList[indx].flag=flag;
+} 
+/*
+void GList :: DeleteWholeMsg(int indx, void *p)
+{
+  int *rc=(int *)((char *)p-sizeof(int));
+  *rc=(int)((char *)p-(char *)(InList[indx].ptr)+2*sizeof(int));
+}
+*/
+
+void GList :: GarbageCollect()
+{
+  for (int i=0;i<InListIndex;i++) {
+       if ((InList[i].refCount <= 0) && (InList[i].flag >0))
+               CmiFree(InList[i].ptr);
+  }
+}
+
+void GList :: Add(void *ptr )
+{
+  InList[InListIndex].ptr=ptr;
+  InList[InListIndex++].refCount=0;
+}
+
+void GList :: Delete()
+{
+  InListIndex=0;
+  /******
+  int counter=0;
+  for (int i=0;i< InListIndex;i++) {
+       if (InList[i].refCount <=0)  {
+               counter++;
+               CmiFree(InList[i].ptr);
+       }
+  }
+  if (counter == InListIndex) InListIndex=0;
+****/
+}
+
diff --git a/src/conv-com/petable.h b/src/conv-com/petable.h
new file mode 100644 (file)
index 0000000..e364833
--- /dev/null
@@ -0,0 +1,79 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#ifndef PETABLE_H
+#define PETABLE_H
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#define MSGQLEN 32
+
+typedef struct ptinfo {
+  int refCount;
+  int magic;
+  int offset;
+  int freelistindex;
+  int msgsize;
+  void *msg;
+  struct ptinfo * next;
+} PTinfo;
+
+typedef struct {
+  int refCount;
+  int flag;
+  void * ptr;
+} InNode;
+
+class GList {
+ private:
+       InNode *InList;
+       int InListIndex;
+ public:
+       GList();
+       ~GList();
+       int AddWholeMsg(void *);
+       void setRefcount(int, int);
+       void DeleteWholeMsg(int);
+       void DeleteWholeMsg(int, int);
+       void GarbageCollect();
+       void Add(void *);
+       void Delete();
+};
+
+class PeTable {
+  private:
+    PTinfo ***PeList;
+    CkVec<PTinfo *> ptrvec;
+
+    PTinfo *PTFreeList;
+    // char * CombBuffer;
+    int *msgnum, *MaxSize;
+    int NumPes;
+    int magic;
+    GList *FreeList;
+    int TotalMsgSize(int, int *, int *, int *);
+ public:
+    PeTable(int n);
+    ~PeTable();
+    void InsertMsgs(int npe, int *pelist, int nmsgs, void **msglist);
+    void InsertMsgs(int npe, int *pelist, int size, void *msg);
+    
+    void ExtractAndDeliverLocalMsgs(int pe);
+    
+    int UnpackAndInsert(void *in);
+    int UnpackAndInsertAll(void *in, int npes, int *pelist);
+    
+    char * ExtractAndPack(comID, int, int, int *pelist, int *length); 
+    char * ExtractAndPackAll(comID id, int ufield, int *length);
+    
+    void GarbageCollect();
+    void Purge();
+};
+
+#endif
diff --git a/src/conv-com/router.h b/src/conv-com/router.h
new file mode 100644 (file)
index 0000000..853c239
--- /dev/null
@@ -0,0 +1,80 @@
+#ifndef ROUTER_H
+#define ROUTER_H
+
+//Base class for routers
+//Imported from Krishnan's Each To Many Communication Framework
+//Modified to suit the new communication library
+//Sameer Kumar 05/14/04
+
+class Router 
+{
+    int doneHandle;
+    
+ public:
+    Router() {};
+    virtual ~Router() {};
+    
+    //Insert messages for the all to all operation
+    //All processors will be multicast the same message here
+    virtual void EachToAllMulticast(comID id, int size, void *msg, 
+                                    int more) 
+        {CmiPrintf("Not impl\n");}
+    
+    //Insert messages for the all to all operation. The destination
+    //processors to which a message is multicast to be can be
+    //specified
+    //id = communication operation identifier
+    //size = size of the message
+    //msg = message to be sent
+    //numPes = number of processors the message has to be sent to
+    //pelist = list of relative proc ids the message has to be multicast to
+    //more = do I get more messages ?
+    virtual void EachToManyMulticast(comID id, int size, void *msg, 
+                                     int numPes, int *pelist, 
+                                     int more) 
+        {CmiPrintf("Not impl\n");}
+    
+    //The first iteration of message combining should call this
+    //entry function
+    virtual void RecvManyMsg(comID, char *) {CmiPrintf("Not Impl\n");}
+    
+    //The send and the rest of the iterations should call this
+    //entry function
+    virtual void ProcManyMsg(comID, char *) {CmiPrintf("Not Impl\n");}
+    
+    //Received a dummy
+    virtual void DummyEP(comID, int )  {CmiPrintf("Base Dummy\n");}
+    
+    //Set the map between processors and virtual processor id's
+    //Useful when only a subset of processors are involved in the
+    //communication operation
+    virtual void SetMap(int *) {;}
+
+    //Utility function
+    void SendDummyMsg(comID id, int pe, int magic) {
+        
+        ComlibPrintf("[%d] Send Dummy to %d\n", pe);
+
+        DummyMsg *m=(DummyMsg *)CmiAlloc(sizeof(DummyMsg));
+        CmiSetHandler(m, CkpvAccess(DummyHandle));
+        m->id=id;
+        m->magic=magic;
+        CmiSyncSendAndFree(pe, sizeof(DummyMsg),(char*) m);
+    }
+
+    void setDoneHandle(int handle) {
+        doneHandle = handle;
+    }
+
+    void Done(comID id) {
+        if(doneHandle == 0)
+            return;
+
+        DummyMsg *m=(DummyMsg *)CmiAlloc(sizeof(DummyMsg));
+        m->id=id;
+        CmiSetHandler(m, doneHandle);
+        CmiSyncSendAndFree(CkMyPe(), sizeof(DummyMsg), (char*)m);
+    }
+};
+
+#endif
diff --git a/src/conv-com/routerstrategy.C b/src/conv-com/routerstrategy.C
new file mode 100644 (file)
index 0000000..681c795
--- /dev/null
@@ -0,0 +1,285 @@
+
+#include "routerstrategy.h"
+
+CkpvDeclare(int, RecvHandle);
+CkpvDeclare(int, ProcHandle);
+CkpvDeclare(int, DummyHandle);
+
+//Handlers that call the entry funtions of routers 
+//Refer to router.h for details on these entry functions
+
+//Correspods to Router::ProcManyMsg
+void procManyCombinedMsg(char *msg)
+{
+    comID id;
+    //  ComlibPrintf("In Recv combined message at %d\n", CkMyPe());
+    memcpy(&id,(msg+CmiReservedHeaderSize+sizeof(int)), sizeof(comID));
+    
+    Strategy *s = ConvComlibGetStrategy(id.instanceID);
+    ((RouterStrategy *)s)->ProcManyMsg(msg);
+}
+
+//Correspods to Router::DummyEP
+void dummyEP(DummyMsg *m)
+{
+    Strategy *s = ConvComlibGetStrategy(m->id.instanceID);
+    
+    ((RouterStrategy *)s)->DummyEP(m);
+}
+
+//Correspods to Router::RecvManyMsg
+void recvManyCombinedMsg(char *msg)
+{
+    comID id;
+    //  ComlibPrintf("In Recv combined message at %d\n", CkMyPe());
+    memcpy(&id,(msg+CmiReservedHeaderSize+sizeof(int)), sizeof(comID));
+    
+    Strategy *s = ConvComlibGetStrategy(id.instanceID);
+    ((RouterStrategy *)s)->RecvManyMsg(msg);
+}
+
+
+void doneHandler(DummyMsg *m){
+    Strategy *s = ConvComlibGetStrategy(m->id.instanceID);
+    
+    ((RouterStrategy *)s)->Done(m);
+}
+
+void RouterStrategy::setReverseMap(){
+    int pcount;
+    for(pcount = 0; pcount < CkNumPes(); pcount++)
+        procMap[pcount] = -1;
+
+    //All processors not in the domain will point to -1
+    for(pcount = 0; pcount < npes; pcount++) {
+        if (pelist[pcount] == CmiMyPe())
+            myPe = pelist[pcount];
+
+        procMap[pelist[pcount]] = pcount;
+    }
+}
+
+RouterStrategy::RouterStrategy(int stratid, int handle, int _npes, 
+                               int *_pelist) 
+    : Strategy(){
+    
+    setType(CONVERSE_STRATEGY);
+
+    CkpvInitialize(int, RecvHandle);
+    CkpvInitialize(int, ProcHandle);
+    CkpvInitialize(int, DummyHandle);
+
+    id.instanceID = 0; //getInstance();
+    id.isAllToAll = 0;
+    id.refno = 0;
+
+    CkpvAccess(RecvHandle) =
+        CkRegisterHandler((CmiHandler)recvManyCombinedMsg);
+    CkpvAccess(ProcHandle) =
+        CkRegisterHandler((CmiHandler)procManyCombinedMsg);
+    CkpvAccess(DummyHandle) = 
+        CkRegisterHandler((CmiHandler)dummyEP);    
+
+    myDoneHandle = CkRegisterHandler((CmiHandler)doneHandler);    
+
+    //Array strategy done handle
+    doneHandle = handle;
+
+    routerID = stratid;
+
+    npes = _npes;
+    pelist = new int[npes];
+    procMap = new int[CmiNumPes()];
+    
+    memcpy(pelist, _pelist, sizeof(int) * npes);    
+    setReverseMap();
+
+    //CkPrintf("Router Strategy : %d, MYPE = %d, NUMPES = %d \n", stratid, 
+    //       myPe, npes);
+
+    switch(stratid) {
+    case USE_TREE: 
+        router = new TreeRouter(npes, myPe);
+        break;
+        
+    case USE_MESH:
+        router = new GridRouter(npes, myPe);
+        break;
+        
+    case USE_HYPERCUBE:
+        router = new DimexRouter(npes, myPe);
+        break;
+        
+    case USE_GRID:
+        router = new D3GridRouter(npes, myPe);
+        break;
+
+    case USE_DIRECT:
+        break;
+        
+    default: CmiAbort("Unknown Strategy\n");
+        break;
+    }
+
+    if(router) {
+        router->SetMap(pelist);
+        router->setDoneHandle(myDoneHandle);
+        //router->SetID(id);
+    }
+
+    //Start with all iterations done
+    doneFlag = 1;
+
+    //No Buffered doneInserting at the begining
+    bufferedDoneInserting = 0;
+}
+
+void RouterStrategy::insertMessage(MessageHolder *cmsg){
+
+    if(routerID == USE_DIRECT) {
+        CmiSyncSendAndFree(cmsg->dest_proc, cmsg->size, cmsg->getMessage());
+        delete cmsg;
+    }
+    else
+        msgQ.enq(cmsg);
+}
+
+void RouterStrategy::doneInserting(){
+    
+    if(routerID == USE_DIRECT)
+        return;
+
+    ComlibPrintf("%d: DoneInserting \n", CkMyPe());
+    
+    if(doneFlag == 0) {
+        ComlibPrintf("%d:Waiting for previous iteration to Finish\n", 
+                     CmiMyPe());
+        bufferedDoneInserting = 1;
+        return;
+    }
+    
+    doneFlag = 0;
+    bufferedDoneInserting = 0;
+
+    id.refno ++;
+
+    if(msgQ.length() == 0) {
+        if(routerID == USE_DIRECT)
+            return;
+
+        DummyMsg * dummymsg = (DummyMsg *)CmiAlloc(sizeof(DummyMsg));
+        ComlibPrintf("[%d] Creating a dummy message\n", CkMyPe());
+        CmiSetHandler(dummymsg, CkpvAccess(RecvdummyHandle));
+        
+        MessageHolder *cmsg = new MessageHolder((char *)dummymsg, 
+                                                     CkMyPe(), 
+                                                     sizeof(DummyMsg));
+        cmsg->isDummy = 1;
+        msgQ.enq(cmsg);
+    }
+
+    int numToDeposit = msgQ.length();
+    
+    while(!msgQ.isEmpty()) {
+        MessageHolder *cmsg = msgQ.deq();
+        char *msg = cmsg->getMessage();
+        
+        if(!cmsg->isDummy)  {            
+            //Assuming list of processors to multicast to is in the
+            //order of relative processors numbering and NOT absolute
+            //processor numbering
+
+            if(cmsg->dest_proc == IS_MULTICAST) {  
+                router->EachToManyMulticast(id, cmsg->size, msg, cmsg->npes, 
+                                            cmsg->pelist, 
+                                            numToDeposit > 1);
+            }            
+            else {                                
+                router->EachToManyMulticast(id, cmsg->size, msg, 1,
+                                            &procMap[cmsg->dest_proc],
+                                            numToDeposit > 1);
+            }            
+        }   
+        else
+            router->EachToManyMulticast(id, cmsg->size, msg, 1, &myPe, 
+                                        numToDeposit > 1);
+        
+        numToDeposit --;        
+    }
+
+    while(!recvQ.isEmpty()) {
+        char *msg = recvQ.deq();
+        RecvManyMsg(msg);
+    }
+
+    while(!procQ.isEmpty()) {
+        char *msg = procQ.deq();
+        ProcManyMsg(msg);
+    }
+
+    while(!dummyQ.isEmpty() > 0) {
+        DummyMsg *m = dummyQ.deq();
+        router->DummyEP(m->id, m->magic);
+        CmiFree(m);
+    }
+}
+
+void RouterStrategy::Done(DummyMsg *m){
+
+    ComlibPrintf("%d: Finished iteration\n", CmiMyPe());
+
+    if(doneHandle > 0) {
+        CmiSetHandler(m, doneHandle);
+        CmiSyncSendAndFree(CkMyPe(), sizeof(DummyMsg), (char*)m);
+    }
+    else
+        CmiFree(m);
+
+    doneFlag = 1;
+
+    if(bufferedDoneInserting)
+        doneInserting();
+}
+
+
+//Implement it later while implementing checkpointing of Comlib
+void RouterStrategy::pup(PUP::er &p){}
+
+//Call the router functions
+void RouterStrategy::RecvManyMsg(char *msg) {
+
+    comID new_id;
+    //  ComlibPrintf("In Recv combined message at %d\n", CkMyPe());
+    memcpy(&new_id,(msg+CmiReservedHeaderSize+sizeof(int)), sizeof(comID));
+
+    ComlibPrintf("REFNO = %d, %d\n", new_id.refno, id.refno);
+
+    if(new_id.refno != id.refno)
+        recvQ.enq(msg);
+    else
+        router->RecvManyMsg(id, msg);
+}
+
+void RouterStrategy::ProcManyMsg(char *msg) {    
+
+    comID new_id;
+    //  ComlibPrintf("In Recv combined message at %d\n", CkMyPe());
+    memcpy(&new_id,(msg+CmiReservedHeaderSize+sizeof(int)), sizeof(comID));
+
+    if(new_id.refno != id.refno)
+        procQ.enq(msg);
+    else
+        router->ProcManyMsg(id, msg);
+}
+
+void RouterStrategy::DummyEP(DummyMsg *m) {
+
+    if(id.refno != m->id.refno)
+        dummyQ.enq(m);
+    else {
+        router->DummyEP(m->id, m->magic);
+        CmiFree(m);
+    }
+}
+
+PUPable_def(RouterStrategy);
diff --git a/src/conv-com/routerstrategy.h b/src/conv-com/routerstrategy.h
new file mode 100644 (file)
index 0000000..3026917
--- /dev/null
@@ -0,0 +1,66 @@
+
+/* Class that calls Krishnan's routers from the new Comlib.
+   Developed to be called from Converse
+   Sameer Kumar 05/14/04
+*/
+   
+
+#ifndef ROUTER_STRATEGY
+#define ROUTER_STRATEGY
+
+#include "convcomlibmanager.h"
+
+#include "gridrouter.h"
+#include "graphrouter.h"
+#include "de.h"
+#include "treerouter.h"
+#include "3dgridrouter.h"
+
+class RouterStrategy : public Strategy {
+
+    Router * router;
+    CkQ<MessageHolder *> msgQ;
+    CkQ<char *> recvQ, procQ;
+    CkQ<DummyMsg *> dummyQ;
+
+    comID id;
+    int *pelist;
+    int npes;
+    int *procMap;
+    int myPe;
+    int routerID;
+
+    int doneHandle;    //Array strategy done handle
+    int myDoneHandle;   //my own done handle, which will inturn call
+                       //array strategy done handle
+   int doneFlag, bufferedDoneInserting;
+
+    void setReverseMap();
+
+ public:
+    //constructor
+    //stratid = which topology to use ? (Mesh?, Grid?, Hypercube ?)
+    //npes = number of processors in the all to all operation
+    //pelist = list of processors
+    RouterStrategy(int stratid, int handle, int npes, int *pelist);
+    RouterStrategy(CkMigrateMessage *m): Strategy(m){}
+
+    //Insert messages
+    void insertMessage(MessageHolder *msg);
+    //Finished inserting
+    void doneInserting();
+
+    //Call Krishnan's router functions
+    void RecvManyMsg(char *msg);
+    void ProcManyMsg(char *msg);
+    void DummyEP(DummyMsg *m);
+    
+    void Done(DummyMsg *m);
+
+    int * getProcMap() {return procMap;}
+
+    virtual void pup(PUP::er &p);
+    PUPable_decl(RouterStrategy);
+};
+#endif
diff --git a/src/conv-com/treerouter.C b/src/conv-com/treerouter.C
new file mode 100644 (file)
index 0000000..8ca73f1
--- /dev/null
@@ -0,0 +1,193 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+/**************************************
+ * File: treerouter.C
+ *
+ * Author: Krishnan V
+ *
+ * Tree based router
+ *********************************************/
+#include "treerouter.h"
+#define DEGREE 4
+#define gmap(pe) (gpes ? gpes[pe] : pe)
+
+
+/**The only communication op used. Modify this to use
+ ** vector send */
+#define TREESENDFN(kid, u, knewmsg, klen, khndl, knextpe)  \
+       {if (knewmsg) {\
+         CmiSetHandler(knewmsg, khndl);\
+         CmiSyncSendAndFree(knextpe, klen, knewmsg);\
+       }\
+       else {\
+         SendDummyMsg(kid, knextpe, u);\
+       }\
+}
+
+/************************************************
+ ************************************************/
+TreeRouter :: TreeRouter(int n, int me)
+{
+  int i;
+  MyPe=me;
+  NumPes=n;
+  gpes=NULL;
+
+  numChildren=0;
+  for (i=0;i<DEGREE;i++) {
+       if ((MyPe*DEGREE+i+1) < NumPes) numChildren++;
+  }
+
+  PeTree = new PeTable(NumPes);
+
+  recvExpected=1+numChildren;
+  InitVars();
+  //CmiPrintf("Tree with n=%d, me=%d\n", n, me);
+}
+
+TreeRouter :: ~TreeRouter()
+{
+  delete PeTree;
+}
+
+void TreeRouter :: InitVars()
+{
+  recvCount=0;
+}
+
+
+void TreeRouter::NumDeposits(comID , int num)
+{
+}
+
+void TreeRouter::EachToAllMulticast(comID id, int size, void *msg, int more)
+{
+  int npe=NumPes;
+  int * destpes=(int *)CmiAlloc(sizeof(int)*npe);
+  for (int i=0;i<npe;i++) destpes[i]=i;
+  EachToManyMulticast(id, size, msg, npe, destpes, more);
+}
+
+void TreeRouter::EachToManyMulticast(comID id, int size, void *msg, int numpes, int *destpes, int more)
+{
+ //Create the message
+  if (size) {
+       PeTree->InsertMsgs(numpes, destpes, size, msg);
+  }
+
+  if (more >0) return;
+
+  //Hand it over to Recv. 
+  RecvManyMsg(id, NULL);
+}
+
+void TreeRouter :: RecvManyMsg(comID id, char *msg)
+{
+  if (msg)
+       PeTree->UnpackAndInsert(msg);
+  recvCount++;
+  //CmiPrintf("%d Tree: recvCount=%d, recvExpected=%d\n",MyPe, recvCount, recvExpected);
+  if (recvCount == recvExpected) {
+       if (MyPe) {
+               int len;
+               int parent=(MyPe-1)/DEGREE;
+               parent=gmap(parent);
+               char *newmsg=SortBufferUp(id, 0, &len);
+               TREESENDFN(id, 0, newmsg, len, CkpvAccess(RecvHandle), parent);
+
+       }
+       else {
+               DownStreamMsg(id);
+       }
+  }
+  if (recvCount > recvExpected) DownStreamMsg(id);
+}
+
+char * TreeRouter :: SortBufferUp(comID id, int ufield, int *len)
+{
+  int np=0, i;
+  int * pelst=(int *)CmiAlloc(sizeof(int)*NumPes);
+
+  for (i=0;i<NumPes;i++) {
+
+       //if (i==MyPe) continue;
+       int pe=i;
+       while (pe!=MyPe && pe>0) pe =(pe-1)/DEGREE;
+       if (pe == MyPe) continue;
+
+       pelst[np++]=i;
+  }
+  char *newmsg=PeTree->ExtractAndPack(id, ufield, np, pelst, len); 
+  CmiFree(pelst);
+  return(newmsg);
+}
+  
+char * TreeRouter :: SortBufferDown(comID id, int ufield, int *len, int s)
+{
+  int np=0, i;
+  int * plist=(int *)CmiAlloc(sizeof(int)*NumPes);
+
+  for (i=0;i<NumPes;i++) {
+       if (i==MyPe) continue;
+       int pe=i;
+       int rep=MyPe*DEGREE+s;
+       while (pe!=rep && pe>0) pe =(pe-1)/DEGREE;
+       if (pe == rep) plist[np++]=i;
+  }
+
+  char * newmsg=PeTree->ExtractAndPack(id, ufield, np, plist, len); 
+  CmiFree(plist);
+  return(newmsg);
+}
+
+void TreeRouter :: DownStreamMsg(comID id)
+{
+  int deg=DEGREE;
+  if (NumPes < deg) deg=NumPes;
+
+  for (int i=0;i<deg;i++) {
+    int len;
+    char *newmsg=SortBufferDown(id, 0, &len, i+1);
+    int child=MyPe*DEGREE+i+1;
+    if (child >=NumPes || child==MyPe) break;
+    child=gmap(child);
+    TREESENDFN(id, 0, newmsg, len, CkpvAccess(RecvHandle), child);
+  }
+
+  LocalProcMsg(id);
+}
+
+void TreeRouter :: ProcManyMsg(comID id, char *m)
+{
+  PeTree->UnpackAndInsert(m);
+  LocalProcMsg(id);
+}
+
+void TreeRouter:: LocalProcMsg(comID id)
+{
+  PeTree->ExtractAndDeliverLocalMsgs(MyPe);
+  PeTree->Purge();
+  InitVars();
+  Done(id);
+}
+
+void TreeRouter :: DummyEP(comID id, int)
+{
+  RecvManyMsg(id, NULL);
+}
+
+Router * newtreeobject(int n, int me)
+{
+  Router * obj=new TreeRouter(n, me);
+  return(obj);
+}
+
+void TreeRouter :: SetMap(int *pes)
+{
+  gpes=pes;
+}
diff --git a/src/conv-com/treerouter.h b/src/conv-com/treerouter.h
new file mode 100644 (file)
index 0000000..620e4dc
--- /dev/null
@@ -0,0 +1,39 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#ifndef TREEROUTER_H
+#define TREEROUTER_H
+#include <converse.h>
+#include "comlib.h"
+#include "petable.h"
+
+class TreeRouter : public Router
+{
+  private:
+       PeTable *PeTree;
+       int numExpect, *gpes;
+       int *totarray;
+       int MyPe, NumPes, numChildren, recvCount, recvExpected;
+       void InitVars();
+       void DownStreamMsg(comID id);
+       void LocalProcMsg(comID);
+       char * SortBufferUp(comID, int, int *);
+       char * SortBufferDown(comID, int, int *, int);
+  public:
+       TreeRouter(int, int);
+       ~TreeRouter();
+       void NumDeposits(comID, int);
+       void EachToAllMulticast(comID, int , void *, int);
+       void EachToManyMulticast(comID, int , void *, int, int *, int);
+       void RecvManyMsg(comID, char *);
+       void ProcManyMsg(comID, char *);
+       void DummyEP(comID, int );
+       void SetMap(int *);
+};
+
+#endif
+