Integrated Prefix Router for alltoall operation. It uses permutation
authorChao Huang <chuang10@uiuc.edu>
Thu, 5 May 2005 21:03:33 +0000 (21:03 +0000)
committerChao Huang <chuang10@uiuc.edu>
Thu, 5 May 2005 21:03:33 +0000 (21:03 +0000)
of dest_proc to avoid contention.

src/ck-com/ComlibManager.h
src/ck-com/EachToManyMulticastStrategy.C
src/conv-com/comlib.h
src/conv-com/prefixrouter.C [new file with mode: 0644]
src/conv-com/prefixrouter.h [new file with mode: 0644]
src/conv-com/routerstrategy.C
src/conv-com/routerstrategy.h
src/libs/ck-libs/ampi/ampi.C
src/scripts/Make.depends
src/scripts/Makefile

index 6fb37d57fd666cb4574e86eea9ee797801375eb7..0e4ab05691608639fd08b1b97a0d4badde9e3970 100644 (file)
@@ -17,6 +17,7 @@
                               //messages without any processing.
 #define USE_GRID 5            //Virtual topology is a 3d grid
 #define USE_LINEAR 6          //Virtual topology is a linear array
+#define USE_PREFIX 7          //Prefix router to avoid contention
 
 #define CHARM_MPI 0 
 #define MAX_NSTRAT 1024
index 61b6aa260b87006b23e3ecbbf2bea09a69cef74b..0ba7d2241a3154a69e8783836461aae4f3a32aca 100644 (file)
@@ -143,6 +143,8 @@ void EachToManyMulticastStrategy::commonInit() {
             routerID = USE_HYPERCUBE;
         else  if(strcmp(router, "USE_DIRECT") == 0)
             routerID = USE_DIRECT;        
+        else  if(strcmp(router, "USE_PREFIX") == 0)
+            routerID = USE_PREFIX;        
 
         //Just for the first step. After learning the learned
         //strategies will be chosen
index aa71193289f370db1329baa3e06bc3e2544615f4..652a40287ffd85711b9e95829abcc5f7dfbae7c4 100644 (file)
@@ -29,6 +29,7 @@ enum{BCAST=0,TREE, GRID, HCUBE};
                               //messages without any processing.
 #define USE_GRID 5            //Virtual topology is a 3d grid
 #define USE_LINEAR 6          //Virtual topology is a linear array
+#define USE_PREFIX 7         //Prefix router to avoid contention
 
 #define IS_BROADCAST -1
 #define IS_SECTION_MULTICAST -2
diff --git a/src/conv-com/prefixrouter.C b/src/conv-com/prefixrouter.C
new file mode 100644 (file)
index 0000000..44a4739
--- /dev/null
@@ -0,0 +1,62 @@
+#include "prefixrouter.h"
+
+#if 0
+#define PREFIXDEB printf
+#else
+#define PREFIXDEB /* printf */
+#endif
+
+void PrefixRouter::EachToManyMulticastQ(comID id, CkQ<MessageHolder *> &msgq) {
+  PREFIXDEB("[%d]Sending through prefix router: ",CmiMyPe());
+  if(!msgq.isEmpty()){
+    MessageHolder *mhdl = msgq[0];
+    if(mhdl->dest_proc<0)  // broadcast or multicast
+        sendMulticast(msgq);
+    else
+        sendPointToPoint(msgq);
+  }
+  Done(id);
+}
+
+void PrefixRouter::sendMulticast(CkQ<MessageHolder *> &msgq) {
+    int count;
+    PREFIXDEB("with a multicast\n");
+    while(!msgq.isEmpty()){
+        MessageHolder *mhdl = msgq.deq();
+        
+        if(mhdl->dest_proc == IS_BROADCAST) {
+            for(count = 0; count < npes; count ++) {
+                int curDest = gpes[MyPe ^ count];
+                char *msg = mhdl->getMessage();
+                CmiSyncSend(curDest, mhdl->size, msg);
+            }
+        }
+        else {
+            CkAbort("Implement later");
+        }
+    }
+}
+
+void PrefixRouter::sendPointToPoint(CkQ<MessageHolder *> &msgq) {
+    int count, i;
+    PREFIXDEB("with a point-to-point\n");
+    int len = msgq.length();
+    for(count = 0; count < npes; count ++) {
+        int curDest = gpes[MyPe ^ count];
+        
+        for(i = 0; i < len; i++) {
+            MessageHolder *mhdl = msgq[i];
+            
+            CkAssert(mhdl->dest_proc >= 0);
+            if(mhdl->dest_proc == curDest) {
+                char *msg = mhdl->getMessage();
+                CmiSyncSendAndFree(curDest, mhdl->size, msg);
+            }
+        }
+    }
+    
+    for(i = 0; i < len; i++) {
+        MessageHolder *mhdl = msgq.deq();
+        delete mhdl;
+    }
+}
diff --git a/src/conv-com/prefixrouter.h b/src/conv-com/prefixrouter.h
new file mode 100644 (file)
index 0000000..44c52df
--- /dev/null
@@ -0,0 +1,27 @@
+#ifndef PREFIX_ROUTER_H
+#define PREFIX_ROUTER_H
+
+#include <math.h>
+#include <converse.h>
+#include "comlib.h"
+
+//Prefix router to avoid contention on m,n-tree networks
+
+class PrefixRouter : public Router {
+    int *gpes;
+    int *prefix_pelist;
+    int MyPe, npes;
+
+ public:
+    PrefixRouter(int _npes, int me) : npes(_npes), MyPe(me) {};
+    virtual ~PrefixRouter() {};
+    
+    virtual void EachToManyMulticastQ(comID id, CkQ<MessageHolder *> &msgq);
+
+    //communication operation
+    virtual void SetMap(int *pelist) {gpes = pelist;}
+    virtual void sendMulticast(CkQ<MessageHolder *> &msgq);
+    virtual void sendPointToPoint(CkQ<MessageHolder *> &msgq);
+};
+
+#endif
index d049deaf5b09dc5b26ad208823f88c987d383c34..e7c641ab8ddebb67bf31166552f523ad4a19fb77 100644 (file)
@@ -147,6 +147,10 @@ RouterStrategy::RouterStrategy(int stratid, int handle, int _npes,
     case USE_GRID:
         router = new D3GridRouter(npes, myPe);
         break;
+       
+    case USE_PREFIX:
+               router = new PrefixRouter(npes, myPe);
+       break;
 
     case USE_DIRECT: router = NULL;
         break;
index c477046d11df9604bc1c4e6c8d152acf55ffbb06..0af0c27ecb7a4dbdd130db64083b368dced046ed 100644 (file)
@@ -15,6 +15,7 @@
 #include "de.h"
 #include "treerouter.h"
 #include "3dgridrouter.h"
+#include "prefixrouter.h"
 
 class RouterStrategy : public Strategy {
 
index b2e7643f1d361ccd7371f89578f1703df06de805..d17911ada342ecd0f5bd4a7f8887c44f3301846b 100644 (file)
@@ -532,7 +532,7 @@ static ampi *ampiInit(char **argv)
        EachToManyMulticastStrategy *sAllgather = new EachToManyMulticastStrategy(USE_HYPERCUBE,arr.ckGetArrayID(),arr.ckGetArrayID());
         ciAllgather.setStrategy(sAllgather);
        
-       EachToManyMulticastStrategy *sAlltoall = new EachToManyMulticastStrategy(USE_MESH, arr.ckGetArrayID(),arr.ckGetArrayID());
+       EachToManyMulticastStrategy *sAlltoall = new EachToManyMulticastStrategy(USE_PREFIX, arr.ckGetArrayID(),arr.ckGetArrayID());
         ciAlltoall.setStrategy(sAlltoall);
 #endif        
   }
index 197f6a57d9f4ceda3827d6ce0a801014476478e6..ece3e89925bee9e7bdccd3e6afdf518ce8d5b4c3 100644 (file)
@@ -1734,7 +1734,7 @@ ComlibManager.o: ComlibManager.C ComlibManager.h charm++.h charm.h \
   charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
   sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
   trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
-  comlib.decl.h EachToManyMulticastStrategy.h routerstrategy.h \
+  comlib.decl.h EachToManyMulticastStrategy.h routerstrategy.h prefixrouter.h \
   gridrouter.h petable.h graphrouter.h de.h treerouter.h 3dgridrouter.h \
   DirectMulticastStrategy.h ComlibSectionInfo.h StreamingStrategy.h \
   DummyStrategy.h MPIStrategy.h NodeMulticast.h MsgPacker.h register.h \
@@ -1839,7 +1839,7 @@ EachToManyMulticastStrategy.o: EachToManyMulticastStrategy.C \
   sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
   trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
   comlib.decl.h routerstrategy.h gridrouter.h petable.h graphrouter.h \
-  de.h treerouter.h 3dgridrouter.h AAPLearner.h AAMLearner.h
+  de.h treerouter.h 3dgridrouter.h AAPLearner.h AAMLearner.h prefixrouter.h 
        $(CHARMC) -c -I. EachToManyMulticastStrategy.C
 
 ComlibSectionInfo.o: ComlibSectionInfo.C ComlibManager.h charm++.h \
@@ -1879,7 +1879,7 @@ AAPLearner.o: AAPLearner.C AAPLearner.h comlib.h converse.h conv-config.h \
   trace-bluegene.h router.h ComlibManager.h convcomlibmanager.h \
   ComlibStats.h comlib.decl.h EachToManyMulticastStrategy.h \
   routerstrategy.h gridrouter.h petable.h graphrouter.h de.h treerouter.h \
-  3dgridrouter.h
+  3dgridrouter.h prefixrouter.h 
        $(CHARMC) -c -I. AAPLearner.C
 
 ComlibStats.o: ComlibStats.C ComlibStats.h charm++.h charm.h converse.h \
@@ -1919,7 +1919,7 @@ AAMLearner.o: AAMLearner.C AAMLearner.h ComlibManager.h charm++.h charm.h \
   trace-bluegene.h comlib.h router.h convcomlibmanager.h ComlibStats.h \
   comlib.decl.h AAPLearner.h EachToManyMulticastStrategy.h \
   routerstrategy.h gridrouter.h petable.h graphrouter.h de.h treerouter.h \
-  3dgridrouter.h
+  3dgridrouter.h prefixrouter.h
        $(CHARMC) -c -I. AAMLearner.C
 
 MsgPacker.o: MsgPacker.C MsgPacker.h charm++.h charm.h converse.h \
@@ -2189,6 +2189,24 @@ graphrouter.o: graphrouter.C graphrouter.h converse.h conv-config.h \
   trace-bluegene.h router.h petable.h hypercubetopology.h
        $(CHARMC) -c -I. graphrouter.C
 
+prefixrouter.o: prefixrouter.C prefixrouter.h converse.h conv-config.h \
+  conv-common.h conv-mach.h conv-autoconfig.h conv-mach-opt.h pup_c.h \
+  conv-cpm.h conv-cpath.h conv-qd.h conv-random.h conv-lists.h \
+  conv-trace.h persistent.h debug-conv.h comlib.h charm++.h charm.h pup.h \
+  middle.h middle-conv.h cklists.h ckbitvector.h ckstream.h init.h \
+  ckhashtable.h debug-charm.h CkMarshall.decl.h cksection.h ckcallback.h \
+  conv-ccs.h sockRoutines.h ccs-server.h ckobjQ.h ckreduction.h \
+  CkReduction.decl.h cknodegroupreduction.h CkArrayReductionMgr.decl.h \
+  ckmemcheckpoint.h CkMemCheckpoint.decl.h readonly.h ckarray.h \
+  cklocation.h LBDatabase.h lbdb.h LBDBManager.h LBObj.h LBOM.h LBComm.h \
+  LBMachineUtil.h LBDatabase.decl.h NullLB.decl.h BaseLB.decl.h \
+  CkLocation.decl.h CkArray.decl.h ComlibArrayListener.h ComlibStrategy.h \
+  convcomlibstrategy.h ComlibLearner.h envelope.h CkFutures.decl.h \
+  charisma.h charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h \
+  sdag.h ckcheckpoint.h CkCheckpoint.decl.h ckarrayreductionmgr.h trace.h \
+  trace-bluegene.h router.h 
+       $(CHARMC) -c -I. prefixrouter.C
+
 hypercubetopology.o: hypercubetopology.C hypercubetopology.h \
   graphrouter.h converse.h conv-config.h conv-common.h conv-mach.h \
   conv-autoconfig.h conv-mach-opt.h pup_c.h conv-cpm.h conv-cpath.h \
@@ -2224,7 +2242,7 @@ routerstrategy.o: routerstrategy.C routerstrategy.h convcomlibmanager.h \
   CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
   ckarrayreductionmgr.h trace.h trace-bluegene.h router.h gridrouter.h \
-  petable.h graphrouter.h de.h treerouter.h 3dgridrouter.h
+  petable.h graphrouter.h de.h treerouter.h 3dgridrouter.h prefixrouter.h
        $(CHARMC) -c -I. routerstrategy.C
 
 convcomlibmanager.o: convcomlibmanager.C convcomlibmanager.h converse.h \
@@ -2244,7 +2262,7 @@ convcomlibmanager.o: convcomlibmanager.C convcomlibmanager.h converse.h \
   waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
   ckarrayreductionmgr.h trace.h trace-bluegene.h router.h \
   routerstrategy.h gridrouter.h petable.h graphrouter.h de.h treerouter.h \
-  3dgridrouter.h
+  3dgridrouter.h prefixrouter.h
        $(CHARMC) -c -I. convcomlibmanager.C
 
 pipebroadcastconverse.o: pipebroadcastconverse.C pipebroadcastconverse.h \
index 960256fe1fa81722e378404e98cd60ae73e2224e..f97f23258cbfea905b2811bff9c595dc2a7f202c 100644 (file)
@@ -161,7 +161,7 @@ CVHEADERS=cpthreads.h converse.h conv-trace.h conv-random.h conv-qd.h \
        convcomlibstrategy.h 3dgridrouter.h de.h hypercubetopology.h    \
        routerstrategy.h comlib.h dummyconversestrategy.h  treerouter.h \
        convcomlibmanager.h graphrouter.h petable.h gridrouter.h router.h \
-       pipebroadcastconverse.h pipelinestrategy.h cmipool.h
+       prefixrouter.h pipebroadcastconverse.h pipelinestrategy.h cmipool.h
 
 # The .c files are there to be #included by clients whole
 # This is a bit unusual, but makes client linking simpler.
@@ -727,7 +727,7 @@ STRATEGY_OBJS = MPIStrategy.o StreamingStrategy.o \
 CONVERSE_COM_OBJS = gridrouter.o de.o treerouter.o petable.o \
        3dgridrouter.o graphrouter.o hypercubetopology.o \
        routerstrategy.o convcomlibmanager.o pipebroadcastconverse.o \
-       pipelinestrategy.o
+       pipelinestrategy.o prefixrouter.o
 
 comlib.def.h: comlib.decl.h