priorities+randomized requests
authorPritish Jetley <pjetley2@illinois.edu>
Sun, 16 Oct 2011 04:01:01 +0000 (23:01 -0500)
committerPritish Jetley <pjetley2@illinois.edu>
Sun, 16 Oct 2011 04:01:01 +0000 (23:01 -0500)
.gitignore
DataManager.cpp
Messages.h
TreePiece.cpp
TreePiece.h
Worker.cpp
Worker.h
barnes.ci
defines.h

index 8cccb50..1a7b922 100644 (file)
@@ -2,3 +2,12 @@
 *.def.h
 charmrun.*
 out*
+*.dat
+.*.swp
+*.o
+*.a
+makefile
+charmrun*
+barnes*
+gen
+plummer
index efeae65..d03d123 100644 (file)
@@ -676,7 +676,9 @@ void DataManager::flushMomentRequests(){
 
 void DataManager::respondToMomentsRequest(Node<ForceData> *node, CkVec<int> &replyTo){
   for(int i = 0; i < replyTo.length(); i++){
-    MomentsMsg *m = new (0) MomentsMsg(node);
+    MomentsMsg *m = new (NUM_PRIORITY_BITS) MomentsMsg(node);
+    *(int *)CkPriorityPtr(m) = RECV_MOMENTS_PRIORITY;
+    CkSetQueueing(m,CK_QUEUEING_IFIFO);
     TB_DEBUG("(%d) responding to %d with node %lu\n", CkMyPe(), replyTo[i], node->getKey());
     thisProxy[replyTo[i]].receiveMoments(m);
   }
@@ -917,7 +919,11 @@ void DataManager::requestParticles(Node<ForceData> *leaf, CutoffWorker<ForceData
   Request &request = particleRequestTable[key];
   if(!request.sent){
     partReqs.incrRequests();
-    RequestMsg *reqMsg = new RequestMsg(key,CkMyPe());
+
+    RequestMsg *reqMsg = new (NUM_PRIORITY_BITS) RequestMsg(key,CkMyPe());
+    *(int *)CkPriorityPtr(reqMsg) = REQUEST_PARTICLES_PRIORITY;
+    CkSetQueueing(reqMsg,CK_QUEUEING_IFIFO);
+
     CkAssert(leaf->getOwnerStart() == leaf->getOwnerEnd());
     int owner = leaf->getOwnerStart();
     treePieceProxy[owner].requestParticles(reqMsg);
@@ -947,7 +953,11 @@ void DataManager::requestParticles(RequestMsg *msg){
 
   Particle *data = bucket->getParticles();
   int np = bucket->getNumParticles();
-  ParticleReplyMsg *pmsg = new (np,0) ParticleReplyMsg;
+
+  ParticleReplyMsg *pmsg = new (np,NUM_PRIORITY_BITS) ParticleReplyMsg;
+  *(int *)CkPriorityPtr(pmsg) = RECV_PARTICLES_PRIORITY;
+  CkSetQueueing(pmsg,CK_QUEUEING_IFIFO);
+
   pmsg->key = msg->key;
   pmsg->np = np;
   for(int i = 0; i < np; i++){
@@ -963,12 +973,16 @@ void DataManager::requestNode(Node<ForceData> *leaf, CutoffWorker<ForceData> *wo
   Request &request = nodeRequestTable[key];
   if(!request.sent){
     nodeReqs.incrRequests();
-    RequestMsg *reqMsg = new RequestMsg(key,CkMyPe());
-    // XXX - which of the owners should i ask? 
-    int owner = leaf->getOwnerStart();
+
+    RequestMsg *reqMsg = new (NUM_PRIORITY_BITS) RequestMsg(key,CkMyPe());
+    *(int *)CkPriorityPtr(reqMsg) = REQUEST_NODE_PRIORITY;
+    CkSetQueueing(reqMsg,CK_QUEUEING_IFIFO);
+
+    int numOwners = leaf->getOwnerEnd()-leaf->getOwnerStart()+1;
+    int requestOwner = leaf->getOwnerStart()+(rand()%numOwners);
     RRDEBUG("(%d) REQUEST node %lu from tp %d\n", 
-            CkMyPe(), key, owner);
-    treePieceProxy[owner].requestNode(reqMsg);
+            CkMyPe(), key, requestOwner);
+    treePieceProxy[requestOwner].requestNode(reqMsg);
     request.sent = true;
     request.parent = leaf;
   }
@@ -999,7 +1013,11 @@ void DataManager::requestNode(RequestMsg *msg){
   fillTrav.topDownTraversal_local(node,&tsz);
 
   int nn = tsz.getNumNodes();
-  NodeReplyMsg *nmsg = new (nn,0) NodeReplyMsg;
+
+  NodeReplyMsg *nmsg = new (nn,NUM_PRIORITY_BITS) NodeReplyMsg;
+  *(int *)CkPriorityPtr(nmsg) = RECV_NODE_PRIORITY;
+  CkSetQueueing(nmsg,CK_QUEUEING_IFIFO);
+
   nmsg->key = msg->key;
   nmsg->nn = nn;
 
index ece0ec7..9f51be8 100644 (file)
@@ -58,4 +58,7 @@ struct MomentsMsg : public CMessage_MomentsMsg {
     data = (*node);
   }
 };
+
+struct RescheduleMsg : public CMessage_RescheduleMsg {
+};
 #endif
index 2b72864..6a527e0 100644 (file)
@@ -71,14 +71,21 @@ void TreePiece::startTraversal(){
 
   remoteTraversalState.reset(myNumBuckets,myBuckets);
   remoteTraversalWorker.reset(&remoteTraversalState,*myBuckets);
-  thisProxy[thisIndex].doRemoteGravity();
+  RescheduleMsg *msg = new (NUM_PRIORITY_BITS) RescheduleMsg;
+  *(int *)CkPriorityPtr(msg) = REMOTE_GRAVITY_PRIORITY;
+  CkSetQueueing(msg, CK_QUEUEING_IFIFO);
+  thisProxy[thisIndex].doRemoteGravity(msg);
 
   localTraversalState.reset(myNumBuckets,myBuckets);
   localTraversalWorker.reset(&localTraversalState,*myBuckets);
-  thisProxy[thisIndex].doLocalGravity();
+
+  msg = new (NUM_PRIORITY_BITS) RescheduleMsg;
+  *(int *)CkPriorityPtr(msg) = LOCAL_GRAVITY_PRIORITY;
+  CkSetQueueing(msg, CK_QUEUEING_IFIFO);
+  thisProxy[thisIndex].doLocalGravity(msg);
 }
 
-void TreePiece::doLocalGravity(){
+void TreePiece::doLocalGravity(RescheduleMsg *msg){
   int i;
   for(i = 0; i < globalParams.yieldPeriod && 
                  localTraversalState.current < myNumBuckets; 
@@ -91,13 +98,14 @@ void TreePiece::doLocalGravity(){
 
   if(localTraversalState.decrPending(i)){
     localGravityDone();
+    delete msg;
   }
   else{
-    thisProxy[thisIndex].doLocalGravity();
+    thisProxy[thisIndex].doLocalGravity(msg);
   }
 }
 
-void TreePiece::doRemoteGravity(){
+void TreePiece::doRemoteGravity(RescheduleMsg *msg){
   int i;
   for(i = 0; i < globalParams.yieldPeriod &&
                  remoteTraversalState.current < myNumBuckets;
@@ -110,9 +118,10 @@ void TreePiece::doRemoteGravity(){
 
   if(remoteTraversalState.decrPending(i)){
     remoteGravityDone();
+    delete msg;
   }
   else{
-    thisProxy[thisIndex].doRemoteGravity();
+    thisProxy[thisIndex].doRemoteGravity(msg);
   }
 }
 
index 22e56d5..b2f6764 100644 (file)
@@ -62,8 +62,8 @@ class TreePiece : public CBase_TreePiece {
   void prepare(Node<ForceData> *_root, Node<ForceData> **buckets, int bucketStart, int bucketEnd);
   void startTraversal();
 
-  void doLocalGravity();
-  void doRemoteGravity();
+  void doLocalGravity(RescheduleMsg *);
+  void doRemoteGravity(RescheduleMsg *);
 
   void requestParticles(RequestMsg *msg);
   void requestNode(RequestMsg *msg);
index a7adf99..ece2a2c 100644 (file)
@@ -69,10 +69,14 @@ void MomentsWorker::setLeafType(Node<ForceData> *leaf){
     // type could have been set to EmptyBucket during construction
     if(leaf->getType() == Invalid){
       leaf->setType(Remote); 
-      // XXX - randomize?
-      int oneOwner = leaf->getOwnerStart();
-      TB_DEBUG("(%d) requestMoments %lu from tree piece %d\n", CkMyPe(), leaf->getKey(), oneOwner);
-      treePieceProxy[oneOwner].requestMoments(leaf->getKey(),CkMyPe());
+      int numOwners = leaf->getOwnerEnd()-leaf->getOwnerStart()+1;
+      int requestOwner = leaf->getOwnerStart()+(rand()%numOwners);
+      TB_DEBUG("(%d) requestMoments %lu from tree piece %d\n", CkMyPe(), leaf->getKey(), requestOwner);
+
+      CkEntryOptions opts;
+      opts.setQueueing(CK_QUEUEING_IFIFO);
+      opts.setPriority(REQUEST_MOMENTS_PRIORITY);
+      treePieceProxy[requestOwner].requestMoments(leaf->getKey(),CkMyPe(),&opts);
     }
     return;
   }
@@ -137,11 +141,6 @@ int TraversalWorker::work(Node<ForceData> *node){
 }
 
 void TraversalWorker::work(ExternalParticle *particle){
-  /*
-  for(int i = 0; i < currentBucket->getNumParticles(); i++){
-    CkAssert(!isnan((currentBucket->getParticles()+i)->position.length()));
-  }
-  */
   int computed = partBucketForce(particle,currentBucket);
   state->incrPartPartInteractions(currentBucket->getKey(),computed);
 }
@@ -158,10 +157,6 @@ bool RemoteTraversalWorker::getKeep(NodeType type){
   return keep[type];
 }
 
-void LocalTraversalWorker::done(){
-  CkAbort("Cache should never call local traversal worker!\n");
-}
-
 void RemoteTraversalWorker::done(){
   ownerTreePiece->remoteGravityDone();
 }
index 7fa2fc9..05de11c 100644 (file)
--- a/Worker.h
+++ b/Worker.h
@@ -120,7 +120,7 @@ class TraversalWorker : public CutoffWorker<ForceData> {
   void work(ExternalParticle *particle);
   void bucketDone(Key k);
   
-  virtual void done() = 0;
+  virtual void done() {}
   virtual bool getKeep(NodeType type) = 0;
 };
 
@@ -129,7 +129,6 @@ class LocalTraversalWorker : public TraversalWorker {
   public:
   LocalTraversalWorker(TreePiece *tp) : TraversalWorker(tp) {}
   LocalTraversalWorker() : TraversalWorker() {}
-  void done();
   bool getKeep(NodeType type);
 };
 
index 8a000aa..42ae91e 100644 (file)
--- a/barnes.ci
+++ b/barnes.ci
@@ -36,6 +36,7 @@ mainmodule barnes {
   };
 
   message MomentsMsg;
+  message RescheduleMsg;
 
   include "Descriptor.h";
 
@@ -54,8 +55,8 @@ mainmodule barnes {
     entry void requestMoments(Key k, int replyTo);
 
     entry void startTraversal();
-    entry void doLocalGravity();
-    entry void doRemoteGravity();
+    entry void doLocalGravity(RescheduleMsg *);
+    entry void doRemoteGravity(RescheduleMsg *);
 
     entry void requestParticles(RequestMsg *msg);
     entry void requestNode(RequestMsg *msg);
index 36ac579..0a75cd5 100644 (file)
--- a/defines.h
+++ b/defines.h
@@ -34,4 +34,15 @@ const double opening_geometry_factor = 2 / sqrt(3.0);
 //#define TB_DEBUG CkPrintf
 #define TB_DEBUG /* empty */
 
+#define NUM_PRIORITY_BITS (sizeof(int)*CHAR_BIT)
+
+#define REQUEST_MOMENTS_PRIORITY (-8)
+#define RECV_MOMENTS_PRIORITY (-7)
+#define REQUEST_NODE_PRIORITY (-6)
+#define REQUEST_PARTICLES_PRIORITY (-5)
+#define RECV_NODE_PRIORITY (-4)
+#define RECV_PARTICLES_PRIORITY (-3)
+#define REMOTE_GRAVITY_PRIORITY (-2)
+#define LOCAL_GRAVITY_PRIORITY (-1)
+
 #endif