Add a workstealing seed balancer
authorYanhua Yanhua <sun51@illinois.edu>
Thu, 29 Jul 2010 04:45:33 +0000 (23:45 -0500)
committerYanhua Yanhua <sun51@illinois.edu>
Thu, 29 Jul 2010 04:45:33 +0000 (23:45 -0500)
examples/charm++/NQueen/Makefile
src/ck-core/init.C
src/conv-ldb/cldb.neighbor.c
src/conv-ldb/cldb.none.c
src/conv-ldb/cldb.prioritycentralized.c
src/conv-ldb/cldb.rand.c
src/conv-ldb/cldb.spray.c
src/conv-ldb/cldb.workstealing.c [new file with mode: 0644]
src/conv-ldb/cldb.workstealing.h [new file with mode: 0644]
src/conv-ldb/topology.C
src/scripts/Makefile

index 485e4c8c268929676ba009daab5c0c3e8a0c3435..e95ef28bb614156e265fb7544834b0f090a37f86 100644 (file)
@@ -1,6 +1,6 @@
 CHARMC = ../../../bin/charmc $(OPTS)
 default: all
-all: nqueen nqueen_prj nqueen_rand  nqueen_neighbor nqueen_neighbor.prj
+all: nqueen nqueen_prj nqueen_rand  nqueen_neighbor nqueen_neighbor.prj nqueen_workstealing
 
 nqueen : main.o nqueen.o counter.o
        $(CHARMC) -language charm++ -o nqueen main.o nqueen.o counter.o
@@ -11,6 +11,9 @@ nqueen_rand : main.o nqueen.o counter.o
 nqueen_neighbor : main.o nqueen.o counter.o
        $(CHARMC) -language charm++ -balance neighbor -o nqueen_neighbor main.o nqueen.o counter.o
 
+nqueen_workstealing : main.o nqueen.o counter.o
+       $(CHARMC) -language charm++ -balance workstealing -o nqueen_workstealing main.o nqueen.o counter.o
+
 nqueen_prj : main.o nqueen.o counter.o
        $(CHARMC) -language charm++ -tracemode projections -balance rand -o nqueen_prj main.o nqueen.o counter.o
 
index 7b333bf46a53618b79bb67d1df8b5c0689a6a5c9..7fd598cc02077ee635494d5c8c341ea29e312153 100644 (file)
@@ -863,6 +863,8 @@ extern "C" void CmiInitCPUTopology(char **argv);
 extern "C" void CmiInitCPUAffinity(char **argv);
 extern "C" void CmiInitMemAffinity(char **argv);
 
+extern "C" void CldCallback();
+
 void _registerInitCall(CkInitCallFn fn, int isNodeCall)
 {
   if (isNodeCall) _initCallTable.initNodeCalls.enq(fn);
@@ -1211,7 +1213,7 @@ void _initCharm(int unused_argc, char **argv)
         }
         CmiInitCPUTopology(argv);
     }
-
+    //CldCallback();
 #if CMK_BLUEGENE_CHARM && CMK_CHARMDEBUG
       // Register the BG handler for CCS. Notice that this is put into a variable shared by
       // the whole real processor. This because converse needs to find it. We check that all
index 99f48eb8db629961733bdb146cf0f34a31536790..2dfb5ffcb85b0e3e2b641ded47b332b3092fd966 100644 (file)
@@ -93,7 +93,7 @@ static void CldStillIdle(void *dummy, double curT)
   CmiBecomeImmediate(&msg);
   for (i=0; i<CpvAccess(numNeighbors); i++) {
     msg.to_rank = CmiRankOf(CpvAccess(neighbors)[i].pe);
-    CmiSyncNodeSend(CpvAccess(neighbors)[i].pe,sizeof(requestmsg),(char *)&msg);
+    CmiSyncSend(CpvAccess(neighbors)[i].pe,sizeof(requestmsg),(char *)&msg);
     //CmiSyncNodeSend(CmiNodeOf(CpvAccess(neighbors)[i].pe),sizeof(requestmsg),(char *)&msg);
   }
   /*
@@ -191,6 +191,10 @@ int CldMinAvg()
   int sum=0, i;
 
   int nNeighbors = CpvAccess(numNeighbors);
+  
+#ifdef YHDEBUG
+  CmiPrintf("Line 196 processor %d, numNeighbor=%d\n", CmiMyPe(), nNeighbors);
+#endif
   if (CpvAccess(start) == -1)
     CpvAccess(start) = CmiMyPe() % nNeighbors;
 
@@ -612,6 +616,11 @@ static void CldComputeNeighborData()
   }
   CpvAccess(neighborGroup) = CmiEstablishGroup(CpvAccess(numNeighbors), pes);
   free(pes);
+
+#ifdef YHDEBUG
+  CmiPrintf("Line 617 On processor=%d npe=%d, numNeighbors=%d\n", CmiMyPe(), npe, CpvAccess(numNeighbors));
+#endif
+
 }
 
 void CldGraphModuleInit(char **argv)
@@ -680,6 +689,8 @@ void CldGraphModuleInit(char **argv)
     CmiNodeBarrier();
 #endif
     CldBalancePeriod(NULL, CmiWallTimer());
+
+
   }
 
 #if 1
@@ -696,6 +707,16 @@ void CldGraphModuleInit(char **argv)
 #endif
 }
 
+void CldCallback()
+{
+    CldComputeNeighborData();
+#if CMK_MULTICORE
+    CmiNodeBarrier();
+#endif
+    CldBalancePeriod(NULL, CmiWallTimer());
+
+}
+
 void CldModuleInit(char **argv)
 {
   CpvInitialize(int, CldHandlerIndex);
index 49d69d930f18431bf9d0b5f3fe3e996d96411819..b19bbff27063a00f898b976f710327b81c107058 100644 (file)
@@ -119,3 +119,5 @@ void CldModuleInit(char **argv)
     CpvAccess(CldMessageChunks) = 0;
   CldModuleGeneralInit(argv);
 }
+
+void CldCallback(){}
index f65797dc96af471f97084e658b14c0277abec527..d34dd4da5e43748a7cb53ce6adec6c117f7fada1 100644 (file)
@@ -474,3 +474,5 @@ void CldEnqueueMulti(int npes, int *pes, void *msg, int infofn)
 }
 
 
+void CldCallback()
+{}
index 29f6298732da0babd28d251a0f8e2818f98c75bd..5077446b343ccb77ff9829b3a9076d913fef9175 100644 (file)
@@ -146,3 +146,6 @@ void CldModuleInit(char **argv)
     CpvAccess(CldMessageChunks) = 0;
   CldModuleGeneralInit(argv);
 }
+
+void CldCallback()
+{}
index 394a72cd33583d27c9bbccb3debde9691dc30260..fc2395d726949d468238f98a08ed743f18ff1a76 100644 (file)
@@ -258,3 +258,5 @@ void CldModuleInit(char **argv)
   CldModuleGeneralInit(argv);
   CldInitiateReduction();
 }
+void CldCallback()
+{}
diff --git a/src/conv-ldb/cldb.workstealing.c b/src/conv-ldb/cldb.workstealing.c
new file mode 100644 (file)
index 0000000..0e2bea7
--- /dev/null
@@ -0,0 +1,281 @@
+#include <stdlib.h>
+
+#include "converse.h"
+#include "cldb.workstealing.h"
+#include "queueing.h"
+#include "cldb.h"
+
+#define IDLE_IMMEDIATE                 1
+#define TRACE_USEREVENTS        0
+
+#define PERIOD 20                /* default: 30 */
+#define MSGDELAY 10
+#define MAXOVERLOAD 1
+
+#define LOADTHRESH       3
+
+
+typedef struct CldProcInfo_s {
+  double lastCheck;
+  int    sent;                 /* flag to disable idle work request */
+  int    balanceEvt;           /* user event for balancing */
+  int    idleEvt;              /* user event for idle balancing */
+  int    idleprocEvt;          /* user event for processing idle req */
+  double lastBalanceTime;
+} *CldProcInfo;
+
+
+CpvStaticDeclare(CldProcInfo, CldData);
+CpvStaticDeclare(int, CldAskLoadHandlerIndex);
+
+void LoadNotifyFn(int l)
+{
+  CldProcInfo  cldData = CpvAccess(CldData);
+  cldData->sent = 0;
+}
+
+char *CldGetStrategy(void)
+{
+  return "work stealing";
+}
+
+/* since I am idle, ask for work from neighbors */
+static void CldBeginIdle(void *dummy)
+{
+  CpvAccess(CldData)->lastCheck = CmiWallTimer();
+}
+
+static void CldEndIdle(void *dummy)
+{
+  CpvAccess(CldData)->lastCheck = -1;
+}
+
+static void CldStillIdle(void *dummy, double curT)
+{
+  int i;
+  double startT;
+  requestmsg msg;
+  int myload;
+  CldProcInfo  cldData = CpvAccess(CldData);
+  int  victim;
+
+  double now = curT;
+  double lt = cldData->lastCheck;
+  /* only ask for work every 20ms */
+  if (cldData->sent && (lt!=-1 && now-lt< PERIOD*0.001)) return;
+  cldData->lastCheck = now;
+
+  myload = CldLoad();
+  if (myload > 0) return;
+
+  msg.from_pe = CmiMyPe();
+
+  victim = (((CrnRand()+CmiMyPe())&0x7FFFFFFF)%CmiNumPes());
+
+  CmiSetHandler(&msg, CpvAccess(CldAskLoadHandlerIndex));
+  /* fixme */
+  //CmiBecomeImmediate(&msg);
+  msg.to_rank = CmiRankOf(victim);
+  CmiSyncSend(victim, sizeof(requestmsg),(char *)&msg);
+  cldData->sent = 1;
+
+#if CMK_TRACE_ENABLED && TRACE_USEREVENTS
+  traceUserBracketEvent(cldData->idleEvt, now, CmiWallTimer());
+#endif
+}
+
+/* immediate message handler, work at node level */
+/* send some work to requested proc */
+static void CldAskLoadHandler(requestmsg *msg)
+{
+  int receiver, rank, recvIdx, i;
+  int myload = CldLoad();
+  double now = CmiWallTimer();
+  CldProcInfo  cldData = CpvAccess(CldData);
+
+  /* only give you work if I have more than 1 */
+  if (myload>0) {
+    int sendLoad;
+    receiver = msg->from_pe;
+    rank = CmiMyRank();
+    if (msg->to_rank != -1) rank = msg->to_rank;
+    sendLoad = myload / 2; 
+    CmiFree(msg);
+    if (sendLoad < 1) return;
+    CldMultipleSend(receiver, sendLoad, rank, 0);
+  }
+}
+
+void CldHandler(void *msg)
+{
+  CldInfoFn ifn; CldPackFn pfn;
+  int len, queueing, priobits; unsigned int *prioptr;
+  
+  CldRestoreHandler(msg);
+  ifn = (CldInfoFn)CmiHandlerToFunction(CmiGetInfo(msg));
+  ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+  /*CsdEnqueueGeneral(msg, CQS_QUEUEING_LIFO, priobits, prioptr); */
+  CsdEnqueueGeneral(msg, queueing, priobits, prioptr);
+}
+
+void CldBalanceHandler(void *msg)
+{
+  CldRestoreHandler(msg);
+  CldPutToken(msg);
+}
+
+void CldEnqueueGroup(CmiGroup grp, void *msg, int infofn)
+{
+  int len, queueing, priobits,i; unsigned int *prioptr;
+  CldInfoFn ifn = (CldInfoFn)CmiHandlerToFunction(infofn);
+  CldPackFn pfn;
+  ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+  if (pfn) {
+    pfn(&msg);
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+  }
+  CldSwitchHandler(msg, CpvAccess(CldHandlerIndex));
+  CmiSetInfo(msg,infofn);
+
+  CmiSyncMulticastAndFree(grp, len, msg);
+}
+
+void CldEnqueueMulti(int npes, int *pes, void *msg, int infofn)
+{
+  int len, queueing, priobits,i; unsigned int *prioptr;
+  CldInfoFn ifn = (CldInfoFn)CmiHandlerToFunction(infofn);
+  CldPackFn pfn;
+  ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+  if (pfn) {
+    pfn(&msg);
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+  }
+  CldSwitchHandler(msg, CpvAccess(CldHandlerIndex));
+  CmiSetInfo(msg,infofn);
+  CmiSyncListSendAndFree(npes, pes, len, msg);
+}
+
+void CldEnqueue(int pe, void *msg, int infofn)
+{
+  int len, queueing, priobits, avg; unsigned int *prioptr;
+  CldInfoFn ifn = (CldInfoFn)CmiHandlerToFunction(infofn);
+  CldPackFn pfn;
+
+  if ((pe == CLD_ANYWHERE) && (CmiNumPes() > 1)) {
+      pe = CmiMyPe();
+    /* always pack the message because the message may be move away
+       to a different processor later by CldGetToken() */
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    if (pfn && CmiNumNodes()>1) {
+       pfn(&msg);
+       ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    }
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    CmiSetInfo(msg,infofn);
+    CldPutToken(msg);
+  } 
+  else if ((pe == CmiMyPe()) || (CmiNumPes() == 1)) {
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    //CsdEnqueueGeneral(msg, CQS_QUEUEING_LIFO, priobits, prioptr);
+    CsdEnqueueGeneral(msg, queueing, priobits, prioptr);
+  }
+  else {
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    if (pfn && CmiNodeOf(pe) != CmiMyNode()) {
+      pfn(&msg);
+      ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    }
+    CldSwitchHandler(msg, CpvAccess(CldHandlerIndex));
+    CmiSetInfo(msg,infofn);
+    if (pe==CLD_BROADCAST) 
+      CmiSyncBroadcastAndFree(len, msg);
+    else if (pe==CLD_BROADCAST_ALL)
+      CmiSyncBroadcastAllAndFree(len, msg);
+    else CmiSyncSendAndFree(pe, len, msg);
+  }
+}
+
+void CldNodeEnqueue(int node, void *msg, int infofn)
+{
+  int len, queueing, priobits, pe, avg; unsigned int *prioptr;
+  CldInfoFn ifn = (CldInfoFn)CmiHandlerToFunction(infofn);
+  CldPackFn pfn;
+  if ((node == CLD_ANYWHERE) && (CmiNumPes() > 1)) {
+      pe = CmiMyPe();
+      node = CmiNodeOf(pe);
+      ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+      CsdNodeEnqueueGeneral(msg, queueing, priobits, prioptr);
+  }
+  else if ((node == CmiMyNode()) || (CmiNumPes() == 1)) {
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    CsdNodeEnqueueGeneral(msg, queueing, priobits, prioptr);
+  } 
+  else {
+    ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    if (pfn) {
+        pfn(&msg);
+        ifn(msg, &pfn, &len, &queueing, &priobits, &prioptr);
+    }
+    CldSwitchHandler(msg, CpvAccess(CldHandlerIndex));
+    CmiSetInfo(msg,infofn);
+    if (node==CLD_BROADCAST) { CmiSyncNodeBroadcastAndFree(len, msg); }
+    else if (node==CLD_BROADCAST_ALL){CmiSyncNodeBroadcastAllAndFree(len,msg);}
+    else CmiSyncNodeSendAndFree(node, len, msg);
+  }
+}
+
+
+void CldGraphModuleInit(char **argv)
+{
+  CpvInitialize(CldProcInfo, CldData);
+  CpvInitialize(int, CldLoadResponseHandlerIndex);
+  CpvInitialize(int, CldAskLoadHandlerIndex);
+  CpvInitialize(int, CldBalanceHandlerIndex);
+
+  CpvAccess(CldData) = (CldProcInfo)CmiAlloc(sizeof(struct CldProcInfo_s));
+  CpvAccess(CldData)->lastCheck = -1;
+  CpvAccess(CldData)->sent = 0;
+#if CMK_TRACE_ENABLED
+  CpvAccess(CldData)->balanceEvt = traceRegisterUserEvent("CldBalance", -1);
+  CpvAccess(CldData)->idleEvt = traceRegisterUserEvent("CldBalanceIdle", -1);
+  CpvAccess(CldData)->idleprocEvt = traceRegisterUserEvent("CldBalanceProcIdle", -1);
+#endif
+
+  CpvAccess(CldBalanceHandlerIndex) = 
+    CmiRegisterHandler(CldBalanceHandler);
+  CpvAccess(CldAskLoadHandlerIndex) = 
+    CmiRegisterHandler((CmiHandler)CldAskLoadHandler);
+
+  /* communication thread */
+  if (CmiMyRank() == CmiMyNodeSize())  return;
+
+#if 1
+  /* register idle handlers - when idle, keep asking work from neighbors */
+  CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE,
+      (CcdVoidFn) CldStillIdle, NULL);
+  CcdCallOnConditionKeep(CcdPROCESSOR_STILL_IDLE,
+      (CcdVoidFn) CldStillIdle, NULL);
+    if (CmiMyPe() == 0) 
+      CmiPrintf("Charm++> Work stealing is enabled. \n");
+#endif
+}
+
+
+void CldModuleInit(char **argv)
+{
+  CpvInitialize(int, CldHandlerIndex);
+  CpvInitialize(int, CldRelocatedMessages);
+  CpvInitialize(int, CldLoadBalanceMessages);
+  CpvInitialize(int, CldMessageChunks);
+  CpvAccess(CldHandlerIndex) = CmiRegisterHandler(CldHandler);
+  CpvAccess(CldRelocatedMessages) = CpvAccess(CldLoadBalanceMessages) = 
+  CpvAccess(CldMessageChunks) = 0;
+
+  CldModuleGeneralInit(argv);
+  CldGraphModuleInit(argv);
+
+  CpvAccess(CldLoadNotify) = 1;
+}
+
+void CldCallback()
+{}
diff --git a/src/conv-ldb/cldb.workstealing.h b/src/conv-ldb/cldb.workstealing.h
new file mode 100644 (file)
index 0000000..5f55e2c
--- /dev/null
@@ -0,0 +1,19 @@
+/*****************************************************************************
+ * $Source$
+ * $Author$
+ * $Date$
+ * $Revision$
+ *****************************************************************************/
+
+#include "cldb.h"
+/* for sqrt() */
+#include <math.h>
+
+
+/* work request message when idle */
+typedef struct requestmsg_s {
+  char header[CmiMsgHeaderSizeBytes];
+  int from_pe;
+  int to_rank;
+} requestmsg;
+
index c6544bdd3df353fa291895c3276aab4d8fbc8210..305812c1c7368d445714392c5f4d0fb1d887aed7 100644 (file)
@@ -762,8 +762,9 @@ public:
     for(i=0;i<dimension;i++) {
       VirtualNodeCount *= Cardinality[i];
     }
-
+#ifdef YHDEBUG
     CmiPrintf(" ppn=%d, NumOfNodes=%d\n", ppn, NumOfNodes);
+#endif
   }
   ~LBTopo_torus_nd_smp() {
     delete[] Cardinality;
@@ -780,8 +781,9 @@ public:
     int node = CmiPhysicalNodeID(mype);
     int _ppn_ = CmiNumPesOnPhysicalNode(node);
     CmiGetPesOnPhysicalNode(node, &nodePeList, &numpes); 
+#ifdef YHDEBUG
     CmiPrintf(" 22222222222222ppn=%d, NumOfNodes=%d, rank=%d, node=%d, numpes=%d\n", _ppn_, NumOfNodes, rank, node, numpes);
-   
+#endif   
     for(int i=0; i<numpes; i++)
     {
         int _pid = nodePeList[i];
@@ -803,7 +805,9 @@ public:
         }
     }
 
+#ifdef YHDEBUG
   CmiPrintf(" Yes my neighbor = %d ppn=%d, NumOfNodes=%d, rank=%d, node=%d, numpes=%d\n", nb, _ppn_, NumOfNodes, rank, node, numpes);
+#endif
   }
   virtual int get_dimension() {
     return dimension;
@@ -1121,6 +1125,39 @@ void LBTopo_graph::neighbors(int mype, int* na, int &nb)
   gengraph(CmiNumPes(), (int)(sqrt(1.0*CmiNumPes())+0.5), 234, na, &nb, 0);
 }
 
+
+template <int dimension>
+class LBTopo_graph_nc: public LBTopology {
+
+public:
+    LBTopo_graph_nc(int p): LBTopology(p) {}
+    int max_neighbors()
+    {
+        return dimension + 1; 
+    }
+
+    void neighbors(int mype, int* na, int &nb)
+    {
+        gengraph(CmiNumPes(), dimension, 234, na, &nb, 0);
+    }
+
+};
+typedef LBTopo_graph_nc<2> LBTopo_graph_nc_2;
+typedef LBTopo_graph_nc<4> LBTopo_graph_nc_4;
+typedef LBTopo_graph_nc<6> LBTopo_graph_nc_6;
+typedef LBTopo_graph_nc<8> LBTopo_graph_nc_8;
+typedef LBTopo_graph_nc<10> LBTopo_graph_nc_10;
+typedef LBTopo_graph_nc<20> LBTopo_graph_nc_20;
+
+LBTOPO_MACRO(LBTopo_graph_nc_2);
+LBTOPO_MACRO(LBTopo_graph_nc_4);
+LBTOPO_MACRO(LBTopo_graph_nc_6);
+LBTOPO_MACRO(LBTopo_graph_nc_8);
+LBTOPO_MACRO(LBTopo_graph_nc_10);
+LBTOPO_MACRO(LBTopo_graph_nc_20);
+
+
+
 // complete graph
 
 class LBTopo_complete: public LBTopology {
@@ -1218,6 +1255,12 @@ public:
     lbTopos.push_back(new LBTopoMap("imesh_nd_6", createLBTopo_imesh_nd_6));
     lbTopos.push_back(new LBTopoMap("imesh_nd_7", createLBTopo_imesh_nd_7));
     lbTopos.push_back(new LBTopoMap("graph", createLBTopo_graph));
+    lbTopos.push_back(new LBTopoMap("graph_nc_2", createLBTopo_graph_nc_2));
+    lbTopos.push_back(new LBTopoMap("graph_nc_4", createLBTopo_graph_nc_4));
+    lbTopos.push_back(new LBTopoMap("graph_nc_6", createLBTopo_graph_nc_6));
+    lbTopos.push_back(new LBTopoMap("graph_nc_8", createLBTopo_graph_nc_8));
+    lbTopos.push_back(new LBTopoMap("graph_nc_10", createLBTopo_graph_nc_10));
+    lbTopos.push_back(new LBTopoMap("graph_nc_20", createLBTopo_graph_nc_20));
     lbTopos.push_back(new LBTopoMap("complete", createLBTopo_complete));
     lbTopos.push_back(new LBTopoMap("2_arytree", createLBTopo_2_arytree));
     lbTopos.push_back(new LBTopoMap("3_arytree", createLBTopo_3_arytree));
index 7cbf3250eeb9680f8907c71951e82ea7179c8ee7..ca7f54bef2504e7526ff8f337a12d816491f52d5 100644 (file)
@@ -327,7 +327,7 @@ dirs+sources:
 # Converse Libraries
 # 
 ###############################################################################
-CLBLIBS=$(L)/libldb-rand.o $(L)/libldb-spray.o $(L)/libldb-prioritycentralized.o     $(L)/libldb-neighbor.o $(L)/libldb-none.o $(L)/libldb-test.o $(L)/libldb-bluegene.o
+CLBLIBS=$(L)/libldb-rand.o $(L)/libldb-spray.o $(L)/libldb-prioritycentralized.o   $(L)/libldb-workstealing.o  $(L)/libldb-neighbor.o $(L)/libldb-none.o $(L)/libldb-test.o $(L)/libldb-bluegene.o
 TRACELIBS=$(L)/libtrace-projections.a  $(L)/libtrace-summary.a   $(L)/libtrace-utilization.a \
          $(L)/libtrace-simple.a \
           $(L)/libtrace-counter.a $(L)/libtrace-bluegene.a \
@@ -506,6 +506,9 @@ $(L)/libldb-rand.o: cldb.rand.c cldb.h $(CVHEADERS)
 $(L)/libldb-neighbor.o: cldb.neighbor.c cldb.neighbor.h graph.h $(CVHEADERS)
        $(CHARMC) -o $@ cldb.neighbor.c
 
+$(L)/libldb-workstealing.o: cldb.workstealing.c cldb.workstealing.h graph.h $(CVHEADERS)
+       $(CHARMC) -o $@ cldb.workstealing.c
+
 $(L)/libldb-spray.o: cldb.spray.c $(CVHEADERS)
        $(CHARMC) -o $@ cldb.spray.c