Feature #1909: Proper support for callbacks to immediate entry methods 96/4196/3
authorNitin Bhat <nbhat4@illinois.edu>
Wed, 16 May 2018 23:14:28 +0000 (18:14 -0500)
committerNitin Bhat <nbhat4@illinois.edu>
Thu, 17 May 2018 19:50:40 +0000 (14:50 -0500)
Currently only immediate entry methods called through proxies are invoked
through the immediate handler. With this feature, callbacks to immediate
entry methods are also behave in the same manner and will bypass the
scheduler to be invoked immediately. (on the comm. thread in SMP mode for a
network message).

Change-Id: Ieff28ca0ca9f27e96bb8d42939730dae3d999100

examples/charm++/Makefile
examples/charm++/immediateEntryMethod/Makefile [new file with mode: 0644]
examples/charm++/immediateEntryMethod/immediateEM.C [new file with mode: 0644]
examples/charm++/immediateEntryMethod/immediateEM.ci [new file with mode: 0644]
src/ck-core/charm.h
src/ck-core/ckcallback.C
src/ck-core/register.C
src/ck-core/register.h
src/xlat-i/xi-Entry.C

index 58072b2c65ce1f8419cd54c07c01d496db81fb25..9c2fcbf33d97b89e4f7debf29bda65ee8bf10f4a 100644 (file)
@@ -6,6 +6,7 @@ completion \
 fib \
 groupsection \
 hello \
+immediateEntryMethod \
 integrate \
 integratePUPable \
 integrateArray \
diff --git a/examples/charm++/immediateEntryMethod/Makefile b/examples/charm++/immediateEntryMethod/Makefile
new file mode 100644 (file)
index 0000000..6a142e3
--- /dev/null
@@ -0,0 +1,21 @@
+-include ../../common.mk
+CHARMC=../../../bin/charmc $(OPTS)
+
+OBJS = immediateEM.o
+
+all: immediateEM
+
+immediateEM: $(OBJS)
+       $(CHARMC) -language charm++ -o immediateEM $(OBJS)
+
+immediateEM.o: immediateEM.C immediateEM.decl.h
+       $(CHARMC) -c immediateEM.C
+
+immediateEM.decl.h: immediateEM.ci
+       $(CHARMC) immediateEM.ci
+
+clean:
+       rm -rf *.decl.h *.def.h conv-host *.o immediateEM charmrun *~ *log *projrc *sts
+
+test: all
+       $(call run, +p4 ./immediateEM)
diff --git a/examples/charm++/immediateEntryMethod/immediateEM.C b/examples/charm++/immediateEntryMethod/immediateEM.C
new file mode 100644 (file)
index 0000000..fa59a08
--- /dev/null
@@ -0,0 +1,72 @@
+#include "immediateEM.decl.h"
+
+CProxy_main mainProxy;
+
+class main : public CBase_main{
+  int counter;
+  CProxy_NG1 ng1Proxy;
+
+  public:
+  main(CkArgMsg *m){
+    CkPrintf("\n[PE: %d][Node: %d][Rank: %d] ****************** Invoking Nodegroup Constructor ****************\n", CkMyPe(), CkMyNode(), CkMyRank());
+    mainProxy = thisProxy;
+    counter = 0;
+    // Create a nodegroup
+    ng1Proxy = CProxy_NG1::ckNew();
+  }
+
+  void invokeRegularEntryMethod() {
+    if(++counter != CkNumNodes()) return;
+    counter = 0;
+    CkPrintf("\n[PE: %d][Node: %d][Rank: %d] ****************** Invoking Regular Entry Method *****************\n", CkMyPe(), CkMyNode(), CkMyRank());
+    ng1Proxy.regularEntryMethod();
+  }
+
+  void invokeImmediateEntryMethod() {
+    if(++counter != CkNumNodes()) return;
+    counter = 0;
+    CkPrintf("\n[PE: %d][Node: %d][Rank: %d] *********** Invoking Immediate Entry Method through proxy ********\n", CkMyPe(), CkMyNode(), CkMyRank());
+    ng1Proxy.immediateEntryMethod();
+  }
+
+  void invokeImmediateEntryMethodCb() {
+    if(++counter != CkNumNodes()) return;
+    counter = 0;
+    CkPrintf("\n[PE: %d][Node: %d][Rank: %d] ******** Invoking Immediate Entry Method Through Callback *******\n", CkMyPe(), CkMyNode(), CkMyRank());
+    CkCallback cb(CkIndex_NG1::immediateEntryMethod(), ng1Proxy);
+    cb.send();
+  }
+
+  void done() {
+    if(++counter != CkNumNodes()) return;
+    CkExit();
+  }
+};
+
+class NG1 : public CBase_NG1{
+  bool immCalled;
+  public:
+  NG1(){
+    immCalled = false;
+    CkPrintf("[PE: %d][Node: %d][Rank: %d] Invoked Nodegroup constructor\n", CkMyPe(), CkMyNode(), CkMyRank());
+    mainProxy.invokeRegularEntryMethod();
+  }
+
+  void regularEntryMethod(){
+    CkPrintf("[PE: %d][Node: %d][Rank: %d] Invoked regular entry method\n", CkMyPe(), CkMyNode(), CkMyRank());
+    mainProxy.invokeImmediateEntryMethod();
+  }
+
+  void immediateEntryMethod(){
+    if(!immCalled) {
+      immCalled = true;
+      CkPrintf("[PE: %d][Node: %d][Rank: %d] Invoked immediate entry method through regular proxy call\n", CkMyPe(), CkMyNode(), CkMyRank());
+      mainProxy.invokeImmediateEntryMethodCb();
+    } else {
+      CkPrintf("[PE: %d][Node: %d][Rank: %d] Invoked immediate entry method through callback\n", CkMyPe(), CkMyNode(), CkMyRank());
+      mainProxy.done();
+    }
+  }
+};
+
+#include "immediateEM.def.h"
diff --git a/examples/charm++/immediateEntryMethod/immediateEM.ci b/examples/charm++/immediateEntryMethod/immediateEM.ci
new file mode 100644 (file)
index 0000000..592a926
--- /dev/null
@@ -0,0 +1,18 @@
+mainmodule immediateEM {
+
+  readonly CProxy_main mainProxy;
+
+  mainchare main {
+    entry main(CkArgMsg *m);
+    entry void invokeRegularEntryMethod();
+    entry void invokeImmediateEntryMethod();
+    entry void invokeImmediateEntryMethodCb();
+    entry void done();
+  };
+
+  nodegroup NG1{
+    entry NG1();
+    entry void regularEntryMethod();
+    entry [immediate] void immediateEntryMethod();
+  };
+};
index 543c9d8e22a0213c657edbe7a8907f92581d2c50..09ad4f60a8bff533b711d35c6d806247e07b8084 100644 (file)
@@ -141,6 +141,7 @@ extern int CkRegisterMsg(const char *name, CkPackFnPtr pack,
 
 #define CK_EP_MEMCRITICAL (1<<5)
 #define CK_EP_APPWORK     (1<<6)
+#define CK_EP_IMMEDIATE   (1<<7)
 
 /** type of a chare */
 #if CMK_MESSAGE_LOGGING
index 0c48fd339e36b06e008eddf947463b07d717faf4..3f47613cab38d6377eeeca97e3783aadeb2cb7c4 100644 (file)
@@ -281,6 +281,10 @@ void CkCallback::send(int length,const void *data) const
 */
 void CkCallback::send(void *msg) const
 {
+  // Variable is set with CK_MSG_IMMEDIATE when callback is pointing to an
+  // immediate entry method
+  int opts = 0;
+
 #if CMK_CHARMPY
   if (isCkExtReductionCb) { // callback target is external
     CkCallbackSendExt(*this, msg);
@@ -321,66 +325,68 @@ void CkCallback::send(void *msg) const
        case sendChare: //Send message to a chare
                if (!msg) msg=CkAllocSysMsg();
                 if (d.chare.hasRefnum) CkSetRefNum(msg, d.chare.refnum);
-               CkSendMsg(d.chare.ep,msg,&d.chare.id);
+               CkSendMsg(d.chare.ep, msg, &d.chare.id);
                break;
        case isendChare: //inline send-to-chare
                if (!msg) msg=CkAllocSysMsg();
                 if (d.chare.hasRefnum) CkSetRefNum(msg, d.chare.refnum);
-               CkSendMsgInline(d.chare.ep,msg,&d.chare.id);
+               CkSendMsgInline(d.chare.ep, msg, &d.chare.id);
                break;
        case sendGroup: //Send message to a group element
                if (!msg) msg=CkAllocSysMsg();
                 if (d.group.hasRefnum) CkSetRefNum(msg, d.group.refnum);
-               CkSendMsgBranch(d.group.ep,msg,d.group.onPE,d.group.id);
+               CkSendMsgBranch(d.group.ep, msg, d.group.onPE, d.group.id);
                break;
        case sendNodeGroup: //Send message to a group element
                if (!msg) msg=CkAllocSysMsg();
                 if (d.group.hasRefnum) CkSetRefNum(msg, d.group.refnum);
-               CkSendMsgNodeBranch(d.group.ep,msg,d.group.onPE,d.group.id);
+                if (_entryTable[d.group.ep]->isImmediate) opts = CK_MSG_IMMEDIATE;
+               CkSendMsgNodeBranch(d.group.ep, msg, d.group.onPE, d.group.id, opts);
                break;
        case isendGroup: //inline send-to-group element
                if (!msg) msg=CkAllocSysMsg();
                 if (d.group.hasRefnum) CkSetRefNum(msg, d.group.refnum);
-               CkSendMsgBranchInline(d.group.ep,msg,d.group.onPE,d.group.id);
+               CkSendMsgBranchInline(d.group.ep, msg, d.group.onPE, d.group.id);
                break;
        case isendNodeGroup: //inline send-to-group element
                if (!msg) msg=CkAllocSysMsg();
                 if (d.group.hasRefnum) CkSetRefNum(msg, d.group.refnum);
-               CkSendMsgNodeBranchInline(d.group.ep,msg,d.group.onPE,d.group.id);
+                if (_entryTable[d.group.ep]->isImmediate) opts = CK_MSG_IMMEDIATE;
+               CkSendMsgNodeBranchInline(d.group.ep, msg, d.group.onPE, d.group.id, opts);
                break;
        case sendArray: //Send message to an array element
                if (!msg) msg=CkAllocSysMsg();
                 if (d.array.hasRefnum) CkSetRefNum(msg, d.array.refnum);
-
                CkSetMsgArrayIfNotThere(msg);
-               CkSendMsgArray(d.array.ep,msg,d.array.id,d.array.idx.asChild());
+               CkSendMsgArray(d.array.ep, msg, d.array.id, d.array.idx.asChild());
                break;
        case isendArray: //inline send-to-array element
                if (!msg) msg=CkAllocSysMsg();
                 if (d.array.hasRefnum) CkSetRefNum(msg, d.array.refnum);
-               CkSendMsgArrayInline(d.array.ep,msg,d.array.id,d.array.idx.asChild());
+               CkSendMsgArrayInline(d.array.ep, msg, d.array.id, d.array.idx.asChild());
                break;
        case bcastGroup:
                if (!msg) msg=CkAllocSysMsg();
                 if (d.group.hasRefnum) CkSetRefNum(msg, d.group.refnum);
-               CkBroadcastMsgBranch(d.group.ep,msg,d.group.id);
+               CkBroadcastMsgBranch(d.group.ep, msg, d.group.id);
                break;
        case bcastNodeGroup:
                if (!msg) msg=CkAllocSysMsg();
                 if (d.group.hasRefnum) CkSetRefNum(msg, d.group.refnum);
-               CkBroadcastMsgNodeBranch(d.group.ep,msg,d.group.id);
+                if (_entryTable[d.group.ep]->isImmediate) opts = CK_MSG_IMMEDIATE;
+               CkBroadcastMsgNodeBranch(d.group.ep, msg, d.group.id, opts);
                break;
        case bcastArray:
                if (!msg) msg=CkAllocSysMsg();
                 if (d.array.hasRefnum) CkSetRefNum(msg, d.array.refnum);
-               CkBroadcastMsgArray(d.array.ep,msg,d.array.id);
+               CkBroadcastMsgArray(d.array.ep, msg, d.array.id);
                break;
        case bcastSection: {
                if(!msg)msg=CkAllocSysMsg();
                 if (d.section.hasRefnum) CkSetRefNum(msg, d.section.refnum);
                 CkSectionInfo sinfo(d.section.sinfo);
                 CkSectionID secID(sinfo, d.section._elems, d.section._nElems, d.section.pelist, d.section.npes);
-               CkBroadcastMsgSection(d.section.ep,msg,secID);
+               CkBroadcastMsgSection(d.section.ep, msg, secID);
                 secID._elems = NULL;
                 secID.pelist = NULL;
                break;
index 11d316c549ef658f38a0fb3e57d7f61382a0f474..ac7867460367471cb015aebeabfd1c6831fca9cb 100644 (file)
@@ -55,6 +55,7 @@ int CkRegisterEp(const char *name, CkCallFnPtr call, int msgIdx, int chareIdx,
   if (ck_ep_flags & CK_EP_INTRINSIC) e->inCharm=true;
   if (ck_ep_flags & CK_EP_TRACEDISABLE) e->traceEnabled=false;
   if (ck_ep_flags & CK_EP_APPWORK) e->appWork=true;
+  if (ck_ep_flags & CK_EP_IMMEDIATE) e->isImmediate=true;
 #if ADAPT_SCHED_MEM
   if (ck_ep_flags & CK_EP_MEMCRITICAL){
      e->isMemCritical=true;
index fd781fcea5d4026c0b7a4675fa7884f6d4193768..4c501a87e0408f8f89ada50105f852e88cb6f1b9 100644 (file)
@@ -82,6 +82,8 @@ class EntryInfo {
     bool traceEnabled;
     /// Method doesn't keep (and delete) message passed in to it.
     bool noKeep;
+    /// Method is an immediate entry method and can bypass scheduler
+    bool isImmediate;
     /// true if this EP is charm internal functions
     bool inCharm;
     bool appWork;
@@ -99,7 +101,7 @@ class EntryInfo {
 #if CMK_CHARMDEBUG
       ,messagePup(0)
 #endif
-      ,traceEnabled(true), noKeep(false), inCharm(false), appWork(false)
+      ,traceEnabled(true), noKeep(false), isImmediate(false), inCharm(false), appWork(false)
       ,name(n)
     { }
 };
index 29a551f9bada45036532b09d3ba711b78281580d..b9366f7f789736cda30e728af0c60807bc38ab2c 100644 (file)
@@ -2792,7 +2792,10 @@ XStr Entry::genRegEp(bool isForRedn) {
   // it to CkReductionMsg not CkMarshallMsg)
   if (!isForRedn && (attribs & SNOKEEP)) str << "+CK_EP_NOKEEP";
   if (attribs & SNOTRACE) str << "+CK_EP_TRACEDISABLE";
-  if (attribs & SIMMEDIATE) str << "+CK_EP_TRACEDISABLE";
+  if (attribs & SIMMEDIATE) {
+     str << "+CK_EP_TRACEDISABLE";
+     str << "+CK_EP_IMMEDIATE";
+  }
   if (attribs & SAPPWORK) str << "+CK_EP_APPWORK";
 
   /*MEICHAO*/