The previous fix dint solve the problem.
[charm.git] / src / ck-core / mpi-interoperate.C
1 #include "mpi-interoperate.h"
2
3 static int   _libExitStarted = 0;
4 int    _libExitHandlerIdx;
5
6 // triger LibExit on PE 0,
7 extern "C"
8 void LibCkExit(void)
9 {
10         // always send to PE 0
11         envelope *env = _allocEnv(StartExitMsg);
12         env->setSrcPe(CkMyPe());
13         CmiSetHandler(env, _libExitHandlerIdx);
14         CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
15 }
16
17 void _libExitHandler(envelope *env)
18 {
19         switch(env->getMsgtype()) {
20                 case StartExitMsg:
21                         CkAssert(CkMyPe()==0);
22                         // else goto next
23                 case ExitMsg:
24                         CkAssert(CkMyPe()==0);
25                         if(_libExitStarted) {
26                                 CmiFree(env);
27                                 return;
28                         }
29                         _libExitStarted = 1;
30                         env->setMsgtype(ReqStatMsg);
31                         env->setSrcPe(CkMyPe());
32                         // if exit in ring, instead of broadcasting, send in ring
33                         if (_ringexit){
34                                 const int stride = CkNumPes()/_ringtoken;
35                                 int pe = 0; while (pe<CkNumPes()) {
36                                         CmiSyncSend(pe, env->getTotalsize(), (char *)env);
37                                         pe += stride;
38                                 }
39                                 CmiFree(env);
40                         }else{
41                                 CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
42                         }       
43                         break;
44                 case ReqStatMsg:
45                         if (_ringexit) {
46                                 int stride = CkNumPes()/_ringtoken;
47                                 int pe = CkMyPe()+1;
48                                 if (pe < CkNumPes() && pe % stride != 0)
49                                         CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
50                                 else
51                                         CmiFree(env);
52                         }
53                         else
54                                 CmiFree(env);
55                         //everyone exits here - there may be issues with leftover messages in the queue
56                         _libExitStarted = 0;
57                         CpvAccess(charmLibExitFlag) = 1;
58                         break;
59                 default:
60                         CmiAbort("Internal Error(_libExitHandler): Unknown-msg-type. Contact Developers.\n");
61         }
62 }
63
64 #if CMK_CONVERSE_MPI
65 void CharmLibInit(MPI_Comm userComm, int argc, char **argv){
66         //note CmiNumNodes and CmiMyNode should just be macros
67   charmComm = userComm;
68   MPI_Comm_size(charmComm, &_Cmi_numnodes);
69   MPI_Comm_rank(charmComm, &_Cmi_mynode);
70
71         CharmLibInterOperate = 1;
72         ConverseInit(argc, argv, (CmiStartFn)_initCharm, 1, 0);
73 }
74 #else
75 void CharmLibInit(int userComm, int argc, char **argv){
76     CmiAbort("mpi-interoperate only supports MPI machine layer");
77 }
78 #endif
79
80 #undef CkExit
81 #define CkExit CkExit
82 void CharmLibExit() {
83         if(CkMyPe() == 0) {
84                 CkExit();
85         }
86         CsdScheduler(-1);
87 }
88