Minor fix
[charm.git] / src / ck-core / mpi-interoperate.C
1 extern "C" void CkExit(void);
2
3 #include "mpi-interoperate.h"
4
5 static int   _libExitStarted = 0;
6 int    _libExitHandlerIdx;
7
8 #if CMK_CONVERSE_MPI
9 extern "C" { MPI_Comm charmComm; }
10 #endif
11
12 extern int _ringexit;               // for charm exit
13 extern int _ringtoken;
14 extern void _initCharm(int unused_argc, char **argv);
15
16 // triger LibExit on PE 0,
17 extern "C"
18 void LibCkExit(void)
19 {
20         // always send to PE 0
21         envelope *env = _allocEnv(StartExitMsg);
22         env->setSrcPe(CkMyPe());
23         CmiSetHandler(env, _libExitHandlerIdx);
24         CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
25 }
26
27 void _libExitHandler(envelope *env)
28 {
29         switch(env->getMsgtype()) {
30                 case StartExitMsg:
31                         CkAssert(CkMyPe()==0);
32                         // else goto next
33                 case ExitMsg:
34                         CkAssert(CkMyPe()==0);
35                         if(_libExitStarted) {
36                                 CmiFree(env);
37                                 return;
38                         }
39                         _libExitStarted = 1;
40                         env->setMsgtype(ReqStatMsg);
41                         env->setSrcPe(CkMyPe());
42                         // if exit in ring, instead of broadcasting, send in ring
43                         if (_ringexit){
44                                 const int stride = CkNumPes()/_ringtoken;
45                                 int pe = 0; while (pe<CkNumPes()) {
46                                         CmiSyncSend(pe, env->getTotalsize(), (char *)env);
47                                         pe += stride;
48                                 }
49                                 CmiFree(env);
50                         }else{
51                                 CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
52                         }       
53                         break;
54                 case ReqStatMsg:
55                         if (_ringexit) {
56                                 int stride = CkNumPes()/_ringtoken;
57                                 int pe = CkMyPe()+1;
58                                 if (pe < CkNumPes() && pe % stride != 0)
59                                         CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
60                                 else
61                                         CmiFree(env);
62                         }
63                         else
64                                 CmiFree(env);
65                         //everyone exits here - there may be issues with leftover messages in the queue
66                         _libExitStarted = 0;
67                         CpvAccess(charmLibExitFlag) = 1;
68                         break;
69                 default:
70                         CmiAbort("Internal Error(_libExitHandler): Unknown-msg-type. Contact Developers.\n");
71         }
72 }
73
74 #if CMK_CONVERSE_MPI
75 extern "C"
76 void CharmLibInit(MPI_Comm userComm, int argc, char **argv){
77         //note CmiNumNodes and CmiMyNode should just be macros
78   charmComm = userComm;
79   MPI_Comm_size(charmComm, &_Cmi_numnodes);
80   MPI_Comm_rank(charmComm, &_Cmi_mynode);
81
82         CharmLibInterOperate = 1;
83         ConverseInit(argc, argv, (CmiStartFn)_initCharm, 1, 0);
84 }
85 #else
86 extern "C"
87 void CharmLibInit(int userComm, int argc, char **argv){
88     CmiAbort("mpi-interoperate only supports MPI machine layer");
89 }
90 #endif
91
92 #undef CkExit
93 #define CkExit CkExit
94 extern "C"
95 void CharmLibExit() {
96         if(CkMyPe() == 0) {
97                 CkExit();
98         }
99         CsdScheduler(-1);
100 }
101