charmxi: Register the debug PUP for templated messages correctly, now that they are...
[charm.git] / src / ck-core / mpi-interoperate.C
1 extern "C" void CkExit(void);
2
3 #include "mpi-interoperate.h"
4
5 static int   _libExitStarted = 0;
6 int    _libExitHandlerIdx;
7
8 extern "C" { MPI_Comm charmComm; }
9 extern int _ringexit;               // for charm exit
10 extern int _ringtoken;
11 extern void _initCharm(int unused_argc, char **argv);
12
13 // triger LibExit on PE 0,
14 extern "C"
15 void LibCkExit(void)
16 {
17         // always send to PE 0
18         envelope *env = _allocEnv(StartExitMsg);
19         env->setSrcPe(CkMyPe());
20         CmiSetHandler(env, _libExitHandlerIdx);
21         CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
22 }
23
24 void _libExitHandler(envelope *env)
25 {
26         switch(env->getMsgtype()) {
27                 case StartExitMsg:
28                         CkAssert(CkMyPe()==0);
29                         // else goto next
30                 case ExitMsg:
31                         CkAssert(CkMyPe()==0);
32                         if(_libExitStarted) {
33                                 CmiFree(env);
34                                 return;
35                         }
36                         _libExitStarted = 1;
37                         env->setMsgtype(ReqStatMsg);
38                         env->setSrcPe(CkMyPe());
39                         // if exit in ring, instead of broadcasting, send in ring
40                         if (_ringexit){
41                                 const int stride = CkNumPes()/_ringtoken;
42                                 int pe = 0; while (pe<CkNumPes()) {
43                                         CmiSyncSend(pe, env->getTotalsize(), (char *)env);
44                                         pe += stride;
45                                 }
46                                 CmiFree(env);
47                         }else{
48                                 CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
49                         }       
50                         break;
51                 case ReqStatMsg:
52                         if (_ringexit) {
53                                 int stride = CkNumPes()/_ringtoken;
54                                 int pe = CkMyPe()+1;
55                                 if (pe < CkNumPes() && pe % stride != 0)
56                                         CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
57                                 else
58                                         CmiFree(env);
59                         }
60                         else
61                                 CmiFree(env);
62                         //everyone exits here - there may be issues with leftover messages in the queue
63                         _libExitStarted = 0;
64                         CpvAccess(charmLibExitFlag) = 1;
65                         break;
66                 default:
67                         CmiAbort("Internal Error(_libExitHandler): Unknown-msg-type. Contact Developers.\n");
68         }
69 }
70
71 #if CMK_CONVERSE_MPI
72 extern "C"
73 void CharmLibInit(MPI_Comm userComm, int argc, char **argv){
74         //note CmiNumNodes and CmiMyNode should just be macros
75   charmComm = userComm;
76   MPI_Comm_size(charmComm, &_Cmi_numnodes);
77   MPI_Comm_rank(charmComm, &_Cmi_mynode);
78
79         CharmLibInterOperate = 1;
80         ConverseInit(argc, argv, (CmiStartFn)_initCharm, 1, 0);
81 }
82 #else
83 extern "C"
84 void CharmLibInit(int userComm, int argc, char **argv){
85     CmiAbort("mpi-interoperate only supports MPI machine layer");
86 }
87 #endif
88
89 #undef CkExit
90 #define CkExit CkExit
91 extern "C"
92 void CharmLibExit() {
93         if(CkMyPe() == 0) {
94                 CkExit();
95         }
96         CsdScheduler(-1);
97 }
98