Merge branch 'charm-mpi-interop' of charmgit:charm into charm
authorNikhil Jain <nikhil@illinois.edu>
Sat, 31 Mar 2012 23:05:38 +0000 (18:05 -0500)
committerNikhil Jain <nikhil@illinois.edu>
Sat, 31 Mar 2012 23:05:38 +0000 (18:05 -0500)
13 files changed:
src/arch/mpi/machine.c
src/arch/util/machine-common-core.c
src/ck-core/init.C
src/ck-core/mpi-interoperate.C [new file with mode: 0644]
src/ck-core/mpi-interoperate.h [new file with mode: 0644]
src/ck-core/mpi-mainmodule.C [new file with mode: 0644]
src/ck-core/mpi-mainmodule.ci [new file with mode: 0644]
src/conv-core/convcore.c
src/conv-core/converse.h
src/scripts/Make.cidepends
src/scripts/Make.depends
src/scripts/Makefile
src/scripts/charmc

index eaa3022bc2e08ab5cb480d2a447525b4c776aa60..6d3f524b353617c3371a7f4005c2815c65099c70 100644 (file)
@@ -1176,11 +1176,13 @@ void MachineExitForMPI() {
 #endif
 #endif
 
+   if(!CharmLibInterOperate) {
 #if ! CMK_AUTOBUILD
-    signal(SIGINT, signal_int);
-    MPI_Finalize();
+      signal(SIGINT, signal_int);
+      MPI_Finalize();
 #endif
-    exit(0);
+      exit(0);
+    }
 }
 
 static int machine_exit_idx;
@@ -1293,33 +1295,40 @@ static void MachineInitForMPI(int *argc, char ***argv, int *numNodes, int *myNod
 #endif
     }
 
+    if(!CharmLibInterOperate) {
 #if CMK_MPI_INIT_THREAD
 #if CMK_SMP
     if (Cmi_smp_mode_setting == COMM_THREAD_SEND_RECV)
-      thread_level = MPI_THREAD_FUNNELED;
-    else
-      thread_level = MPI_THREAD_MULTIPLE;
+        thread_level = MPI_THREAD_FUNNELED;
+      else
+        thread_level = MPI_THREAD_MULTIPLE;
 #else
-    thread_level = MPI_THREAD_SINGLE;
+      thread_level = MPI_THREAD_SINGLE;
 #endif
-    MPI_Init_thread(argc, argv, thread_level, &provided);
-    _thread_provided = provided;
+      MPI_Init_thread(argc, argv, thread_level, &provided);
+      _thread_provided = provided;
 #else
-    MPI_Init(argc, argv);
-    thread_level = 0;
-    _thread_provided = -1;
+      MPI_Init(argc, argv);
+      thread_level = 0;
+      _thread_provided = -1;
 #endif
+    }
+
     largc = *argc;
     largv = *argv;
-    MPI_Comm_size(MPI_COMM_WORLD, numNodes);
-    MPI_Comm_rank(MPI_COMM_WORLD, myNodeID);
+    if(!CharmLibInterOperate) {
+      MPI_Comm_size(MPI_COMM_WORLD, numNodes);
+      MPI_Comm_rank(MPI_COMM_WORLD, myNodeID);
+    }
 
     myNID = *myNodeID;
 
     MPI_Get_version(&ver, &subver);
-    if (myNID == 0) {
+    if(!CharmLibInterOperate) {
+      if (myNID == 0) {
         printf("Charm++> Running on MPI version: %d.%d\n", ver, subver);
         printf("Charm++> level of thread support used: %s (desired: %s)\n", thread_level_tostring(_thread_provided), thread_level_tostring(thread_level));
+      }
     }
 
 #if CMK_SMP
index c2727170e8d2823e94703b1ebd992402c209d0f9..59cfbd6fd4cfdb1edac87e9da80b1a2d045faa20 100644 (file)
@@ -378,7 +378,7 @@ void CmiPushPE(int rank,void *msg) {
     }
 #endif
 
-    PCQueuePush(cs->recv,msg);
+    PCQueuePush(cs->recv,(char*)msg);
 
 #if CMK_SHARED_VARS_POSIX_THREADS_SMP
   if (_Cmi_noprocforcommthread)
@@ -800,16 +800,23 @@ static void ConverseRunPE(int everReturn) {
        node barrier previously should take care of the node synchronization */
     _immediateReady = 1;
 
-    /* communication thread */
-    if (CmiMyRank() == CmiMyNodeSize()) {
+    if(CharmLibInterOperate) {
+       /* !!! Not considering SMP mode now */
+       /* TODO: make interoperability working in SMP!!! */
+       Cmi_startfn(CmiGetArgc(CmiMyArgv), CmiMyArgv);
+       CsdScheduler(-1);
+    } else {
+      /* communication thread */
+      if (CmiMyRank() == CmiMyNodeSize()) {
         Cmi_startfn(CmiGetArgc(CmiMyArgv), CmiMyArgv);
         while (1) CommunicationServerThread(5);
-    } else { /* worker thread */
+      } else { /* worker thread */
         if (!everReturn) {
-            Cmi_startfn(CmiGetArgc(CmiMyArgv), CmiMyArgv);
-            if (Cmi_usrsched==0) CsdScheduler(-1);
-            ConverseExit();
+          Cmi_startfn(CmiGetArgc(CmiMyArgv), CmiMyArgv);
+          if (Cmi_usrsched==0) CsdScheduler(-1);
+          ConverseExit();
         }
+      }
     }
 }
 /* ##### End of Functions Related with Machine Startup ##### */
index 7b8828bf0179183928d1945a612e11a280eff9d7..776e8525d7fa81e00742f5fb4fbaa6a8e6db6461 100644 (file)
@@ -160,6 +160,9 @@ CkpvStaticDeclare(int,  _numInitsRecd);
 CkpvStaticDeclare(PtrQ*, _buffQ);
 CkpvStaticDeclare(PtrVec*, _bocInitVec);
 
+//for interoperability
+extern void _libExitHandler(envelope *env);
+extern int _libExitHandlerIdx;
 
 /*
        FAULT_EVAC
@@ -503,15 +506,14 @@ static void _exitHandler(envelope *env)
       break;
     case ReqStatMsg:
 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
-        _messageLoggingExit();
+      _messageLoggingExit();
 #endif
       DEBUGF(("ReqStatMsg on %d\n", CkMyPe()));
       CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
       CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
-       /*FAULT_EVAC*/
+      /*FAULT_EVAC*/
       if(CmiNodeAlive(CkMyPe())){
          _sendStats();
-      }        
       _mainDone = 1; // This is needed because the destructors for
                      // readonly variables will be called when the program
                     // exits. If the destructor is called while _mainDone
@@ -521,6 +523,7 @@ static void _exitHandler(envelope *env)
 #if CMK_TRACE_ENABLED
       if (_ringexit) traceClose();
 #endif
+    }
       if (_ringexit) {
         int stride = CkNumPes()/_ringtoken;
         int pe = CkMyPe()+1;
@@ -531,10 +534,13 @@ static void _exitHandler(envelope *env)
       }
       else
         CmiFree(env);
+      //everyone exits here - there may be issues with leftover messages in the queue
       if(CkMyPe()){
-       DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
+        DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
         ConverseExit();
-      }        
+                               if(CharmLibInterOperate)
+                                       CpvAccess(charmLibExitFlag) = 1;
+      }
       break;
     case StatMsg:
       CkAssert(CkMyPe()==0);
@@ -546,8 +552,10 @@ static void _exitHandler(envelope *env)
                        /*FAULT_EVAC*/
       if(_numStatsRecd==CkNumValidPes()) {
         _printStats();
-       DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
+        DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
         ConverseExit();
+                               if(CharmLibInterOperate)
+                                       CpvAccess(charmLibExitFlag) = 1;
       }
       break;
     default:
@@ -834,6 +842,7 @@ void CkExit(void)
 #if ! CMK_BIGSIM_THREAD
   _TRACE_END_EXECUTE();
   //Wait for stats, which will call ConverseExit when finished:
+       if(!CharmLibInterOperate)
   CsdScheduler(-1);
 #endif
 }
@@ -1035,6 +1044,8 @@ void _initCharm(int unused_argc, char **argv)
        CkNumberHandlerEx(_initHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
        _roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler);
        _exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler);
+       //added for interoperabilitY
+       _libExitHandlerIdx = CkRegisterHandler((CmiHandler)_libExitHandler);
        _bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
        CkNumberHandlerEx(_bocHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
 
diff --git a/src/ck-core/mpi-interoperate.C b/src/ck-core/mpi-interoperate.C
new file mode 100644 (file)
index 0000000..62e1043
--- /dev/null
@@ -0,0 +1,81 @@
+#include "mpi-interoperate.h"
+
+static int   _libExitStarted = 0;
+int    _libExitHandlerIdx;
+
+// triger LibExit on PE 0,
+extern "C"
+void LibCkExit(void)
+{
+       // always send to PE 0
+       envelope *env = _allocEnv(StartExitMsg);
+       env->setSrcPe(CkMyPe());
+       CmiSetHandler(env, _libExitHandlerIdx);
+       CmiSyncSendAndFree(0, env->getTotalsize(), (char *)env);
+}
+
+void _libExitHandler(envelope *env)
+{
+       switch(env->getMsgtype()) {
+               case StartExitMsg:
+                       CkAssert(CkMyPe()==0);
+                       // else goto next
+               case ExitMsg:
+                       CkAssert(CkMyPe()==0);
+                       if(_libExitStarted) {
+                               CmiFree(env);
+                               return;
+                       }
+                       _libExitStarted = 1;
+                       env->setMsgtype(ReqStatMsg);
+                       env->setSrcPe(CkMyPe());
+                       // if exit in ring, instead of broadcasting, send in ring
+                       if (_ringexit){
+                               const int stride = CkNumPes()/_ringtoken;
+                               int pe = 0; while (pe<CkNumPes()) {
+                                       CmiSyncSend(pe, env->getTotalsize(), (char *)env);
+                                       pe += stride;
+                               }
+                               CmiFree(env);
+                       }else{
+                               CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
+                       }       
+                       break;
+               case ReqStatMsg:
+                       if (_ringexit) {
+                               int stride = CkNumPes()/_ringtoken;
+                               int pe = CkMyPe()+1;
+                               if (pe < CkNumPes() && pe % stride != 0)
+                                       CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
+                               else
+                                       CmiFree(env);
+                       }
+                       else
+                               CmiFree(env);
+                       //everyone exits here - there may be issues with leftover messages in the queue
+                       _libExitStarted = 0;
+                       CpvAccess(charmLibExitFlag) = 1;
+                       break;
+               default:
+                       CmiAbort("Internal Error(_libExitHandler): Unknown-msg-type. Contact Developers.\n");
+       }
+}
+
+void CharmLibInit(int peid, int numpes, int argc, char **argv){
+       //note CmiNumNodes and CmiMyNode should just be macros
+       _Cmi_numnodes = numpes;
+       _Cmi_mynode = peid;
+
+       CharmLibInterOperate = 1;
+       ConverseInit(argc, argv, (CmiStartFn)_initCharm, 1, 0);
+}
+
+#undef CkExit
+#define CkExit CkExit
+void CharmLibExit() {
+       if(CkMyPe() == 0) {
+               CkExit();
+       }
+       CsdScheduler(-1);
+}
+
diff --git a/src/ck-core/mpi-interoperate.h b/src/ck-core/mpi-interoperate.h
new file mode 100644 (file)
index 0000000..2d8126e
--- /dev/null
@@ -0,0 +1,20 @@
+#ifndef _MPI_INTEROPERATE_
+#define _MPI_INTEROPERATE_
+
+#include "converse.h"
+#include "ck.h"
+#include "trace.h"
+
+void CharmLibInit(int peid, int numpes, int argc, char **argv);
+void CharmLibExit();
+
+extern "C" void LibCkExit(void);
+
+extern int _ringexit;              // for charm exit
+extern int _ringtoken;
+extern void _initCharm(int unused_argc, char **argv);
+extern void CkExit(void);
+
+#define CkExit LibCkExit
+
+#endif //_MPI_INTEROPERATE_
diff --git a/src/ck-core/mpi-mainmodule.C b/src/ck-core/mpi-mainmodule.C
new file mode 100644 (file)
index 0000000..e604d13
--- /dev/null
@@ -0,0 +1,15 @@
+#include "mpi_main.decl.h"
+#include "mpi-interoperate.h"
+
+/*mainchare of mainmodule for interoperability*/
+class mpi_main : public CBase_mpi_main
+{
+public:
+  mpi_main(CkArgMsg* m) 
+  { 
+    delete m;
+    CkExit();
+  };
+};
+#include "mpi_main.def.h"
+
diff --git a/src/ck-core/mpi-mainmodule.ci b/src/ck-core/mpi-mainmodule.ci
new file mode 100644 (file)
index 0000000..cd2fb63
--- /dev/null
@@ -0,0 +1,6 @@
+mainmodule mpi_main {
+  mainchare mpi_main {
+    entry mpi_main(CkArgMsg *m);
+  };
+}
+
index 58af533bbe797480d9b43816faa6f69b7cc0192d..27886a21056aafc98fc4c040aaad09ec9600baec 100644 (file)
@@ -135,6 +135,9 @@ extern void CldModuleInit(char **);
 CpvDeclare(int,_curRestartPhase);
 static int CsdLocalMax = CSD_LOCAL_MAX_DEFAULT;
 
+int CharmLibInterOperate = 0;
+CpvDeclare(int,charmLibExitFlag);
+
 CpvStaticDeclare(int, CmiMainHandlerIDP); /* Main handler for _CmiMultipleSend that is run on every node */
 
 #if CMK_MEM_CHECKPOINT
@@ -1811,6 +1814,13 @@ void CsdScheduleForever(void)
   int isIdle=0;
   SCHEDULE_TOP
   while (1) {
+    /* The interoperation will cost this little overhead in scheduling */
+    if(CharmLibInterOperate) {
+      if(CpvAccess(charmLibExitFlag)) {
+        CpvAccess(charmLibExitFlag) = 0;
+        break;
+      }
+    }
     msg = CsdNextMessage(&state);
     if (msg!=NULL) { /*A message is available-- process it*/
       if (isIdle) {isIdle=0;CsdEndIdle();}
@@ -3498,6 +3508,9 @@ void ConverseCommonInit(char **argv)
   CpvInitialize(int, cmiArgDebugFlag);
   CpvAccess(cmiArgDebugFlag) = 0;
 #endif
+  CpvInitialize(int,charmLibExitFlag);
+  CpvAccess(charmLibExitFlag) = 0;
+
   CpvInitialize(int,_curRestartPhase);
   CpvAccess(_curRestartPhase)=1;
   CmiArgInit(argv);
index 7d322c78cee1c695a3397cdef6bd182b181316da..34673d222043d7587bf03ab98ff8be8f32b71126 100644 (file)
@@ -130,6 +130,8 @@ extern int CmiMyRank_();
 extern int _Cmi_mype;
 extern int _Cmi_numpes;
 extern int _Cmi_myrank; /* Normally zero; only 1 during SIGIO handling */
+extern int _Cmi_mynode;
+extern int _Cmi_numnodes;
 
 #define CmiMyPe()           _Cmi_mype
 #define CmiMyRank()         0
@@ -1919,4 +1921,10 @@ EXTERN void CmiNotifyCommThd(CmiNotifyCommThdMsg *msg);
 CpvCExtern(int, _urgentSend);
 #define CmiEnableUrgentSend(yn)   CpvAccess(_urgentSend)=(yn)
 
+/* CharmLibInterOperate should be a global variable as it will be
+ * set only once by MPI ranks respectively.
+ */
+extern int CharmLibInterOperate;
+CpvExtern(int,charmLibExitFlag);
+
 #endif /* CONVERSE_H */
index a3d844263109e7bd23956b33ca8642fe3f62cdc6..81627b0e64e0572f59b0c78a4fb89cdf38988703 100644 (file)
@@ -37,6 +37,7 @@ HybridBaseLB.decl.h HybridBaseLB.def.h: HybridBaseLB.ci.stamp
 HybridLB.decl.h HybridLB.def.h: HybridLB.ci.stamp
 LBDatabase.decl.h LBDatabase.def.h: LBDatabase.ci.stamp
 MetisLB.decl.h MetisLB.def.h: MetisLB.ci.stamp
+mpi_main.decl.h mpi_main.def.h: mpi-mainmodule.ci.stamp
 NborBaseLB.decl.h NborBaseLB.def.h: NborBaseLB.ci.stamp
 NeighborCommLB.decl.h NeighborCommLB.def.h: NeighborCommLB.ci.stamp
 NeighborLB.decl.h NeighborLB.def.h: NeighborLB.ci.stamp
@@ -50,6 +51,7 @@ RecBisectBfLB.decl.h RecBisectBfLB.def.h: RecBisectBfLB.ci.stamp
 RefineCommLB.decl.h RefineCommLB.def.h: RefineCommLB.ci.stamp
 RefineKLB.decl.h RefineKLB.def.h: RefineKLB.ci.stamp
 RefineLB.decl.h RefineLB.def.h: RefineLB.ci.stamp
+RefineSwapLB.decl.h RefineSwapLB.def.h: RefineSwapLB.ci.stamp
 RefineTopoLB.decl.h RefineTopoLB.def.h: RefineTopoLB.ci.stamp
 RotateLB.decl.h RotateLB.def.h: RotateLB.ci.stamp
 ScotchLB.decl.h ScotchLB.def.h: ScotchLB.ci.stamp
@@ -69,3 +71,4 @@ TraceUtilization.decl.h TraceUtilization.def.h: trace-utilization.ci.stamp
 TreeMatchLB.decl.h TreeMatchLB.def.h: TreeMatchLB.ci.stamp
 waitqd.decl.h waitqd.def.h: waitqd.ci.stamp
 WSLB.decl.h WSLB.def.h: WSLB.ci.stamp
+ZoltanLB.decl.h ZoltanLB.def.h: ZoltanLB.ci.stamp
index afc185f8ef01508c50addfdce772af9423091328..b9215c2cf6aa23856f44ba41901c3424bdd4e2be 100644 (file)
@@ -89,7 +89,7 @@ conv-ccs.o: conv-ccs.c converse.h conv-config.h conv-autoconfig.h \
  conv-common.h conv-mach.h conv-mach-opt.h pup_c.h queueing.h conv-cpm.h \
  conv-cpath.h conv-qd.h conv-random.h conv-lists.h conv-trace.h \
  persistent.h debug-conv.h conv-ccs.h sockRoutines.h ccs-server.h \
- ckhashtable.h pup.h
+ ckhashtable.h pup.h ccs-server.c ccs-auth.h ccs-auth.c
        $(CHARMC) -c -I. conv-ccs.c
 
 ccs-builtins.o: ccs-builtins.C converse.h conv-config.h conv-autoconfig.h \
@@ -326,10 +326,10 @@ trace-projector.o: trace-projector.C charm++.h charm.h converse.h \
  CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
  waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
- trace-projector.h ck.h qd.h register.h stats.h allEvents.h charmEvents.h \
- charmProjections.h traceCoreAPI.h converseEvents.h converseProjections.h \
machineEvents.h machineProjections.h traceCore.h threadEvents.h \
- traceCoreCommon.h trace-common.h trace-projections.h
+ trace-projector.h ck.h qd.h register.h stats.h TopoManager.h allEvents.h \
+ charmEvents.h charmProjections.h traceCoreAPI.h converseEvents.h \
converseProjections.h machineEvents.h machineProjections.h traceCore.h \
+ threadEvents.h traceCoreCommon.h trace-common.h trace-projections.h
        $(CHARMC) -c -I. trace-projector.C
 
 trace-converse.o: trace-converse.c conv-trace.h converse.h conv-config.h \
@@ -418,7 +418,8 @@ init.o: init.C ck.h charm++.h charm.h converse.h conv-config.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h qd.h register.h stats.h
+ trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
+ TopoManager.h
        $(CHARMC) -c -I. init.C
 
 register.o: register.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -435,7 +436,8 @@ register.o: register.C ck.h charm++.h charm.h converse.h conv-config.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h qd.h register.h stats.h
+ trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
+ TopoManager.h
        $(CHARMC) -c -I. register.C
 
 qd.o: qd.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -452,7 +454,8 @@ qd.o: qd.C ck.h charm++.h charm.h converse.h conv-config.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h qd.h register.h stats.h
+ trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
+ TopoManager.h
        $(CHARMC) -c -I. qd.C
 
 ck.o: ck.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -469,8 +472,9 @@ ck.o: ck.C ck.h charm++.h charm.h converse.h conv-config.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h qd.h register.h stats.h crc32.h \
- trace-common.h BaseLB.h ckliststring.h CkMarshall.def.h
+ trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
+ TopoManager.h crc32.h trace-common.h BaseLB.h ckliststring.h \
+ CkMarshall.def.h
        $(CHARMC) -c -I. ck.C
 
 main.o: main.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -487,7 +491,8 @@ main.o: main.C ck.h charm++.h charm.h converse.h conv-config.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h qd.h register.h stats.h
+ trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
+ TopoManager.h
        $(CHARMC) -c -I. main.C
 
 msgalloc.o: msgalloc.C ck.h charm++.h charm.h converse.h conv-config.h \
@@ -504,7 +509,8 @@ msgalloc.o: msgalloc.C ck.h charm++.h charm.h converse.h conv-config.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h qd.h register.h stats.h
+ trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
+ TopoManager.h
        $(CHARMC) -c -I. msgalloc.C
 
 ckfutures.o: ckfutures.C charm++.h charm.h converse.h conv-config.h \
@@ -522,7 +528,7 @@ ckfutures.o: ckfutures.C charm++.h charm.h converse.h conv-config.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
  trace.h trace-bluegene.h envelope.h ck.h qd.h register.h stats.h \
- CkFutures.def.h
TopoManager.h CkFutures.def.h
        $(CHARMC) -c -I. ckfutures.C
 
 ckIgetControl.o: ckIgetControl.C charm++.h charm.h converse.h \
@@ -558,7 +564,7 @@ debug-message.o: debug-message.C ck.h charm++.h charm.h converse.h \
  CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
  waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
- qd.h register.h stats.h envelope.h queueing.h
+ qd.h register.h stats.h TopoManager.h envelope.h queueing.h
        $(CHARMC) -c -I. debug-message.C
 
 debug-charm.o: debug-charm.C converse.h conv-config.h conv-autoconfig.h \
@@ -575,7 +581,7 @@ debug-charm.o: debug-charm.C converse.h conv-config.h conv-autoconfig.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h qd.h stats.h
+ trace.h trace-bluegene.h envelope.h qd.h stats.h TopoManager.h
        $(CHARMC) -c -I. debug-charm.C
 
 ckcallback.o: ckcallback.C charm++.h charm.h converse.h conv-config.h \
@@ -629,7 +635,8 @@ ckarray.o: ckarray.C charm++.h charm.h converse.h conv-config.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
  trace.h trace-bluegene.h envelope.h register.h ck.h qd.h stats.h \
- pathHistory.h PathHistory.decl.h pup_stl.h pup.h CkArray.def.h
+ TopoManager.h pathHistory.h PathHistory.decl.h pup_stl.h pup.h \
+ CkArray.def.h
        $(CHARMC) -c -I. ckarray.C
 
 ckreduction.o: ckreduction.C charm++.h charm.h converse.h conv-config.h \
@@ -647,7 +654,7 @@ ckreduction.o: ckreduction.C charm++.h charm.h converse.h conv-config.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
  trace.h trace-bluegene.h envelope.h ck.h qd.h register.h stats.h \
- CkReduction.def.h
TopoManager.h CkReduction.def.h
        $(CHARMC) -c -I. ckreduction.C
 
 ckarrayreductionmgr.o: ckarrayreductionmgr.C charm++.h charm.h converse.h \
@@ -665,7 +672,7 @@ ckarrayreductionmgr.o: ckarrayreductionmgr.C charm++.h charm.h converse.h \
  CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
  waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
- ck.h qd.h register.h stats.h CkArrayReductionMgr.def.h
+ ck.h qd.h register.h stats.h TopoManager.h CkArrayReductionMgr.def.h
        $(CHARMC) -c -I. ckarrayreductionmgr.C
 
 tempo.o: tempo.C charm++.h charm.h converse.h conv-config.h \
@@ -841,7 +848,7 @@ ckcheckpoint.o: ckcheckpoint.C charm++.h charm.h converse.h conv-config.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
  trace.h trace-bluegene.h envelope.h ck.h qd.h register.h stats.h \
- CkCheckpoint.def.h
TopoManager.h CkCheckpoint.def.h
        $(CHARMC) -c -I. ckcheckpoint.C
 
 ckmemcheckpoint.o: ckmemcheckpoint.C charm++.h charm.h converse.h \
@@ -859,7 +866,7 @@ ckmemcheckpoint.o: ckmemcheckpoint.C charm++.h charm.h converse.h \
  CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
  waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
- ck.h qd.h register.h stats.h CkMemCheckpoint.def.h
+ ck.h qd.h register.h stats.h TopoManager.h CkMemCheckpoint.def.h
        $(CHARMC) -c -I. ckmemcheckpoint.C
 
 ckevacuation.o: ckevacuation.C charm++.h charm.h converse.h conv-config.h \
@@ -876,7 +883,8 @@ ckevacuation.o: ckevacuation.C charm++.h charm.h converse.h conv-config.h \
  CkLocation.decl.h CkArray.decl.h ckfutures.h CkFutures.decl.h charisma.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
- trace.h trace-bluegene.h envelope.h ck.h qd.h register.h stats.h
+ trace.h trace-bluegene.h envelope.h ck.h qd.h register.h stats.h \
+ TopoManager.h
        $(CHARMC) -c -I. ckevacuation.C
 
 ckmessagelogging.o: ckmessagelogging.C charm.h converse.h conv-config.h \
@@ -894,8 +902,8 @@ ckmessagelogging.o: ckmessagelogging.C charm.h converse.h conv-config.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
  trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
ckmessagelogging.h ckobjid.h CentralLB.h BaseLB.h CentralLB.decl.h \
- CentralLBMsg.h
TopoManager.h ckmessagelogging.h ckobjid.h CentralLB.h BaseLB.h \
+ CentralLB.decl.h CentralLBMsg.h
        $(CHARMC) -c -I. ckmessagelogging.C
 
 ckcausalmlog.o: ckcausalmlog.C charm.h converse.h conv-config.h \
@@ -913,8 +921,8 @@ ckcausalmlog.o: ckcausalmlog.C charm.h converse.h conv-config.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
  trace.h trace-bluegene.h envelope.h qd.h register.h stats.h \
ckmessagelogging.h ckobjid.h CentralLB.h BaseLB.h CentralLB.decl.h \
- CentralLBMsg.h
TopoManager.h ckcausalmlog.h ckobjid.h CentralLB.h BaseLB.h \
+ CentralLB.decl.h CentralLBMsg.h
        $(CHARMC) -c -I. ckcausalmlog.C
 
 LBDBManager.o: LBDBManager.C charm++.h charm.h converse.h conv-config.h \
@@ -1028,10 +1036,10 @@ CentralLB.o: CentralLB.C charm++.h charm.h converse.h conv-config.h \
  charisma.decl.h tempo.h tempo.decl.h waitqd.h waitqd.decl.h sdag.h \
  ckcheckpoint.h CkCheckpoint.decl.h ckevacuation.h ckarrayreductionmgr.h \
  trace.h trace-bluegene.h envelope.h ck.h charm++.h envelope.h qd.h \
- register.h stats.h ckfutures.h charisma.h CentralLB.h BaseLB.h \
LBDatabase.h CentralLB.decl.h BaseLB.decl.h LBDatabase.decl.h \
CentralLBMsg.h LBDBManager.h LBSimulation.h ComlibStrategy.h \
ckhashtable.h pup.h convcomlibstrategy.h converse.h cklists.h \
+ register.h stats.h ckfutures.h charisma.h TopoManager.h converse.h \
CentralLB.h BaseLB.h LBDatabase.h CentralLB.decl.h BaseLB.decl.h \
LBDatabase.decl.h CentralLBMsg.h LBDBManager.h LBSimulation.h \
ComlibStrategy.h ckhashtable.h pup.h convcomlibstrategy.h cklists.h \
  ComlibLearner.h CentralLB.def.h
        $(CHARMC) -c -I. CentralLB.C
 
@@ -1219,7 +1227,8 @@ NullLB.o: NullLB.C NullLB.h BaseLB.h LBDatabase.h lbdb.h converse.h \
  waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
  NullLB.decl.h BaseLB.decl.h NullLB.decl.h ck.h charm++.h envelope.h qd.h \
- register.h stats.h ckfutures.h charisma.h NullLB.def.h
+ register.h stats.h ckfutures.h charisma.h TopoManager.h converse.h \
+ NullLB.def.h
        $(CHARMC) -c -I. NullLB.C
 
 LBSimulation.o: LBSimulation.C LBSimulation.h charm++.h charm.h \
@@ -1292,7 +1301,7 @@ charmProjections.o: charmProjections.C converse.h conv-config.h \
  CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
  waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
- qd.h register.h stats.h trace-common.h
+ qd.h register.h stats.h TopoManager.h trace-common.h
        $(CHARMC) -c -I. charmProjections.C
 
 cktiming.o: cktiming.C charm++.h charm.h converse.h conv-config.h \
@@ -1367,9 +1376,9 @@ controlPoints.o: controlPoints.C charm++.h charm.h converse.h \
  controlPoints.h conv-config.h ControlPoints.decl.h charm++.h pup_stl.h \
  charm-api.h LBDatabase.h arrayRedistributor.h pathHistory.h \
  PathHistory.decl.h envelope.h cp_effects.h ck.h qd.h register.h stats.h \
ckarray.h trace-controlPoints.h trace.h register.h trace-common.h \
- trace-projections.h ckhashtable.h pup.h pathHistory.h cp_effects.h \
- ControlPoints.def.h
TopoManager.h ckarray.h trace-controlPoints.h trace.h register.h \
+ trace-common.h trace-projections.h ckhashtable.h pup.h pathHistory.h \
cp_effects.h ControlPoints.def.h
        $(CHARMC) -c -I. controlPoints.C
 
 arrayRedistributor.o: arrayRedistributor.C charm++.h charm.h converse.h \
@@ -1390,7 +1399,8 @@ arrayRedistributor.o: arrayRedistributor.C charm++.h charm.h converse.h \
  trace-controlPoints.h trace.h envelope.h register.h trace-common.h \
  LBDatabase.h controlPoints.h conv-config.h ControlPoints.decl.h \
  charm++.h pup_stl.h charm-api.h arrayRedistributor.h pathHistory.h \
- PathHistory.decl.h cp_effects.h ck.h qd.h register.h stats.h ckarray.h
+ PathHistory.decl.h cp_effects.h ck.h qd.h register.h stats.h \
+ TopoManager.h ckarray.h
        $(CHARMC) -c -I. arrayRedistributor.C
 
 cp_effects.o: cp_effects.C cp_effects.h charm++.h charm.h converse.h \
@@ -1408,7 +1418,7 @@ cp_effects.o: cp_effects.C cp_effects.h charm++.h charm.h converse.h \
  CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
  waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
  ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
- ck.h qd.h register.h stats.h ckarray.h
+ ck.h qd.h register.h stats.h TopoManager.h ckarray.h
        $(CHARMC) -c -I. cp_effects.C
 
 trace-controlPoints.o: trace-controlPoints.C charm++.h charm.h converse.h \
@@ -1430,6 +1440,24 @@ trace-controlPoints.o: trace-controlPoints.C charm++.h charm.h converse.h \
  TraceControlPoints.decl.h TraceControlPoints.def.h
        $(CHARMC) -c -I. trace-controlPoints.C
 
+mpi-interoperate.o: mpi-interoperate.C mpi-interoperate.h converse.h \
+ conv-config.h conv-autoconfig.h conv-common.h conv-mach.h \
+ conv-mach-opt.h pup_c.h queueing.h conv-cpm.h conv-cpath.h conv-qd.h \
+ conv-random.h conv-lists.h conv-trace.h persistent.h debug-conv.h ck.h \
+ charm++.h charm.h pup.h middle.h middle-conv.h cklists.h ckbitvector.h \
+ ckstream.h init.h ckhashtable.h debug-charm.h debug-conv++.h simd.h \
+ CkMarshall.decl.h ckarrayindex.h cksection.h ckcallback.h conv-ccs.h \
+ sockRoutines.h ccs-server.h ckobjQ.h ckreduction.h CkReduction.decl.h \
+ CkArrayReductionMgr.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h \
+ readonly.h ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h \
+ LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h \
+ NullLB.decl.h BaseLB.decl.h CkLocation.decl.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
+ waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
+ ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
+ qd.h register.h stats.h TopoManager.h
+       $(CHARMC) -c -I. mpi-interoperate.C
+
 EveryLB.o: EveryLB.C LBDatabase.h lbdb.h converse.h conv-config.h \
  conv-autoconfig.h conv-common.h conv-mach.h conv-mach-opt.h pup_c.h \
  queueing.h conv-cpm.h conv-cpath.h conv-qd.h conv-random.h conv-lists.h \
@@ -2345,9 +2373,9 @@ ChunkMulticastStrategy.o: ChunkMulticastStrategy.C \
  trace.h conv-trace.h trace-bluegene.h envelope.h convcomlibstrategy.h \
  ComlibLearner.h ComlibArrayListener.h ComlibStats.h comlib.decl.h \
  ComlibSectionInfo.h queueing.h ck.h qd.h register.h stats.h \
spanningTreeStrategy.h spanningTreeVertex.h treeStrategy_topoUnaware.h \
- treeStrategy_nodeAware_minGens.h treeStrategy_nodeAware_minBytes.h \
- treeStrategy_3dTorus_minHops.h TopoManager.h \
TopoManager.h spanningTreeStrategy.h spanningTreeVertex.h \
+ treeStrategy_topoUnaware.h treeStrategy_nodeAware_minGens.h \
+ treeStrategy_nodeAware_minBytes.h treeStrategy_3dTorus_minHops.h \
  treeStrategy_3dTorus_minBytesHops.h
        $(CHARMC) -c -I. ChunkMulticastStrategy.C
 
@@ -2684,7 +2712,7 @@ ComlibManager.o: ComlibManager.C ComlibManager.h convcomlibmanager.h \
  EachToManyMulticastStrategy.h routerstrategy.h router.h \
  DirectMulticastStrategy.h MulticastStrategy.h RingMulticastStrategy.h \
  MultiRingMulticast.h RectMulticastStrategy.h ck.h qd.h register.h \
- stats.h comlib.def.h
+ stats.h TopoManager.h comlib.def.h
        $(CHARMC) -c -I. ComlibManager.C
 
 ComlibStrategy.o: ComlibStrategy.C ComlibStrategy.h charm++.h charm.h \
@@ -2925,3 +2953,22 @@ prefixrouter.o: prefixrouter.C prefixrouter.h converse.h conv-config.h \
  ComlibLearner.h
        $(CHARMC) -c -I. prefixrouter.C
 
+mpi-mainmodule.o: mpi-mainmodule.C mpi_main.decl.h charm++.h charm.h \
+ converse.h conv-config.h conv-autoconfig.h conv-common.h conv-mach.h \
+ conv-mach-opt.h pup_c.h queueing.h conv-cpm.h conv-cpath.h conv-qd.h \
+ conv-random.h conv-lists.h conv-trace.h persistent.h debug-conv.h pup.h \
+ middle.h middle-conv.h cklists.h ckbitvector.h ckstream.h init.h \
+ ckhashtable.h debug-charm.h debug-conv++.h simd.h CkMarshall.decl.h \
+ ckarrayindex.h cksection.h ckcallback.h conv-ccs.h sockRoutines.h \
+ ccs-server.h ckobjQ.h ckreduction.h CkReduction.decl.h \
+ CkArrayReductionMgr.decl.h ckmemcheckpoint.h CkMemCheckpoint.decl.h \
+ readonly.h ckarray.h cklocation.h LBDatabase.h lbdb.h LBDBManager.h \
+ LBObj.h LBOM.h LBComm.h LBMachineUtil.h lbdb++.h LBDatabase.decl.h \
+ NullLB.decl.h BaseLB.decl.h CkLocation.decl.h CkArray.decl.h ckfutures.h \
+ CkFutures.decl.h charisma.h charisma.decl.h tempo.h tempo.decl.h \
+ waitqd.h waitqd.decl.h sdag.h ckcheckpoint.h CkCheckpoint.decl.h \
+ ckevacuation.h ckarrayreductionmgr.h trace.h trace-bluegene.h envelope.h \
+ mpi-interoperate.h ck.h qd.h register.h stats.h TopoManager.h \
+ mpi_main.def.h
+       $(CHARMC) -c -I. mpi-mainmodule.C
+
index 1f96f1936e638fdb145fb123b9e4fa5635334275..ad34d34bf4cce6e96a73740cdbc3ca16a00322c4 100644 (file)
@@ -65,7 +65,7 @@ ARMCI: charm++ LIBS
 
 f90charm: charm++ $(L)/libf90charm.a
 
-default_libs: completion multicast pythonCCS
+default_libs: completion multicast pythonCCS mpi-main
 
 ChaNGa: charm++ cache liveViz
 
@@ -84,6 +84,9 @@ pythonCCS: charmxi
 completion: charmxi
        cd libs/ck-libs/completion && $(MAKE)
 
+mpi-main: mpi-mainmodule.o
+       cp mpi-mainmodule.o ../lib
+
 translators: charmxi conv-cpm
 
 pose:  charm++
@@ -251,6 +254,7 @@ CKHEADERS=ck.h ckstream.h envelope.h init.h qd.h charm.h charm++.h \
          pathHistory.h envelope-path.h \
           controlPoints.h controlPointsf.h arrayRedistributor.h cp_effects.h register.h stats.h   \
          cksequence_internal.h cksequence_factory.h random_sequence.h strided_sequence.h \
+               mpi-interoperate.h \
          $(CVHEADERS) $(CONVCOMHEADERS)
 
 ALLHEADERS=$(CKHEADERS) \
@@ -713,14 +717,15 @@ LIBCK_CORE=trace-common.o tracec.o tracef.o init.o register.o qd.o ck.o main.o \
            RefinerComm.o bitvecset.o ckset.o ckheap.o CommLBHeap.o \
           NullLB.o LBSimulation.o $(COMLIB_CORE_OBJS) modifyScheduler.o \
           charmProjections.o cktiming.o ckbitvector.o \
-           pathHistory.o controlPoints.o arrayRedistributor.o cp_effects.o trace-controlPoints.o
+           pathHistory.o controlPoints.o arrayRedistributor.o cp_effects.o \
+                                        trace-controlPoints.o mpi-interoperate.o
 
 charm-target: converse $(L)/libck.a $(L)/libckf.a loadbalancers default_libs comlib_objs 
 
-CHARMLIBS: charm++ CONVLIBS
+CHARMLIBS: charm++ CONVLIBS 
        cd libs; $(MAKE) charmlibs OPTS='$(OPTS)'
 
-$(L)/libck.a: $(LIBCK_CORE)
+$(L)/libck.a: $(LIBCK_CORE) 
        $(CHARMC) -o $@ $(LIBCK_CORE)
 
 ## Charm fortran AMPI
@@ -821,7 +826,6 @@ TRACE_OBJS =  trace-projections.o  trace-summary.o  trace-simple.o \
              trace-bluegene.o trace-projector.o trace-converse.o trace-all.o \
           trace-memory.o
 
-
 ###############################################################################
 #
 # Dynamic Load Balancers
@@ -999,6 +1003,13 @@ $(L)/libf90charm.a: charmxi f90main.ci f90main.h f90main.C futil.f90
        -$(CHARMC) -c futil.f90
        -$(CHARMC) -o $@ f90main.o futil.o
 
+##############################################################################
+# adding this entry for files which do not belong to any of other locations
+
+OTHER_OBJS= mpi-mainmodule.o
+
+##############################################################################
+
 ###############################################################################
 #
 # Make clean
@@ -1071,7 +1082,7 @@ depends: cidepends commitid.c
        echo '#generated by make depends' > $(DEPENDFILE); \
         touch metis.h scotch.h;   \
        processFiles='';        \
-        for i in $(LIBCONV_CORE) $(TRACE_OBJS) $(LIBCK_CORE) $(ALL_LB_OBJS) $(LIBBLUE_OBJS) $(COMLIB_OBJS) ; do \
+        for i in $(LIBCONV_CORE) $(TRACE_OBJS) $(LIBCK_CORE) $(ALL_LB_OBJS) $(LIBBLUE_OBJS) $(COMLIB_OBJS) $(OTHER_OBJS) ; do \
              processed='false';        \
              for f in $$processFiles; do       \
                if test $$i = $$f ; then        \
@@ -1098,7 +1109,7 @@ dependsCaseSensitive: cidepends
        echo '#generated by make depends' > $(DEPENDFILE); \
         touch metis.h scotch.h;   \
        processFiles='';        \
-        for i in $(LIBCONV_CORE) $(TRACE_OBJS) $(LIBCK_CORE) $(ALL_LB_OBJS) $(LIBBLUE_OBJS) $(COMLIB_OBJS); do \
+        for i in $(LIBCONV_CORE) $(TRACE_OBJS) $(LIBCK_CORE) $(ALL_LB_OBJS) $(LIBBLUE_OBJS) $(COMLIB_OBJS) $(OTHER_OBJS); do \
              processed='false';        \
              for f in $$processFiles; do       \
                if test $$i = $$f ; then        \
index a2edbe1a6e096fbf41c3ce7c4583964e459232f5..630e0370c37589286d369b1b4b90b9acc3c13fa9 100755 (executable)
@@ -96,6 +96,8 @@ CMK_CELL=0
 SPU_USE_CPP=0
 SPU_EXTRA_LIBS=""
 
+MPI_INTEROPERATE=""
+
 ####################################################################
 #
 #  Utility routines used below
@@ -763,6 +765,9 @@ do
        -nof90main)
                F90_MAIN=
                ;;
+       -mpi)
+               MPI_INTEROPERATE="yes"
+               ;;
        -*|+*)
 #              Warning "passing unrecognized option $arg to all compilers and linkers"
                OPTS="$OPTS  $arg"
@@ -1510,6 +1515,11 @@ then
         fi
 fi
 
+if [ "$MPI_INTEROPERATE" != "" ]
+then
+       OBJECTFILES="$CHARMLIB/mpi-mainmodule.o $OBJECTFILES"
+fi
+
 # check if we're linking a sequential object
 case "$LANGUAGE" in
 "c"|"C"|"f90"|"f77")