Make the case that work threads send msgs optional if MPI_THREAD_MULTIPLE is provided...
authorChao Mei <chaomei2@illinois.edu>
Tue, 3 Jan 2012 06:36:29 +0000 (00:36 -0600)
committerChao Mei <chaomei2@illinois.edu>
Tue, 3 Jan 2012 06:51:02 +0000 (00:51 -0600)
src/arch/bluegenep/machine.c
src/arch/gemini_gni/machine.c
src/arch/mpi/machine.c
src/arch/util/machine-common-core.c

index 1a0413ca4e673aaf688e45ae11b119147f54e3b6..735747c22ff84d205532147e6174f3286351fc59 100644 (file)
@@ -732,6 +732,7 @@ static void MachineInitForDCMF(int *argc, char ***argv, int *numNodes, int *myNo
 
     DCMF_Messager_configure(&config_in, &config_out);
     //assert (config_out.thread_level == DCMF_THREAD_MULTIPLE); //not supported in vn mode
+    Cmi_smp_mode_setting = COMM_THREAD_ONLY_RECV;
 #endif
 
     DCMF_Send_Configuration_t short_config, eager_config, rzv_config;
index 76d42d89404c629158591aee7e552e64635559f4..1c379631b2e006bb43392ddc3da8ae4498b94cd4 100644 (file)
@@ -2246,6 +2246,11 @@ void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID)
     *myNodeID = myrank;
     *numNodes = mysize;
   
+#if !COMM_THREAD_SEND
+    /* Currently, we only consider the case that comm. thread will only recv msgs */
+    Cmi_smp_mode_setting = COMM_THREAD_ONLY_RECV;
+#endif
+
     useDynamicSMSG = CmiGetArgFlag(*argv, "+useDynamicSmsg");
     CmiGetArgIntDesc(*argv, "+smsgConnection", &avg_smsg_connection,"Initial number of SMSGS connection per code");
     if (avg_smsg_connection>mysize) avg_smsg_connection = mysize;
index 54d0f8162ef453bfb603dcc303ecbd76b3c37dc8..20ca297ac986c3360f23f175d8fb2c3f81ca6c9c 100644 (file)
@@ -388,7 +388,7 @@ static CmiCommHandle MachineSpecificSendForMPI(int destNode, int size, char *msg
 
     CmiAssert(destNode != CmiMyNode());
 #if CMK_SMP
-    if (_thread_provided != MPI_THREAD_MULTIPLE) {
+    if (Cmi_smp_mode_setting == COMM_THREAD_SEND_RECV) {
       EnqueueMsg(msg, size, destNode, mode);
       return 0;
     }
@@ -860,9 +860,12 @@ static void MachinePostNonLocalForMPI() {
     }
 #endif
 #else
-  if (_thread_provided == MPI_THREAD_MULTIPLE) {
-        CmiReleaseSentMessages();
-        SendMsgBuf();
+  if (Cmi_smp_mode_setting == COMM_THREAD_ONLY_RECV) {
+        CmiReleaseSentMessages();       
+        /* ??? SendMsgBuf is a not a thread-safe function. If it is put
+         * here and this function will be called in CmiNotifyStillIdle,
+         * then a data-race problem occurs */
+        /*SendMsgBuf();*/
   }
 #endif
 }
@@ -899,10 +902,16 @@ void DrainResourcesForMPI() {
         CmiReleaseSentMessages();
     }
 #else
-    while (!MsgQueueEmpty() || !CmiAllAsyncMsgsSent()) {
-        CmiReleaseSentMessages();
-        SendMsgBuf();
-        PumpMsgs();
+    if(Cmi_smp_mode_setting == COMM_THREAD_SEND_RECV){
+        while (!MsgQueueEmpty() || !CmiAllAsyncMsgsSent()) {
+           CmiReleaseSentMessages();
+            SendMsgBuf();
+            PumpMsgs();
+        }
+    }else if(Cmi_smp_mode_setting == COMM_THREAD_ONLY_RECV) {
+        while(!CmiAllAsyncMsgsSent()) {
+            CmiReleaseSentMessages();
+        }
     }
 #endif
 #if CMK_MEM_CHECKPOINT
@@ -1072,6 +1081,23 @@ static void MachineInitForMPI(int *argc, char ***argv, int *numNodes, int *myNod
 
     myNID = *myNodeID;
 
+#if CMK_SMP
+    if (provided == MPI_THREAD_MULTIPLE) {
+        int smpmode =  0;
+        Cmi_smp_mode_setting = COMM_THREAD_SEND_RECV; /* the default value */
+        smpmode = CmiGetArgFlag(largv, "+comm_thread_only_recv");
+        if (smpmode) {
+            Cmi_smp_mode_setting = COMM_THREAD_ONLY_RECV;
+        } else {
+#if 0
+            /* TODO: Not supported yet! Needs some re-design -Chao Mei */
+            smpmode = CmiGetArgFlag(largv, "+no_comm_thread");
+            if (smpmode) Cmi_smp_mode_setting = COMM_THREAD_NOT_EXIST;
+#endif
+        }
+    }
+#endif
+
     MPI_Get_version(&ver, &subver);
     if (myNID == 0) {
         printf("Charm++> Running on MPI version: %d.%d multi-thread support: %s (max supported: %s)\n", ver, subver, thread_level_tostring(thread_level), thread_level_tostring(provided));
@@ -1303,7 +1329,7 @@ static void MachinePostCommonInitForMPI(int everReturn) {
 #if CMK_SMP
     CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE,(CcdVoidFn)CmiNotifyBeginIdle,(void *)s);
     CcdCallOnConditionKeep(CcdPROCESSOR_STILL_IDLE,(CcdVoidFn)CmiNotifyStillIdle,(void *)s);
-    if (_thread_provided == MPI_THREAD_MULTIPLE)
+    if (Cmi_smp_mode_setting == COMM_THREAD_ONLY_RECV)
       CcdCallOnConditionKeep(CcdPERIODIC,(CcdVoidFn)LrtsPostNonLocal,NULL);
 #else
     CcdCallOnConditionKeep(CcdPROCESSOR_STILL_IDLE,(CcdVoidFn)CmiNotifyIdleForMPI,NULL);
index 019ff54f255f67d5b2f02b5f26c994244cf8edad..eca5119bc4b3513eefb63a8926525f47bb6fd86e 100644 (file)
@@ -144,6 +144,16 @@ CpvDeclare(void*, CmiLocalQueue);
 #define BCAST_SYNC 0x3
 #define BCAST_ASYNC 0x4
 
+enum MACHINE_SMP_MODE {
+    INVALID_MODE,
+    COMM_THREAD_SEND_RECV = 0,
+    COMM_THREAD_ONLY_RECV, /* work threads will do the send */
+    COMM_THREAD_NOT_EXIST /* work threads will do both send and recv */
+};
+/* The default mode of smp charm runtime */
+static enum MACHINE_SMP_MODE Cmi_smp_mode_setting = COMM_THREAD_SEND_RECV;
+
+
 #if CMK_SMP
 static volatile int commThdExit = 0;
 static CmiNodeLock  commThdExitLock = 0;
@@ -611,12 +621,22 @@ if (  MSG_STATISTIC)
 
     LrtsInit(&argc, &argv, &_Cmi_numnodes, &_Cmi_mynode);
    
-if (_Cmi_mynode==0) 
+       if (_Cmi_mynode==0) {
 #if !CMK_SMP 
-    printf("Charm++> Running on Non-smp mode\n");
+               printf("Charm++> Running on Non-smp mode\n");
 #else
-    printf("Charm++> Running on SMP mode, %d worker threads per process\n", _Cmi_mynodesize);
-#endif
+               printf("Charm++> Running on SMP mode, %d worker threads per process\n", _Cmi_mynodesize);
+               if (Cmi_smp_mode_setting == COMM_THREAD_SEND_RECV) {
+                       printf("Charm++ SMP>The comm. thread will both send and receive messages\n");
+               } else if (Cmi_smp_mode_setting == COMM_THREAD_ONLY_RECV) {
+                       printf("Charm++ SMP>The comm. thread will only receive messages, while work threads will send messages\n");
+               } else if (Cmi_smp_mode_setting == COMM_THREAD_NOT_EXIST) {
+                       printf("Charm++ SMP> There's no comm. thread. Work threads will both send and receive messages\n");
+               } else {
+                       CmiAbort("Charm++ SMP> Invalid SMP mode setting\n");
+               }
+#endif
+       }
 
     _Cmi_numpes = _Cmi_numnodes * _Cmi_mynodesize;
     Cmi_nodestart = _Cmi_mynode * _Cmi_mynodesize;
@@ -785,6 +805,10 @@ void ConverseExit(void) {
     int i;
 #if !CMK_SMP
     LrtsDrainResources();
+#else
+       if(Cmi_smp_mode_setting == COMM_THREAD_ONLY_RECV
+          || Cmi_smp_mode_setting == COMM_THREAD_NOT_EXIST)
+               LrtsDrainResources();
 #endif
 
     ConverseCommonExit();