b03fba224b2cc6f252d72d71888bd41a9ed491c3
[charm.git] / src / ck-com / ComlibManager.h
1 #ifndef COMMLIBMANAGER_H
2 #define COMMLIBMANAGER_H
3
4 #include "charm++.h"
5 #include "cksection.h"
6 #include "envelope.h"
7 #include "convcomlib.h"
8 #include <math.h>
9
10 #include "charm++.h"
11 #include "convcomlibmanager.h"
12
13 #define CHARM_MPI 0 
14 #define MAX_NSTRAT 1024
15 #define LEARNING_PERIOD 1000 //Number of iterations after which the
16                              //learning framework will discover 
17                              //the appropriate strategy, not completely 
18                              //implemented
19 PUPbytes(comID);
20
21 #include "ComlibStats.h"
22
23 #include "comlib.decl.h"
24
25 //Dummy message to be sent incase there are no messages to send. 
26 //Used by only the EachToMany strategy!
27 class ComlibDummyMsg: public CMessage_ComlibDummyMsg {
28     int dummy;
29 };
30
31 /*
32 //Priority message to call end iteration
33 class PrioMsg: public CMessage_PrioMsg {
34  public:
35     int instID;
36 };
37 */
38
39 /**
40  * Structure used to hold a count of the indeces associated to each pe in a multicast message.
41  */
42 struct ComlibMulticastIndexCount {
43   int pe;
44   int count;
45 };
46
47 ///for use of qsort
48 inline int indexCountCompare(const void *a, const void *b) {
49   ComlibMulticastIndexCount a1 = *(ComlibMulticastIndexCount*) a;
50   ComlibMulticastIndexCount b1 = *(ComlibMulticastIndexCount*) b;
51
52   if(a1.pe < b1.pe)
53     return -1;
54   
55   if(a1.pe == b1.pe)
56     return 0;
57   
58   if(a1.pe > b1.pe)
59     return 1;
60   
61   return 0;
62 }
63
64 class ComlibMulticastMsg : public CkMcastBaseMsg, 
65                public CMessage_ComlibMulticastMsg {
66     
67   public:
68     int nPes;
69     ComlibMulticastIndexCount *indicesCount;
70     CkArrayIndexMax *indices;
71     char *usrMsg;        
72 };
73
74 class ComlibManager;
75
76 //An Instance of the communication library.
77 class ComlibInstanceHandle : public CkDelegateData {
78  private:    
79     int _instid;
80     CkGroupID _dmid;
81     int _srcPe;
82     int toForward;
83
84  public:
85     ComlibInstanceHandle();
86     ComlibInstanceHandle(const ComlibInstanceHandle &h);
87     ComlibInstanceHandle(int instid, CkGroupID dmid);    
88    
89     ComlibInstanceHandle &operator=(const ComlibInstanceHandle &h);
90
91     void setForwardingOnMigration(){toForward = 1;} 
92     void beginIteration();
93     void endIteration();
94     
95     CkGroupID getComlibManagerID();
96     void setStrategy(CharmStrategy *);
97     CharmStrategy *getStrategy();        
98     int getSourcePe() {return _srcPe;}
99
100     void setSourcePe() {_srcPe = CkMyPe();}
101
102     friend class ComlibManager;
103     void pup(PUP::er &p) {
104
105         if(p.isUnpacking())
106              reset();        
107
108         p | _instid;
109         p | _dmid;
110         p | _srcPe;
111         p | toForward;
112     }
113 };
114
115 class LBMigrateMsg;
116
117 class ComlibManager: public CkDelegateMgr {
118     friend class ComlibInstanceHandle;
119
120     int *bcast_pelist;  //Pelist passed to all broadcast operations
121
122     int section_send_event;
123
124     int remotePe;
125     CmiBool isRemote;
126     CmiBool strategyCreated;
127
128     int npes;
129     int *pelist;
130
131     CkArrayIndexMax dummyArrayIndex;
132
133     //For compatibility and easier use!
134     int strategyID; //Identifier of the strategy
135
136     //Pointer to the converse comm lib strategy table
137     StrategyTable *strategyTable;
138
139     CkQ<CharmStrategy *> ListOfStrategies; //temporary list of strategies
140     
141     CkQ<CharmMessageHolder *> remoteQ;  //list of remote messages
142                                         //after the object has
143                                         //migrated
144
145     //The number of strategies created by the user
146     //int nstrats; //now part of conv comlib
147     
148     int curStratID, prevStratID;      
149     //Number of strategies created by the user.
150
151     //flags
152     int receivedTable, setupComplete, barrierReached, barrier2Reached;
153     CmiBool lbUpdateReceived;
154
155     int bcount , b2count;
156     //int totalMsgCount, totalBytes, nIterations;
157
158     ComlibArrayListener *alistener;
159     int prioEndIterationFlag;
160
161     ComlibGlobalStats clib_gstats; 
162     int    numStatsReceived;
163
164     int curComlibController;   //Processor where strategies are  recreated
165     int clibIteration;         //Number of such learning iterations,
166                                //each of which is triggered by a
167                                //loadbalancing operation
168
169     void init(); //initialization function
170
171     //charm_message for multicast for a section of that group
172     void multicast(CharmMessageHolder *cmsg); //charm message holder here
173     //void multicast(void *charm_msg, int npes, int *pelist);
174
175     //The following funtions can be accessed only from ComlibInstanceHandle
176     void beginIteration();     //Notify begining of a bracket with
177                                //strategy identifier
178
179     void endIteration();       //Notify end, endIteration must be
180                                //called if a beginIteration is
181                                //called. Otherwise end of the entry
182                                //method is assumed to be the end of
183                                //the bracket.
184     
185     void setInstance(int id); 
186
187     //void prioEndIteration(PrioMsg *pmsg);
188     void registerStrategy(int pos, CharmStrategy *s);
189
190  public:
191
192     ComlibLocalStats clib_stats;   //To store statistics of
193                                    //communication operations
194     
195     ComlibManager();  //Recommended constructor
196
197     ComlibManager(CkMigrateMessage *m) { }
198     int useDefCtor(void){ return 1; } //Use default constructor should
199     //be pupped and store all the strategies.
200     
201     void barrier(void);
202     void barrier2(void);
203     void resumeFromBarrier2(void);
204
205     //Receive table of strategies.
206     void receiveTable(StrategyWrapper &sw, 
207                       CkHashtableT <ClibGlobalArrayIndex, int>&); 
208
209     void ArraySend(CkDelegateData *pd,int ep, void *msg, 
210                    const CkArrayIndexMax &idx, CkArrayID a);
211
212     void receiveRemoteSend(CkQ<CharmMessageHolder*> &rq, int id);
213     void sendRemote();
214
215     void GroupSend(CkDelegateData *pd, int ep, void *msg, int onpe, 
216                    CkGroupID gid);
217     
218     virtual void ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a);
219     virtual void GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g);
220     virtual void ArraySectionSend(CkDelegateData *pd, int ep ,void *m, 
221                                   CkArrayID a, CkSectionID &s, int opts);
222
223     CharmStrategy *getStrategy(int instid)
224         {return (CharmStrategy *)(* strategyTable)[instid].strategy;}
225
226     StrategyTableEntry *getStrategyTableEntry(int instid)
227         {return &((*strategyTable)[instid]);}
228
229     //To create a new strategy, returns handle to the strategy table;
230     ComlibInstanceHandle createInstance();  
231     void broadcastStrategies();             //Done creating instances
232
233     void AtSync();           //User program called loadbalancer
234     void lbUpdate(LBMigrateMsg *); //loadbalancing updates
235
236     //Learning functions
237     //void learnPattern(int totalMessageCount, int totalBytes);
238     //void switchStrategy(int strat);
239
240     void setRemote(int remotePe);
241
242     void collectStats(ComlibLocalStats &s, int src,CkVec<ClibGlobalArrayIndex>&);
243
244     //Returns the processor on which the comlib sees the array element
245     //belonging to
246     inline int getLastKnown(CkArrayID a, CkArrayIndexMax &idx) {
247         return ComlibGetLastKnown(a, idx);
248     }
249
250     CkDelegateData* ckCopyDelegateData(CkDelegateData *data); 
251     CkDelegateData *DelegatePointerPup(PUP::er &p,CkDelegateData *pd);
252 };
253
254 void ComlibDelegateProxy(CProxy *proxy);
255 void ComlibAssociateProxy(ComlibInstanceHandle *cinst, CProxy &proxy);
256 void ComlibAssociateProxy(CharmStrategy *strat, CProxy &proxy); 
257 ComlibInstanceHandle ComlibRegister(CharmStrategy *strat);
258 void ComlibBegin(CProxy &proxy);    
259 void ComlibEnd(CProxy &proxy);    
260
261 ComlibInstanceHandle CkCreateComlibInstance();
262 ComlibInstanceHandle CkGetComlibInstance();
263 ComlibInstanceHandle CkGetComlibInstance(int id);
264
265 void ComlibResetSectionProxy(CProxySection_ArrayBase *sproxy);
266
267 inline void ComlibResetProxy(CProxy *aproxy) {
268   ComlibInstanceHandle *handle = 
269     (ComlibInstanceHandle *) aproxy->ckDelegatedPtr();
270   handle->setSourcePe();
271 }
272
273 //Only Called when the strategies are not being created in main::main
274 void ComlibDoneCreating(); 
275
276 void ComlibInitSectionID(CkSectionID &sid);
277
278 void ComlibAtSync(void *msg);
279 void ComlibNotifyMigrationDoneHandler(void *msg);
280 void ComlibLBMigrationUpdate(LBMigrateMsg *);
281
282 #define RECORD_SEND_STATS(sid, bytes, dest) {             \
283         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));               \
284         cgproxy.ckLocalBranch()->clib_stats.recordSend(sid, bytes, dest); \
285 }\
286
287 #define RECORD_RECV_STATS(sid, bytes, src) { \
288         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); \
289         cgproxy.ckLocalBranch()->clib_stats.recordRecv(sid, bytes, src); \
290 }\
291
292 #define RECORD_SENDM_STATS(sid, bytes, dest_arr, ndest) {       \
293         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); \
294         cgproxy.ckLocalBranch()->clib_stats.recordSendM(sid, bytes, dest_arr, ndest); \
295 }\
296
297 #define RECORD_RECVM_STATS(sid, bytes, src_arr, nsrc) {        \
298         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); \
299         cgproxy.ckLocalBranch()->clib_stats.recordRecvM(sid, bytes, src_arr, nsrc); \
300 }\
301
302 #endif