563503a0c149413ee4da4677fa0f1907d2c078d0
[charm.git] / src / ck-com / ComlibManager.h
1 #ifndef COMMLIBMANAGER_H
2 #define COMMLIBMANAGER_H
3
4 #include "charm++.h"
5 #include "cksection.h"
6 #include "envelope.h"
7 #include "comlib.h"
8 #include <math.h>
9
10 #include "charm++.h"
11 #include "convcomlibmanager.h"
12
13 #define USE_TREE 1            //Organizes the all to all as a tree
14 #define USE_MESH 2            //Virtual topology is a mesh here
15 #define USE_HYPERCUBE 3       //Virtual topology is a hypercube
16 #define USE_DIRECT 4          //A dummy strategy that directly forwards 
17                               //messages without any processing.
18 #define USE_GRID 5            //Virtual topology is a 3d grid
19 #define USE_LINEAR 6          //Virtual topology is a linear array
20
21 #define CHARM_MPI 0 
22 #define MAX_NSTRAT 1024
23 #define LEARNING_PERIOD 1000 //Number of iterations after which the
24                              //learning framework will discover 
25                              //the appropriate strategy, not completely 
26                              //implemented
27 PUPbytes(comID);
28
29 #include "ComlibStats.h"
30
31 #include "comlib.decl.h"
32
33 //Dummy message to be sent incase there are no messages to send. 
34 //Used by only the EachToMany strategy!
35 class ComlibDummyMsg: public CMessage_ComlibDummyMsg {
36     int dummy;
37 };
38
39 /*
40 //Priority message to call end iteration
41 class PrioMsg: public CMessage_PrioMsg {
42  public:
43     int instID;
44 };
45 */
46
47 class ComlibMulticastMsg : public CkMcastBaseMsg, 
48                public CMessage_ComlibMulticastMsg {
49     
50   public:
51     int nIndices;
52     char *usrMsg;        
53     CkArrayIndexMax *indices;
54 };
55
56 class ComlibManager;
57
58 //An Instance of the communication library.
59 class ComlibInstanceHandle : public CkDelegateData {
60  private:    
61     int _instid;
62     CkGroupID _dmid;
63     int _srcPe;
64     int toForward;
65     
66  public:
67     ComlibInstanceHandle();
68     ComlibInstanceHandle(const ComlibInstanceHandle &h);
69     ComlibInstanceHandle(int instid, CkGroupID dmid);    
70    
71     ComlibInstanceHandle &operator=(ComlibInstanceHandle &h);
72
73     void setForwardingOnMigration(){toForward = 1;} 
74     void beginIteration();
75     void endIteration();
76     
77     CkGroupID getComlibManagerID();
78     void setStrategy(CharmStrategy *);
79     CharmStrategy *getStrategy();        
80     int getSourcePe() {return _srcPe;}
81
82     void setSourcePe() {_srcPe = CkMyPe();}
83
84     friend class ComlibManager;
85     void pup(PUP::er &p) {
86
87         if(p.isUnpacking())
88              reset();        
89
90         p | _instid;
91         p | _dmid;
92         p | _srcPe;
93         p | toForward;
94     }
95 };
96
97 class LBMigrateMsg;
98
99 class ComlibManager: public CkDelegateMgr {
100     friend class ComlibInstanceHandle;
101
102     int *bcast_pelist;  //Pelist passed to all broadcast operations
103
104     int section_send_event;
105
106     int remotePe;
107     CmiBool isRemote;
108     CmiBool strategyCreated;
109
110     int npes;
111     int *pelist;
112
113     CkArrayIndexMax dummyArrayIndex;
114
115     //For compatibility and easier use!
116     int strategyID; //Identifier of the strategy
117
118     //Pointer to the converse comm lib strategy table
119     StrategyTable *strategyTable;
120
121     CkQ<CharmStrategy *> ListOfStrategies; //temporary list of strategies
122     
123     CkQ<CharmMessageHolder *> remoteQ;  //list of remote messages
124                                         //after the object has
125                                         //migrated
126
127     //The number of strategies created by the user
128     //int nstrats; //now part of conv comlib
129     
130     int curStratID, prevStratID;      
131     //Number of strategies created by the user.
132
133     //flags
134     int receivedTable, flushTable, barrierReached, barrier2Reached;
135     CmiBool lbUpdateReceived;
136
137     int bcount , b2count;
138     //int totalMsgCount, totalBytes, nIterations;
139
140     ComlibArrayListener *alistener;
141     int prioEndIterationFlag;
142
143     ComlibGlobalStats clib_gstats; 
144     int    numStatsReceived;
145
146     int curComlibController;   //Processor where strategies are  recreated
147     int clibIteration;         //Number of such learning iterations,
148                                //each of which is triggered by a
149                                //loadbalancing operation
150
151     void init(); //initialization function
152
153     //charm_message for multicast for a section of that group
154     void multicast(CharmMessageHolder *cmsg); //charm message holder here
155     //void multicast(void *charm_msg, int npes, int *pelist);
156
157     //The following funtions can be accessed only from ComlibInstanceHandle
158     void beginIteration();     //Notify begining of a bracket with
159                                //strategy identifier
160
161     void endIteration();       //Notify end, endIteration must be
162                                //called if a beginIteration is
163                                //called. Otherwise end of the entry
164                                //method is assumed to be the end of
165                                //the bracket.
166     
167     void setInstance(int id); 
168
169     //void prioEndIteration(PrioMsg *pmsg);
170     void registerStrategy(int pos, CharmStrategy *s);
171
172  public:
173
174     ComlibLocalStats clib_stats;   //To store statistics of
175                                    //communication operations
176     
177     ComlibManager();  //Recommended constructor
178
179     ComlibManager(CkMigrateMessage *m) { }
180     int useDefCtor(void){ return 1; } //Use default constructor should
181     //be pupped and store all the strategies.
182     
183     void barrier(void);
184     void barrier2(void);
185     void resumeFromBarrier2(void);
186
187     //Receive table of strategies.
188     void receiveTable(StrategyWrapper &sw, 
189                       CkHashtableT <ClibGlobalArrayIndex, int>&); 
190
191     void ArraySend(CkDelegateData *pd,int ep, void *msg, 
192                    const CkArrayIndexMax &idx, CkArrayID a);
193
194     void receiveRemoteSend(CkQ<CharmMessageHolder*> &rq, int id);
195     void sendRemote();
196
197     void GroupSend(CkDelegateData *pd, int ep, void *msg, int onpe, 
198                    CkGroupID gid);
199     
200     virtual void ArrayBroadcast(CkDelegateData *pd,int ep,void *m,CkArrayID a);
201     virtual void GroupBroadcast(CkDelegateData *pd,int ep,void *m,CkGroupID g);
202     virtual void ArraySectionSend(CkDelegateData *pd, int ep ,void *m, 
203                                   CkArrayID a, CkSectionID &s);
204
205     CharmStrategy *getStrategy(int instid)
206         {return (CharmStrategy *)(* strategyTable)[instid].strategy;}
207
208     StrategyTableEntry *getStrategyTableEntry(int instid)
209         {return &((*strategyTable)[instid]);}
210
211     //To create a new strategy, returns handle to the strategy table;
212     ComlibInstanceHandle createInstance();  
213     void broadcastStrategies();             //Done creating instances
214
215     void AtSync();           //User program called loadbalancer
216     void lbUpdate(LBMigrateMsg *); //loadbalancing updates
217
218     //Learning functions
219     //void learnPattern(int totalMessageCount, int totalBytes);
220     //void switchStrategy(int strat);
221
222     void setRemote(int remotePe);
223
224     void collectStats(ComlibLocalStats &s, int src,CkVec<ClibGlobalArrayIndex>&);
225
226     //Returns the processor on which the comlib sees the array element
227     //belonging to
228     inline int getLastKnown(CkArrayID a, CkArrayIndexMax &idx) {
229         return ComlibGetLastKnown(a, idx);
230     }
231
232     CkDelegateData* ckCopyDelegateData(CkDelegateData *data); 
233     CkDelegateData *DelegatePointerPup(PUP::er &p,CkDelegateData *pd);
234 };
235
236 void ComlibDelegateProxy(CProxy *proxy);
237 void ComlibAssociateProxy(CharmStrategy *strat, CProxy &proxy); 
238 void ComlibBegin(CProxy &proxy);    
239 void ComlibEnd(CProxy &proxy);    
240
241 ComlibInstanceHandle CkCreateComlibInstance();
242 ComlibInstanceHandle CkGetComlibInstance();
243 ComlibInstanceHandle CkGetComlibInstance(int id);
244 void ComlibResetSectionProxy(CProxySection_ArrayBase *sproxy);
245
246 //Only Called when the strategies are not being created in main::main
247 void ComlibDoneCreating(); 
248
249 void ComlibInitSectionID(CkSectionID &sid);
250
251 void ComlibAtSync(void *msg);
252 void ComlibNotifyMigrationDoneHandler(void *msg);
253 void ComlibLBMigrationUpdate(LBMigrateMsg *);
254
255 #define RECORD_SEND_STATS(sid, bytes, dest) {             \
256         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID));               \
257         cgproxy.ckLocalBranch()->clib_stats.recordSend(sid, bytes, dest); \
258 }\
259
260 #define RECORD_RECV_STATS(sid, bytes, src) { \
261         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); \
262         cgproxy.ckLocalBranch()->clib_stats.recordRecv(sid, bytes, src); \
263 }\
264
265 #define RECORD_SENDM_STATS(sid, bytes, dest_arr, ndest) {       \
266         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); \
267         cgproxy.ckLocalBranch()->clib_stats.recordSendM(sid, bytes, dest_arr, ndest); \
268 }\
269
270 #define RECORD_RECVM_STATS(sid, bytes, src_arr, nsrc) {        \
271         CProxy_ComlibManager cgproxy(CkpvAccess(cmgrID)); \
272         cgproxy.ckLocalBranch()->clib_stats.recordRecvM(sid, bytes, src_arr, nsrc); \
273 }\
274
275 #endif