d501150be9fc65fe186a1e242b8b286c31ad01dd
[charm.git] / src / ck-core / ckcheckpoint.C
1 /*
2 Charm++ File: Checkpoint Library
3 added 01/03/2003 by Chao Huang, chuang10@uiuc.edu
4
5 More documentation goes here...
6 --- Updated 12/14/2003 by Gengbin, gzheng@uiuc.edu
7     see ckcheckpoint.h for change log
8 */
9
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include "charm++.h"
14 #include "ck.h"
15 #include "ckcheckpoint.h"
16
17 #define DEBCHK  // CkPrintf
18
19 #define DEBUGC(x) x
20 //#define DEBUGC(x) 
21
22 CkGroupID _sysChkptMgr;
23
24 typedef struct _GroupInfo{
25         CkGroupID gID;
26         int MigCtor, DefCtor;
27         char name[256];
28 } GroupInfo;
29 PUPbytes(GroupInfo)
30 PUPmarshall(GroupInfo)
31
32 // help class to find how many array elements
33 class ElementCounter : public CkLocIterator {
34 private:
35         int count;
36 public:
37         ElementCounter():count(0){};
38         void addLocation(CkLocation &loc)  { count++; }
39         int getCount() { return count; }
40 };
41
42 // helper class to pup all elements that belong to same ckLocMgr
43 class ElementCheckpointer : public CkLocIterator {
44 private:
45         CkLocMgr *locMgr;
46         PUP::er &p;
47 public:
48         ElementCheckpointer(CkLocMgr* mgr_, PUP::er &p_):locMgr(mgr_),p(p_){};
49         void addLocation(CkLocation &loc) {
50                 CkArrayIndexMax idx=loc.getIndex();
51                 CkGroupID gID = locMgr->ckGetGroupID();
52                 p|gID;      // store loc mgr's GID as well for easier restore
53                 p|idx;
54                 p|loc;
55                 //CkPrintf("[%d] addLocation: ", CkMyPe()), idx.print();
56         }
57 };
58
59
60 extern void _initDone();
61
62 static void bdcastRO(void){
63         int i;
64         //Determine the size of the RODataMessage
65         PUP::sizer ps;
66         for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
67
68         //Allocate and fill out the RODataMessage
69         envelope *env = _allocEnv(RODataMsg, ps.size());
70         PUP::toMem pp((char *)EnvToUsr(env));
71         for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
72         
73         env->setCount(++_numInitMsgs);
74         env->setSrcPe(CkMyPe());
75         CmiSetHandler(env, _roRestartHandlerIdx);
76         CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
77 }
78
79 // Print out an array index to this string as decimal fields
80 // separated by underscores.
81 void printIndex(const CkArrayIndex &idx,char *dest) {
82         const int *idxData=idx.data();
83         for (int i=0;i<idx.nInts;i++) {
84                 sprintf(dest,"%s%d",i==0?"":"_", idxData[i]);
85                 dest+=strlen(dest);
86         }
87 }
88
89 static void checkpointOne(const char* dirname, CkCallback& cb);
90
91 // broadcast
92 void CkCheckpointMgr::Checkpoint(const char *dirname, CkCallback& cb){
93         chkptStartTimer = CmiWallTimer();
94         // every body make dir in case it is local directory
95         CmiMkdir(dirname);
96
97         if (CkMyPe() == 0) {
98           checkpointOne(dirname, cb);
99         }
100
101         char fileName[1024];
102         // save groups into Groups.dat
103         // content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
104         sprintf(fileName,"%s/Groups_%d.dat",dirname,CkMyPe());
105         FILE* fGroups = fopen(fileName,"wb");
106         if(!fGroups) CkAbort("Failed to create checkpoint file for group table!");
107         PUP::toDisk pGroups(fGroups);
108         CkPupGroupData(pGroups);
109         fclose(fGroups);
110
111         // save nodegroups into NodeGroups.dat
112         // content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
113         if (CkMyRank() == 0) {
114           sprintf(fileName,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
115           FILE* fNodeGroups = fopen(fileName,"wb");
116           if(!fNodeGroups) 
117             CkAbort("Failed to create checkpoint file for nodegroup table!");
118           PUP::toDisk pNodeGroups(fNodeGroups);
119           CkPupNodeGroupData(pNodeGroups);
120           fclose(fNodeGroups);
121         }
122
123         //DEBCHK("[%d]CkCheckpointMgr::Checkpoint called dirname={%s}\n",CkMyPe(),dirname);
124         sprintf(fileName,"%s/arr_%d.dat",dirname, CkMyPe());
125         FILE *datFile=fopen(fileName,"wb");
126         if (datFile==NULL) CkAbort("Could not create data file");
127         PUP::toDisk  p(datFile);
128         CkPupArrayElementsData(p);
129         fclose(datFile);
130
131 #if CMK_HAS_SYNC && ! CMK_DISABLE_SYNC
132         system("sync");
133 #endif
134
135         restartCB = cb;
136         DEBCHK("[%d]restartCB installed\n",CkMyPe());
137         CkCallback localcb(CkIndex_CkCheckpointMgr::SendRestartCB(NULL),0,thisgroup);
138         contribute(0,NULL,CkReduction::sum_int,localcb);
139 }
140
141 void CkCheckpointMgr::SendRestartCB(CkReductionMsg *m){ 
142         delete m; 
143         DEBCHK("[%d]Sending out the cb\n",CkMyPe());
144         CkPrintf("Checkpoint to disk finished in %fs, sending out the cb...\n", CmiWallTimer() - chkptStartTimer);
145         restartCB.send(); 
146 }
147
148 void CkPupROData(PUP::er &p)
149 {
150         int _numReadonlies;
151         if (!p.isUnpacking()) _numReadonlies=_readonlyTable.size();
152         p|_numReadonlies;
153         if (p.isUnpacking()) {
154           if (_numReadonlies != _readonlyTable.size())
155             CkAbort("You cannot add readonlies and restore from checkpoint...");
156         }
157         for(int i=0;i<_numReadonlies;i++) _readonlyTable[i]->pupData(p);
158 }
159
160 // handle main chare
161 void CkPupMainChareData(PUP::er &p)
162 {
163         int nMains=_mainTable.size();
164         DEBCHK("[%d] CkPupMainChareData %s: nMains = %d\n", CkMyPe(),p.typeString(),nMains);
165         for(int i=0;i<nMains;i++){  /* Create all mainchares */
166                 ChareInfo *entry = _chareTable[_mainTable[i]->chareIdx];
167                 int entryMigCtor = entry->getMigCtor();
168                 if(entryMigCtor!=-1) {
169                         Chare* obj;
170                         if (p.isUnpacking()) {
171                                 int size = entry->size;
172                                 DEBCHK("MainChare PUP'ed: name = %s, idx = %d, size = %d\n", entry->name, i, size);
173                                 obj = (Chare*)malloc(size);
174                                 _MEMCHECK(obj);
175                                 _mainTable[i]->setObj(obj);
176                                 void *m = CkAllocSysMsg();
177                                 _entryTable[entryMigCtor]->call(m, obj);
178                         }
179                         else 
180                                 obj = (Chare *)_mainTable[i]->getObj();
181                         obj->pup(p);
182                 }
183         }
184         // to update mainchare proxy
185         // only readonly variables of Chare Proxy is taken care of here;
186         // in general, if chare proxy is contained in some data structure
187         // for example CkCallback, it is user's responsibility to
188         // update them after restarting
189         if (p.isUnpacking() && CkMyPe()==0)
190                 bdcastRO();
191 }
192
193 // handle GroupTable and data
194 void CkPupGroupData(PUP::er &p)
195 {
196         int numGroups, i;
197
198         if (!p.isUnpacking()) {
199           numGroups = CkpvAccess(_groupIDTable)->size();
200         }
201         p|numGroups;
202         if (p.isUnpacking()) {
203           if(CkMyPe()==0)  
204             CkpvAccess(_numGroups) = numGroups+1; 
205           else 
206             CkpvAccess(_numGroups) = 1;
207         }
208         DEBCHK("[%d] CkPupGroupData %s: numGroups = %d\n", CkMyPe(),p.typeString(),numGroups);
209
210         GroupInfo *tmpInfo = new GroupInfo [numGroups];
211         if (!p.isUnpacking()) {
212           for(i=0;i<numGroups;i++) {
213                 tmpInfo[i].gID = (*CkpvAccess(_groupIDTable))[i];
214                 TableEntry ent = CkpvAccess(_groupTable)->find(tmpInfo[i].gID);
215                 tmpInfo[i].MigCtor = _chareTable[ent.getcIdx()]->migCtor;
216                 tmpInfo[i].DefCtor = _chareTable[ent.getcIdx()]->defCtor;
217                 strncpy(tmpInfo[i].name,_chareTable[ent.getcIdx()]->name,255);
218                 DEBCHK("[%d] CkPupGroupData: %s group %s \n",
219                         CkMyPe(), p.typeString(), tmpInfo[i].name);
220
221                 if(tmpInfo[i].MigCtor==-1) {
222                         char buf[512];
223                         sprintf(buf,"Group %s needs a migration constructor and PUP'er routine for restart.\n", tmpInfo[i].name);
224                         CkAbort(buf);
225                 }
226           }
227         }
228         for (i=0; i<numGroups; i++) p|tmpInfo[i];
229
230         for(i=0;i<numGroups;i++) 
231         {
232           CkGroupID gID = tmpInfo[i].gID;
233           if (p.isUnpacking()) {
234             //CkpvAccess(_groupIDTable)->push_back(gID);
235             int eIdx = tmpInfo[i].MigCtor;
236             // error checking
237             if (eIdx == -1) {
238               CkPrintf("[%d] ERROR> Group %s's migration constructor is not defined!\n", CkMyPe(), tmpInfo[i].name); CkAbort("Abort");
239             }
240             void *m = CkAllocSysMsg();
241             envelope* env = UsrToEnv((CkMessage *)m);
242             CkCreateLocalGroup(gID, eIdx, env);
243           }   // end of unPacking
244           IrrGroup *gobj = CkpvAccess(_groupTable)->find(gID).getObj();
245           // if using migration constructor, you'd better have a pup
246           gobj->pup(p);
247           DEBCHK("Group PUP'ed: gid = %d, name = %s\n",
248                         gobj->ckGetGroupID().idx, tmpInfo[i].name);
249         }
250         delete [] tmpInfo;
251 }
252
253 // handle NodeGroupTable and data
254 void CkPupNodeGroupData(PUP::er &p)
255 {
256         int numNodeGroups, i;
257         if (!p.isUnpacking()) {
258           numNodeGroups = CksvAccess(_nodeGroupIDTable).size();
259         }
260         p|numNodeGroups;
261         if (p.isUnpacking()) {
262           if(CkMyPe()==0){ CksvAccess(_numNodeGroups) = numNodeGroups+1; }
263           else { CksvAccess(_numNodeGroups) = 1; }
264         }
265         DEBCHK("[%d] CkPupNodeGroupData %s: numNodeGroups = %d\n",CkMyPe(),p.typeString(),numNodeGroups);
266
267         GroupInfo *tmpInfo = new GroupInfo [numNodeGroups];
268         if (!p.isUnpacking()) {
269           for(i=0;i<numNodeGroups;i++) {
270                 tmpInfo[i].gID = CksvAccess(_nodeGroupIDTable)[i];
271                 TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(tmpInfo[i].gID);
272                 tmpInfo[i].MigCtor = _chareTable[ent2.getcIdx()]->migCtor;
273                 if(tmpInfo[i].MigCtor==-1) {
274                         char buf[512];
275                         sprintf(buf,"NodeGroup %s either need a migration constructor and\n\
276                                      declared as [migratable] in .ci to be able to checkpoint.",\
277                                      _chareTable[ent2.getcIdx()]->name);
278                         CkAbort(buf);
279                 }
280           }
281         }
282         for (i=0; i<numNodeGroups; i++) p|tmpInfo[i];
283         for (i=0;i<numNodeGroups;i++) {
284                 CkGroupID gID = tmpInfo[i].gID;
285                 if (p.isUnpacking()) {
286                         //CksvAccess(_nodeGroupIDTable).push_back(gID);
287                         int eIdx = tmpInfo[i].MigCtor;
288                         void *m = CkAllocSysMsg();
289                         envelope* env = UsrToEnv((CkMessage *)m);
290                         CkCreateLocalNodeGroup(gID, eIdx, env);
291                 }
292                 TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(gID);
293                 IrrGroup *obj = ent2.getObj();
294                 obj->pup(p);
295                 DEBCHK("Nodegroup PUP'ed: gid = %d, name = %s\n",
296                         obj->ckGetGroupID().idx,
297                         _chareTable[ent2.getcIdx()]->name);
298         }
299         delete [] tmpInfo;
300 }
301
302
303 // handle chare array elements for this processor
304 void CkPupArrayElementsData(PUP::er &p, int notifyListeners)
305 {
306         int i;
307         // safe in both packing/unpakcing at this stage
308         int numGroups = CkpvAccess(_groupIDTable)->size();
309
310         // number of array elements on this processor
311         int numElements;
312         if (!p.isUnpacking()) {
313           ElementCounter  counter;
314           CKLOCMGR_LOOP(mgr->iterate(counter););
315           numElements = counter.getCount();
316         }
317         p|numElements;
318
319         DEBCHK("[%d] CkPupArrayElementsData %s numGroups:%d numElements:%d \n",CkMyPe(),p.typeString(), numGroups, numElements);
320
321         if (!p.isUnpacking())
322         {
323           // let CkLocMgr to iterate and store every array elements
324           CKLOCMGR_LOOP(ElementCheckpointer chk(mgr, p); mgr->iterate(chk););
325         }
326         else {
327           // loop and create all array elements ourselves
328           for (int i=0; i<numElements; i++) {
329                 CkGroupID gID;
330                 CkArrayIndexMax idx;
331                 p|gID;
332                 p|idx;
333                 CkLocMgr *mgr = (CkLocMgr*)CkpvAccess(_groupTable)->find(gID).getObj();
334                 if (notifyListeners)
335                   mgr->resume(idx,p);
336                 else
337                   mgr->restore(idx,p);
338           }
339         }
340         // finish up
341         if (notifyListeners)
342         for(i=0;i<numGroups;i++) {
343                 IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
344                 obj->ckJustMigrated();
345         }
346 }
347
348 void CkPupProcessorData(PUP::er &p)
349 {
350     // save readonlys, and callback BTW
351     if(CkMyRank()==0) {
352         CkPupROData(p);
353     }
354
355     // save mainchares into MainChares.dat
356     if(CkMyPe()==0) {
357         CkPupMainChareData(p);
358     }
359         
360     // save groups into Groups.dat
361     CkPupGroupData(p);
362
363     // save nodegroups into NodeGroups.dat
364     if(CkMyRank()==0) {
365         CkPupNodeGroupData(p);
366     }
367
368     // pup array elements
369     CkPupArrayElementsData(p);
370 }
371
372 // called only on pe 0
373 static void checkpointOne(const char* dirname, CkCallback& cb){
374         CmiAssert(CkMyPe()==0);
375         int i;
376         char filename[1024];
377         
378         // save readonlys, and callback BTW
379         sprintf(filename,"%s/RO.dat",dirname);
380         FILE* fRO = fopen(filename,"wb");
381         if(!fRO) CkAbort("Failed to create checkpoint file for readonly data!");
382         PUP::toDisk pRO(fRO);
383         int _numPes = CkNumPes();
384         pRO|_numPes;
385         CkPupROData(pRO);
386         pRO((char *)&cb, sizeof(cb));
387         fclose(fRO);
388
389         // save mainchares into MainChares.dat
390         {
391                 sprintf(filename,"%s/MainChares.dat",dirname);
392                 FILE* fMain = fopen(filename,"wb");
393                 if(!fMain) CkAbort("Failed to open checkpoint file for mainchare data!");
394                 PUP::toDisk pMain(fMain);
395                 CkPupMainChareData(pMain);
396                 fclose(fMain);
397         }
398 }
399
400 void CkRemoveArrayElements()
401 {
402   int i;
403   int numGroups = CkpvAccess(_groupIDTable)->size();
404   CKLOCMGR_LOOP(mgr->flushAllRecs(););
405 }
406
407 void CkStartCheckpoint(char* dirname,const CkCallback& cb)
408 {
409         CkPrintf("[%d] Checkpoint starting in %s\n", CkMyPe(), dirname);
410         
411         // hand over to checkpoint managers for per-processor checkpointing
412         CProxy_CkCheckpointMgr(_sysChkptMgr).Checkpoint((char *)dirname, cb);
413 }
414
415 /**
416   * Restart: There's no such object as restart manager is created
417   *          because a group cannot restore itself anyway.
418   *          The mechanism exists as converse code and get invoked by
419   *          broadcast message.
420   **/
421
422 void CkRestartMain(const char* dirname){
423         int i;
424         char filename[1024];
425         CkCallback cb;
426         
427         // restore readonlys
428         sprintf(filename,"%s/RO.dat",dirname);
429         FILE* fRO = fopen(filename,"rb");
430         if(!fRO) CkAbort("Failed to open checkpoint file for readonly data!");
431         int _numPes = -1;
432         PUP::fromDisk pRO(fRO);
433         pRO|_numPes;
434         CkPupROData(pRO);
435         pRO|cb;
436         fclose(fRO);
437         DEBCHK("[%d]CkRestartMain: readonlys restored\n",CkMyPe());
438
439         // restore mainchares
440         sprintf(filename,"%s/MainChares.dat",dirname);
441         FILE* fMain = fopen(filename,"rb");
442         if(fMain && CkMyPe()==0){ // only main chares have been checkpointed, we restart on PE0
443                 PUP::fromDisk pMain(fMain);
444                 CkPupMainChareData(pMain);
445                 fclose(fMain);
446                 DEBCHK("[%d]CkRestartMain: mainchares restored\n",CkMyPe());
447                 //bdcastRO(); // moved to CkPupMainChareData()
448         }
449         
450         // restore groups
451         // content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
452         // restore from PE0's copy if shrink/expand
453         if(CkNumPes() != _numPes)
454                 sprintf(filename,"%s/Groups_0.dat",dirname);
455         else
456                 sprintf(filename,"%s/Groups_%d.dat",dirname,CkMyPe());
457         FILE* fGroups = fopen(filename,"rb");
458         if(!fGroups) CkAbort("Failed to open checkpoint file for group table!");
459         PUP::fromDisk pGroups(fGroups);
460         CkPupGroupData(pGroups);
461         fclose(fGroups);
462
463         // restore nodegroups
464         // content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
465         if(CkMyRank()==0){
466                 if(CkNumPes() != _numPes)
467                         sprintf(filename,"%s/NodeGroups_0.dat",dirname);
468                 else
469                         sprintf(filename,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
470                 FILE* fNodeGroups = fopen(filename,"rb");
471                 if(!fNodeGroups) CkAbort("Failed to open checkpoint file for nodegroup table!");
472                 PUP::fromDisk pNodeGroups(fNodeGroups);
473                 CkPupNodeGroupData(pNodeGroups);
474                 fclose(fNodeGroups);
475         }
476
477         // for each location, restore arrays
478         //DEBCHK("[%d]Trying to find location manager\n",CkMyPe());
479         DEBCHK("[%d]Number of PE: %d -> %d\n",CkMyPe(),_numPes,CkNumPes());
480         if(CkMyPe() < _numPes)  // in normal range: restore, otherwise, do nothing
481           for (i=0; i<_numPes;i++) {
482             if (i%CkNumPes() == CkMyPe()) {
483               sprintf(filename,"%s/arr_%d.dat",dirname, i);
484               FILE *datFile=fopen(filename,"rb");
485               if (datFile==NULL) CkAbort("Could not read data file");
486               PUP::fromDisk  p(datFile);
487               CkPupArrayElementsData(p);
488               fclose(datFile);
489             }
490           }
491
492         _initDone();
493
494         if(CkMyPe()==0) {
495                 CmiPrintf("[%d]CkRestartMain done. sending out callback.\n",CkMyPe());
496                 cb.send();
497         }
498 }
499
500 // Main chare: initialize system checkpoint manager
501 class CkCheckpointInit : public Chare {
502 public:
503   CkCheckpointInit(CkArgMsg *msg) {
504     _sysChkptMgr = CProxy_CkCheckpointMgr::ckNew();
505     delete msg;
506   }
507   CkCheckpointInit(CkMigrateMessage *m) {delete m;}
508 };
509
510 #include "CkCheckpoint.def.h"
511