A few bug fixes related to the mlogft machine.
[charm.git] / src / ck-core / ckcheckpoint.C
1 /*
2 Charm++ File: Checkpoint Library
3 added 01/03/2003 by Chao Huang, chuang10@uiuc.edu
4
5 More documentation goes here...
6 --- Updated 12/14/2003 by Gengbin, gzheng@uiuc.edu
7     see ckcheckpoint.h for change log
8 */
9
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include "charm++.h"
14 #include "ck.h"
15 #include "ckcheckpoint.h"
16
17 #define DEBCHK  // CkPrintf
18
19 #define DEBUGC(x) x
20 //#define DEBUGC(x) 
21
22 CkGroupID _sysChkptMgr;
23
24 typedef struct _GroupInfo{
25         CkGroupID gID;
26         int MigCtor, DefCtor;
27         char name[256];
28 } GroupInfo;
29 PUPbytes(GroupInfo)
30 PUPmarshall(GroupInfo)
31
32 int _inrestart = 0;
33
34 // help class to find how many array elements
35 class ElementCounter : public CkLocIterator {
36 private:
37         int count;
38 public:
39         ElementCounter():count(0){};
40         void addLocation(CkLocation &loc)  { count++; }
41         int getCount() { return count; }
42 };
43
44 // helper class to pup all elements that belong to same ckLocMgr
45 class ElementCheckpointer : public CkLocIterator {
46 private:
47         CkLocMgr *locMgr;
48         PUP::er &p;
49 public:
50         ElementCheckpointer(CkLocMgr* mgr_, PUP::er &p_):locMgr(mgr_),p(p_){};
51         void addLocation(CkLocation &loc) {
52                 CkArrayIndexMax idx=loc.getIndex();
53                 CkGroupID gID = locMgr->ckGetGroupID();
54                 p|gID;      // store loc mgr's GID as well for easier restore
55                 p|idx;
56                 p|loc;
57                 //CkPrintf("[%d] addLocation: ", CkMyPe()), idx.print();
58         }
59 };
60
61
62 extern void _initDone();
63
64 static void bdcastRO(void){
65         int i;
66         //Determine the size of the RODataMessage
67         PUP::sizer ps;
68         for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
69
70         //Allocate and fill out the RODataMessage
71         envelope *env = _allocEnv(RODataMsg, ps.size());
72         PUP::toMem pp((char *)EnvToUsr(env));
73         for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
74         
75         env->setCount(++_numInitMsgs);
76         env->setSrcPe(CkMyPe());
77         CmiSetHandler(env, _roRestartHandlerIdx);
78         CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
79 }
80
81 // Print out an array index to this string as decimal fields
82 // separated by underscores.
83 void printIndex(const CkArrayIndex &idx,char *dest) {
84         const int *idxData=idx.data();
85         for (int i=0;i<idx.nInts;i++) {
86                 sprintf(dest,"%s%d",i==0?"":"_", idxData[i]);
87                 dest+=strlen(dest);
88         }
89 }
90
91 static void checkpointOne(const char* dirname, CkCallback& cb);
92
93 // broadcast
94 void CkCheckpointMgr::Checkpoint(const char *dirname, CkCallback& cb){
95         chkptStartTimer = CmiWallTimer();
96         // every body make dir in case it is local directory
97         CmiMkdir(dirname);
98
99         if (CkMyPe() == 0) {
100           checkpointOne(dirname, cb);
101         }
102
103         char fileName[1024];
104         // save groups into Groups.dat
105         // content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
106         sprintf(fileName,"%s/Groups_%d.dat",dirname,CkMyPe());
107         FILE* fGroups = fopen(fileName,"wb");
108         if(!fGroups) CkAbort("Failed to create checkpoint file for group table!");
109         PUP::toDisk pGroups(fGroups);
110         CkPupGroupData(pGroups);
111         fclose(fGroups);
112
113         // save nodegroups into NodeGroups.dat
114         // content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
115         if (CkMyRank() == 0) {
116           sprintf(fileName,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
117           FILE* fNodeGroups = fopen(fileName,"wb");
118           if(!fNodeGroups) 
119             CkAbort("Failed to create checkpoint file for nodegroup table!");
120           PUP::toDisk pNodeGroups(fNodeGroups);
121           CkPupNodeGroupData(pNodeGroups);
122           fclose(fNodeGroups);
123         }
124
125         //DEBCHK("[%d]CkCheckpointMgr::Checkpoint called dirname={%s}\n",CkMyPe(),dirname);
126         sprintf(fileName,"%s/arr_%d.dat",dirname, CkMyPe());
127         FILE *datFile=fopen(fileName,"wb");
128         if (datFile==NULL) CkAbort("Could not create data file");
129         PUP::toDisk  p(datFile);
130         CkPupArrayElementsData(p);
131         fclose(datFile);
132
133 #if CMK_HAS_SYNC && ! CMK_DISABLE_SYNC
134         system("sync");
135 #endif
136
137         restartCB = cb;
138         DEBCHK("[%d]restartCB installed\n",CkMyPe());
139         CkCallback localcb(CkIndex_CkCheckpointMgr::SendRestartCB(NULL),0,thisgroup);
140         contribute(0,NULL,CkReduction::sum_int,localcb);
141 }
142
143 void CkCheckpointMgr::SendRestartCB(CkReductionMsg *m){ 
144         delete m; 
145         DEBCHK("[%d]Sending out the cb\n",CkMyPe());
146         CkPrintf("Checkpoint to disk finished in %fs, sending out the cb...\n", CmiWallTimer() - chkptStartTimer);
147         restartCB.send(); 
148 }
149
150 void CkPupROData(PUP::er &p)
151 {
152         int _numReadonlies;
153         if (!p.isUnpacking()) _numReadonlies=_readonlyTable.size();
154         p|_numReadonlies;
155         if (p.isUnpacking()) {
156           if (_numReadonlies != _readonlyTable.size())
157             CkAbort("You cannot add readonlies and restore from checkpoint...");
158         }
159         for(int i=0;i<_numReadonlies;i++) _readonlyTable[i]->pupData(p);
160 }
161
162 // handle main chare
163 void CkPupMainChareData(PUP::er &p, CkArgMsg *args)
164 {
165         int nMains=_mainTable.size();
166         DEBCHK("[%d] CkPupMainChareData %s: nMains = %d\n", CkMyPe(),p.typeString(),nMains);
167         for(int i=0;i<nMains;i++){  /* Create all mainchares */
168                 ChareInfo *entry = _chareTable[_mainTable[i]->chareIdx];
169                 int entryMigCtor = entry->getMigCtor();
170                 if(entryMigCtor!=-1) {
171                         Chare* obj;
172                         if (p.isUnpacking()) {
173                                 int size = entry->size;
174                                 DEBCHK("MainChare PUP'ed: name = %s, idx = %d, size = %d\n", entry->name, i, size);
175                                 obj = (Chare*)malloc(size);
176                                 _MEMCHECK(obj);
177                                 _mainTable[i]->setObj(obj);
178                                 //void *m = CkAllocSysMsg();
179                                 _entryTable[entryMigCtor]->call(args, obj);
180                         }
181                         else 
182                                 obj = (Chare *)_mainTable[i]->getObj();
183                         obj->pup(p);
184                 }
185         }
186         // to update mainchare proxy
187         // only readonly variables of Chare Proxy is taken care of here;
188         // in general, if chare proxy is contained in some data structure
189         // for example CkCallback, it is user's responsibility to
190         // update them after restarting
191         if (p.isUnpacking() && CkMyPe()==0)
192                 bdcastRO();
193 }
194
195 // handle GroupTable and data
196 void CkPupGroupData(PUP::er &p)
197 {
198         int numGroups, i;
199
200         if (!p.isUnpacking()) {
201           numGroups = CkpvAccess(_groupIDTable)->size();
202         }
203         p|numGroups;
204         if (p.isUnpacking()) {
205           if(CkMyPe()==0)  
206             CkpvAccess(_numGroups) = numGroups+1; 
207           else 
208             CkpvAccess(_numGroups) = 1;
209         }
210         DEBCHK("[%d] CkPupGroupData %s: numGroups = %d\n", CkMyPe(),p.typeString(),numGroups);
211
212         GroupInfo *tmpInfo = new GroupInfo [numGroups];
213         if (!p.isUnpacking()) {
214           for(i=0;i<numGroups;i++) {
215                 tmpInfo[i].gID = (*CkpvAccess(_groupIDTable))[i];
216                 TableEntry ent = CkpvAccess(_groupTable)->find(tmpInfo[i].gID);
217                 tmpInfo[i].MigCtor = _chareTable[ent.getcIdx()]->migCtor;
218                 tmpInfo[i].DefCtor = _chareTable[ent.getcIdx()]->defCtor;
219                 strncpy(tmpInfo[i].name,_chareTable[ent.getcIdx()]->name,255);
220                 DEBCHK("[%d] CkPupGroupData: %s group %s \n",
221                         CkMyPe(), p.typeString(), tmpInfo[i].name);
222
223                 if(tmpInfo[i].MigCtor==-1) {
224                         char buf[512];
225                         sprintf(buf,"Group %s needs a migration constructor and PUP'er routine for restart.\n", tmpInfo[i].name);
226                         CkAbort(buf);
227                 }
228           }
229         }
230         for (i=0; i<numGroups; i++) p|tmpInfo[i];
231
232         for(i=0;i<numGroups;i++) 
233         {
234           CkGroupID gID = tmpInfo[i].gID;
235           if (p.isUnpacking()) {
236             //CkpvAccess(_groupIDTable)->push_back(gID);
237             int eIdx = tmpInfo[i].MigCtor;
238             // error checking
239             if (eIdx == -1) {
240               CkPrintf("[%d] ERROR> Group %s's migration constructor is not defined!\n", CkMyPe(), tmpInfo[i].name); CkAbort("Abort");
241             }
242             void *m = CkAllocSysMsg();
243             envelope* env = UsrToEnv((CkMessage *)m);
244             CkCreateLocalGroup(gID, eIdx, env);
245           }   // end of unPacking
246           IrrGroup *gobj = CkpvAccess(_groupTable)->find(gID).getObj();
247           // if using migration constructor, you'd better have a pup
248           gobj->pup(p);
249           DEBCHK("Group PUP'ed: gid = %d, name = %s\n",
250                         gobj->ckGetGroupID().idx, tmpInfo[i].name);
251         }
252         delete [] tmpInfo;
253 }
254
255 // handle NodeGroupTable and data
256 void CkPupNodeGroupData(PUP::er &p)
257 {
258         int numNodeGroups, i;
259         if (!p.isUnpacking()) {
260           numNodeGroups = CksvAccess(_nodeGroupIDTable).size();
261         }
262         p|numNodeGroups;
263         if (p.isUnpacking()) {
264           if(CkMyPe()==0){ CksvAccess(_numNodeGroups) = numNodeGroups+1; }
265           else { CksvAccess(_numNodeGroups) = 1; }
266         }
267         DEBCHK("[%d] CkPupNodeGroupData %s: numNodeGroups = %d\n",CkMyPe(),p.typeString(),numNodeGroups);
268
269         GroupInfo *tmpInfo = new GroupInfo [numNodeGroups];
270         if (!p.isUnpacking()) {
271           for(i=0;i<numNodeGroups;i++) {
272                 tmpInfo[i].gID = CksvAccess(_nodeGroupIDTable)[i];
273                 TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(tmpInfo[i].gID);
274                 tmpInfo[i].MigCtor = _chareTable[ent2.getcIdx()]->migCtor;
275                 if(tmpInfo[i].MigCtor==-1) {
276                         char buf[512];
277                         sprintf(buf,"NodeGroup %s either need a migration constructor and\n\
278                                      declared as [migratable] in .ci to be able to checkpoint.",\
279                                      _chareTable[ent2.getcIdx()]->name);
280                         CkAbort(buf);
281                 }
282           }
283         }
284         for (i=0; i<numNodeGroups; i++) p|tmpInfo[i];
285         for (i=0;i<numNodeGroups;i++) {
286                 CkGroupID gID = tmpInfo[i].gID;
287                 if (p.isUnpacking()) {
288                         //CksvAccess(_nodeGroupIDTable).push_back(gID);
289                         int eIdx = tmpInfo[i].MigCtor;
290                         void *m = CkAllocSysMsg();
291                         envelope* env = UsrToEnv((CkMessage *)m);
292                         CkCreateLocalNodeGroup(gID, eIdx, env);
293                 }
294                 TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(gID);
295                 IrrGroup *obj = ent2.getObj();
296                 obj->pup(p);
297                 DEBCHK("Nodegroup PUP'ed: gid = %d, name = %s\n",
298                         obj->ckGetGroupID().idx,
299                         _chareTable[ent2.getcIdx()]->name);
300         }
301         delete [] tmpInfo;
302 }
303
304
305 // handle chare array elements for this processor
306 void CkPupArrayElementsData(PUP::er &p, int notifyListeners)
307 {
308         int i;
309         // safe in both packing/unpakcing at this stage
310         int numGroups = CkpvAccess(_groupIDTable)->size();
311
312         // number of array elements on this processor
313         int numElements;
314         if (!p.isUnpacking()) {
315           ElementCounter  counter;
316           CKLOCMGR_LOOP(mgr->iterate(counter););
317           numElements = counter.getCount();
318         }
319         p|numElements;
320
321         DEBCHK("[%d] CkPupArrayElementsData %s numGroups:%d numElements:%d \n",CkMyPe(),p.typeString(), numGroups, numElements);
322
323         if (!p.isUnpacking())
324         {
325           // let CkLocMgr to iterate and store every array elements
326           CKLOCMGR_LOOP(ElementCheckpointer chk(mgr, p); mgr->iterate(chk););
327         }
328         else {
329           // loop and create all array elements ourselves
330           //CkPrintf("total chare array cnts: %d\n", numElements);
331           for (int i=0; i<numElements; i++) {
332                 CkGroupID gID;
333                 CkArrayIndexMax idx;
334                 p|gID;
335                 p|idx;
336                 CkLocMgr *mgr = (CkLocMgr*)CkpvAccess(_groupTable)->find(gID).getObj();
337                 if (notifyListeners){
338                   mgr->resume(idx,p);
339                 }
340                 else{
341                   mgr->restore(idx,p);
342                 }
343           }
344         }
345         // finish up
346         if (notifyListeners)
347         for(i=0;i<numGroups;i++) {
348                 IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
349                 obj->ckJustMigrated();
350         }
351 }
352
353 #ifdef _FAULT_MLOG_
354 int  CkCountArrayElements(){
355     int numGroups = CkpvAccess(_groupIDTable)->size();
356     int i;
357     ElementCounter  counter;
358     CKLOCMGR_LOOP(mgr->iterate(counter););
359   int numElements = counter.getCount();
360     return numElements;
361 }
362 #endif
363
364 void CkPupProcessorData(PUP::er &p)
365 {
366     // save readonlys, and callback BTW
367     if(CkMyRank()==0) {
368         CkPupROData(p);
369     }
370
371     // save mainchares into MainChares.dat
372     if(CkMyPe()==0) {
373       CkPupMainChareData(p, NULL);
374     }
375         
376     // save groups into Groups.dat
377     CkPupGroupData(p);
378
379     // save nodegroups into NodeGroups.dat
380     if(CkMyRank()==0) {
381         CkPupNodeGroupData(p);
382     }
383
384     // pup array elements
385     CkPupArrayElementsData(p);
386 }
387
388 // called only on pe 0
389 static void checkpointOne(const char* dirname, CkCallback& cb){
390         CmiAssert(CkMyPe()==0);
391         int i;
392         char filename[1024];
393         
394         // save readonlys, and callback BTW
395         sprintf(filename,"%s/RO.dat",dirname);
396         FILE* fRO = fopen(filename,"wb");
397         if(!fRO) CkAbort("Failed to create checkpoint file for readonly data!");
398         PUP::toDisk pRO(fRO);
399         int _numPes = CkNumPes();
400         pRO|_numPes;
401         CkPupROData(pRO);
402         pRO((char *)&cb, sizeof(cb));
403         fclose(fRO);
404
405         // save mainchares into MainChares.dat
406         {
407                 sprintf(filename,"%s/MainChares.dat",dirname);
408                 FILE* fMain = fopen(filename,"wb");
409                 if(!fMain) CkAbort("Failed to open checkpoint file for mainchare data!");
410                 PUP::toDisk pMain(fMain);
411                 CkPupMainChareData(pMain, NULL);
412                 fclose(fMain);
413         }
414 }
415
416 void CkRemoveArrayElements()
417 {
418   int i;
419   int numGroups = CkpvAccess(_groupIDTable)->size();
420   CKLOCMGR_LOOP(mgr->flushAllRecs(););
421 /*  GroupTable *gTbl = CkpvAccess(_groupTable);
422   for(i=0; i<numGroups; i++){
423     IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
424     if(obj->isLocMgr()) {
425         CkLocMgr *mgr = (CkLocMgr *)obj;
426         mgr->flushAllRecs();
427     }
428   }*/
429 }
430
431 /*
432 void CkTestArrayElements()
433 {
434   int i;
435   int numGroups = CkpvAccess(_groupIDTable)->size();
436   //CKLOCMGR_LOOP(mgr->flushAllRecs(););
437   GroupTable *gTbl = CkpvAccess(_groupTable);
438   for(i=0; i<numGroups; i++){
439     IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
440     CkPrintf("An object at [%d]: %p | isLocMgr: %d\n", i, obj, obj->isLocMgr());
441   }
442 }
443 */
444
445 void CkStartCheckpoint(char* dirname,const CkCallback& cb)
446 {
447         CkPrintf("[%d] Checkpoint starting in %s\n", CkMyPe(), dirname);
448         
449         // hand over to checkpoint managers for per-processor checkpointing
450         CProxy_CkCheckpointMgr(_sysChkptMgr).Checkpoint((char *)dirname, cb);
451 }
452
453 /**
454   * Restart: There's no such object as restart manager is created
455   *          because a group cannot restore itself anyway.
456   *          The mechanism exists as converse code and get invoked by
457   *          broadcast message.
458   **/
459
460 void CkRestartMain(const char* dirname, CkArgMsg *args){
461         int i;
462         char filename[1024];
463         CkCallback cb;
464         
465         _inrestart = 1;
466
467         // restore readonlys
468         sprintf(filename,"%s/RO.dat",dirname);
469         FILE* fRO = fopen(filename,"rb");
470         if(!fRO) CkAbort("Failed to open checkpoint file for readonly data!");
471         int _numPes = -1;
472         PUP::fromDisk pRO(fRO);
473         pRO|_numPes;
474         CkPupROData(pRO);
475         pRO|cb;
476         fclose(fRO);
477         DEBCHK("[%d]CkRestartMain: readonlys restored\n",CkMyPe());
478
479         CmiNodeBarrier();
480
481         // restore mainchares
482         sprintf(filename,"%s/MainChares.dat",dirname);
483         FILE* fMain = fopen(filename,"rb");
484         if(fMain && CkMyPe()==0){ // only main chares have been checkpointed, we restart on PE0
485                 PUP::fromDisk pMain(fMain);
486                 CkPupMainChareData(pMain, args);
487                 fclose(fMain);
488                 DEBCHK("[%d]CkRestartMain: mainchares restored\n",CkMyPe());
489                 //bdcastRO(); // moved to CkPupMainChareData()
490         }
491         
492         // restore groups
493         // content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
494         // restore from PE0's copy if shrink/expand
495         if(CkNumPes() != _numPes)
496                 sprintf(filename,"%s/Groups_0.dat",dirname);
497         else
498                 sprintf(filename,"%s/Groups_%d.dat",dirname,CkMyPe());
499         FILE* fGroups = fopen(filename,"rb");
500         if(!fGroups) CkAbort("Failed to open checkpoint file for group table!");
501         PUP::fromDisk pGroups(fGroups);
502         CkPupGroupData(pGroups);
503         fclose(fGroups);
504
505         // restore nodegroups
506         // content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
507         if(CkMyRank()==0){
508                 if(CkNumPes() != _numPes)
509                         sprintf(filename,"%s/NodeGroups_0.dat",dirname);
510                 else
511                         sprintf(filename,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
512                 FILE* fNodeGroups = fopen(filename,"rb");
513                 if(!fNodeGroups) CkAbort("Failed to open checkpoint file for nodegroup table!");
514                 PUP::fromDisk pNodeGroups(fNodeGroups);
515                 CkPupNodeGroupData(pNodeGroups);
516                 fclose(fNodeGroups);
517         }
518
519         // for each location, restore arrays
520         //DEBCHK("[%d]Trying to find location manager\n",CkMyPe());
521         DEBCHK("[%d]Number of PE: %d -> %d\n",CkMyPe(),_numPes,CkNumPes());
522         if(CkMyPe() < _numPes)  // in normal range: restore, otherwise, do nothing
523           for (i=0; i<_numPes;i++) {
524             if (i%CkNumPes() == CkMyPe()) {
525               sprintf(filename,"%s/arr_%d.dat",dirname, i);
526               FILE *datFile=fopen(filename,"rb");
527               if (datFile==NULL) CkAbort("Could not read data file");
528               PUP::fromDisk  p(datFile);
529               CkPupArrayElementsData(p);
530               fclose(datFile);
531             }
532           }
533
534         _inrestart = 0;
535
536         _initDone();
537
538         if(CkMyPe()==0) {
539                 CmiPrintf("[%d]CkRestartMain done. sending out callback.\n",CkMyPe());
540                 cb.send();
541         }
542 }
543
544 // Main chare: initialize system checkpoint manager
545 class CkCheckpointInit : public Chare {
546 public:
547   CkCheckpointInit(CkArgMsg *msg) {
548     _sysChkptMgr = CProxy_CkCheckpointMgr::ckNew();
549     delete msg;
550   }
551   CkCheckpointInit(CkMigrateMessage *m) {delete m;}
552 };
553
554 #include "CkCheckpoint.def.h"
555