5e154fae46c73befb1348b9ec893c325be518f71
[charm.git] / src / ck-core / ckcheckpoint.C
1 /*
2 Charm++ File: Checkpoint Library
3 added 01/03/2003 by Chao Huang, chuang10@uiuc.edu
4
5 More documentation goes here...
6 --- Updated 12/14/2003 by Gengbin, gzheng@uiuc.edu
7     see ckcheckpoint.h for change log
8 */
9
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include "charm++.h"
14 #include "ck.h"
15 #include "ckcheckpoint.h"
16
17 #define DEBCHK  // CkPrintf
18
19 #define DEBUGC(x) x
20 //#define DEBUGC(x) 
21
22 CkGroupID _sysChkptMgr;
23
24 typedef struct _GroupInfo{
25         CkGroupID gID;
26         int MigCtor, DefCtor;
27         char name[256];
28 } GroupInfo;
29 PUPbytes(GroupInfo)
30 PUPmarshall(GroupInfo)
31
32 int _inrestart = 0;
33
34 // help class to find how many array elements
35 class ElementCounter : public CkLocIterator {
36 private:
37         int count;
38 public:
39         ElementCounter():count(0){};
40         void addLocation(CkLocation &loc)  { count++; }
41         int getCount() { return count; }
42 };
43
44 // helper class to pup all elements that belong to same ckLocMgr
45 class ElementCheckpointer : public CkLocIterator {
46 private:
47         CkLocMgr *locMgr;
48         PUP::er &p;
49 public:
50         ElementCheckpointer(CkLocMgr* mgr_, PUP::er &p_):locMgr(mgr_),p(p_){};
51         void addLocation(CkLocation &loc) {
52                 CkArrayIndexMax idx=loc.getIndex();
53                 CkGroupID gID = locMgr->ckGetGroupID();
54                 p|gID;      // store loc mgr's GID as well for easier restore
55                 p|idx;
56                 p|loc;
57                 //CkPrintf("[%d] addLocation: ", CkMyPe()), idx.print();
58         }
59 };
60
61
62 extern void _initDone();
63
64 static void bdcastRO(void){
65         int i;
66         //Determine the size of the RODataMessage
67         PUP::sizer ps;
68         for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
69
70         //Allocate and fill out the RODataMessage
71         envelope *env = _allocEnv(RODataMsg, ps.size());
72         PUP::toMem pp((char *)EnvToUsr(env));
73         for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
74         
75         env->setCount(++_numInitMsgs);
76         env->setSrcPe(CkMyPe());
77         CmiSetHandler(env, _roRestartHandlerIdx);
78         CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
79 }
80
81 // Print out an array index to this string as decimal fields
82 // separated by underscores.
83 void printIndex(const CkArrayIndex &idx,char *dest) {
84         const int *idxData=idx.data();
85         for (int i=0;i<idx.nInts;i++) {
86                 sprintf(dest,"%s%d",i==0?"":"_", idxData[i]);
87                 dest+=strlen(dest);
88         }
89 }
90
91 static void checkpointOne(const char* dirname, CkCallback& cb);
92
93 // broadcast
94 void CkCheckpointMgr::Checkpoint(const char *dirname, CkCallback& cb){
95         chkptStartTimer = CmiWallTimer();
96         // every body make dir in case it is local directory
97         CmiMkdir(dirname);
98
99         if (CkMyPe() == 0) {
100           checkpointOne(dirname, cb);
101         }
102
103         char fileName[1024];
104         // save groups into Groups.dat
105         // content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
106         sprintf(fileName,"%s/Groups_%d.dat",dirname,CkMyPe());
107         FILE* fGroups = fopen(fileName,"wb");
108         if(!fGroups) CkAbort("Failed to create checkpoint file for group table!");
109         PUP::toDisk pGroups(fGroups);
110         CkPupGroupData(pGroups);
111         fclose(fGroups);
112
113         // save nodegroups into NodeGroups.dat
114         // content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
115         if (CkMyRank() == 0) {
116           sprintf(fileName,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
117           FILE* fNodeGroups = fopen(fileName,"wb");
118           if(!fNodeGroups) 
119             CkAbort("Failed to create checkpoint file for nodegroup table!");
120           PUP::toDisk pNodeGroups(fNodeGroups);
121           CkPupNodeGroupData(pNodeGroups);
122           fclose(fNodeGroups);
123         }
124
125         //DEBCHK("[%d]CkCheckpointMgr::Checkpoint called dirname={%s}\n",CkMyPe(),dirname);
126         sprintf(fileName,"%s/arr_%d.dat",dirname, CkMyPe());
127         FILE *datFile=fopen(fileName,"wb");
128         if (datFile==NULL) CkAbort("Could not create data file");
129         PUP::toDisk  p(datFile);
130         CkPupArrayElementsData(p);
131         fclose(datFile);
132
133 #if CMK_HAS_SYNC && ! CMK_DISABLE_SYNC
134         system("sync");
135 #endif
136
137         restartCB = cb;
138         DEBCHK("[%d]restartCB installed\n",CkMyPe());
139         CkCallback localcb(CkIndex_CkCheckpointMgr::SendRestartCB(NULL),0,thisgroup);
140         contribute(0,NULL,CkReduction::sum_int,localcb);
141 }
142
143 void CkCheckpointMgr::SendRestartCB(CkReductionMsg *m){ 
144         delete m; 
145         DEBCHK("[%d]Sending out the cb\n",CkMyPe());
146         CkPrintf("Checkpoint to disk finished in %fs, sending out the cb...\n", CmiWallTimer() - chkptStartTimer);
147         restartCB.send(); 
148 }
149
150 void CkPupROData(PUP::er &p)
151 {
152         int _numReadonlies;
153         if (!p.isUnpacking()) _numReadonlies=_readonlyTable.size();
154         p|_numReadonlies;
155         if (p.isUnpacking()) {
156           if (_numReadonlies != _readonlyTable.size())
157             CkAbort("You cannot add readonlies and restore from checkpoint...");
158         }
159         for(int i=0;i<_numReadonlies;i++) _readonlyTable[i]->pupData(p);
160 }
161
162 // handle main chare
163 void CkPupMainChareData(PUP::er &p, CkArgMsg *args)
164 {
165         int nMains=_mainTable.size();
166         DEBCHK("[%d] CkPupMainChareData %s: nMains = %d\n", CkMyPe(),p.typeString(),nMains);
167         for(int i=0;i<nMains;i++){  /* Create all mainchares */
168                 ChareInfo *entry = _chareTable[_mainTable[i]->chareIdx];
169                 int entryMigCtor = entry->getMigCtor();
170                 if(entryMigCtor!=-1) {
171                         Chare* obj;
172                         if (p.isUnpacking()) {
173                                 int size = entry->size;
174                                 DEBCHK("MainChare PUP'ed: name = %s, idx = %d, size = %d\n", entry->name, i, size);
175                                 obj = (Chare*)malloc(size);
176                                 _MEMCHECK(obj);
177                                 _mainTable[i]->setObj(obj);
178                                 //void *m = CkAllocSysMsg();
179                                 _entryTable[entryMigCtor]->call(args, obj);
180                         }
181                         else 
182                                 obj = (Chare *)_mainTable[i]->getObj();
183                         obj->pup(p);
184                 }
185         }
186         // to update mainchare proxy
187         // only readonly variables of Chare Proxy is taken care of here;
188         // in general, if chare proxy is contained in some data structure
189         // for example CkCallback, it is user's responsibility to
190         // update them after restarting
191         if (p.isUnpacking() && CkMyPe()==0)
192                 bdcastRO();
193 }
194
195 #if CMK_FT_CHARE
196
197 CpvExtern(CkVec<void *>, chare_objs);
198 CpvExtern(CkVec<VidBlock *>, vidblocks);
199
200 // handle plain non-migratable chare
201 void CkPupChareData(PUP::er &p)
202 {
203   int i, n;
204   if (!p.isUnpacking()) n = CpvAccess(chare_objs).size();
205   p|n;
206   for (i=0; i<n; i++) {
207         Chare* obj;
208         int size;
209         if (!p.isUnpacking()) {
210                 PUP::sizer ps;
211                 obj = (Chare*)CpvAccess(chare_objs)[i];
212                 obj->pup(ps);
213                 size = ps.size();
214         }
215         p | size;
216         if (p.isUnpacking()) {
217                 //DEBCHK("Chare PUP'ed: name = %s, idx = %d, size = %d\n", entry->name, i, size);
218                 obj = (Chare*)malloc(size);
219                 _MEMCHECK(obj);
220                 CpvAccess(chare_objs).push_back(obj);
221         }
222         obj->pup(p);
223   }
224
225   if (!p.isUnpacking()) n = CpvAccess(vidblocks).size();
226   p|n;
227   for (i=0; i<n; i++) {
228         VidBlock *v;
229         if (p.isUnpacking()) {
230                 v = new VidBlock();
231                 CpvAccess(vidblocks).push_back(v);
232         }
233         else
234                 v = CpvAccess(vidblocks)[i];
235         v->pup(p);
236   }
237 }
238 #else
239 void CkPupChareData(PUP::er &p)
240 {
241    // not implemented
242 }
243 #endif
244
245 // handle GroupTable and data
246 void CkPupGroupData(PUP::er &p)
247 {
248         int numGroups, i;
249
250         if (!p.isUnpacking()) {
251           numGroups = CkpvAccess(_groupIDTable)->size();
252         }
253         p|numGroups;
254         if (p.isUnpacking()) {
255           if(CkMyPe()==0)  
256             CkpvAccess(_numGroups) = numGroups+1; 
257           else 
258             CkpvAccess(_numGroups) = 1;
259         }
260         DEBCHK("[%d] CkPupGroupData %s: numGroups = %d\n", CkMyPe(),p.typeString(),numGroups);
261
262         GroupInfo *tmpInfo = new GroupInfo [numGroups];
263         if (!p.isUnpacking()) {
264           for(i=0;i<numGroups;i++) {
265                 tmpInfo[i].gID = (*CkpvAccess(_groupIDTable))[i];
266                 TableEntry ent = CkpvAccess(_groupTable)->find(tmpInfo[i].gID);
267                 tmpInfo[i].MigCtor = _chareTable[ent.getcIdx()]->migCtor;
268                 tmpInfo[i].DefCtor = _chareTable[ent.getcIdx()]->defCtor;
269                 strncpy(tmpInfo[i].name,_chareTable[ent.getcIdx()]->name,255);
270                 DEBCHK("[%d] CkPupGroupData: %s group %s \n",
271                         CkMyPe(), p.typeString(), tmpInfo[i].name);
272
273                 if(tmpInfo[i].MigCtor==-1) {
274                         char buf[512];
275                         sprintf(buf,"Group %s needs a migration constructor and PUP'er routine for restart.\n", tmpInfo[i].name);
276                         CkAbort(buf);
277                 }
278           }
279         }
280         for (i=0; i<numGroups; i++) p|tmpInfo[i];
281
282         for(i=0;i<numGroups;i++) 
283         {
284           CkGroupID gID = tmpInfo[i].gID;
285           if (p.isUnpacking()) {
286             //CkpvAccess(_groupIDTable)->push_back(gID);
287             int eIdx = tmpInfo[i].MigCtor;
288             // error checking
289             if (eIdx == -1) {
290               CkPrintf("[%d] ERROR> Group %s's migration constructor is not defined!\n", CkMyPe(), tmpInfo[i].name); CkAbort("Abort");
291             }
292             void *m = CkAllocSysMsg();
293             envelope* env = UsrToEnv((CkMessage *)m);
294             CkCreateLocalGroup(gID, eIdx, env);
295           }   // end of unPacking
296           IrrGroup *gobj = CkpvAccess(_groupTable)->find(gID).getObj();
297           // if using migration constructor, you'd better have a pup
298           gobj->pup(p);
299           DEBCHK("Group PUP'ed: gid = %d, name = %s\n",
300                         gobj->ckGetGroupID().idx, tmpInfo[i].name);
301         }
302         delete [] tmpInfo;
303 }
304
305 // handle NodeGroupTable and data
306 void CkPupNodeGroupData(PUP::er &p)
307 {
308         int numNodeGroups, i;
309         if (!p.isUnpacking()) {
310           numNodeGroups = CksvAccess(_nodeGroupIDTable).size();
311         }
312         p|numNodeGroups;
313         if (p.isUnpacking()) {
314           if(CkMyPe()==0){ CksvAccess(_numNodeGroups) = numNodeGroups+1; }
315           else { CksvAccess(_numNodeGroups) = 1; }
316         }
317         DEBCHK("[%d] CkPupNodeGroupData %s: numNodeGroups = %d\n",CkMyPe(),p.typeString(),numNodeGroups);
318
319         GroupInfo *tmpInfo = new GroupInfo [numNodeGroups];
320         if (!p.isUnpacking()) {
321           for(i=0;i<numNodeGroups;i++) {
322                 tmpInfo[i].gID = CksvAccess(_nodeGroupIDTable)[i];
323                 TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(tmpInfo[i].gID);
324                 tmpInfo[i].MigCtor = _chareTable[ent2.getcIdx()]->migCtor;
325                 if(tmpInfo[i].MigCtor==-1) {
326                         char buf[512];
327                         sprintf(buf,"NodeGroup %s either need a migration constructor and\n\
328                                      declared as [migratable] in .ci to be able to checkpoint.",\
329                                      _chareTable[ent2.getcIdx()]->name);
330                         CkAbort(buf);
331                 }
332           }
333         }
334         for (i=0; i<numNodeGroups; i++) p|tmpInfo[i];
335         for (i=0;i<numNodeGroups;i++) {
336                 CkGroupID gID = tmpInfo[i].gID;
337                 if (p.isUnpacking()) {
338                         //CksvAccess(_nodeGroupIDTable).push_back(gID);
339                         int eIdx = tmpInfo[i].MigCtor;
340                         void *m = CkAllocSysMsg();
341                         envelope* env = UsrToEnv((CkMessage *)m);
342                         CkCreateLocalNodeGroup(gID, eIdx, env);
343                 }
344                 TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(gID);
345                 IrrGroup *obj = ent2.getObj();
346                 obj->pup(p);
347                 DEBCHK("Nodegroup PUP'ed: gid = %d, name = %s\n",
348                         obj->ckGetGroupID().idx,
349                         _chareTable[ent2.getcIdx()]->name);
350         }
351         delete [] tmpInfo;
352 }
353
354
355 // handle chare array elements for this processor
356 void CkPupArrayElementsData(PUP::er &p, int notifyListeners)
357 {
358         int i;
359         // safe in both packing/unpakcing at this stage
360         int numGroups = CkpvAccess(_groupIDTable)->size();
361
362         // number of array elements on this processor
363         int numElements;
364         if (!p.isUnpacking()) {
365           ElementCounter  counter;
366           CKLOCMGR_LOOP(mgr->iterate(counter););
367           numElements = counter.getCount();
368         }
369         p|numElements;
370
371         DEBCHK("[%d] CkPupArrayElementsData %s numGroups:%d numElements:%d \n",CkMyPe(),p.typeString(), numGroups, numElements);
372
373         if (!p.isUnpacking())
374         {
375           // let CkLocMgr to iterate and store every array elements
376           CKLOCMGR_LOOP(ElementCheckpointer chk(mgr, p); mgr->iterate(chk););
377         }
378         else {
379           // loop and create all array elements ourselves
380           //CkPrintf("total chare array cnts: %d\n", numElements);
381           for (int i=0; i<numElements; i++) {
382                 CkGroupID gID;
383                 CkArrayIndexMax idx;
384                 p|gID;
385                 p|idx;
386                 CkLocMgr *mgr = (CkLocMgr*)CkpvAccess(_groupTable)->find(gID).getObj();
387                 if (notifyListeners){
388                   mgr->resume(idx,p);
389                 }
390                 else{
391                   mgr->restore(idx,p);
392                 }
393           }
394         }
395         // finish up
396         if (notifyListeners)
397         for(i=0;i<numGroups;i++) {
398                 IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
399                 obj->ckJustMigrated();
400         }
401 }
402
403 #ifdef _FAULT_MLOG_
404 int  CkCountArrayElements(){
405     int numGroups = CkpvAccess(_groupIDTable)->size();
406     int i;
407     ElementCounter  counter;
408     CKLOCMGR_LOOP(mgr->iterate(counter););
409   int numElements = counter.getCount();
410     return numElements;
411 }
412 #endif
413
414 void CkPupProcessorData(PUP::er &p)
415 {
416     // save readonlys, and callback BTW
417     if(CkMyRank()==0) {
418         CkPupROData(p);
419     }
420
421     // save mainchares into MainChares.dat
422     if(CkMyPe()==0) {
423       CkPupMainChareData(p, NULL);
424     }
425         
426     // save non-migratable chare
427     CkPupChareData(p);
428
429     // save groups 
430     CkPupGroupData(p);
431
432     // save nodegroups
433     if(CkMyRank()==0) {
434         CkPupNodeGroupData(p);
435     }
436
437     // pup array elements
438     CkPupArrayElementsData(p);
439 }
440
441 // called only on pe 0
442 static void checkpointOne(const char* dirname, CkCallback& cb){
443         CmiAssert(CkMyPe()==0);
444         int i;
445         char filename[1024];
446         
447         // save readonlys, and callback BTW
448         sprintf(filename,"%s/RO.dat",dirname);
449         FILE* fRO = fopen(filename,"wb");
450         if(!fRO) CkAbort("Failed to create checkpoint file for readonly data!");
451         PUP::toDisk pRO(fRO);
452         int _numPes = CkNumPes();
453         pRO|_numPes;
454         CkPupROData(pRO);
455         pRO((char *)&cb, sizeof(cb));
456         fclose(fRO);
457
458         // save mainchares into MainChares.dat
459         {
460                 sprintf(filename,"%s/MainChares.dat",dirname);
461                 FILE* fMain = fopen(filename,"wb");
462                 if(!fMain) CkAbort("Failed to open checkpoint file for mainchare data!");
463                 PUP::toDisk pMain(fMain);
464                 CkPupMainChareData(pMain, NULL);
465                 fclose(fMain);
466         }
467 }
468
469 void CkRemoveArrayElements()
470 {
471   int i;
472   int numGroups = CkpvAccess(_groupIDTable)->size();
473   CKLOCMGR_LOOP(mgr->flushAllRecs(););
474 /*  GroupTable *gTbl = CkpvAccess(_groupTable);
475   for(i=0; i<numGroups; i++){
476     IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
477     if(obj->isLocMgr()) {
478         CkLocMgr *mgr = (CkLocMgr *)obj;
479         mgr->flushAllRecs();
480     }
481   }*/
482 }
483
484 /*
485 void CkTestArrayElements()
486 {
487   int i;
488   int numGroups = CkpvAccess(_groupIDTable)->size();
489   //CKLOCMGR_LOOP(mgr->flushAllRecs(););
490   GroupTable *gTbl = CkpvAccess(_groupTable);
491   for(i=0; i<numGroups; i++){
492     IrrGroup *obj = CkpvAccess(_groupTable)->find((*CkpvAccess(_groupIDTable))[i]).getObj();
493     CkPrintf("An object at [%d]: %p | isLocMgr: %d\n", i, obj, obj->isLocMgr());
494   }
495 }
496 */
497
498 void CkStartCheckpoint(char* dirname,const CkCallback& cb)
499 {
500         CkPrintf("[%d] Checkpoint starting in %s\n", CkMyPe(), dirname);
501         
502         // hand over to checkpoint managers for per-processor checkpointing
503         CProxy_CkCheckpointMgr(_sysChkptMgr).Checkpoint((char *)dirname, cb);
504 }
505
506 /**
507   * Restart: There's no such object as restart manager is created
508   *          because a group cannot restore itself anyway.
509   *          The mechanism exists as converse code and get invoked by
510   *          broadcast message.
511   **/
512
513 void CkRestartMain(const char* dirname, CkArgMsg *args){
514         int i;
515         char filename[1024];
516         CkCallback cb;
517         
518         _inrestart = 1;
519
520         // restore readonlys
521         sprintf(filename,"%s/RO.dat",dirname);
522         FILE* fRO = fopen(filename,"rb");
523         if(!fRO) CkAbort("Failed to open checkpoint file for readonly data!");
524         int _numPes = -1;
525         PUP::fromDisk pRO(fRO);
526         pRO|_numPes;
527         CkPupROData(pRO);
528         pRO|cb;
529         fclose(fRO);
530         DEBCHK("[%d]CkRestartMain: readonlys restored\n",CkMyPe());
531
532         CmiNodeBarrier();
533
534         // restore mainchares
535         sprintf(filename,"%s/MainChares.dat",dirname);
536         FILE* fMain = fopen(filename,"rb");
537         if(fMain && CkMyPe()==0){ // only main chares have been checkpointed, we restart on PE0
538                 PUP::fromDisk pMain(fMain);
539                 CkPupMainChareData(pMain, args);
540                 fclose(fMain);
541                 DEBCHK("[%d]CkRestartMain: mainchares restored\n",CkMyPe());
542                 //bdcastRO(); // moved to CkPupMainChareData()
543         }
544         
545         // restore groups
546         // content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
547         // restore from PE0's copy if shrink/expand
548         if(CkNumPes() != _numPes)
549                 sprintf(filename,"%s/Groups_0.dat",dirname);
550         else
551                 sprintf(filename,"%s/Groups_%d.dat",dirname,CkMyPe());
552         FILE* fGroups = fopen(filename,"rb");
553         if(!fGroups) CkAbort("Failed to open checkpoint file for group table!");
554         PUP::fromDisk pGroups(fGroups);
555         CkPupGroupData(pGroups);
556         fclose(fGroups);
557
558         // restore nodegroups
559         // content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
560         if(CkMyRank()==0){
561                 if(CkNumPes() != _numPes)
562                         sprintf(filename,"%s/NodeGroups_0.dat",dirname);
563                 else
564                         sprintf(filename,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
565                 FILE* fNodeGroups = fopen(filename,"rb");
566                 if(!fNodeGroups) CkAbort("Failed to open checkpoint file for nodegroup table!");
567                 PUP::fromDisk pNodeGroups(fNodeGroups);
568                 CkPupNodeGroupData(pNodeGroups);
569                 fclose(fNodeGroups);
570         }
571
572         // for each location, restore arrays
573         //DEBCHK("[%d]Trying to find location manager\n",CkMyPe());
574         DEBCHK("[%d]Number of PE: %d -> %d\n",CkMyPe(),_numPes,CkNumPes());
575         if(CkMyPe() < _numPes)  // in normal range: restore, otherwise, do nothing
576           for (i=0; i<_numPes;i++) {
577             if (i%CkNumPes() == CkMyPe()) {
578               sprintf(filename,"%s/arr_%d.dat",dirname, i);
579               FILE *datFile=fopen(filename,"rb");
580               if (datFile==NULL) CkAbort("Could not read data file");
581               PUP::fromDisk  p(datFile);
582               CkPupArrayElementsData(p);
583               fclose(datFile);
584             }
585           }
586
587         _inrestart = 0;
588
589         _initDone();
590
591         if(CkMyPe()==0) {
592                 CmiPrintf("[%d]CkRestartMain done. sending out callback.\n",CkMyPe());
593                 cb.send();
594         }
595 }
596
597 // Main chare: initialize system checkpoint manager
598 class CkCheckpointInit : public Chare {
599 public:
600   CkCheckpointInit(CkArgMsg *msg) {
601     _sysChkptMgr = CProxy_CkCheckpointMgr::ckNew();
602     delete msg;
603   }
604   CkCheckpointInit(CkMigrateMessage *m) {delete m;}
605 };
606
607 #include "CkCheckpoint.def.h"
608