Made data print out in tables. Much easier to analyze. Also I added some more data...
authorIsaac Dooley <idooley2@illinois.edu>
Wed, 4 Apr 2007 20:03:53 +0000 (20:03 +0000)
committerIsaac Dooley <idooley2@illinois.edu>
Wed, 4 Apr 2007 20:03:53 +0000 (20:03 +0000)
examples/bigsim/tools/rewritelog/EventInterpolator.C
examples/bigsim/tools/rewritelog/EventInterpolator.h
examples/bigsim/tools/rewritelog/interpolatelog.C

index e909ee61a6c08b0550d0d5083a2134744d186396..3d82c9ea3193b6fdccd460ff55e1516e6d58ce8c 100644 (file)
@@ -1,10 +1,47 @@
-
 #include <EventInterpolator.h>
 
-#define DEBUG
+//#define DEBUG
+#define PRETEND_NO_ENERGY
 
 using namespace std;
 
+
+/** Create a vector such that "count" items out of "length" in the vector are set to true */
+vector<bool> random_select(int count, int length){
+  vector<bool> v(length);
+  int c=0;
+  // Initialize all to false
+  for(int i=0;i<length;++i)
+       v[i] = false;
+  while(c < count){
+       // find a random entry int the vector
+       int r = rand() % length;
+       // mark it as true, incrementing c if this is the first time we mark it
+       if(v[r] == false){
+         c++;
+         v[r] = true;
+       }
+  }
+  return v;
+}
+
+template <typename T, typename T2>
+multimap<T,T2> random_select_multimap(multimap<T,T2> input_map , int count){
+    assert(input_map.size() >= count);
+    vector<bool> which_to_keep = random_select(count, input_map.size());
+    multimap<T,T2> output_map;
+    int len = which_to_keep.size();
+    typename multimap<T, T2>::iterator itr=input_map.begin();
+    for(int i=0; i<len; ++i, ++itr){
+        if(which_to_keep[i] == true){
+          // Do something like this to keep them around:
+            output_map.insert(make_pair((*itr).first, (*itr).second));
+        }
+    }
+//     assert(output_map.size()==count);
+    return output_map;
+}
+
 int EventInterpolator::numCoefficients(const string &funcname){
 // We create a dummy input stringstream and pass it to parseParameters.
 // Then we count how many parameters that function creates
@@ -37,6 +74,7 @@ pair<int,vector<double> > EventInterpolator::parseParameters(const string &funcn
 }
 
 
+/** First parameter is a category or subclass for a particular timed region. For example, we wish to consider computation for adjacent patches differently than patches that are neighbors of adjacent patches. This allows a single timed region to be broken out and treated as a number of different cases. */
 pair<int,vector<double> > EventInterpolator::parseParameters(const string &funcname, istringstream &line, double &time, const bool log){
     vector<double> params;
     int category=0;
@@ -90,8 +128,8 @@ pair<int,vector<double> > EventInterpolator::parseParameters(const string &funcn
 
     }
     else if(funcname == string("*integrate*")){
-        double d1, d2, d3, d4, d5, d6, d7, t1;
-        line >> d1 >> d2 >> d3 >> d4 >> d5 >> d6 >> d7 >> t1;
+        double d1, d2, d3, d4, d5, d6, d7, d8, t1;
+        line >> d1 >> d2 >> d3 >> d4 >> d5 >> d6 >> d7 >> d8 >> t1;
 
         if(log)
             log1 << funcname << "\t" << t1 << "\t" << d1 << "\t" << d2 << "\t" << d3 << "\t" << d4 << "\t" << d5 << "\t" << d6 << "\t" << d7 << endl;
@@ -107,18 +145,20 @@ pair<int,vector<double> > EventInterpolator::parseParameters(const string &funcn
         throw new runtime_error("unknown function");
     }
 
-    return pair<int,vector<double> >(category,params);
+    return make_pair(category,params);
 }
 
 
 
 
 
-EventInterpolator::EventInterpolator(char *table_filename){
-    exact_matches=0;
-    exact_positive_matches=0;
-    approx_matches=0;
-    approx_positive_matches=0;
+EventInterpolator::EventInterpolator(char *table_filename, double sample_rate){
+  std::map<string,double> cycle_accurate_time_sum;
+
+  exact_matches=0;
+  exact_positive_matches=0;
+  approx_matches=0;
+  approx_positive_matches=0;
 
        log1.open("log1");
 
@@ -151,14 +191,65 @@ EventInterpolator::EventInterpolator(char *table_filename){
                        fullParams params = parseParameters(funcname,line,time,false);
             funcIdentifier func(funcname,params.first);
             sample_count[func]++;
+                       cycle_accurate_time_sum[funcname] += time;
+
+            accurateTimings.insert(make_pair(make_pair(func,params.second),time));
+
         }
     }
 
+
+       std::map<string,double>::iterator cycle_i;
+       cout << "The following table displays the total cycle count for each event in the \n" 
+            "cycle-accurate timings file. This may be useful for back of the envelope \n"
+            "calculations for expected performance " << endl << endl;
+       cout << "\t|=========================|===================|" << endl;
+       cout << "\t|                  event  | total cycle count |" << endl;
+       cout << "\t|-------------------------|-------------------|" << endl;
+       for(cycle_i= cycle_accurate_time_sum.begin();cycle_i!=cycle_accurate_time_sum.end();++cycle_i){
+         cout << "\t|";
+         for(int i=0;i<24-(*cycle_i).first.length();++i) 
+               cout << " ";
+         cout << (*cycle_i).first << " | ";
+         cout.width(17);
+         cout << (*cycle_i).second;
+         cout << " |" << endl;
+       }       
+       cout << "\t|=========================|===================|" << endl << endl;
+
+
+
+    unsigned long sample_keep = (unsigned long) ceil(sample_rate * accurateTimings.size());
+    if (sample_rate < 1.0){
+        cout << "Randomly dropping " << (1.0-sample_rate)*100 << "% of the cycle accurate timings" << endl;
+        accurateTimings = random_select_multimap(accurateTimings, sample_keep);
+    }
+
+       analyzeExactVariances();
+
+       cout << "The following table displays the number of timing samples from \n the cycle-accurate file for each event." << endl << endl;
+       cout << "\t|=========================|==========================|" << endl;
+       cout << "\t|                  event  | number of timing samples |" << endl;
+       cout << "\t|-------------------------|--------------------------|" << endl;
+
+
     // Create a gsl interpolator workspace for each event/function
     for(map<funcIdentifier,unsigned long>::iterator i=sample_count.begin(); i!=sample_count.end();++i){
         funcIdentifier func = (*i).first;
         unsigned long samples = (*i).second;
-        cout << "     > " << func.first << "," << func.second << " has " << samples << " sampled timings" << endl;
+               
+        cout << "\t|";
+               
+               for(int i=0;i<20-func.first.length();++i)
+                 cout << " ";
+               cout << func.first << ",";
+               cout.width(3);
+               cout << func.second;
+               
+               cout << " | ";
+               cout.width(24);
+               cout << samples << " |" << endl;
+
 
         if(samples < numCoefficients(func.first) ){
             cerr << "FATAL ERROR: Not enough input timing samples for " << func.first << "," << func.second << " which has " << numCoefficients(func.first) << " coefficients" << endl;
@@ -177,77 +268,90 @@ EventInterpolator::EventInterpolator(char *table_filename){
 
         }
     }
+       cout << "\t|=========================|==========================|" << endl << endl;
 
     accurateTimeTable.close();
-    ifstream accurateTimeTable2(table_filename);
+
 
 #ifdef WRITESTATS
     ofstream statfile("stats-out");
 #endif
 
-    // Second pass, scan through cycle accurate time file
-    while(accurateTimeTable2.good()){
-        string line_s;
-        getline(accurateTimeTable2,line_s);
-        istringstream line(line_s);
+    // Fill in values for matrix X and vector Y which will be fed into the least square fit algorithm
+       // The data is currently in  accurateTimings[pair<funcIdentifier,vector<double> >(func,params.second)]=time;
 
-        string temp("");
-        while(temp != string("TRACEBIGSIM") && line.good() && accurateTimeTable2.good() ){
-            line >> temp;
-        }
-        line >> temp; // gobble up one more worthless bit of input line
+       //iterate through accurateTimings
+       cout << "accurateTimings.size() = " << accurateTimings.size() << endl;
+       map< pair<funcIdentifier,vector<double> >, double >::iterator itr;
+    for(itr=accurateTimings.begin();itr!=accurateTimings.end();++itr){
+         const double &time = (*itr).second;
+         const vector<double> &paramsSecond =(*itr).first.second;
+         const funcIdentifier func = (*itr).first.first;
 
-        if(line.good() && accurateTimeTable2.good()){
-            string funcname;
-            line >> funcname;
+         // lookup the next unused entry in X
+         unsigned next = Xcount[func] ++;
+         gsl_matrix * x = X[func];
 
-            double time;
-            fullParams params = parseParameters(funcname,line,time,false);
+         // copy data into array X and Y
+         for(int param_index=0;param_index<paramsSecond.size();++param_index){
+               gsl_matrix_set(x,next,param_index, paramsSecond[param_index]);
+         }
+         gsl_vector_set(y[func],next,time);
 
-            funcIdentifier func(funcname,params.first);
+       }
 
-            unsigned i = Xcount[func] ++;
-            gsl_matrix * x = X[func];
-            accurateTimings[pair<funcIdentifier,vector<double> >(func,params.second)]=time;
-
-#ifdef WRITESTATS
-            statfile << funcname << "\t" << time << endl;
-#endif
-
-            for(int param_index=0;param_index<params.second.size();++param_index){
-                gsl_matrix_set(x,i,param_index, params.second[param_index]);
-            }
-            gsl_vector_set(y[func],i,time);
-
-        }
-    }
 
 #ifdef WRITESTATS
     statfile.close();
 #endif
 
     // Perform a sanity check now
-
+       int count1=0, count2=0;
     for(map<funcIdentifier, gsl_multifit_linear_workspace *>::iterator i=work.begin();i!=work.end();++i){
-        if(sample_count[(*i).first]!=Xcount[(*i).first]){
-          cerr << "FATAL ERROR: sanity check failed: " << sample_count[(*i).first] << "!=" << Xcount[(*i).first] << "  :(" << endl;
-       throw new runtime_error("sanity check failed");
-        }
-    }
+#if DEBUG
+         cout << "sample count vs Xcount (" << (*i).first.first << "): " << sample_count[(*i).first] << "?=" << Xcount[(*i).first] << " " << endl;
+#endif
+         count1 += sample_count[(*i).first];
+         count2 += Xcount[(*i).first] ;
+       }
+       cout << "Samples from cycle accurate file : " << count1 << ".  Keeping only " << count2 << " of them " << endl;
 
     cout << "Performing Least Squared Fit to sampled time data" << endl;
 
     //  Now do Least Square Fit: Find C where y=Xc
+       cout << "The following table displays the chi^2 measure for how well the model fits the input times." << endl << endl;
+       cout << "\t|=========================|=========|=========|" << endl;
+       cout << "\t|                  event  |   chi^2 |     chi |" << endl;
+       cout << "\t|-------------------------|---------|---------|" << endl;
+
+       cout.setf(ios_base::scientific, ios_base::floatfield);
+       cout.precision(1);
+
     map<funcIdentifier, gsl_multifit_linear_workspace *>::iterator i;
     for(i=work.begin();i!=work.end();++i){
         funcIdentifier func = (*i).first;
         assert(! gsl_multifit_linear(X[func],y[func],c[func],cov[func],&chisqr[func],work[func]));
         gsl_matrix_free(X[func]);
         gsl_vector_free(y[func]);
-        cout << "     > " << func.first << "," << func.second << " has chisqr=" << chisqr[func] << endl;
-    }
 
+               cout << "\t|";
+               
+               for(int i=0;i<20-func.first.length();++i)
+                 cout << " ";
+               cout << func.first << ",";
+               cout.width(3);
+               cout << func.second;
+               
+               cout << " | ";
+               cout.width(7);
+               cout << chisqr[func];
+
+               cout << " | ";
+               cout.width(7);
+               cout << sqrt(chisqr[func]) << " |" << endl;
 
+    }
+       cout << "\t|=========================|=========|=========|" << endl << endl;
 
     // Load in Parameter File which maps event id to function name and parameters
     cout << "Loading parameter files" << endl;
@@ -277,10 +381,19 @@ EventInterpolator::EventInterpolator(char *table_filename){
                     line >> funcname;
 
                     if(t1 == string("TRACEBIGSIM")){
+#ifdef PRETEND_NO_ENERGY
+                                           if(funcname == string("calc_pair_energy")){\
+                                                 funcname = "calc_pair";
+                                               }
+                                               else if(funcname == string("calc_self_energy")){
+                                                 funcname = "calc_self";
+                                               }
+#endif
+
                         fullParams params = parseParameters(funcname,line,false);
                         funcIdentifier func(funcname,params.first);
                         Xcount[func] ++;
-                        eventparams[pair<unsigned,unsigned>(i,eventid)] = pair<funcIdentifier,vector<double> >(func,params.second);
+                        eventparams[make_pair(i,eventid)] = make_pair(func,params.second);
                     }
                 }
             }
@@ -295,9 +408,8 @@ EventInterpolator::EventInterpolator(char *table_filename){
 }
 
 
-
 bool EventInterpolator::haveExactTime(const unsigned pe, const unsigned eventid){
-    return haveExactTime(eventparams[pair<unsigned,unsigned>(pe,eventid)]);
+    return haveExactTime(eventparams[make_pair(pe,eventid)]);
 }
 
 bool EventInterpolator::haveExactTime(const pair<funcIdentifier,vector<double> > &p) {
@@ -305,11 +417,11 @@ bool EventInterpolator::haveExactTime(const pair<funcIdentifier,vector<double> >
 }
 
 bool EventInterpolator::haveExactTime(const funcIdentifier& func, const vector<double> &p){
-    return (accurateTimings.find(pair<funcIdentifier,vector<double> >(func,p)) != accurateTimings.end());
+  return (accurateTimings.find(make_pair(func,p)) != accurateTimings.end());
 }
 
 double EventInterpolator::lookupExactTime(const unsigned pe, const unsigned eventid){
-    return lookupExactTime(eventparams[pair<unsigned,unsigned>(pe,eventid)]);
+    return lookupExactTime(eventparams[make_pair(pe,eventid)]);
 }
 
 double EventInterpolator::lookupExactTime(const pair<funcIdentifier,vector<double> > &p) {
@@ -317,22 +429,156 @@ double EventInterpolator::lookupExactTime(const pair<funcIdentifier,vector<doubl
 }
 
 double EventInterpolator::lookupExactTime(const funcIdentifier& func, const vector<double> &p){
-    double val = accurateTimings[pair<funcIdentifier,vector<double> >(func,p)];
+    const  pair<funcIdentifier,vector<double> > key(func,p);
+    const int count = accurateTimings.count(key);
+
+       pair<timings_type::iterator, timings_type::iterator> itrs = accurateTimings.equal_range(key);
+       double time_sum=0;
+       for(timings_type::iterator itr=itrs.first; itr!=itrs.second; ++itr){
+         time_sum += (*itr).second;
+       }
+       double time_average = time_sum / count;
+
     exact_matches++;
-    if(val>=0.0)
-        exact_positive_matches++;
-       else
-         cout << "exact negative match = " << val << endl;
-    return val;
+       assert(time_average >= 0.0 );
+    return time_average;
 }
 
+
+/** Print out some statistics about the timings for any parameter that has multiple associated cycle-accurate timings */
+void EventInterpolator::analyzeExactVariances(){
+
+    map< pair<funcIdentifier,vector<double> >, double > variances;
+    map< pair<funcIdentifier,vector<double> >, double > means;
+       map< funcIdentifier, double > max_std_dev;
+       map< funcIdentifier, double > associated_mean;
+
+       // Iterate through items in accurateTimings and store variances
+       timings_type::iterator itr;
+       for(itr=accurateTimings.begin();itr!=accurateTimings.end();++itr){
+         variances[(*itr).first] = lookupExactVariance( (*itr).first );
+         means[(*itr).first] = lookupExactMean( (*itr).first );
+       }
+
+       // Display the variances
+       map< pair<funcIdentifier,vector<double> >, double >::iterator vItr;
+       for(vItr = variances.begin(); vItr!=variances.end(); ++vItr){
+         double var = (*vItr).second;
+         double stddev = sqrt(var);
+         double mean = means[(*vItr).first];
+
+         if(var > 0.0){
+               if(stddev > max_std_dev[(*vItr).first.first]){
+                 max_std_dev[(*vItr).first.first] = stddev;
+                 associated_mean[(*vItr).first.first] = mean;
+               }
+         }
+       }
+       
+       // Display the max std dev for any given event
+       cout << "The following events have differing exact times for one or more parameter set.\n"
+               "Each line corresponds to whichever parameter list produced the greatest variance\n"
+               "(and equivalently std dev). The mean is the mean timing value associated with the\n"
+            " same parameter list of greatest variance" << endl << endl;
+
+
+       cout << "\t|=========================|=================|=================|==============|" << endl;
+       cout << "\t|                  event  |     max std dev |            mean | sd % of mean |" << endl;
+       cout << "\t|-------------------------|-----------------|-----------------|--------------|" << endl;
+
+       cout.setf(ios_base::fixed, ios_base::floatfield);
+       cout.precision(1);
+
+       int func_name_field_width=20;
+       map< funcIdentifier, double >::iterator sItr;
+       for(sItr=max_std_dev.begin(); sItr!=max_std_dev.end(); ++sItr) {
+         double stddev = (*sItr).second;
+         double mean = associated_mean[(*sItr).first];
+         cout << "\t|";
+         for(int i=0;i<func_name_field_width-(*sItr).first.first.length();++i) 
+               cout << " ";
+         cout << (*sItr).first.first ;
+         cout << ",";
+         cout.width(3);
+         cout << (*sItr).first.second;
+         cout << " | ";
+         cout.width(15);
+         cout <<  stddev;
+         cout << " | ";
+         cout.width(15);
+         cout << mean << " | ";
+         cout.width(9);
+         cout << (stddev * 100.0 / mean) << "    | " << endl;
+       }
+       cout.setf(ios_base::fmtflags(0), ios_base::floatfield);
+       cout << "\t|=========================|=================|=================|==============|" << endl << endl;
+
+}
+
+/** Lookup the average timing for a given event and parameter list */
+double EventInterpolator::lookupExactVariance(const unsigned pe, const unsigned eventid){
+    return lookupExactVariance(eventparams[make_pair(pe,eventid)]);
+}
+
+/** Lookup the variance in the timings for a given event and parameter list */
+double EventInterpolator::lookupExactVariance(const pair<funcIdentifier,vector<double> > &p) {
+    return lookupExactVariance(p.first,p.second);
+}
+
+/** Lookup the variance in the timings for a given event and parameter list */
+double EventInterpolator::lookupExactVariance(const funcIdentifier& func, const vector<double> &p){
+    const  pair<funcIdentifier,vector<double> > key(func,p);
+    const int count = accurateTimings.count(key);
+
+       // Compute mean
+       pair<timings_type::iterator, timings_type::iterator> itrs = accurateTimings.equal_range(key);
+       double time_sum=0;
+       for(timings_type::iterator itr=itrs.first; itr!=itrs.second; ++itr){
+         time_sum += (*itr).second;
+       }
+       const double time_average = time_sum / count;
+
+       // Compute Variance
+       double V_sum=0;
+       for(timings_type::iterator itr=itrs.first; itr!=itrs.second; ++itr){
+         const double d = (*itr).second-time_average;
+         V_sum += d*d;
+       }
+       const double Var = V_sum / count;
+
+    return Var;
+}
+
+/** Lookup the average timing for a given event and parameter list */
+double EventInterpolator::lookupExactMean(const unsigned pe, const unsigned eventid){
+    return lookupExactMean(eventparams[make_pair(pe,eventid)]);
+}
+/** Lookup the average timing for a given event and parameter list */
+double EventInterpolator::lookupExactMean(const pair<funcIdentifier,vector<double> > &p) {
+    return lookupExactMean(p.first,p.second);
+}
+/** Lookup the average timing for a given event and parameter list */
+double EventInterpolator::lookupExactMean(const funcIdentifier& func, const vector<double> &p){
+    const  pair<funcIdentifier,vector<double> > key(func,p);
+    const int count = accurateTimings.count(key);
+
+       // Compute mean
+       pair<timings_type::iterator, timings_type::iterator> itrs = accurateTimings.equal_range(key);
+       double time_sum=0;
+       for(timings_type::iterator itr=itrs.first; itr!=itrs.second; ++itr){
+         time_sum += (*itr).second;
+       }
+       return time_sum / count;
+}
+
+
 /** If we have a parameterfile entry for the requested pe,eventid pair */
 double EventInterpolator::haveNewTiming(const unsigned pe, const unsigned eventid) {
-    return eventparams.find( pair<unsigned,unsigned>(pe,eventid) ) != eventparams.end();
+    return eventparams.find( make_pair(pe,eventid) ) != eventparams.end();
 }
 
 double EventInterpolator::predictTime(const unsigned pe, const unsigned eventid) {
-    return predictTime(eventparams[pair<unsigned,unsigned>(pe,eventid)]);
+    return predictTime(eventparams[make_pair(pe,eventid)]);
 }
 
 double EventInterpolator::predictTime(const pair<funcIdentifier,vector<double> > &p) {
@@ -349,7 +595,7 @@ double EventInterpolator::getNewTiming(const unsigned pe, const unsigned eventid
        return lookupExactTime(pe,eventid);
   else
        return predictTime(pe,eventid);
-}                        
+}
 
 double EventInterpolator::predictTime(const funcIdentifier &func, const vector<double> &params) {
 
index 195a5e4ed2d07973e02c7bcebae509487cc5381e..db882aa70efcd1edd6e2dee3d4be9c50075b3cd8 100644 (file)
@@ -1,11 +1,13 @@
 
 #include <gsl/gsl_multifit.h>
 #include <iostream>
+#include <iomanip>
 #include <fstream>
 #include <string>
 #include <sstream>
 #include <cassert>
 #include <stdexcept>
+#include <stdlib.h> // for rand()
 
 #include <map>
 #include <string>
@@ -29,10 +31,11 @@ using namespace std;
 
 typedef pair<string,int> funcIdentifier;
 typedef pair<int,vector<double> > fullParams;
+typedef multimap< pair<funcIdentifier,vector<double> >, double > timings_type;
 
 class EventInterpolator{
 private:
-  ofstream log1;
+    ofstream log1;
 
     // For each interpolatable function we record things in these maps:
 
@@ -47,7 +50,7 @@ private:
     map<funcIdentifier, gsl_vector *>y;  // vector of cycle accurate times for each input parameter set
 
     map<pair<unsigned,unsigned>,pair<funcIdentifier,vector<double> > > eventparams;
-    map< pair<funcIdentifier,vector<double> >, double > accurateTimings;
+    timings_type accurateTimings;
 
 
     bool canInterpolateFunc(const funcIdentifier& name);
@@ -66,16 +69,36 @@ public:
 
     double predictTime(const unsigned pe, const unsigned eventid);
     double predictTime(const pair<funcIdentifier,vector<double> > &p);
-    double predictTime(const funcIdentifier &name, const vector<double> &params);
+    double predictTime(const funcIdentifier &f, const vector<double> &p);
 
     bool haveExactTime(const unsigned pe, const unsigned eventid);
     bool haveExactTime(const pair<funcIdentifier,vector<double> > &p);
-    bool haveExactTime(const funcIdentifier& name, const vector<double> &p);
+    bool haveExactTime(const funcIdentifier& f, const vector<double> &p);
 
+
+       /** Compute average time for the provided event signature(name&parameters) */
     double lookupExactTime(const unsigned pe, const unsigned eventid);
+       /** Compute average time for the provided event signature(name&parameters) */
     double lookupExactTime(const pair<funcIdentifier,vector<double> > &p);
+       /** Compute average time for the provided event signature(name&parameters) */
     double lookupExactTime(const funcIdentifier& name, const vector<double> &p);
 
+       /** Compute variance of the timings for the provided event signature(name&parameters) */
+    double lookupExactVariance(const unsigned pe, const unsigned eventid);
+       /** Compute variance of the timings for the provided event signature(name&parameters) */
+    double lookupExactVariance(const pair<funcIdentifier,vector<double> > &p);
+       /** Compute variance of the timings for the the provided event signature(name&parameters)*/
+       double lookupExactVariance(const funcIdentifier& func, const vector<double> &p);
+
+       /** Compute average timing for the provided event signature(name&parameters) */
+    double lookupExactMean(const unsigned pe, const unsigned eventid);
+       /** Compute average timing for the provided event signature(name&parameters) */
+    double lookupExactMean(const pair<funcIdentifier,vector<double> > &p);
+       /** Compute average timing for the the provided event signature(name&parameters)*/
+       double lookupExactMean(const funcIdentifier& func, const vector<double> &p);
+
+       void analyzeExactVariances();
+
        /** Get the new timing exactly if possible otherwise predict it */
        double getNewTiming(const unsigned pe, const unsigned eventid);
 
@@ -91,8 +114,14 @@ public:
 
     void printMatches();
 
+       /**
+               Setup an EventInterpolator object from the data in the specified file
+               @params sample_rate specifies what fraction of the cycle accurate times
+               to keep. The rest will be dropped prior to constructing the best fit
+               interpolating model
+       */
+    EventInterpolator(char *table_filename, double sample_rate=1.0);
 
-    EventInterpolator(char *table_filename);
     ~EventInterpolator();
 
 };
index 2d37ab6d18e67a483ad3505bbb8a327075ff52aa..7c077726b0ef4afbf0e3360efeaaac42955255b5 100644 (file)
@@ -6,9 +6,7 @@
        email : idooley2@uiuc.edu
        date  : Jan 2007
 
-    This uses the gsl library for least-square fitting. Gnu Scientific Library is available under GPL
-
-    Currently we hard code in two parameters some places.
+    This uses the gsl library for least-square fitting. GNU Scientific Library is available under GPL
 
 */
 
 #include <utility> // for std::pair
 #include <vector>
 
-#define OUTPUTDIR "newtraces/"
+#define KEEP_RATIO 1.0
+
+#define OUTPUTDIR "newtraces-keep1.0/"
 #define CYCLE_TIMES_FILE "nopme"
 #define sec_per_cycle 0.00000000025
 
 // Scale the duration of all unknown events by this factor
-#define time_factor 0.2
+//#define time_factor 0.2
+//#define time_factor (1.0/4.418)
+#define time_factor (1.0/6.05)
 
 // Set these for more output:
-#define DEBUG
+//#define DEBUG
 #undef PRINT_NEW_TIMES
 #define WRITE_OUTPUT_FILES
 
@@ -48,7 +50,7 @@ extern BgTimeLineRec* currTline;
 extern int currTlineIdx;
 
 
-/** linearly map a point from an interval with sepcified bounds to a new interval linearly 
+/** linearly map a point from an interval with sepcified bounds to a new interval linearly
     @return a value between new_lower and new_upper
 */
 double map_linearly_to_interval(double val, double old_lower, double old_upper, double new_lower, double new_upper){
@@ -61,20 +63,36 @@ double map_linearly_to_interval(double val, double old_lower, double old_upper,
   else
        new_val = new_lower + ((val-old_lower)*(new_upper-new_lower))/(old_upper-old_lower);
 
+  assert(new_upper >= new_lower);
   assert(new_val <= new_upper);
   assert(new_val >= new_lower);
 
+  assert(old_upper >= old_lower);
+  assert(val <= old_upper);
+  assert(val >= old_lower);
+
   return new_val;
 }
 
 int main()
 {
     // Load in Mambo Times
-    EventInterpolator interpolator(CYCLE_TIMES_FILE);
+    EventInterpolator interpolator(CYCLE_TIMES_FILE, KEEP_RATIO);
 
     int totalProcs, numX, numY, numZ, numCth, numWth, numPes;
        double ratio_sum=0.0;
        unsigned long ratio_count= 0;
+
+       double ratio_bracketed_sum=0.0;
+       unsigned long ratio_bracketed_count = 0;
+
+       double old_times_sum=0.0;
+       double old_bracketed_times_sum=0.0;
+       double new_bracketed_times_sum=0.0;
+
+       ofstream of_ratio_all("ratio_all");
+       ofstream of_ratio_bracketed("ratio_bracketed");
+
        interpolator.printCoefficients();
 
     // load bg trace summary file
@@ -101,7 +119,7 @@ int main()
 
 
     for (int fileNum=0; fileNum<numPes; fileNum++){
-        BgTimeLineRec *tlinerecs = new BgTimeLineRec[totalProcs/numPes];
+        BgTimeLineRec *tlinerecs = new BgTimeLineRec[totalProcs/numPes+1];
         int rec_count = 0;
 
         for(int procNum=fileNum;procNum<totalProcs;procNum+=numPes){
@@ -134,11 +152,8 @@ int main()
                 double old_end   = timeLog->endTime;
                 double old_duration = old_end-old_start;
 
-                               double old_bracket_start = -1.0;
-                               double old_bracket_end;
-
-                               double new_bracket_duration;
-                               double new_other_duration;
+                               double old_bracket_start=0.0;
+                               double old_bracket_end=0.0;
 
                                int have_bracket_start=0;
                                int have_bracket_end=0;
@@ -147,51 +162,76 @@ int main()
                                double middle_piece=old_duration;
                                double end_piece=0.0;
 
-                               //                      assert(old_end > 0.0);
+                               assert(old_duration >= 0.0);
+                               assert(old_end > 0.0);
+
                                if(old_end > old_start){
+                                 old_times_sum += old_duration;
+
+                                 //FIXME: check only the right kind of events.
 
                                  // Look for BG_EVENT_PRINT 'events' inside this event.
                                  for(int i=0;i<timeLog->evts.length();i++){
                                        char *data = (char*)timeLog->evts[i]->data;
-                                       printf("Event ->%s<-\n",data);
                                        if(strncmp(data,"startTraceBigSim",16)==0){
                                          old_bracket_start = old_start+timeLog->evts[i]->rTime;
                                          have_bracket_start = 1;
-                                         printf("\t\tfound startTraceBigSim!!!\n");
                                        }
                                        else if(strncmp(data,"endTraceBigSim",14)==0){
                                          old_bracket_end = old_start+timeLog->evts[i]->rTime;
-                                         have_bracket_end = 1;  
-                                         printf("\t\tfound endTraceBigSim!!!\n");
+                                         have_bracket_end = 1;
                                        }
                                  }
 
-                                 // If we have bracketed timings, the middle part will be the old 
-                                 // bracketed time region, and the begin and end pieces will be non-zero                               
+                                 // If we have bracketed timings, the middle part will be the old
+                                 // bracketed time region, and the begin and end pieces will be non-zero
                                  if(have_bracket_end && have_bracket_start){
                                        begin_piece = old_bracket_start - old_start;
                                        middle_piece = old_bracket_end - old_bracket_start;
-                                       end_piece = (old_end - old_start) - begin_piece - middle_piece;
+                                       end_piece = old_duration - begin_piece - middle_piece;
+                                       old_bracketed_times_sum += middle_piece;
+                                       assert(begin_piece >= 0.0);
+                                       assert(middle_piece >= 0.0);
+                                       assert(end_piece >= 0.0);
+                                 }
+                                 else{
+                                       old_bracket_start = old_start;
+                                       old_bracket_end = old_end;
+                                       assert(old_bracket_end - old_bracket_start >= 0.0);
                                  }
 
 
                                  // If this event occurs in the paramerter file and cycle-accurate simulations, use that data to predict its runtime
-                               
+
                                  if( interpolator.haveNewTiming(procNum,timeLog->seqno) ) {
+                                       double old_middle_piece = middle_piece;
                                        middle_piece = interpolator.getNewTiming(procNum,timeLog->seqno) * sec_per_cycle;
                                        found_event_count ++;
+
+                                       double ratio =  old_middle_piece / middle_piece;
+                                       if(ratio > 1e-10 && ratio < 1e10){
+                                         ratio_bracketed_sum += ratio;
+                                         ratio_bracketed_count ++;
+                                         of_ratio_bracketed << ratio << endl;
+                                       }
+
+
                                  }
                                  // If event is not in parameter file then we just scale its duration by a simple constant
                                  else {
                                        middle_piece = middle_piece*time_factor ;
                                  }
 
-                               
+
                                  if(middle_piece < 0.0) {
                                        middle_piece=0.0;
                                        negative_durations_occured=true;
                                  }
 
+                                 if(have_bracket_end && have_bracket_start){
+                                       new_bracketed_times_sum += middle_piece;
+                                 }
+
                                  // Scale the begin and end pieces by time_factor;
                                  begin_piece = begin_piece*time_factor;
                                  end_piece = end_piece*time_factor;
@@ -201,8 +241,8 @@ int main()
                                  assert(end_piece >= 0.0);
 
                                  double new_start    = old_start;
-                                 double new_end      = new_start + begin_piece + middle_piece + end_piece;
-                                 double new_duration =             begin_piece + middle_piece + end_piece;
+                                 double new_duration = begin_piece + middle_piece + end_piece;
+                                 double new_end      = new_start + new_duration;
 
                                  timeLog->startTime = new_start;
                                  timeLog->endTime   = new_end;
@@ -215,47 +255,48 @@ int main()
 #ifdef PRINT_NEW_TIMES
                                  printf("Rewriting duration of event %d name=%s from [%.10lf , %.10lf] (%.10lf) to [%.10lf , %.10lf] (%.10lf) ratio=%.10lf\n", j, timeLog->name, old_start,old_end,old_end-old_start,new_start,new_end,new_end-new_start,(old_end-old_start)/(new_end-new_start));
 #endif
-                               
+
                                  double ratio = (old_duration)/(new_duration);
-                                 if(ratio >= 0.5 && ratio <= 50.0){
+                                 if(ratio >= 1e-10 && ratio <= 1e10){
                                        ratio_sum += ratio;
                                        ratio_count ++;
+                                       of_ratio_all << ratio << endl;
                                  }
 
                                  // Rewrite times of messages sent from this event
                                  for(int m=0;m<timeLog->msgs.length();m++){
                                        double old_send = timeLog->msgs[m]->sendTime;
                                        double new_send;
-                                 
+
                                        assert(old_send <= old_end);
                                        assert(old_send >= old_start);
-                                 
+
                                        // We have three places where the message is coming from
                                        // We linearly map the value into the beginning, middle, or end piece
                                        if(old_send < old_bracket_start){
                                          new_send = map_linearly_to_interval(old_send, old_start,old_bracket_start,new_start,new_bracket_start);
-                                       } 
+                                       }
                                        else if(old_send < old_bracket_end){
                                          new_send = map_linearly_to_interval(old_send, old_bracket_start,old_bracket_end,new_bracket_start,new_bracket_end);
                                        }
                                        else {
                                          new_send = map_linearly_to_interval(old_send, old_bracket_end,old_end,new_bracket_end,new_end);
-                                       }                                 
-                                 
+                                       }
+
                                        timeLog->msgs[m]->sendTime = new_send;
-                                 
+
 #ifdef PRINT_NEW_TIMES
                                        printf("pe=%d changing message %d send time from %.10lf to %.10lf\n", procNum, m, old_send, new_send);
 #endif
-                                 
+
                                        assert(new_send <= new_end);
-                                       assert(new_send >= new_start);                            
-                                 
+                                       assert(new_send >= new_start);
+
                                  }
                                }
-                               
+
             }
-                       
+
 
         }
 
@@ -279,7 +320,14 @@ int main()
 
     printf("Writing new bgTrace files ...\n");
 
-       printf("average duration ratio: %.15lf\n", ratio_sum / (double)ratio_count);
+       printf("average duration speedup(including unbracketed pieces): %.8lf\n", ratio_sum / (double)ratio_count);
+
+       printf("average bracketed speedup: %.8lf\n", ratio_bracketed_sum / (double)ratio_bracketed_count);
+
+       cout << "Sum of times of bracketed portions of input logs: " << old_bracketed_times_sum << endl;
+       cout << "Sum of times of bracketed portions of output logs: " << new_bracketed_times_sum << endl;
+
+
 
 #ifdef WRITE_OUTPUT_FILES
     // Write out the timelines to the same number of files as we started with.
@@ -288,11 +336,14 @@ int main()
 
     delete [] allNodeOffsets;
 
-    std::cout << "Of the " << total_count << " events found in the bgTrace files, " << found_event_count << " were found in the param files" << endl;
+    cout << "Of the " << total_count << " events found in the bgTrace files, " << found_event_count << " were found in the param files" << endl;
 
     interpolator.printMatches();
 
-    std::cout << "End of program" << std::endl;
+       cout << "The sum of all positive duration events from the original bgTrace files is: " << old_times_sum << endl;
+
+
+    cout << "End of program" << endl;
 
 }