JsonRideFile XDATA support

.. added a new 'XDATA' element for the JsonRideFile and
   RideFile classes.

.. this allows ride file readers to extract and load any
   time series data that doesn't neccessarily fit into
   the usual data points.

.. this was added to support weather data from FIT files
   and also to support 3rd party merging data with the
   GC supported data.

.. ** IMPORTANT ** the XDATA segment is added at the END
   of the JSON format. So older parsers will fail to read
   but will have loaded all data already (i.e. they will
   fail gracefully enough)

   This means files with XDATA can be read by earlier
   versions of GC, but the XDATA will be discarded.

src/FileIO/JsonRideFile.l

@@ -117,6 +117,8 @@ void JsonRideFilefree (void * ptr , yyscan_t /*scanner*/)
 \"STOP\"            return STOP;
 \"CALIBRATIONS\"    return CALIBRATIONS;
 \"VALUE\"           return VALUE;
+\"VALUES\"          return VALUES;
+\"XDATA\"           return XDATA;
 \"REFERENCES\"      return REFERENCES;
 \"SAMPLES\"         return SAMPLES;
 \"SECS\"            return SECS;

src/FileIO/JsonRideFile.y

@@ -59,6 +59,11 @@ struct JsonContext {
     QStringList JsonRideFileerrors;
     QMap <QString, QString> JsonOverrides;
 
+    XDataSeries xdataseries;
+    XDataPoint xdatapoint;
+    QStringList stringlist;
+    QVector<double> numberlist;
+
 };
 
 #define YYSTYPE QString
@@ -109,9 +114,10 @@ static QString protect(const QString string)
 %token RIDE STARTTIME RECINTSECS DEVICETYPE IDENTIFIER
 %token OVERRIDES
 %token TAGS INTERVALS NAME START STOP
-%token CALIBRATIONS VALUE
+%token CALIBRATIONS VALUE VALUES
 %token REFERENCES
-%token SAMPLES SECS KM WATTS NM CAD KPH HR ALTITUDE LAT LON HEADWIND SLOPE TEMP 
+%token XDATA
+%token SAMPLES SECS KM WATTS NM CAD KPH HR ALTITUDE LAT LON HEADWIND SLOPE TEMP
 %token LRBALANCE LTE RTE LPS RPS THB SMO2 RVERT RCAD RCON
 %token LPCO RPCO LPPB RPPB LPPE RPPE LPPPB RPPPB LPPPE RPPPE
 
@@ -143,6 +149,7 @@ rideelement: starttime
             | calibrations
             | references
             | samples
+            | xdata
             ;
 
 /*
@@ -233,6 +240,56 @@ reference_list: reference | reference_list ',' reference;
 reference: '{' series '}'               { jc->JsonRide->appendReference(jc->JsonPoint);
                                           jc->JsonPoint = RideFilePoint();
                                         }
+/*
+ * XData series
+ */
+
+xdata: XDATA ':' '[' xdata_list ']'
+xdata_list: xdata_series
+            | xdata_list ',' xdata_series
+            ;
+
+xdata_series: '{' xdata_items '}'              { XDataSeries *add = new XDataSeries;
+                                                 add->name=jc->xdataseries.name;
+                                                 add->datapoints=jc->xdataseries.datapoints;
+                                                 add->valuename=jc->xdataseries.valuename;
+                                                 jc->JsonRide->addXData(add->name, add);
+
+                                                 // clear for next one
+                                                 jc->xdataseries = XDataSeries();
+                                               }
+
+
+xdata_items: xdata_item
+            | xdata_items ',' xdata_item
+            ;
+
+xdata_item: NAME ':' string                     { jc->xdataseries.name = $3; }
+          | VALUE ':' string                    { jc->xdataseries.valuename << $3; }
+          | VALUES ':' '[' string_list ']'      { jc->xdataseries.valuename = jc->stringlist;
+                                                  jc->stringlist.clear(); }
+          | SAMPLES ':' '[' xdata_samples ']'
+          ;
+
+xdata_samples: xdata_sample
+         | xdata_samples ',' xdata_sample
+         ;
+xdata_sample: '{' xdata_value_list '}'          { jc->xdataseries.datapoints.append(new XDataPoint(jc->xdatapoint));
+                                                  jc->xdatapoint = XDataPoint();
+                                                }
+          ;
+
+xdata_value_list: xdata_value | xdata_value_list ',' xdata_value
+xdata_value:
+        SECS ':' number                         { jc->xdatapoint.secs = jc->JsonNumber; }
+        | KM ':' number                         { jc->xdatapoint.km = jc->JsonNumber; }
+        | VALUE ':' number                      { jc->xdatapoint.number[0] = jc->JsonNumber; }
+        | VALUES ':' '[' number_list ']'        { for(int i=0; i<jc->numberlist.count() && i<8; i++)
+                                                      jc->xdatapoint.number[i]= jc->numberlist[i];
+                                                  jc->numberlist.clear(); }
+        | string ':' number                     { /* ignored for future compatibility */ }
+        | string ':' string                     { /* ignored for future compatibility */ }
+        ;
 
 /*
  * Ride datapoints
@@ -307,6 +364,14 @@ number: JS_INTEGER                         { jc->JsonNumber = QString($1).toInt(
 
 string: JS_STRING                          { jc->JsonString = $1; }
         ;
+
+ string_list: string                       { jc->stringlist << $1; }
+            | string_list ',' string       { jc->stringlist << $3; }
+            ;
+
+ number_list: number                       { jc->numberlist << QString($1).toDouble(); }
+            | number_list ',' number       { jc->numberlist << QString($3).toDouble(); }
+
 %%
 
 
@@ -361,7 +426,7 @@ JsonFileReader::openRideFile(QFile &file, QStringList &errors, QList<RideFile*>*
     // set to non-zero if you want to
     // to debug the yyparse() state machine
     // sending state transitions to stderr
-    //yydebug = 0;
+    //yydebug = 1;
 
     // parse it
     JsonRideFileparse(jc);
@@ -593,6 +658,90 @@ JsonFileReader::writeRideFile(Context *, const RideFile *ride, QFile &file) cons
         out <<"\n\t\t]";
     }
 
+    //
+    // XDATA
+    //
+    if (ride->xdata().count()) {
+        // output the xdata series
+        out << ",\n\t\t\"XDATA\":[\n";
+
+        bool first = true;
+        QMapIterator<QString,XDataSeries*> xdata(ride->xdata());
+        xdata.toFront();
+        while(xdata.hasNext()) {
+
+            // iterate
+            xdata.next();
+
+            XDataSeries *series = xdata.value();
+
+            // does it have values names?
+            if (series->valuename.isEmpty()) continue;
+
+            if (!first) out<<",\n";
+            out << "\t\t{\n";
+
+            // series name
+            out << "\t\t\t\"NAME\" : \"" << xdata.key() << "\",\n";
+
+            // value names
+            if (series->valuename.count() > 1) {
+                out << "\t\t\t\"VALUES\" : [ ";
+                bool firstv=true;
+                foreach(QString x, series->valuename) {
+                    if (!firstv) out << ", ";
+                    out << "\"" << x << "\"";
+                    firstv=false;
+                }
+                out << " ]";
+            } else {
+                out << "\t\t\t\"VALUE\" : \"" << series->valuename[0] << "\"";
+            }
+
+            // samples
+            if (series->datapoints.count()) {
+                out << ",\n\t\t\t\"SAMPLES\" : [\n";
+
+                bool firsts=true;
+                foreach(XDataPoint *p, series->datapoints) {
+                    if (!firsts) out<< ",\n";
+
+                    // multi value sample
+                    if (series->valuename.count()>1) {
+
+                        out << "\t\t\t\t{ \"SECS\":"<<QString("%1").arg(p->secs) <<", "
+                            << "\"KM\":"<<QString("%1").arg(p->km) << ", "
+                            << "\"VALUES\":[ ";
+
+                        bool firstvv=true;
+                        for(int i=0; i<series->valuename.count(); i++) {
+                            if (!firstvv) out << ", ";
+                            out << QString("%1").arg(p->number[i]);
+                            firstvv=false;
+                         }
+                         out << " ] }";
+
+                    } else {
+
+                        out << "\t\t\t\t{ \"SECS\":"<<QString("%1").arg(p->secs) << ", "
+                            << "\"KM\":"<<QString("%1").arg(p->km) << ", "
+                            << "\"VALUE\":" << QString("%1").arg(p->number[0]) << " }";
+                    }
+                    firsts = false;
+                }
+
+                out << "\n\t\t\t]\n";
+            }
+
+            out << "\t\t}";
+
+            // now do next
+            first = false;
+        }
+
+        out << "\n\t\t]";
+    }
+
     // end of ride and document
     out << "\n\t}\n}\n";
 

src/FileIO/RideFile.cpp

@@ -768,11 +768,68 @@ RideFile *RideFileFactory::openRideFile(Context *context, QFile &file,
 
         //foreach(RideFile::seriestype x, result->arePresent()) qDebug()<<"present="<<x;
 
+        // sample code for using XDATA, left here temporarily till we have an
+        // example of using it in a ride file reader
+#if 0
+
+        // ADD XDATA TO RIDEFILE
+
+        // For testing xdata, this code just adds an xdata series
+        // XDataSeries *xdata = new XDataSeries();
+        // xdata->name = "SPEED";
+        // xdata->valuename << "SPEED";
+        // for(int i=0; i<100; i++) {
+        // XDataPoint *p = new XDataPoint();
+        // p->km = i;
+        // p->secs = i;
+        // p->number[0] = i;
+        // xdata->datapoints.append(p);
+        // }
+        // result->addXData("SPEED", xdata);
+
+        // DEBUG OUTPUT TO SHOW XDATA LOADED FROM RIDEFILE
+        // for testing, print out what we loaded
+        if (result->xdata_.count()) {
+
+            // output the xdata series
+            qDebug()<<"XDATA";
+
+            QMapIterator<QString,XDataSeries*> xdata(result->xdata());
+            xdata.toFront();
+            while(xdata.hasNext()) {
+
+                // iterate
+                xdata.next();
+
+                XDataSeries *series = xdata.value();
+
+                // does it have values names?
+                if (series->valuename.isEmpty()) {
+                    qDebug()<<"empty xdata"<<series->name;
+                    continue;
+                } else {
+                    qDebug()<<"xdata" <<series->name<<series->valuename<<series->datapoints.count();
+                }
+
+                // samples
+                if (series->datapoints.count()) {
+                    foreach(XDataPoint *p, series->datapoints)
+                        qDebug()<<"sample:"<<p->secs<<p->km<<p->number[0]<<p->number[1];
+                }
+            }
+        }
+#endif
     }
 
     return result;
 }
 
+void
+RideFile::addXData(QString name, XDataSeries *series)
+{
+    xdata_.insert(name, series);
+}
+
 QStringList RideFileFactory::listRideFiles(const QDir &dir) const
 {
     QStringList filters;

src/FileIO/RideFile.h

@@ -34,6 +34,8 @@ class Specification;
 class IntervalItem;
 class WPrime;
 class RideFile;
+class XDataSeries;
+class XDataPoint;
 struct RideFilePoint;
 struct RideFileDataPresent;
 class RideFileInterval;
@@ -325,6 +327,11 @@ class RideFile : public QObject // QObject to emit signals
  
         WPrime *wprimeData(); // return wprime, init/refresh if needed
 
+        // XDATA
+        XDataSeries *xdata(QString name);
+        void addXData(QString name, XDataSeries *series);
+        const QMap<QString,XDataSeries*> &xdata() const { return xdata_; }
+
         // METRIC OVERRIDES
         QMap<QString,QMap<QString,QString> > metricOverrides;
 
@@ -401,6 +408,9 @@ class RideFile : public QObject // QObject to emit signals
 
         bool dstale; // is derived data up to date?
 
+        // xdata series
+        QMap<QString, XDataSeries*> xdata_;
+
         // data required to compute headwind based on weather broadcast
         double windSpeed_, windHeading_;
 };
@@ -505,6 +515,37 @@ class RideFileIterator {
         int start, stop, index;
 };
 
+class XDataSeries {
+public:
+    QString name;
+    QStringList valuename;
+    QVector<XDataPoint*> datapoints;
+};
+
+// each sample has up to 8 strings or values
+class XDataPoint {
+public:
+    XDataPoint() {
+        secs=km=0;
+        for(int i=0; i<8; i++) {
+            number[i]=0;
+            string[i]="";
+        }
+    }
+    XDataPoint (const XDataPoint &other) {
+        this->secs=other.secs;
+        this->km=other.km;
+        for(int i=0; i<8; i++) {
+            this->number[i]= other.number[i];
+            this->string[i]= other.string[i];
+        }
+    }
+
+    double secs, km;
+    double number[8];
+    QString string[8];
+};
+
 struct RideFileReader {
     virtual ~RideFileReader() {}
     virtual RideFile *openRideFile(QFile &file, QStringList &errors, QList<RideFile*>* = 0) const = 0;