nmelone/GoldenCheetah
1
0
Fork 0
mirror of https://github.com/GoldenCheetah/GoldenCheetah.git synced 2026-02-13 16:18:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Data Filter (Part 1 of 3)

Browse Source 
Part 1 of Data filtering, this patch adds the ability
to enter and parse filter statements in the search box
(by clicking on the magnifying glass it becomes a filter
box).

The statements can reference metrics and metadata fields
allowing the user to define boolean expressions to filter
data. An example of the syntax;

    Average_Power > 200 and Duration > 3600

This references the metric Average_Power and ride Duration. But
will also support operations on metadata fields, for example;

    Workout_Code endsWith "SST"

The operators are;
    = <> > >= < <= matches contains endswith beginswith
    ( ) && and || or

Filters are syntactically and semantically validated. But at
this point the resulting tree is not evaluated, i.e. we can
parse the filters, but do not execute them.

Two further updates are pending (once written and tested):
- Part 2 of 3 : Execute filters and apply to the ride list
- Part 3 of 3 : Allow named filters and apply to LTM charts

Further updates will support a visual editor and allowing filters
to be applied to CP and Histogram charts and affect the PMC stress
calculators.
This commit is contained in:
Mark Liversedge
2012-10-28 20:05:12 +00:00
parent 41850aa6be
commit 4c60268245
9 changed files with 542 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/DataFilter.h Normal file
View File

@@ -0,0 +1,96 @@
/*
* Copyright (c) 2012 Mark Liversedge (liversedge@gmail.com)
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <QString>
#include <QObject>
#include <QDebug>
#include <QList>
#include <QStringList>
class MainWindow;
class RideMetric;
class FieldDefinition;
class SummaryMetrics;
class SymbolDef {
public:
enum { Float, Integer, String } type;
union {
RideMetric *metric; // into ride factory
FieldDefinition *meta; // into field definitions in MainWindow
} def;
};
class DataFilter;
class Leaf {
public:
Leaf() : type(none) { }
// evaluate against a SummaryMetric
bool eval(Leaf *, SummaryMetrics);
// tree traversal etc
void print(Leaf *); // print leaf and all children
void validateFilter(DataFilter *, Leaf*); // validate
bool isNumber(DataFilter *df, Leaf *leaf);
void clear(Leaf*);
enum { none, Float, Integer, String, Symbol, Logical, Operation } type;
union value {
float f;
int i;
QString *s;
QString *n;
Leaf *l;
} lvalue, rvalue;
int op;
private:
SymbolDef symbol; // hold information about symbols
};
class DataFilter : public QObject
{
Q_OBJECT
public:
DataFilter(QObject *parent, MainWindow *main);
QMap<QString,QString> lookupMap;
QMap<QString,bool> lookupType; // true if a number, false if a string
public slots:
QStringList parseFilter(QString query);
void clearFilter();
void configUpdate();
//void setData(); // set the file list from the current filter
private:
MainWindow *main;
Leaf *treeRoot;
QStringList errors;
QStringList files;
};

71
src/DataFilter.l Normal file
View File

@@ -0,0 +1,71 @@
%{
/*
* Copyright (c) 2010 Mark Liversedge (liversedge@gmail.com)
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "DataFilter.h"
// need to get rid of this and use a string...
#include <stdio.h>
// tokens
#include "DataFilter_yacc.h"/* generated by the scanner */
%}
%option noyywrap
%option nounput
%%
"=" DataFilterlval.op = EQ; return EQ;
"<>" DataFilterlval.op = NEQ; return NEQ;
"<" DataFilterlval.op = LT; return LT;
"<=" DataFilterlval.op = LTE; return LTE;
">" DataFilterlval.op = GT; return GT;
">=" DataFilterlval.op = GTE; return GTE;
[Mm][Aa][Tt][Cc][Hh][Ee][Ss] DataFilterlval.op = MATCHES; return MATCHES;
[Bb][Ee][Gg][Ii][Nn][Ss][Ww][Ii][Tt][Hh] DataFilterlval.op = BEGINSWITH; return BEGINSWITH;
[Ee][Nn][Dd][Ss][Ww][Ii][Tt][Hh] DataFilterlval.op = ENDSWITH; return ENDSWITH;
[Cc][Oo][Nn][Tt][Aa][Ii][Nn][Ss] DataFilterlval.op = CONTAINS; return CONTAINS;
"&&" DataFilterlval.op = AND; return AND;
[Aa][nN][Dd] DataFilterlval.op = AND; return AND;
"||" DataFilterlval.op = OR; return OR;
[Oo][Rr] DataFilterlval.op = OR; return OR;
[-+]?[0-9]+ return INTEGER;
[-+]?[0-9]+e-[0-9]+ return FLOAT;
[-+]?[0-9]+\.[-e0-9]* return FLOAT;
\"([^\"]|\\\")*\" return STRING; /* contains non-quotes or escaped-quotes */
[a-zA-Z0-9][a-zA-Z0-9_]+ return SYMBOL; /* symbols can start with 0-9 */
[ \n\t\r] ; /* we just ignore whitespace */
%%
void DataFilter_setString(QString p)
{
DataFilter_scan_string(p.toLatin1().data());
}
void DataFilter_clearString()
{
DataFilterlex_destroy();
}

347
src/DataFilter.y Normal file
View File

@@ -0,0 +1,347 @@
%{
/*
* Copyright (c) 2012 Mark Liversedge (liversedge@gmail.com)
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
// This grammar should work with yacc and bison, but has
// only been tested with bison. In addition, since qmake
// uses the -p flag to rename all the yy functions to
// enable multiple grammars in a single executable you
// should make sure you use the very latest bison since it
// has been known to be problematic in the past. It is
// know to work well with bison v2.4.1.
//
// To make the grammar readable I have placed the code
// for each nterm at column 40, this source file is best
// edited / viewed in an editor which is at least 120
// columns wide (e.g. vi in xterm of 120x40)
//
//
#include "DataFilter.h"
#include "MainWindow.h"
#include "RideNavigator.h"
#include <QDebug>
// LEXER VARIABLES WE INTERACT WITH
// Standard yacc/lex variables / functions
extern int DataFilterlex(); // the lexer aka yylex()
extern char *DataFiltertext; // set by the lexer aka yytext
extern void DataFilter_setString(QString);
extern void DataFilter_clearString();
// PARSER STATE VARIABLES
static QStringList DataFiltererrors;
void DataFiltererror(const char *error) { DataFiltererrors << QString(error);}
static Leaf *root; // root node for parsed statement
%}
// Symbol can be meta or metric name
%token <leaf> SYMBOL
// Constants can be a string or a number
%token <leaf> STRING INTEGER FLOAT
// comparative operators
%token <op> EQ NEQ LT LTE GT GTE
%token <op> MATCHES ENDSWITH BEGINSWITH CONTAINS
// logical operators
%token <op> AND OR
%union {
Leaf *leaf;
int op;
}
%type <leaf> value lexpr;
%type <op> lop op;
%start filter;
%%
filter: lexpr { root = $1; }
;
lexpr : '(' lexpr ')' { $$ = new Leaf();
$$->type = Leaf::Logical;
$$->lvalue.l = $2;
$$->op = 0; }
| lexpr lop lexpr { $$ = new Leaf();
$$->type = Leaf::Logical;
$$->lvalue.l = $1;
$$->op = $2;
$$->rvalue.l = $3; }
| value op value { $$ = new Leaf();
$$->type = Leaf::Operation;
$$->lvalue.l = $1;
$$->op = $2;
$$->rvalue.l = $3; }
;
op : EQ
| NEQ
| LT
| LTE
| GT
| GTE
| MATCHES
| ENDSWITH
| BEGINSWITH
| CONTAINS
;
lop : AND
| OR
;
value : SYMBOL { $$ = new Leaf(); $$->type = Leaf::Symbol;
$$->lvalue.n = new QString(DataFiltertext); }
| STRING { $$ = new Leaf(); $$->type = Leaf::String;
$$->lvalue.s = new QString(DataFiltertext); }
| FLOAT { $$ = new Leaf(); $$->type = Leaf::Float;
$$->lvalue.f = QString(DataFiltertext).toFloat(); }
| INTEGER { $$ = new Leaf(); $$->type = Leaf::Integer;
$$->lvalue.i = QString(DataFiltertext).toInt(); }
;
%%
void Leaf::print(Leaf *leaf)
{
switch(leaf->type) {
case Leaf::Float : qDebug()<<"float"<<leaf->lvalue.f; break;
case Leaf::Integer : qDebug()<<"integer"<<leaf->lvalue.i; break;
case Leaf::String : qDebug()<<"string"<<*leaf->lvalue.s; break;
case Leaf::Symbol : qDebug()<<"symbol"<<*leaf->lvalue.n; break;
case Leaf::Logical : qDebug()<<"logical"<<leaf->op;
leaf->print(leaf->lvalue.l);
leaf->print(leaf->rvalue.l);
break;
case Leaf::Operation : qDebug()<<"compare"<<leaf->op;
leaf->print(leaf->lvalue.l);
leaf->print(leaf->rvalue.l);
break;
default:
break;
}
}
bool Leaf::isNumber(DataFilter *df, Leaf *leaf)
{
switch(leaf->type) {
case Leaf::Float : return true;
case Leaf::Integer : return true;
case Leaf::String : return false;
case Leaf::Symbol : return df->lookupType.value(*(leaf->lvalue.n), false);
case Leaf::Logical : return false; // not possible!
case Leaf::Operation : return false;
default:
return false;
break;
}
}
void Leaf::clear(Leaf *leaf)
{
switch(leaf->type) {
case Leaf::String : delete leaf->lvalue.s; break;
case Leaf::Symbol : delete leaf->lvalue.n; break;
case Leaf::Logical :
case Leaf::Operation : clear(leaf->lvalue.l);
clear(leaf->rvalue.l);
delete(leaf->lvalue.l);
delete(leaf->rvalue.l);
break;
default:
break;
}
}
void Leaf::validateFilter(DataFilter *df, Leaf *leaf)
{
switch(leaf->type) {
case Leaf::Symbol :
{
// are the symbols correct?
// if so set the type to meta or metric
// and save the technical name used to do
// a lookup at execution time
QString lookup = df->lookupMap.value(*(leaf->lvalue.n), "");
if (lookup == "") {
DataFiltererrors << QString("Unknown: %1").arg(*(leaf->lvalue.n));
}
}
break;
case Leaf::Operation :
{
// first lets make sure the lhs and rhs are of the same type
bool lhsType = Leaf::isNumber(df, leaf->lvalue.l);
bool rhsType = Leaf::isNumber(df, leaf->rvalue.l);
if (lhsType != rhsType) {
DataFiltererrors << QString("Type mismatch");
}
// what about using string operations on a lhs/rhs that
// are numeric?
if ((lhsType || rhsType) && leaf->op >= MATCHES && leaf->op <= CONTAINS) {
DataFiltererrors << "Mixing string operations with numbers";
}
validateFilter(df, leaf->lvalue.l);
validateFilter(df, leaf->rvalue.l);
}
break;
case Leaf::Logical : validateFilter(df, leaf->lvalue.l);
validateFilter(df, leaf->rvalue.l);
break;
default:
break;
}
}
DataFilter::DataFilter(QObject *parent, MainWindow *main) : QObject(parent), main(main), treeRoot(NULL)
{
configUpdate();
connect(main, SIGNAL(configChanged()), this, SLOT(configUpdate()));
}
QStringList DataFilter::parseFilter(QString query)
{
//DataFilterdebug = 0; // no debug -- needs bison -t in src.pro
root = NULL;
// if something was left behind clear it up now
clearFilter();
// Parse from string
DataFiltererrors.clear(); // clear out old errors
DataFilter_setString(query);
DataFilterparse();
DataFilter_clearString();
// save away the results
treeRoot = root;
// if it passed syntax lets check semantics
if (treeRoot && DataFiltererrors.count() == 0) treeRoot->validateFilter(this, treeRoot);
// ok, did it pass all tests?
if (DataFiltererrors.count() > 0) { // nope
// Bzzzt, malformed
qDebug()<<"parse filter errors:"<<DataFiltererrors;
clearFilter();
} else { // yep! .. we have a winner!
// successfuly parsed, lets check semantics
//treeRoot->print(treeRoot);
}
errors = DataFiltererrors;
return errors;
}
void DataFilter::clearFilter()
{
if (treeRoot) {
treeRoot->clear(treeRoot);
treeRoot = NULL;
}
}
void DataFilter::configUpdate()
{
lookupMap.clear();
lookupType.clear();
// create lookup map from 'friendly name' to name used in smmaryMetrics
// to enable a quick lookup && the lookup for the field type (number, text)
const RideMetricFactory &factory = RideMetricFactory::instance();
for (int i=0; i<factory.metricCount(); i++) {
QString symbol = factory.metricName(i);
QString name = factory.rideMetric(symbol)->name();
lookupMap.insert(name.replace(" ","_"), symbol);
lookupType.insert(name.replace(" ","_"), true);
}
// now add the ride metadata fields -- should be the same generally
foreach(FieldDefinition field, main->rideMetadata()->getFields()) {
QString underscored = field.name;
lookupMap.insert(field.name.replace(" ","_"), field.name);
lookupType.insert(field.name.replace(" ","_"), (field.type > 2)); // true if is number
}
#if 0
QMapIterator<QString, QString>r(lookupMap);
while(r.hasNext()) {
r.next();
qDebug()<<"Lookup"<<r.key()<<"to get"<<r.value();
}
#endif
}
bool Leaf::eval(Leaf *leaf, SummaryMetrics m)
{
switch(leaf->type) {
case Leaf::Logical :
switch (leaf->op) {
case AND :
return (eval(leaf->lvalue.l, m) && eval(leaf->rvalue.l, m));
case OR :
return (eval(leaf->lvalue.l, m) || eval(leaf->rvalue.l, m));
}
case Leaf::Operation :
{
switch (leaf->op) {
case EQ:
case NEQ:
case LT:
case LTE:
case GT:
case GTE:
case MATCHES:
case ENDSWITH:
case BEGINSWITH:
case CONTAINS:
default:
break;
}
}
break;
default:
break;
}
return false;
}

4
src/MainWindow.cpp
View File

@@ -92,6 +92,7 @@
#ifdef GC_HAVE_LUCENE
#include "SearchBox.h"
#include "Lucene.h"
#include "DataFilter.h"
#endif
#include <assert.h>
@@ -203,6 +204,7 @@ MainWindow::MainWindow(const QDir &home) :
_rideMetadata->hide();
#ifdef GC_HAVE_LUCENE
lucene = new Lucene(this); // before metricDB attempts to refresh
datafilter = new DataFilter(this, this); // before metricDB attempts to refresh
#endif
metricDB = new MetricAggregator(this, home, zones(), hrZones()); // just to catch config updates!
metricDB->refreshMetrics();
@@ -462,7 +464,9 @@ MainWindow::MainWindow(const QDir &home) :
toolbuttons->addWidget(searchBox);
//toolbuttons->addStretch();
connect(searchBox, SIGNAL(submitQuery(QString)), this, SLOT(searchSubmitted(QString)));
connect(searchBox, SIGNAL(submitFilter(QString)), datafilter, SLOT(parseFilter(QString)));
connect(searchBox, SIGNAL(clearQuery()), this, SLOT(searchCleared()));
connect(searchBox, SIGNAL(clearFilter()), datafilter, SLOT(clearFilter()));
#endif

2
src/MainWindow.h
View File

@@ -63,6 +63,7 @@ class LionFullScreen;
class QTFullScreen;
class TrainTool;
class Lucene;
class DataFilter;
class SearchBox;
extern QList<MainWindow *> mainwindows; // keep track of all the MainWindows we have open
@@ -157,6 +158,7 @@ class MainWindow : public QMainWindow
#ifdef GC_HAVE_LUCENE
SearchBox *searchBox;
Lucene *lucene;
DataFilter *datafilter;
#endif
// *********************************************

2
src/RideNavigator.h
View File

@@ -37,6 +37,7 @@ class GroupByModel;
class SearchFilter;
class DiaryWindow;
class BUGFIXQSortFilterProxyModel;
class DataFilter;
//
// The RideNavigator
@@ -60,6 +61,7 @@ class RideNavigator : public GcWindow
friend class ::GroupByModel;
friend class ::DiaryWindow;
friend class ::GcCalendar;
friend class ::DataFilter;
public:
RideNavigator(MainWindow *);

16
src/SearchBox.cpp
View File

@@ -35,7 +35,7 @@ SearchBox::SearchBox(QWidget *parent)
clearButton->setStyleSheet("QToolButton { border: none; padding: 0px; }");
clearButton->hide();
connect(clearButton, SIGNAL(clicked()), this, SLOT(clear()));
connect(clearButton, SIGNAL(clicked()), this, SIGNAL(clearQuery()));
connect(clearButton, SIGNAL(clicked()), this, SLOT(clearClicked()));
connect(this, SIGNAL(textChanged(const QString&)), this, SLOT(updateCloseButton(const QString&)));
// search button
@@ -92,6 +92,7 @@ void SearchBox::resizeEvent(QResizeEvent *)
void SearchBox::toggleMode()
{
clear(); // clear whatever is there first
if (mode == Search) setMode(Filter);
else setMode(Search);
}
@@ -124,20 +125,25 @@ void SearchBox::setMode(SearchBoxMode mode)
void SearchBox::updateCloseButton(const QString& text)
{
if (clearButton->isVisible() && text.isEmpty()) clearQuery();
if (clearButton->isVisible() && text.isEmpty()) mode == Search ? clearQuery() : clearFilter();
clearButton->setVisible(!text.isEmpty());
if (mode == Search) searchSubmit();
if (mode == Search) searchSubmit(); // only do search as you type in search mode
}
void SearchBox::searchSubmit()
{
// return hit / key pressed
if (text() != "") {
submitQuery(text());
mode == Search ? submitQuery(text()) : submitFilter(text());
}
}
void SearchBox::clearClicked()
{
mode == Search ? clearQuery() : clearFilter();
}
// Drag and drop columns from the chooser...
void
SearchBox::dragEnterEvent(QDragEnterEvent *event)
@@ -157,5 +163,5 @@ SearchBox::dropEvent(QDropEvent *event)
// we do very little to the name, just space to _ and lower case it for now...
name.replace(' ', '_');
insert(name + ":\"\"");
insert(name + (mode == Search ? ":\"\"" : ""));
}

6
src/SearchBox.h
View File

@@ -45,15 +45,21 @@ private slots:
void updateCloseButton(const QString &text);
void searchSubmit();
void toggleMode();
void clearClicked();
// drop column headings from column chooser
void dragEnterEvent(QDragEnterEvent *event);
void dropEvent(QDropEvent *event);
signals:
// text search mode
void submitQuery(QString);
void clearQuery();
// db filter mode
void submitFilter(QString);
void clearFilter();
private:
QToolButton *clearButton, *searchButton;
SearchBoxMode mode;

5
src/src.pro
View File

@@ -222,6 +222,7 @@ HEADERS += \
CpintPlot.h \
CriticalPowerWindow.h \
CsvRideFile.h \
DataFilter.h \
DataProcessor.h \
DBAccess.h \
DatePickerDialog.h \
@@ -365,8 +366,8 @@ HEADERS += \
Zones.h \
ZoneScaleDraw.h
YACCSOURCES = JsonRideFile.y WithingsParser.y
LEXSOURCES = JsonRideFile.l WithingsParser.l
YACCSOURCES = JsonRideFile.y WithingsParser.y DataFilter.y
LEXSOURCES = JsonRideFile.l WithingsParser.l DataFilter.l
#-t turns on debug, use with caution
#QMAKE_YACCFLAGS = -t -d