From 4c60268245ef1b7bd558aaf6a39ff90efa2b9250 Mon Sep 17 00:00:00 2001 From: Mark Liversedge Date: Sun, 28 Oct 2012 20:05:12 +0000 Subject: [PATCH] Data Filter (Part 1 of 3) Part 1 of Data filtering, this patch adds the ability to enter and parse filter statements in the search box (by clicking on the magnifying glass it becomes a filter box). The statements can reference metrics and metadata fields allowing the user to define boolean expressions to filter data. An example of the syntax; Average_Power > 200 and Duration > 3600 This references the metric Average_Power and ride Duration. But will also support operations on metadata fields, for example; Workout_Code endsWith "SST" The operators are; = <> > >= < <= matches contains endswith beginswith ( ) && and || or Filters are syntactically and semantically validated. But at this point the resulting tree is not evaluated, i.e. we can parse the filters, but do not execute them. Two further updates are pending (once written and tested): - Part 2 of 3 : Execute filters and apply to the ride list - Part 3 of 3 : Allow named filters and apply to LTM charts Further updates will support a visual editor and allowing filters to be applied to CP and Histogram charts and affect the PMC stress calculators. --- src/DataFilter.h | 96 ++++++++++++ src/DataFilter.l | 71 +++++++++ src/DataFilter.y | 347 ++++++++++++++++++++++++++++++++++++++++++++ src/MainWindow.cpp | 4 + src/MainWindow.h | 2 + src/RideNavigator.h | 2 + src/SearchBox.cpp | 16 +- src/SearchBox.h | 6 + src/src.pro | 5 +- 9 files changed, 542 insertions(+), 7 deletions(-) create mode 100644 src/DataFilter.h create mode 100644 src/DataFilter.l create mode 100644 src/DataFilter.y diff --git a/src/DataFilter.h b/src/DataFilter.h new file mode 100644 index 000000000..6e15b1384 --- /dev/null +++ b/src/DataFilter.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2012 Mark Liversedge (liversedge@gmail.com) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include + +class MainWindow; +class RideMetric; +class FieldDefinition; +class SummaryMetrics; + +class SymbolDef { + + public: + + enum { Float, Integer, String } type; + union { + + RideMetric *metric; // into ride factory + FieldDefinition *meta; // into field definitions in MainWindow + + } def; +}; + +class DataFilter; +class Leaf { + + public: + + Leaf() : type(none) { } + + // evaluate against a SummaryMetric + bool eval(Leaf *, SummaryMetrics); + + // tree traversal etc + void print(Leaf *); // print leaf and all children + void validateFilter(DataFilter *, Leaf*); // validate + bool isNumber(DataFilter *df, Leaf *leaf); + void clear(Leaf*); + + enum { none, Float, Integer, String, Symbol, Logical, Operation } type; + union value { + float f; + int i; + QString *s; + QString *n; + Leaf *l; + } lvalue, rvalue; + int op; + + private: + + SymbolDef symbol; // hold information about symbols +}; + +class DataFilter : public QObject +{ + Q_OBJECT + + public: + DataFilter(QObject *parent, MainWindow *main); + QMap lookupMap; + QMap lookupType; // true if a number, false if a string + + public slots: + QStringList parseFilter(QString query); + void clearFilter(); + void configUpdate(); + + //void setData(); // set the file list from the current filter + + private: + MainWindow *main; + Leaf *treeRoot; + QStringList errors; + + QStringList files; +}; diff --git a/src/DataFilter.l b/src/DataFilter.l new file mode 100644 index 000000000..24020b2ec --- /dev/null +++ b/src/DataFilter.l @@ -0,0 +1,71 @@ +%{ +/* + * Copyright (c) 2010 Mark Liversedge (liversedge@gmail.com) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "DataFilter.h" + +// need to get rid of this and use a string... +#include + +// tokens +#include "DataFilter_yacc.h"/* generated by the scanner */ + +%} +%option noyywrap +%option nounput +%% + +"=" DataFilterlval.op = EQ; return EQ; +"<>" DataFilterlval.op = NEQ; return NEQ; +"<" DataFilterlval.op = LT; return LT; +"<=" DataFilterlval.op = LTE; return LTE; +">" DataFilterlval.op = GT; return GT; +">=" DataFilterlval.op = GTE; return GTE; + +[Mm][Aa][Tt][Cc][Hh][Ee][Ss] DataFilterlval.op = MATCHES; return MATCHES; +[Bb][Ee][Gg][Ii][Nn][Ss][Ww][Ii][Tt][Hh] DataFilterlval.op = BEGINSWITH; return BEGINSWITH; +[Ee][Nn][Dd][Ss][Ww][Ii][Tt][Hh] DataFilterlval.op = ENDSWITH; return ENDSWITH; +[Cc][Oo][Nn][Tt][Aa][Ii][Nn][Ss] DataFilterlval.op = CONTAINS; return CONTAINS; + +"&&" DataFilterlval.op = AND; return AND; +[Aa][nN][Dd] DataFilterlval.op = AND; return AND; +"||" DataFilterlval.op = OR; return OR; +[Oo][Rr] DataFilterlval.op = OR; return OR; + + +[-+]?[0-9]+ return INTEGER; +[-+]?[0-9]+e-[0-9]+ return FLOAT; +[-+]?[0-9]+\.[-e0-9]* return FLOAT; +\"([^\"]|\\\")*\" return STRING; /* contains non-quotes or escaped-quotes */ + +[a-zA-Z0-9][a-zA-Z0-9_]+ return SYMBOL; /* symbols can start with 0-9 */ + +[ \n\t\r] ; /* we just ignore whitespace */ + +%% + +void DataFilter_setString(QString p) +{ + DataFilter_scan_string(p.toLatin1().data()); +} + +void DataFilter_clearString() +{ + DataFilterlex_destroy(); +} + diff --git a/src/DataFilter.y b/src/DataFilter.y new file mode 100644 index 000000000..77edd8f1c --- /dev/null +++ b/src/DataFilter.y @@ -0,0 +1,347 @@ +%{ +/* + * Copyright (c) 2012 Mark Liversedge (liversedge@gmail.com) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +// This grammar should work with yacc and bison, but has +// only been tested with bison. In addition, since qmake +// uses the -p flag to rename all the yy functions to +// enable multiple grammars in a single executable you +// should make sure you use the very latest bison since it +// has been known to be problematic in the past. It is +// know to work well with bison v2.4.1. +// +// To make the grammar readable I have placed the code +// for each nterm at column 40, this source file is best +// edited / viewed in an editor which is at least 120 +// columns wide (e.g. vi in xterm of 120x40) +// +// + +#include "DataFilter.h" +#include "MainWindow.h" +#include "RideNavigator.h" +#include + +// LEXER VARIABLES WE INTERACT WITH +// Standard yacc/lex variables / functions +extern int DataFilterlex(); // the lexer aka yylex() +extern char *DataFiltertext; // set by the lexer aka yytext + +extern void DataFilter_setString(QString); +extern void DataFilter_clearString(); + +// PARSER STATE VARIABLES +static QStringList DataFiltererrors; +void DataFiltererror(const char *error) { DataFiltererrors << QString(error);} + +static Leaf *root; // root node for parsed statement + +%} + +// Symbol can be meta or metric name +%token SYMBOL + +// Constants can be a string or a number +%token STRING INTEGER FLOAT + +// comparative operators +%token EQ NEQ LT LTE GT GTE +%token MATCHES ENDSWITH BEGINSWITH CONTAINS + +// logical operators +%token AND OR + +%union { + Leaf *leaf; + int op; +} + +%type value lexpr; +%type lop op; + +%start filter; +%% + +filter: lexpr { root = $1; } + ; + +lexpr : '(' lexpr ')' { $$ = new Leaf(); + $$->type = Leaf::Logical; + $$->lvalue.l = $2; + $$->op = 0; } + + | lexpr lop lexpr { $$ = new Leaf(); + $$->type = Leaf::Logical; + $$->lvalue.l = $1; + $$->op = $2; + $$->rvalue.l = $3; } + + | value op value { $$ = new Leaf(); + $$->type = Leaf::Operation; + $$->lvalue.l = $1; + $$->op = $2; + $$->rvalue.l = $3; } + ; + + +op : EQ + | NEQ + | LT + | LTE + | GT + | GTE + | MATCHES + | ENDSWITH + | BEGINSWITH + | CONTAINS + ; + +lop : AND + | OR + ; + +value : SYMBOL { $$ = new Leaf(); $$->type = Leaf::Symbol; + $$->lvalue.n = new QString(DataFiltertext); } + | STRING { $$ = new Leaf(); $$->type = Leaf::String; + $$->lvalue.s = new QString(DataFiltertext); } + | FLOAT { $$ = new Leaf(); $$->type = Leaf::Float; + $$->lvalue.f = QString(DataFiltertext).toFloat(); } + | INTEGER { $$ = new Leaf(); $$->type = Leaf::Integer; + $$->lvalue.i = QString(DataFiltertext).toInt(); } + ; + +%% + +void Leaf::print(Leaf *leaf) +{ + switch(leaf->type) { + case Leaf::Float : qDebug()<<"float"<lvalue.f; break; + case Leaf::Integer : qDebug()<<"integer"<lvalue.i; break; + case Leaf::String : qDebug()<<"string"<<*leaf->lvalue.s; break; + case Leaf::Symbol : qDebug()<<"symbol"<<*leaf->lvalue.n; break; + case Leaf::Logical : qDebug()<<"logical"<op; + leaf->print(leaf->lvalue.l); + leaf->print(leaf->rvalue.l); + break; + case Leaf::Operation : qDebug()<<"compare"<op; + leaf->print(leaf->lvalue.l); + leaf->print(leaf->rvalue.l); + break; + default: + break; + + } +} + +bool Leaf::isNumber(DataFilter *df, Leaf *leaf) +{ + switch(leaf->type) { + case Leaf::Float : return true; + case Leaf::Integer : return true; + case Leaf::String : return false; + case Leaf::Symbol : return df->lookupType.value(*(leaf->lvalue.n), false); + case Leaf::Logical : return false; // not possible! + case Leaf::Operation : return false; + default: + return false; + break; + + } +} + +void Leaf::clear(Leaf *leaf) +{ + switch(leaf->type) { + case Leaf::String : delete leaf->lvalue.s; break; + case Leaf::Symbol : delete leaf->lvalue.n; break; + case Leaf::Logical : + case Leaf::Operation : clear(leaf->lvalue.l); + clear(leaf->rvalue.l); + delete(leaf->lvalue.l); + delete(leaf->rvalue.l); + break; + default: + break; + } +} + +void Leaf::validateFilter(DataFilter *df, Leaf *leaf) +{ + switch(leaf->type) { + case Leaf::Symbol : + { + // are the symbols correct? + // if so set the type to meta or metric + // and save the technical name used to do + // a lookup at execution time + + QString lookup = df->lookupMap.value(*(leaf->lvalue.n), ""); + if (lookup == "") { + DataFiltererrors << QString("Unknown: %1").arg(*(leaf->lvalue.n)); + } + } + break; + + case Leaf::Operation : + { + // first lets make sure the lhs and rhs are of the same type + bool lhsType = Leaf::isNumber(df, leaf->lvalue.l); + bool rhsType = Leaf::isNumber(df, leaf->rvalue.l); + if (lhsType != rhsType) { + DataFiltererrors << QString("Type mismatch"); + } + + // what about using string operations on a lhs/rhs that + // are numeric? + if ((lhsType || rhsType) && leaf->op >= MATCHES && leaf->op <= CONTAINS) { + DataFiltererrors << "Mixing string operations with numbers"; + } + + validateFilter(df, leaf->lvalue.l); + validateFilter(df, leaf->rvalue.l); + } + break; + + case Leaf::Logical : validateFilter(df, leaf->lvalue.l); + validateFilter(df, leaf->rvalue.l); + break; + default: + break; + } +} + +DataFilter::DataFilter(QObject *parent, MainWindow *main) : QObject(parent), main(main), treeRoot(NULL) +{ + configUpdate(); + connect(main, SIGNAL(configChanged()), this, SLOT(configUpdate())); +} + +QStringList DataFilter::parseFilter(QString query) +{ + //DataFilterdebug = 0; // no debug -- needs bison -t in src.pro + root = NULL; + + // if something was left behind clear it up now + clearFilter(); + + // Parse from string + DataFiltererrors.clear(); // clear out old errors + DataFilter_setString(query); + DataFilterparse(); + DataFilter_clearString(); + + // save away the results + treeRoot = root; + + // if it passed syntax lets check semantics + if (treeRoot && DataFiltererrors.count() == 0) treeRoot->validateFilter(this, treeRoot); + + // ok, did it pass all tests? + if (DataFiltererrors.count() > 0) { // nope + + // Bzzzt, malformed + qDebug()<<"parse filter errors:"<print(treeRoot); + } + + errors = DataFiltererrors; + return errors; +} + +void DataFilter::clearFilter() +{ + if (treeRoot) { + treeRoot->clear(treeRoot); + treeRoot = NULL; + } +} + +void DataFilter::configUpdate() +{ + lookupMap.clear(); + lookupType.clear(); + + // create lookup map from 'friendly name' to name used in smmaryMetrics + // to enable a quick lookup && the lookup for the field type (number, text) + const RideMetricFactory &factory = RideMetricFactory::instance(); + for (int i=0; iname(); + lookupMap.insert(name.replace(" ","_"), symbol); + lookupType.insert(name.replace(" ","_"), true); + } + + // now add the ride metadata fields -- should be the same generally + foreach(FieldDefinition field, main->rideMetadata()->getFields()) { + QString underscored = field.name; + lookupMap.insert(field.name.replace(" ","_"), field.name); + lookupType.insert(field.name.replace(" ","_"), (field.type > 2)); // true if is number + } + +#if 0 + QMapIteratorr(lookupMap); + while(r.hasNext()) { + + r.next(); + qDebug()<<"Lookup"<type) { + + case Leaf::Logical : + switch (leaf->op) { + case AND : + return (eval(leaf->lvalue.l, m) && eval(leaf->rvalue.l, m)); + + case OR : + return (eval(leaf->lvalue.l, m) || eval(leaf->rvalue.l, m)); + } + + case Leaf::Operation : + { + switch (leaf->op) { + + case EQ: + case NEQ: + case LT: + case LTE: + case GT: + case GTE: + case MATCHES: + case ENDSWITH: + case BEGINSWITH: + case CONTAINS: + default: + break; + } + } + break; + default: + break; + } + return false; +} diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 287ae7fe8..44cc35d84 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -92,6 +92,7 @@ #ifdef GC_HAVE_LUCENE #include "SearchBox.h" #include "Lucene.h" +#include "DataFilter.h" #endif #include @@ -203,6 +204,7 @@ MainWindow::MainWindow(const QDir &home) : _rideMetadata->hide(); #ifdef GC_HAVE_LUCENE lucene = new Lucene(this); // before metricDB attempts to refresh + datafilter = new DataFilter(this, this); // before metricDB attempts to refresh #endif metricDB = new MetricAggregator(this, home, zones(), hrZones()); // just to catch config updates! metricDB->refreshMetrics(); @@ -462,7 +464,9 @@ MainWindow::MainWindow(const QDir &home) : toolbuttons->addWidget(searchBox); //toolbuttons->addStretch(); connect(searchBox, SIGNAL(submitQuery(QString)), this, SLOT(searchSubmitted(QString))); + connect(searchBox, SIGNAL(submitFilter(QString)), datafilter, SLOT(parseFilter(QString))); connect(searchBox, SIGNAL(clearQuery()), this, SLOT(searchCleared())); + connect(searchBox, SIGNAL(clearFilter()), datafilter, SLOT(clearFilter())); #endif diff --git a/src/MainWindow.h b/src/MainWindow.h index 12d11f5d1..49a3442c4 100644 --- a/src/MainWindow.h +++ b/src/MainWindow.h @@ -63,6 +63,7 @@ class LionFullScreen; class QTFullScreen; class TrainTool; class Lucene; +class DataFilter; class SearchBox; extern QList mainwindows; // keep track of all the MainWindows we have open @@ -157,6 +158,7 @@ class MainWindow : public QMainWindow #ifdef GC_HAVE_LUCENE SearchBox *searchBox; Lucene *lucene; + DataFilter *datafilter; #endif // ********************************************* diff --git a/src/RideNavigator.h b/src/RideNavigator.h index 09b8edb8f..445f28b56 100644 --- a/src/RideNavigator.h +++ b/src/RideNavigator.h @@ -37,6 +37,7 @@ class GroupByModel; class SearchFilter; class DiaryWindow; class BUGFIXQSortFilterProxyModel; +class DataFilter; // // The RideNavigator @@ -60,6 +61,7 @@ class RideNavigator : public GcWindow friend class ::GroupByModel; friend class ::DiaryWindow; friend class ::GcCalendar; + friend class ::DataFilter; public: RideNavigator(MainWindow *); diff --git a/src/SearchBox.cpp b/src/SearchBox.cpp index 105067c4b..c35554d05 100644 --- a/src/SearchBox.cpp +++ b/src/SearchBox.cpp @@ -35,7 +35,7 @@ SearchBox::SearchBox(QWidget *parent) clearButton->setStyleSheet("QToolButton { border: none; padding: 0px; }"); clearButton->hide(); connect(clearButton, SIGNAL(clicked()), this, SLOT(clear())); - connect(clearButton, SIGNAL(clicked()), this, SIGNAL(clearQuery())); + connect(clearButton, SIGNAL(clicked()), this, SLOT(clearClicked())); connect(this, SIGNAL(textChanged(const QString&)), this, SLOT(updateCloseButton(const QString&))); // search button @@ -92,6 +92,7 @@ void SearchBox::resizeEvent(QResizeEvent *) void SearchBox::toggleMode() { + clear(); // clear whatever is there first if (mode == Search) setMode(Filter); else setMode(Search); } @@ -124,20 +125,25 @@ void SearchBox::setMode(SearchBoxMode mode) void SearchBox::updateCloseButton(const QString& text) { - if (clearButton->isVisible() && text.isEmpty()) clearQuery(); + if (clearButton->isVisible() && text.isEmpty()) mode == Search ? clearQuery() : clearFilter(); clearButton->setVisible(!text.isEmpty()); - if (mode == Search) searchSubmit(); + if (mode == Search) searchSubmit(); // only do search as you type in search mode } void SearchBox::searchSubmit() { // return hit / key pressed if (text() != "") { - submitQuery(text()); + mode == Search ? submitQuery(text()) : submitFilter(text()); } } +void SearchBox::clearClicked() +{ + mode == Search ? clearQuery() : clearFilter(); +} + // Drag and drop columns from the chooser... void SearchBox::dragEnterEvent(QDragEnterEvent *event) @@ -157,5 +163,5 @@ SearchBox::dropEvent(QDropEvent *event) // we do very little to the name, just space to _ and lower case it for now... name.replace(' ', '_'); - insert(name + ":\"\""); + insert(name + (mode == Search ? ":\"\"" : "")); } diff --git a/src/SearchBox.h b/src/SearchBox.h index d2fb8ada6..864e38c28 100644 --- a/src/SearchBox.h +++ b/src/SearchBox.h @@ -45,15 +45,21 @@ private slots: void updateCloseButton(const QString &text); void searchSubmit(); void toggleMode(); + void clearClicked(); // drop column headings from column chooser void dragEnterEvent(QDragEnterEvent *event); void dropEvent(QDropEvent *event); signals: + // text search mode void submitQuery(QString); void clearQuery(); + // db filter mode + void submitFilter(QString); + void clearFilter(); + private: QToolButton *clearButton, *searchButton; SearchBoxMode mode; diff --git a/src/src.pro b/src/src.pro index c363e3c30..5b05f58e3 100644 --- a/src/src.pro +++ b/src/src.pro @@ -222,6 +222,7 @@ HEADERS += \ CpintPlot.h \ CriticalPowerWindow.h \ CsvRideFile.h \ + DataFilter.h \ DataProcessor.h \ DBAccess.h \ DatePickerDialog.h \ @@ -365,8 +366,8 @@ HEADERS += \ Zones.h \ ZoneScaleDraw.h -YACCSOURCES = JsonRideFile.y WithingsParser.y -LEXSOURCES = JsonRideFile.l WithingsParser.l +YACCSOURCES = JsonRideFile.y WithingsParser.y DataFilter.y +LEXSOURCES = JsonRideFile.l WithingsParser.l DataFilter.l #-t turns on debug, use with caution #QMAKE_YACCFLAGS = -t -d