From 0f5f44053ff631f833c6d28e8a6cf2317beb05b8 Mon Sep 17 00:00:00 2001 From: Mark Liversedge Date: Sun, 15 Mar 2020 20:22:42 +0000 Subject: [PATCH] Datafilter vectors - smooth() .. part of a few updates to add some smoothing algorithms to apply to vector data, this first one is just a simple moving average. .. smooth(list, sma, centered|forward|backward, window) will apply smoothing to the list. Note that centered doesn't use multiple windows when the window size is even, so recommend always using an odd number for this parameter. .. will add ewma and maybe some others over the next few commits. --- src/Core/DataFilter.cpp | 74 +++++++++++++++++++++++++++++++++++++++++ src/Core/Utils.cpp | 55 ++++++++++++++++++++++++++++++ src/Core/Utils.h | 5 +++ 3 files changed, 134 insertions(+) diff --git a/src/Core/DataFilter.cpp b/src/Core/DataFilter.cpp index 6457787fb..9353b6f29 100644 --- a/src/Core/DataFilter.cpp +++ b/src/Core/DataFilter.cpp @@ -166,8 +166,11 @@ static struct { { "sapply", 2 }, // sapply(vector, expr) - returns a vector where expr has been applied to every element. x and i // are both available in the expr for element value and index position. + { "lr", 2 }, // lr(xlist, ylist) - linear regression on x,y co-ords returns vector [slope, intercept, r2, see] + { "smooth", 0 }, // smooth(list, algorithm, ... parameters) - returns smoothed data. + // add new ones above this line { "", -1 } }; @@ -1220,6 +1223,7 @@ void Leaf::validateFilter(Context *context, DataFilterRuntime *df, Leaf *leaf) QRegExp constValidSymbols("^(e|pi)$", Qt::CaseInsensitive); // just do basics for now QRegExp dateRangeValidSymbols("^(start|stop)$", Qt::CaseInsensitive); // date range QRegExp pmcValidSymbols("^(stress|lts|sts|sb|rr|date)$", Qt::CaseInsensitive); + QRegExp smoothAlgos("^(sma|ewma)$", Qt::CaseInsensitive); if (leaf->series) { // old way of hand crafting each function in the lexer including support for literal parameter e.g. (power, 1) @@ -1468,6 +1472,49 @@ void Leaf::validateFilter(Context *context, DataFilterRuntime *df, Leaf *leaf) } } + } else if (leaf->function == "smooth") { + + if (leaf->fparms.count() < 2 || leaf->fparms[1]->type != Leaf::Symbol) { + + leaf->inerror = true; + DataFiltererrors << QString(tr("smooth(list, algorithm [,parameters]) need at least 2 parameters.")); + + } else { + + QString algo = *(leaf->fparms[1]->lvalue.n); + if (!smoothAlgos.exactMatch(algo)) { + leaf->inerror = true; + DataFiltererrors << QString(tr("smoothing algorithm '%1' not available").arg(algo)); + } else { + + if (algo == "sma") { + // smooth(list, sma, centred|forward|backward, window) + if (leaf->fparms.count() != 4 || leaf->fparms[2]->type != Leaf::Symbol) { + leaf->inerror = true; + DataFiltererrors << QString(tr("smooth(list, sma, forward|centered|backward, windowsize")); + + } else { + QRegExp parms("^(forward|centered|backward)$"); + QString parm1 = *(leaf->fparms[2]->lvalue.n); + if (!parms.exactMatch(parm1)) { + + leaf->inerror = true; + DataFiltererrors << QString(tr("smooth(list, sma, forward|centered|backward, windowsize")); + } + + // check list and windowsize + validateFilter(context, df, leaf->fparms[0]); + validateFilter(context, df, leaf->fparms[3]); + } + + } else if (algo == "ewma") { + // smooth(list, ewma, alpha) + // TODO + } + } + + } + } else if (leaf->function == "lr") { if (leaf->fparms.count() != 2) { @@ -2654,6 +2701,33 @@ Result Leaf::eval(DataFilterRuntime *df, Leaf *leaf, float x, long it, RideItem return returning; } + // smooth + if (leaf->function == "smooth") { + + Result returning(0); + + // moving average + if (*(leaf->fparms[1]->lvalue.n) == "sma") { + + QString type = *(leaf->fparms[2]->lvalue.n); + int window = eval(df,leaf->fparms[3],x, it, m, p, c, s, d).number; + Result data = eval(df,leaf->fparms[0],x, it, m, p, c, s, d); + int pos=2; // fallback + + if (type=="backward") pos=0; + if (type=="forward") pos=1; + if (type=="centered") pos=2; + + // smooth in different ways.... + returning.vector = Utils::smooth_sma(data.vector, pos, window); + + // sum. ugh. + for(int i=0; ifunction == "lr") { Result returning(0); diff --git a/src/Core/Utils.cpp b/src/Core/Utils.cpp index 39b90f057..c9da2edd4 100644 --- a/src/Core/Utils.cpp +++ b/src/Core/Utils.cpp @@ -297,5 +297,60 @@ argsort(QVector &v, bool ascending) return returning; } +// simple moving average +static double mean(QVector&data, int start, int end) +{ + double sum=0; + double count=0; + + // add em up and handle out of bounds + for (int i=start; i=data.count()) sum += data[data.count()-1]; + else sum += data[i]; + count ++; + } + return sum/count; +} + +QVector +smooth_sma(QVector&data, int pos, int window) +{ + QVector returning; + + int window_start=0, window_end=0; + int index=0; + double ma=0; + + // window is offset from index depending upon the forward/backward/centred position + switch (pos) { + case GC_SMOOTH_FORWARD: + window_start=0; + window_end=window; + break; + + case GC_SMOOTH_BACKWARD: + window_end=0; + window_start=window *-1; + break; + + case GC_SMOOTH_CENTERED: // we should handle odd/even size better + window_start = (window*-1)/2; + window_end = (window)/2; + } + + while (index < data.count()) { + + returning << mean(data, window_start, window_end); + + index ++; + window_start++; + window_end++; + } + + return returning; + +} + }; diff --git a/src/Core/Utils.h b/src/Core/Utils.h index 68ddca548..a06d05c95 100644 --- a/src/Core/Utils.h +++ b/src/Core/Utils.h @@ -30,6 +30,10 @@ class QString; class QStringList; +#define GC_SMOOTH_FORWARD 0 +#define GC_SMOOTH_BACKWARD 1 +#define GC_SMOOTH_CENTERED 2 + namespace Utils { QString xmlprotect(const QString &string); @@ -39,6 +43,7 @@ namespace Utils QStringList searchPath(QString path, QString binary, bool isexec=true); QString removeDP(QString); QVector argsort(QVector&, bool ascending=false); + QVector smooth_sma(QVector&, int pos, int window); };