From 924ef2a8f1b5412ced56e45352b236d98c6f3f02 Mon Sep 17 00:00:00 2001 From: Mark Liversedge Date: Thu, 27 Mar 2014 16:20:33 +0000 Subject: [PATCH] Improve compatibility with Dropbox .. by checking the ride file CRC before recomputing either the metrics or the ridefilecache. .. this means that users that routinely copy or backup or use dropbox to keep things in sync won't see any unneccessary metric computations. --- src/DBAccess.cpp | 14 +++++++++----- src/DBAccess.h | 1 + src/MetricAggregator.cpp | 41 +++++++++++++++++++++++++--------------- src/RideFileCache.cpp | 23 +++++++++++++++------- src/RideFileCache.h | 5 ++++- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/DBAccess.cpp b/src/DBAccess.cpp index f049c23f7..7f0ecfc48 100644 --- a/src/DBAccess.cpp +++ b/src/DBAccess.cpp @@ -88,6 +88,7 @@ // 67 22 Mar 2014 Mark Liversedge Added Anaerobic TISS prototype // 68 22 Mar 2014 Mark Liversedge Added dTISS prototype // 69 23 Mar 2014 Mark Liversedge Updated Gompertz constansts for An-TISS sigmoid +// 70 27 Mar 2014 Mark Liversedge Add file CRC to refresh only if contents change (not just timestamps) int DBSchemaVersion = 69; @@ -136,8 +137,8 @@ DBAccess::initDatabase(QDir home) } } -static int -computeFileCRC(QString filename) +unsigned int +DBAccess::computeFileCRC(QString filename) { QFile file(filename); QFileInfo fileinfo(file); @@ -180,6 +181,7 @@ bool DBAccess::createMetricsTable() QString createMetricTable = "create table metrics (filename varchar primary key," "identifier varchar," "timestamp integer," + "crc integer," "ride_date date," "color varchar," "fingerprint integer"; @@ -421,7 +423,7 @@ bool DBAccess::importRide(SummaryMetrics *summaryMetrics, RideFile *ride, QColor } // construct an insert statement - QString insertStatement = "insert into metrics ( filename, identifier, timestamp, ride_date, color, fingerprint "; + QString insertStatement = "insert into metrics ( filename, identifier, crc, timestamp, ride_date, color, fingerprint "; const RideMetricFactory &factory = RideMetricFactory::instance(); for (int i=0; iathlete->rideMetadata()->getFields()) { @@ -451,9 +453,11 @@ bool DBAccess::importRide(SummaryMetrics *summaryMetrics, RideFile *ride, QColor query.prepare(insertStatement); - // filename, timestamp, ride date + // filename, crc, timestamp, ride date + QString fullPath = QString(context->athlete->home.absolutePath()) + "/" + summaryMetrics->getFileName(); query.addBindValue(summaryMetrics->getFileName()); query.addBindValue(summaryMetrics->getId()); + query.addBindValue((int)computeFileCRC(fullPath)); query.addBindValue(timestamp.toTime_t()); query.addBindValue(summaryMetrics->getRideDate()); query.addBindValue(color.name()); diff --git a/src/DBAccess.h b/src/DBAccess.h index 5551b9b18..1da016689 100644 --- a/src/DBAccess.h +++ b/src/DBAccess.h @@ -53,6 +53,7 @@ class DBAccess // get schema version int getDBVersion(); + static unsigned int computeFileCRC(QString); QList &getMetadataFields() { return mfieldDefinitions; } // create and drop connections diff --git a/src/MetricAggregator.cpp b/src/MetricAggregator.cpp index f16f3f677..e9e704af4 100644 --- a/src/MetricAggregator.cpp +++ b/src/MetricAggregator.cpp @@ -58,8 +58,8 @@ MetricAggregator::~MetricAggregator() * the ride file no longer exists *----------------------------------------------------------------------*/ -// used to store timestamp and fingerprint used in database -struct status { unsigned long timestamp, fingerprint; }; +// used to store timestamp, file crc and schema fingerprint used in database +struct status { unsigned long timestamp, crc, fingerprint; }; // Refresh not up to date metrics void MetricAggregator::refreshMetrics() @@ -101,12 +101,13 @@ void MetricAggregator::refreshMetrics(QDateTime forceAfterThisDate) // get a Hash map of statistic records and timestamps QSqlQuery query(dbaccess->connection()); QHash dbStatus; - bool rc = query.exec("SELECT filename, timestamp, fingerprint FROM metrics ORDER BY ride_date;"); + bool rc = query.exec("SELECT filename, crc, timestamp, fingerprint FROM metrics ORDER BY ride_date;"); while (rc && query.next()) { status add; QString filename = query.value(0).toString(); - add.timestamp = query.value(1).toInt(); - add.fingerprint = query.value(2).toInt(); + add.crc = query.value(1).toInt(); + add.timestamp = query.value(2).toInt(); + add.fingerprint = query.value(3).toInt(); dbStatus.insert(filename, add); } @@ -151,6 +152,7 @@ void MetricAggregator::refreshMetrics(QDateTime forceAfterThisDate) // if it s missing or out of date then update it! status current = dbStatus.value(name); unsigned long dbTimeStamp = current.timestamp; + unsigned long crc = current.crc; unsigned long fingerprint = current.fingerprint; RideFile *ride = NULL; @@ -185,20 +187,28 @@ void MetricAggregator::refreshMetrics(QDateTime forceAfterThisDate) (!forceAfterThisDate.isNull() && name >= forceAfterThisDate.toString("yyyy_MM_dd_hh_mm_ss"))) { QStringList errors; - // log - out << "Opening ride: " << name << "\r\n"; + // ooh we have one to update -- lets check the CRC in case + // its actually unchanged since last time and the timestamps + // have been mucked up by dropbox / file copying / backups etc + QString fullPath = QString(context->athlete->home.absolutePath()) + "/" + name; + if (crc == 0 || crc != DBAccess::computeFileCRC(fullPath)) { - // read file and process it if we didn't already... - if (ride == NULL) ride = RideFileFactory::instance().openRideFile(context, file, errors); + // log + out << "Opening ride: " << name << "\r\n"; - out << "File open completed: " << name << "\r\n"; + // read file and process it if we didn't already... + if (ride == NULL) ride = RideFileFactory::instance().openRideFile(context, file, errors); - if (ride != NULL) { + out << "File open completed: " << name << "\r\n"; - out << "Getting weight: " << name << "\r\n"; - ride->getWeight(); - out << "Updating statistics: " << name << "\r\n"; - importRide(context->athlete->home, ride, name, zoneFingerPrint, (dbTimeStamp > 0)); + if (ride != NULL) { + + out << "Getting weight: " << name << "\r\n"; + ride->getWeight(); + out << "Updating statistics: " << name << "\r\n"; + importRide(context->athlete->home, ride, name, zoneFingerPrint, (dbTimeStamp > 0)); + + } } } @@ -207,6 +217,7 @@ void MetricAggregator::refreshMetrics(QDateTime forceAfterThisDate) // if ride wasn't opened it will do it itself // we only want to check so passing check=true // because we don't actually want the results now + // it will also check the file CRC as well as timestamps RideFileCache updater(context, context->athlete->home.absolutePath() + "/" + name, ride, true); // free memory - if needed diff --git a/src/RideFileCache.cpp b/src/RideFileCache.cpp index 1ce94a7c9..4a8c2ad99 100644 --- a/src/RideFileCache.cpp +++ b/src/RideFileCache.cpp @@ -76,8 +76,8 @@ RideFileCache::RideFileCache(Context *context, QString fileName, RideFile *passe QFileInfo cacheFileInfo(cacheFileName); // is it up-to-date? - if (cacheFileInfo.exists() && rideFileInfo.lastModified() <= cacheFileInfo.lastModified() && - cacheFileInfo.size() >= (int)sizeof(struct RideFileCacheHeader)) { + if (cacheFileInfo.exists() && cacheFileInfo.size() >= (int)sizeof(struct RideFileCacheHeader)) { + // we have a file, it is more recent than the ride file // but is it the latest version? RideFileCacheHeader head; @@ -89,12 +89,17 @@ RideFileCache::RideFileCache(Context *context, QString fileName, RideFile *passe inFile.readRawData((char *) &head, sizeof(head)); cacheFile.close(); - // is it as recent as we are? - if (head.version == RideFileCacheVersion) { + // its more recent -or- the crc is the same + if (rideFileInfo.lastModified() <= cacheFileInfo.lastModified() || + head.crc == DBAccess::computeFileCRC(rideFileName)) { + + // it is the same ? + if (head.version == RideFileCacheVersion) { - // WE'RE GOOD - if (check == false) readCache(); // if check is false we aren't just checking - return; + // WE'RE GOOD + if (check == false) readCache(); // if check is false we aren't just checking + return; + } } } } @@ -429,6 +434,9 @@ RideFileCache::refreshCache() { static bool writeerror=false; + // set head crc + crc = DBAccess::computeFileCRC(rideFileName); + // update cache! QFile cacheFile(cacheFileName); @@ -1261,6 +1269,7 @@ RideFileCache::serialize(QDataStream *out) // write header head.version = RideFileCacheVersion; + head.crc = crc; head.CP = CP; head.LTHR = LTHR; diff --git a/src/RideFileCache.h b/src/RideFileCache.h index e4b33020d..463521e88 100644 --- a/src/RideFileCache.h +++ b/src/RideFileCache.h @@ -40,7 +40,7 @@ typedef double data_t; // arrays when plotting CP curves and histograms. It is precoputed // to save time and cached in a file .cpx // -static const unsigned int RideFileCacheVersion = 15; +static const unsigned int RideFileCacheVersion = 16; // revision history: // version date description // 1 29-Apr-11 Initial - header, mean-max & distribution data blocks @@ -57,6 +57,7 @@ static const unsigned int RideFileCacheVersion = 15; // 12 21-Feb-14 Added Acceleration (speed) // 12 22-Feb-14 Acceleration precision way too high! // 13-15 24-Feb-14 Add hr, cad, watts, nm Δ data series +// 13-15 24-Feb-14 Add crc to the header // The cache file (.cpx) has a binary format: // 1 x Header data - describing the version and contents of the cache @@ -71,6 +72,7 @@ static const unsigned int RideFileCacheVersion = 15; struct RideFileCacheHeader { unsigned int version; + unsigned int crc; unsigned int wattsMeanMaxCount, hrMeanMaxCount, @@ -127,6 +129,7 @@ class RideFileCache enum cachetype { meanmax, distribution, none }; typedef enum cachetype CacheType; QDate start, end; + unsigned int crc; // Construct from a ridefile or its filename // will reference cache if it exists, and create it