From bea79092abbe25832f731d9163dae96c5b7ed7d9 Mon Sep 17 00:00:00 2001 From: Rainer Clasen Date: Fri, 17 Jun 2011 15:34:18 +0200 Subject: [PATCH] Fit: handle unknown fields gracefully So far the FIT parser bailed out, whenever it found something unknown/uninterested to GC. This is quite orthogonal to the FIT design, as it's supposed to be extended. renamed read_ functions to match the FIT base_type names. unified handling of "unavailable/invalid" values - i.e. if sensor data is temporary unavailable. This allows easier and consistent handling - especially for the uintXz base_types, which only differ by a different "invalid" value. Had to change the type of the "values" list to int64 to fit uint32/int32, as well. added proper support for signed integer types. I'm wondering, why lon, lat + temperature were decoded properly... added support for currently unsupported base types by just skipping their bytes. This allows us to continue reading. --- src/FitRideFile.cpp | 316 +++++++++++++++++++++++++++++--------------- 1 file changed, 213 insertions(+), 103 deletions(-) diff --git a/src/FitRideFile.cpp b/src/FitRideFile.cpp index c0bc7d022..27ddecfd9 100644 --- a/src/FitRideFile.cpp +++ b/src/FitRideFile.cpp @@ -21,9 +21,11 @@ #include #include #include +#include #include #include #include +#include #define RECORD_TYPE 20 @@ -31,29 +33,11 @@ static int fitFileReaderRegistered = RideFileFactory::instance().registerReader( "fit", "Garmin FIT", new FitFileReader()); -/*static const char *base_type_names[] = { - "enum", // 0 - "int8", // 1 - "uint8", // 2 - "int16", // 3 - "uint16", // 4 - "int32", // 5 - "uint32", // 6 - "string", // 7 - "float32", // 8 - "float64", // 9 - "uint8z", // 10 - "uint16z", // 11 - "uint32z", // 12 - "byte" // 13 -}; */ -static const int base_type_names_size(14); - static const QDateTime qbase_time(QDate(1989, 12, 31), QTime(0, 0, 0), Qt::UTC); struct FitField { int num; - int type; + int type; // FIT base_type int size; // in bytes }; @@ -63,6 +47,13 @@ struct FitDefinition { std::vector fields; }; +/* FIT has uint32 as largest integer type. So qint64 is large enough to + * store all integer types - no matter if they're signed or not */ +// XXX this needs to get changed to support non-integer values +typedef qint64 fit_value_t; +#define NA_VALUE std::numeric_limits::max() + + struct FitFileReaderState { QFile &file; @@ -71,7 +62,7 @@ struct FitFileReaderState time_t start_time; time_t last_time; QMap local_msg_types; - QSet unknown_record_fields, unknown_global_msg_nums; + QSet unknown_record_fields, unknown_global_msg_nums, unknown_base_type; int interval; int devices; bool stopped; @@ -88,42 +79,139 @@ struct FitFileReaderState struct TruncatedRead {}; - int read_byte(int *count = NULL) { - char c; - if (file.read(&c, 1) != 1) + void read_unknown( int size, int *count = NULL ){ + char c[size+1]; + + // XXX: just seek instead of read? + if (file.read(c, size ) != size) + throw TruncatedRead(); + if (count) + (*count) += size; + } + + fit_value_t read_int8(int *count = NULL) { + qint8 i; + if (file.read(reinterpret_cast( &i), 1) != 1) throw TruncatedRead(); if (count) (*count) += 1; - return 0xff & c; + + return i == 0x7f ? NA_VALUE : i; } - int read_short(bool is_big_endian, int *count = NULL) { - uint16_t s; - if (file.read(reinterpret_cast(&s), 2) != 2) + fit_value_t read_uint8(int *count = NULL) { + quint8 i; + if (file.read(reinterpret_cast( &i), 1) != 1) + throw TruncatedRead(); + if (count) + (*count) += 1; + + return i == 0xff ? NA_VALUE : i; + } + + fit_value_t read_uint8z(int *count = NULL) { + quint8 i; + if (file.read(reinterpret_cast( &i), 1) != 1) + throw TruncatedRead(); + if (count) + (*count) += 1; + + return i == 0x00 ? NA_VALUE : i; + } + + fit_value_t read_int16(bool is_big_endian, int *count = NULL) { + qint16 i; + if (file.read(reinterpret_cast(&i), 2) != 2) throw TruncatedRead(); if (count) (*count) += 2; - return 0xffff & ( is_big_endian - ? qFromBigEndian( s ) - : qFromLittleEndian( s ) ); + + i = is_big_endian + ? qFromBigEndian( i ) + : qFromLittleEndian( i ); + + return i == 0x7fff ? NA_VALUE : i; } - int read_long(bool is_big_endian, int *count = NULL) { - uint32_t l; - if (file.read(reinterpret_cast(&l), 4) != 4) + fit_value_t read_uint16(bool is_big_endian, int *count = NULL) { + quint16 i; + if (file.read(reinterpret_cast(&i), 2) != 2) + throw TruncatedRead(); + if (count) + (*count) += 2; + + i = is_big_endian + ? qFromBigEndian( i ) + : qFromLittleEndian( i ); + + return i == 0xffff ? NA_VALUE : i; + } + + fit_value_t read_uint16z(bool is_big_endian, int *count = NULL) { + quint16 i; + if (file.read(reinterpret_cast(&i), 2) != 2) + throw TruncatedRead(); + if (count) + (*count) += 2; + + i = is_big_endian + ? qFromBigEndian( i ) + : qFromLittleEndian( i ); + + return i == 0x0000 ? NA_VALUE : i; + } + + fit_value_t read_int32(bool is_big_endian, int *count = NULL) { + qint32 i; + if (file.read(reinterpret_cast(&i), 4) != 4) throw TruncatedRead(); if (count) (*count) += 4; - return is_big_endian - ? qFromBigEndian( l ) - : qFromLittleEndian( l ); + + i = is_big_endian + ? qFromBigEndian( i ) + : qFromLittleEndian( i ); + + return i == 0x7fffffff ? NA_VALUE : i; } - void decodeFileId(const FitDefinition &def, int, const std::vector values) { + fit_value_t read_uint32(bool is_big_endian, int *count = NULL) { + quint32 i; + if (file.read(reinterpret_cast(&i), 4) != 4) + throw TruncatedRead(); + if (count) + (*count) += 4; + + i = is_big_endian + ? qFromBigEndian( i ) + : qFromLittleEndian( i ); + + return i == 0xffffffff ? NA_VALUE : i; + } + + fit_value_t read_uint32z(bool is_big_endian, int *count = NULL) { + quint32 i; + if (file.read(reinterpret_cast(&i), 4) != 4) + throw TruncatedRead(); + if (count) + (*count) += 4; + + i = is_big_endian + ? qFromBigEndian( i ) + : qFromLittleEndian( i ); + + return i == 0x00000000 ? NA_VALUE : i; + } + + void decodeFileId(const FitDefinition &def, int, const std::vector values) { int i = 0; int manu = -1, prod = -1; foreach(const FitField &field, def.fields) { - int value = values[i++]; + fit_value_t value = values[i++]; + + if( value == NA_VALUE ) + continue; + switch (field.num) { case 1: manu = value; break; case 2: prod = value; break; @@ -146,13 +234,17 @@ struct FitFileReaderState } } - void decodeEvent(const FitDefinition &def, int, const std::vector values) { + void decodeEvent(const FitDefinition &def, int, const std::vector values) { time_t time = 0; int event = -1; int event_type = -1; int i = 0; foreach(const FitField &field, def.fields) { - int value = values[i++]; + fit_value_t value = values[i++]; + + if( value == NA_VALUE ) + continue; + switch (field.num) { case 253: time = value + qbase_time.toTime_t(); break; case 0: event = value; break; @@ -192,7 +284,7 @@ struct FitFileReaderState last_event_type = event_type; } - void decodeLap(const FitDefinition &def, int time_offset, const std::vector values) { + void decodeLap(const FitDefinition &def, int time_offset, const std::vector values) { time_t time = 0; if (time_offset > 0) time = last_time + time_offset; @@ -200,7 +292,11 @@ struct FitFileReaderState time_t this_start_time = 0; ++interval; foreach(const FitField &field, def.fields) { - int value = values[i++]; + fit_value_t value = values[i++]; + + if( value == NA_VALUE ) + continue; + switch (field.num) { case 253: time = value + qbase_time.toTime_t(); break; case 2: this_start_time = value + qbase_time.toTime_t(); break; @@ -215,30 +311,36 @@ struct FitFileReaderState } } - void decodeRecord(const FitDefinition &def, int time_offset, const std::vector values) { + void decodeRecord(const FitDefinition &def, int time_offset, const std::vector values) { time_t time = 0; if (time_offset > 0) time = last_time + time_offset; double alt = 0, cad = 0, km = 0, grade = 0, hr = 0, lat = 0, lng = 0, badgps = 0; double resistance = 0, kph = 0, temperature = 0, time_from_course = 0, watts = 0; - int lati = 0x7fffffff, lngi = 0x7fffffff; + fit_value_t lati = NA_VALUE, lngi = NA_VALUE; int i = 0; foreach(const FitField &field, def.fields) { - int value = values[i++]; + fit_value_t value = values[i++]; + + if( value == NA_VALUE ) + continue; + switch (field.num) { case 253: time = value + qbase_time.toTime_t(); break; case 0: lati = value; break; case 1: lngi = value; break; - case 2: alt = (value == 0xffff) ? 0 : (value / 5.0 - 500.0); break; - case 3: hr = (value == 0xff) ? 0 : value; break; - case 4: cad = (value == 0xff) ? 0 : value; break; - case 5: km = ((uint32_t) value == 0xffffffff) ? 0 : value / 100000.0; break; - case 6: kph = (value == 0xffff) ? 0 : value * 3.6 / 1000.0; break; - case 7: watts = (value == 0xffff) ? 0 : value; break; - case 9: grade = (value == 0x7fff) ? 0 : value / 100.0; break; - case 10: resistance = (value == 0xff) ? 0 : value; break; - case 11: time_from_course = (value == 0x7fffffff) ? 0 : value / 1000.0; break; - case 13: temperature = (value == 0x7f) ? 0 : value; break; + case 2: alt = value / 5.0 - 500.0; break; + case 3: hr = value; break; + case 4: cad = value; break; + case 5: km = value / 100000.0; break; + case 6: kph = value * 3.6 / 1000.0; break; + case 7: watts = value; break; + case 8: break; // XXX packed speed/dist + case 9: grade = value / 100.0; break; + case 10: resistance = value; break; + case 11: time_from_course = value / 1000.0; break; + case 12: break; // XXX "cycle_length" + case 13: temperature = value; break; default: unknown_record_fields.insert(field.num); } } @@ -254,7 +356,7 @@ struct FitFileReaderState return; */ } - if (lati != 0x7fffffff && lngi != 0x7fffffff) { + if (lati != NA_VALUE && lngi != NA_VALUE) { lat = lati * 180.0 / 0x7fffffff; lng = lngi * 180.0 / 0x7fffffff; } else @@ -270,6 +372,14 @@ struct FitFileReaderState t.setTime_t(start_time); rideFile->setStartTime(t); } + + //printf( "point time=%d lat=%.2lf lon=%.2lf alt=%.1lf hr=%.0lf " + // "cad=%.0lf km=%.1lf kph=%.1lf watts=%.0lf grade=%.1lf " + // "resist=%.1lf off=%.1lf temp=%.1lf\n", + // time, lat, lng, alt, hr, + // cad, km, kph, watts, grade, + // resistance, time_from_course, temperature ); + double secs = time - start_time; double nm = 0; double headwind = 0.0; @@ -320,7 +430,7 @@ struct FitFileReaderState int read_record(bool &stop, QStringList &errors) { stop = false; int count = 0; - int header_byte = read_byte(&count); + int header_byte = read_uint8(&count); if (!(header_byte & 0x80) && (header_byte & 0x40)) { // Definition record int local_msg_type = header_byte & 0xf; @@ -328,30 +438,24 @@ struct FitFileReaderState local_msg_types.insert(local_msg_type, FitDefinition()); FitDefinition &def = local_msg_types[local_msg_type]; - int reserved = read_byte(&count); (void) reserved; // unused - def.is_big_endian = read_byte(&count); - def.global_msg_num = read_short(def.is_big_endian, &count); - int num_fields = read_byte(&count); - /*printf("definition: local type %d\n", - local_msg_type);*/ - // printf(", %d num_fields:\n", num_fields); + int reserved = read_uint8(&count); (void) reserved; // unused + def.is_big_endian = read_uint8(&count); + def.global_msg_num = read_uint16(def.is_big_endian, &count); + int num_fields = read_uint8(&count); + //printf("definition: local type=%d global=%d arch=%d fields=%d\n", + // local_msg_type, def.global_msg_num, def.is_big_endian, + // num_fields ); for (int i = 0; i < num_fields; ++i) { def.fields.push_back(FitField()); FitField &field = def.fields.back(); - field.num = read_byte(&count); - field.size = read_byte(&count); - int base_type = read_byte(&count); + field.num = read_uint8(&count); + field.size = read_uint8(&count); + int base_type = read_uint8(&count); field.type = base_type & 0x1f; - if (field.type >= base_type_names_size) { - errors << QString("unknown base type %1").arg(field.type); - stop = true; - return count; - } - /*printf(" field %d: %d bytes, num %d, type %s, %s endianness\n", - i, field_size, field_def_num, base_type_names[base_type_num], - (base_type & 0x80) ? "with" : "without");*/ + //printf(" field %d: %d bytes, num %d, type %d\n", + // i, field.size, field.num, field.type ); } } else { @@ -368,31 +472,43 @@ struct FitFileReaderState } if (!local_msg_types.contains(local_msg_type)) { - errors << QString("unrecognized local type %1").arg(local_msg_type); + errors << QString("local type %1 without previous definition").arg(local_msg_type); stop = true; return count; } const FitDefinition &def = local_msg_types[local_msg_type]; - std::vector values; + //printf( "message local=%d global=%d\n", local_msg_type, + // def.global_msg_num ); + + std::vector values; foreach(const FitField &field, def.fields) { - int v; - switch (field.size) { - case 1: v = read_byte(&count); break; - case 2: v = read_short(def.is_big_endian, &count); break; - case 4: v = read_long(def.is_big_endian, &count); break; + fit_value_t v; + switch (field.type) { + case 0: v = read_uint8(&count); break; + case 1: v = read_int8(&count); break; + case 2: v = read_uint8(&count); break; + case 3: v = read_int16(def.is_big_endian, &count); break; + case 4: v = read_uint16(def.is_big_endian, &count); break; + case 5: v = read_int32(def.is_big_endian, &count); break; + case 6: v = read_uint32(def.is_big_endian, &count); break; + case 10: v = read_uint8z(&count); break; + case 11: v = read_uint16z(def.is_big_endian, &count); break; + case 12: v = read_uint32z(def.is_big_endian, &count); break; + //XXX: support float, string + byte base types default: - errors << QString("unsupported field size %1").arg(field.size); - stop = true; - return count; + read_unknown( field.size, &count ); + v = NA_VALUE; + unknown_base_type.insert(field.num); } values.push_back(v); + //printf( " field: type=%d num=%d value=%lld\n", + // field.type, field.num, v ); } // Most of the record types in the FIT format aren't actually all // that useful. FileId, Lap, and Record clearly are. The one // other one that might be useful is DeviceInfo, but it doesn't // seem to be filled in properly. Sean's Cinqo, for example, // shows up as manufacturer #65535, even though it should be #7. - // printf("msg: %d\n", global_msg_num); switch (def.global_msg_num) { case 0: decodeFileId(def, time_offset, values); break; case 19: decodeLap(def, time_offset, values); break; @@ -406,14 +522,6 @@ struct FitFileReaderState case 49: /* file creator */ break; default: - /* - int i = 0; - printf("----------------\n"); - foreach(const FitField &field, def.fields) { - int value = values[i++]; - printf("msg %d: field %d = %d\n", def.global_msg_num, field.num, value); - } - */ unknown_global_msg_nums.insert(def.global_msg_num); } last_msg_type = def.global_msg_num; @@ -429,17 +537,17 @@ struct FitFileReaderState delete rideFile; return NULL; } - int header_size = read_byte(); + int header_size = read_uint8(); if (header_size != 12) { errors << QString("bad header size: %1").arg(header_size); delete rideFile; return NULL; } - int protocol_version = read_byte(); + int protocol_version = read_uint8(); (void) protocol_version; - int profile_version = read_short(false); // always littleEndian + int profile_version = read_uint16(false); // always littleEndian (void) profile_version; // not sure what to do with this - int data_size = read_long(false); // always littleEndian + int data_size = read_uint32(false); // always littleEndian char fit_str[5]; if (file.read(fit_str, 4) != 4) { errors << "truncated header"; @@ -468,13 +576,15 @@ struct FitFileReaderState return NULL; } else { - int crc = read_short( false ); // always littleEndian + int crc = read_uint16( false ); // always littleEndian (void) crc; - foreach(int num, unknown_global_msg_nums) { - errors << QString("unknown global message number %1; ignoring it").arg(num); - } + foreach(int num, unknown_global_msg_nums) + qDebug() << QString("FitRideFile: unknown global message number %1; ignoring it").arg(num); foreach(int num, unknown_record_fields) - errors << QString("unknown record field %1; ignoring it").arg(num); + qDebug() << QString("FitRideFile: unknown record field %1; ignoring it").arg(num); + foreach(int num, unknown_base_type) + qDebug() << QString("FitRideFile: unknown base type %1; skipped").arg(num); + return rideFile; } }