From 6621352551166e0cf963f9be1385640412d1ce6d Mon Sep 17 00:00:00 2001
From: Joern <joern.rm@gmail.com>
Date: Thu, 4 Sep 2014 19:44:56 +0200
Subject: [PATCH] Fix "JSON char > 127 codepage problem""

[cherry-picked from master]
---
 src/JsonRideFile.l |  5 ++++-
 src/JsonRideFile.y | 19 ++++++++++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/JsonRideFile.l b/src/JsonRideFile.l
index 3d8eeb68f..c37bbab98 100644
--- a/src/JsonRideFile.l
+++ b/src/JsonRideFile.l
@@ -91,5 +91,8 @@ int JsonRideFilelex_destroy(void) { return 0; }
 
 void JsonRideFile_setString(QString p)
 {
-    JsonRideFile_scan_string(p.toLocal8Bit().data());
+    // internally work with UTF-8 encoding
+    // this works for FLEX, since the multi-byte characters only appear WITHIN a "String",
+    // but not as part of the grammar - this is important since a char in UTF-8 can have up to 4 bytes
+    JsonRideFile_scan_string(p.toUtf8().data());
 }
diff --git a/src/JsonRideFile.y b/src/JsonRideFile.y
index 3970878fc..42cda45dd 100644
--- a/src/JsonRideFile.y
+++ b/src/JsonRideFile.y
@@ -84,7 +84,8 @@ static QString protect(const QString string)
 // Un-Escape special characters (JSON compliance)
 static QString unprotect(const char * string)
 {
-    QString string2 = QString::fromLocal8Bit(string);
+    // sending UTF-8 to FLEX demands symetric conversion back to QString
+    QString string2 = QString::fromUtf8(string);
 
     // this is a lexer string so it will be enclosed
     // in quotes. Lets strip those first
@@ -293,9 +294,22 @@ JsonFileReader::openRideFile(QFile &file, QStringList &errors, QList<RideFile*>*
 
         // read in the whole thing
         QTextStream in(&file);
+        // GC .JSON is stored in UTF-8 with BOM(Byte order mark) for identification
+        in.setCodec ("UTF-8");
         contents = in.readAll();
         file.close();
 
+        // check if the text string contains the replacement character for UTF-8 encoding
+        // if yes, try to read with Latin1/ISO 8859-1 (assuming this is an "old" non-UTF-8 Json file)
+        if (contents.contains(QChar::ReplacementCharacter)) {
+           if (file.exists() && file.open(QFile::ReadOnly | QFile::Text)) {
+             QTextStream in(&file);
+             in.setCodec ("ISO 8859-1");
+             contents = in.readAll();
+             file.close();
+           }
+         }
+
     } else {
 
         errors << "unable to open file" + file.fileName();
@@ -340,6 +354,9 @@ JsonFileReader::writeRideFile(Context *, const RideFile *ride, QFile &file) cons
 
     // setup streamer
     QTextStream out(&file);
+    // unified codepage and BOM for identification on all platforms
+    out.setCodec("UTF-8");
+    out.setGenerateByteOrderMark(true);
 
     // start of document and ride
     out << "{\n\t\"RIDE\":{\n";