t.mettler/giesCANne
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed a precision bug that caused signed 32 bit integers to not work

Browse Source 
properly.
This commit is contained in:
Collin Kidder
2015-11-23 09:19:47 -05:00
parent 1d89fe576e
commit 70ebfc19c7
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
graphingwindow.cpp
+2 -2
View File
@@ -960,7 +960,7 @@ void GraphingWindow::createGraph(GraphParams &params, bool createGraphParam)
float tempValue; float tempValue;
int64_t tempValInt; int64_t tempValInt;
int numBytes = (params.endByte - params.startByte) + 1; int numBytes = (params.endByte - params.startByte) + 1;
int64_t shiftRef = 1 << (numBytes * 8); int64_t shiftRef = (uint64_t)1 << (numBytes * 8);
uint64_t maskShifter; uint64_t maskShifter;
uint8_t tempByte; uint8_t tempByte;
for (int j = 0; j < numEntries; j++) for (int j = 0; j < numEntries; j++)
@@ -1010,7 +1010,7 @@ void GraphingWindow::createGraph(GraphParams &params, bool createGraphParam)
float tempValue; float tempValue;
int64_t tempValInt; int64_t tempValInt;
int numBytes = (params.startByte - params.endByte) + 1; int numBytes = (params.startByte - params.endByte) + 1;
int64_t shiftRef = 1 << (numBytes * 8); int64_t shiftRef = (uint64_t)1 << (numBytes * 8);
uint64_t maskShifter; uint64_t maskShifter;
uint8_t tempByte; uint8_t tempByte;
for (int j = 0; j < numEntries; j++) for (int j = 0; j < numEntries; j++)