Fixing issue with reporting diff tokens in 'ignore-line-ends' mode (a…

…nd adding a test to cover for the discovered problem).

judges/recodex_token_judge/reader.hpp

@@ -28,7 +28,7 @@ template <typename CHAR = char, typename OFFSET = std::uint32_t> class Reader
 
 
 	/**
-	 * Internal structure that hold references to tokens.s
+	 * Internal structure that hold references to tokens.
 	 */
 	class TokenRef
 	{
@@ -302,9 +302,16 @@ template <typename CHAR = char, typename OFFSET = std::uint32_t> class Reader
 	}
 
 public:
-	Reader(bool ignoreEmptyLines, bool allowComments, bool ignoreLineEnds, bool ignoreTrailingWhitespace)
-		: mIgnoreEmptyLines(ignoreEmptyLines), mAllowComments(allowComments), mIgnoreLineEnds(ignoreLineEnds),
-		  mIgnoreTrailingWhitespace(ignoreTrailingWhitespace), mData(nullptr), mOffset(0), mLength(0)
+	Reader(bool ignoreEmptyLines, bool allowComments, bool ignoreLineEnds, bool ignoreTrailingWhitespace) :
+		mIgnoreEmptyLines(ignoreEmptyLines),
+		mAllowComments(allowComments),
+		mIgnoreLineEnds(ignoreLineEnds),
+		mIgnoreTrailingWhitespace(ignoreTrailingWhitespace),
+		mData(nullptr),
+		mOffset(0),
+		mLength(0),
+		mLineNumber(0),
+		mLineOffset(0)
 	{
 	}
 
@@ -375,6 +382,7 @@ template <typename CHAR = char, typename OFFSET = std::uint32_t> class Reader
 		if (eof()) { return std::unique_ptr<Line>(); }
 
 		auto line = bpp::make_unique<Line>(*this, mLineNumber, mData + mOffset);
+		auto startOffset = mOffset;
 		while (!eof()) {
 			skipWhitespace();
 
@@ -398,15 +406,17 @@ template <typename CHAR = char, typename OFFSET = std::uint32_t> class Reader
 			// If we got here, an empty line or a comment line was read (which we skipped).
 			line->mLineNumber = mLineNumber;
 			line->mRawData = mData + mOffset;
+			startOffset = mOffset;
 		}
 
 		if (line->mTokens.empty() && mIgnoreEmptyLines) {
 			// The last line of the file was empty, we should skip it as well ...
 			return std::unique_ptr<Line>();
 		}
 
-		line->mRawLength =
-			line->mTokens.size() > 0 ? line->mTokens.back().charNumber() + line->mTokens.back().length() - 1 : 0;
+		line->mRawLength = line->mTokens.size() > 0
+			? line->mTokens.back().offset() - startOffset + line->mTokens.back().length()
+			: 0;
 		return line;
 	}
 };

judges/recodex_token_judge/tests/031-ignore-eol-wrong.bats

@@ -0,0 +1,9 @@
+#!/usr/bin/env bats
+
+load bats-shared
+
+@test "ignore line ends (wrong answer)" {
+	run $EXE_FILE --ignore-line-ends $CORRECT_FILE $RESULT_FILE
+	[ "$status" -eq 1 ]
+	echo "$output" | diff -abB - $ERROR_FILE
+}

judges/recodex_token_judge/tests/031.correct.in

@@ -0,0 +1,8 @@
+alfa
+bravo
+charlie
+delta
+echo
+foxtrot
+golf
+hotel

judges/recodex_token_judge/tests/031.error.out

@@ -0,0 +1,11 @@
+0
+-1: alfa
+bravo
+charlie
+delta
+echo
+foxtrot
+golf
+hotel
++1: alfa bravo cola
+echo firework hotel

judges/recodex_token_judge/tests/031.result.in

@@ -0,0 +1,2 @@
+alfa bravo cola
+echo firework hotel