Fixes bug #736

papaparse.js

@@ -682,7 +682,10 @@ License: MIT
 				return;
 			}
 
-			this._start += xhr.responseText.length;
+			// The length of the responseText doesn't corresponds to the actual size we requested due to potential encoding
+			// of some characters that has more than 1 byte. As such, better to just increment the start index by the chunk
+			// size if it was given
+			this._start += this._config.chunkSize ? this._config.chunkSize : xhr.responseText.length;
 			this._finished = !this._config.chunkSize || this._start >= getFileSize(xhr);
 			this.parseChunk(xhr.responseText);
 		};
@@ -1100,7 +1103,10 @@ License: MIT
 		{
 			_paused = true;
 			_parser.abort();
-			_input = _input.substring(_parser.getCharIndex());
+
+			// If it is streaming via "chunking", the reader will start appending correctly already so no need to substring,
+			// otherwise we can get duplicate content within a row
+			_input = isFunction(_config.chunk) ? "" : _input.substring(_parser.getCharIndex());
 		};
 
 		this.resume = function()

tests/test-cases.js

@@ -1897,6 +1897,114 @@ var CUSTOM_TESTS = [
 			});
 		}
 	},
+	{
+		description: "Pause and resume works for chunks with NetworkStreamer (for Bug #736)",
+		disabled: !XHR_ENABLED,
+		timeout: 30000,
+		expected: ["SembCorp Industries Ltd", "Singapore", "SCIL.SI", "10%", "Yes", "0.30%"],
+		run: function(callback) {
+			var chunkNum = 0;
+			var actual = [];
+			Papa.parse(BASE_PATH + "duplicate.csv", {
+				download: true,
+				chunkSize: 250000,
+				chunk: function(results, parser) {
+					chunkNum++;
+					parser.pause();
+
+					if (chunkNum === 2) {
+						callback(results.data[0]);
+					} else {
+						parser.resume();
+					}
+				},
+				complete: function() {
+					callback(new Error("Should have more than 2 chunks"));
+				}
+			});
+		}
+	},
+	{
+		description: "Pause and resume works for chunks with FileStreamer (for Bug #736)",
+		disabled: !XHR_ENABLED,
+		timeout: 30000,
+		expected: ["SembCorp Industries Ltd", "Singapore", "SCIL.SI", "10%", "Yes", "0.30%"],
+		run: function(callback) {
+			var chunkNum = 0;
+			var actual = [];
+
+			// A little bit of a hack but this allows us to test the FileStreamer for local files. Essentially, this uses the
+			// AJAX request to get the full content and fake the local file.
+			var xhr = new XMLHttpRequest();
+			xhr.onload = function() {
+				Papa.parse(new File([xhr.responseText], './duplicate.csv'), {
+					chunkSize: 250000,
+					chunk: function(results, parser) {
+						chunkNum++;
+						parser.pause();
+
+						if (chunkNum === 2) {
+							callback(results.data[0]);
+						} else {
+							parser.resume();
+						}
+					},
+					complete: function() {
+						callback(new Error("Should have more than 2 chunks"));
+					}
+				});
+			}
+
+			xhr.open("GET", BASE_PATH + "duplicate.csv");
+			try {
+				xhr.send();
+			} catch (err) {
+				callback(err);
+			}
+		}
+	},
+	{
+		description: "Pause and resume works for chunks with StringStreamer (for Bug #736)",
+		disabled: !XHR_ENABLED,
+		timeout: 30000,
+		// For those wondering why this is different than the two above, reading by byte size isn't exactly the same as a
+		// string's length (a string with a length of 10 can have a byte size of 12 for example)
+		expected: ["SembCorp Marine Ltd", "Singapore", "SCMN.SI", "15%", "Yes", "0.30%"],
+		run: function(callback) {
+			var chunkNum = 0;
+			var actual = [];
+
+			// Same hack for testing FileStreamer but this time, we just provide the content
+			var xhr = new XMLHttpRequest();
+			xhr.onload = function() {
+				debugger;
+				Papa.parse(xhr.responseText, {
+					chunkSize: 250000,
+					chunk: function(results, parser) {
+						debugger;
+						chunkNum++;
+						parser.pause();
+
+						if (chunkNum === 2) {
+							callback(results.data[0]);
+						} else {
+							parser.resume();
+						}
+					},
+					complete: function() {
+						callback(new Error("Should have more than 2 chunks"));
+					}
+				});
+			}
+
+			xhr.open("GET", BASE_PATH + "duplicate.csv");
+			try {
+				xhr.send();
+			} catch (err) {
+				callback(err);
+			}
+		}
+	},
 	{
 		description: "Complete is called with all results if neither step nor chunk is defined",
 		expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']],