more refactoring: StringStreamer

papaparse.js

@@ -52,6 +52,7 @@
 	global.Papa.ParserHandle = ParserHandle;
 	global.Papa.NetworkStreamer = NetworkStreamer;
 	global.Papa.FileStreamer = FileStreamer;
+	global.Papa.StringStreamer = StringStreamer;
 
 	if (global.jQuery)
 	{
@@ -194,58 +195,22 @@
 				config: config,
 				workerId: w.id
 			});
+
+			return;
 		}
-		else
-		{
-			if (typeof _input === 'string')
-			{
-				if (config.download)
-				{
-					var streamer = new NetworkStreamer(config);
-					streamer.stream(_input);
-				}
-				else
-				{
-					var ph = new ParserHandle(config);
-					var results = ph.parse(_input);
-					return results;
-				}
-			}
-			else if ((global.File && _input instanceof File) || _input instanceof Object)	// ...Safari. (see issue #106)
-			{
-				if (config.step || config.chunk)
-				{
-					var streamer = new FileStreamer(config);
-					streamer.stream(_input);
-				}
-				else
-				{
-					var ph = new ParserHandle(config);
 
-					if (IS_WORKER)
-					{
-						var reader = new FileReaderSync();
-						var input = reader.readAsText(_input, config.encoding);
-						return ph.parse(input);
-					}
-					else
-					{
-						reader = new FileReader();
-						reader.onload = function(event)
-						{
-							var ph = new ParserHandle(config);
-							var results = ph.parse(event.target.result);
-						};
-						reader.onerror = function()
-						{
-							if (isFunction(config.error))
-								config.error(reader.error, _input);
-						};
-						reader.readAsText(_input, config.encoding);
-					}
-				}
-			}
+		var streamer = null;
+		if (typeof _input === 'string')
+		{
+			if (config.download)
+				streamer = new NetworkStreamer(config);
+			else
+				streamer = new StringStreamer(config);
 		}
+		else if ((global.File && _input instanceof File) || _input instanceof Object)	// ...Safari. (see issue #106)
+			streamer = new FileStreamer(config);
+
+		return streamer.stream(_input);
 	}
 
 
@@ -448,12 +413,13 @@
 
 		this._nextChunk = null;
 
-		this._parseChunk = function(chunk) {
+		this.parseChunk = function(chunk) {
 			// Rejoin the line we likely just split in two by chunking the file
 			var aggregate = this._partialLine + chunk;
 			this._partialLine = "";
 
 			var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
+			if (this._handle.paused()) return;
 			var lastIndex = results.meta.cursor;
 			if (!this._finished)
 			{
@@ -486,6 +452,8 @@
 
 			if (!finishedIncludingPreview && (!results || !results.meta.paused))
 				this._nextChunk();
+
+			return results;
 		};
 
 		this._sendError = function(error)
@@ -600,7 +568,7 @@
 			}
 
 			this._finished = (!this._config.step && !this._config.chunk) || this._start > getFileSize(xhr);
-			this._parseChunk(xhr.responseText);
+			this.parseChunk(xhr.responseText);
 		}
 
 		this._chunkError = function(errorMessage)
@@ -669,7 +637,7 @@
 			// Very important to increment start each time before handling results
 			this._start += this._config.chunkSize;
 			this._finished = this._start >= this._input.size;
-			this._parseChunk(event.target.result);
+			this.parseChunk(event.target.result);
 		}
 
 		this._chunkError = function()
@@ -682,6 +650,31 @@
 	FileStreamer.prototype.constructor = FileStreamer;
 
 
+	function StringStreamer(config)
+	{
+		config = config || {};
+		ChunkStreamer.call(this, config);
+
+		var string;
+		var remaining;
+		this.stream = function(s)
+		{
+			string = s;
+			remaining = s;
+			return this._nextChunk();
+		}
+		this._nextChunk = function()
+		{
+			if (this._finished) return;
+			var size = this._config.chunkSize;
+			var chunk = size ? remaining.substr(0, size) : remaining;
+			remaining = size ? remaining.substr(size) : '';
+			this._finished = !remaining;
+			return this.parseChunk(chunk);
+		}
+	}
+	StringStreamer.prototype = Object.create(StringStreamer.prototype);
+	StringStreamer.prototype.constructor = StringStreamer;
 
 
 
@@ -757,11 +750,16 @@
 			_parser = new Parser(parserConfig);
 			_results = _parser.parse(_input, baseIndex, ignoreLastRow);
 			processResults();
-			if (isFunction(_config.complete) && !_paused && (!self.streamer || self.streamer.finished()))
+			if (isFunction(_config.complete) && !_paused && self.streamer.finished())
 				_config.complete(_results);
 			return _paused ? { meta: { paused: true } } : (_results || { meta: { paused: false } });
 		};
 
+		this.paused = function()
+		{
+			return _paused;
+		};
+
 		this.pause = function()
 		{
 			_paused = true;
@@ -772,15 +770,7 @@
 		this.resume = function()
 		{
 			_paused = false;
-			_parser = new Parser(_config);
-			_parser.parse(_input);
-			if (!_paused)
-			{
-				if (self.streamer && !self.streamer.finished())
-					self.streamer.resume();		// more of the file yet to come
-				else if (isFunction(_config.complete))
-					_config.complete(_results);
-			}
+			self.streamer.parseChunk(_input);
 		};
 
 		this.abort = function()

tests/test-cases.js

@@ -2,8 +2,8 @@ var RECORD_SEP = String.fromCharCode(30);
 var UNIT_SEP = String.fromCharCode(31);
 var FILES_ENABLED = false;
 try {
-  new File([""], "");
-  FILES_ENABLED = true;
+	new File([""], "");
+	FILES_ENABLED = true;
 } catch (e) {} // safari, ie
 
 // Tests for the core parser using new Papa.Parser().parse() (CSV to JSON)
@@ -851,19 +851,21 @@ var PARSE_ASYNC_TESTS = [
 			errors: []
 		}
 	},
-  {
-    description: "Simple file",
-    input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z"], "sample.csv") : false,
+	{
+		description: "Simple file",
+		disabled: !FILES_ENABLED,
+		input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z"], "sample.csv") : false,
 		config: {
 		},
 		expected: {
 			data: [['A','B','C'],['X','Y','Z']],
 			errors: []
 		}
-  },
-  {
-    description: "Simple file + worker",
-    input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z"], "sample.csv") : false,
+	},
+	{
+		description: "Simple file + worker",
+		disabled: !FILES_ENABLED,
+		input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z"], "sample.csv") : false,
 		config: {
 			worker: true,
 		},
@@ -871,7 +873,7 @@ var PARSE_ASYNC_TESTS = [
 			data: [['A','B','C'],['X','Y','Z']],
 			errors: []
 		}
-  }
+	}
 ];
 
 
@@ -1207,11 +1209,10 @@ var CUSTOM_TESTS = [
 				step: function(response, handle) {
 					updates.push(response.data[0]);
 					handle.abort();
+				}, complete: function() {
+					callback(updates);
 				}
 			});
-			setTimeout(function() {
-				callback(updates);
-			}, 100);
 		}
 	},
 	{
@@ -1223,11 +1224,11 @@ var CUSTOM_TESTS = [
 				step: function(response, handle) {
 					updates.push(response.data[0]);
 					handle.pause();
+					callback(updates);
+				}, complete: function() {
+					callback('incorrect complete callback');
 				}
 			});
-			setTimeout(function() {
-				callback(updates);
-			}, 100);
 		}
 	},
 	{
@@ -1250,7 +1251,7 @@ var CUSTOM_TESTS = [
 			});
 			setTimeout(function() {
 				handle.resume();
-			}, 100);
+			}, 500);
 		}
 	},
 	{

tests/test-runner.js

@@ -91,10 +91,13 @@ function runParseTests(asyncDone)
 			failCount++;
 	}
 
-	var asyncRemaining = PARSE_ASYNC_TESTS.length;
+	var asyncRemaining = 0;
 
 	PARSE_ASYNC_TESTS.forEach(function(test)
 	{
+		if (test.disabled)
+			return;
+		asyncRemaining++;
 		var config = test.config;
 		config.complete = function(actual)
 		{
@@ -321,9 +324,7 @@ function runCustomTests(asyncDone)
 		asyncRemaining++;
 		try
 		{
-			test.run(function(actual) {
-				displayResults(test, actual);
-			});
+			displayAsyncTest(test);
 		}
 		catch (e)
 		{
@@ -331,10 +332,10 @@ function runCustomTests(asyncDone)
 		}
 	}
 
-	function displayResults(test, actual)
+	function displayAsyncTest(test)
 	{
 		var testId = testCount++;
-		var results = compare(actual, test.expected);
+		test.testId = testId;
 
 		var testDescription = (test.description || "");
 		if (testDescription.length > 0)
@@ -345,13 +346,44 @@ function runCustomTests(asyncDone)
 		var tr = '<tr class="collapsed" id="test-'+testId+'">'
 				+ '<td class="rvl">+</td>'
 				+ '<td>' + testDescription + '</td>'
-				+ passOrFailTd(results)
+				+ '<td class="status pending">pending</td>'
 				+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">' + test.expected + '</div></td>'
-				+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">' + actual + '</div></td>'
+				+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden actual"></div></td>'
 				+ '</tr>';
 
 		$('#custom-tests .results').append(tr);
 
+		test.run(function(actual)
+		{
+			displayAsyncResults(test, actual);
+		});
+
+		setTimeout(function()
+		{
+			if (test.complete) return;
+			displayAsyncResults(test, '(incomplete)');
+		}, 2000);
+	}
+
+	function displayAsyncResults(test, actual)
+	{
+		var testId = test.testId;
+		if (test.complete)
+		{
+			asyncRemaining++;
+			actual = '(multiple results from test)';
+		}
+		test.complete = true;
+		var results = compare(actual, test.expected);
+
+		var tr = $('#test-'+testId);
+		tr.find('.actual').text(actual);
+
+		var status = $(passOrFailTd(results));
+		var oldStatus = tr.find('.status');
+		oldStatus.attr('class', status.attr('class'));
+		oldStatus.text(status.text());
+
 		if (!results.passed)
 			$('#test-' + testId + ' td.rvl').click();
 
@@ -385,9 +417,9 @@ function runCustomTests(asyncDone)
 function passOrFailTd(result)
 {
 	if (result.passed)
-		return '<td class="ok">OK</td>';
+		return '<td class="status ok">OK</td>';
 	else
-		return '<td class="fail">FAIL</td>';
+		return '<td class="status fail">FAIL</td>';
 }
 
 

tests/tests.css

@@ -88,8 +88,7 @@ table td {
 	font-size: 14px;
 }
 
-td.ok,
-td.fail {
+td.status {
 	text-transform: uppercase;
 	font-weight: 300;
 	vertical-align: middle;
@@ -105,6 +104,10 @@ td.fail {
 	background: rgb(255, 192, 192);
 }
 
+td.pending {
+	background: rgb(255, 255, 150);
+}
+
 td.rvl {
 	background: #444;
 	color: #999;
@@ -167,4 +170,4 @@ td.pre {
 
 .whitespace-char {
 	background: #D5FCFA;
-}
+}