From 22d321095c7070082934a11cc213946ec48e81ce Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 12 Jan 2015 17:41:55 -0700 Subject: [PATCH] Cleanup and various minor fixes galore! (#136 and #138) --- papaparse.js | 91 ++++++++++++++++++-------------------------- player/player.js | 4 +- tests/test-cases.js | 36 ++++++++++++------ tests/test-runner.js | 9 ++--- 4 files changed, 66 insertions(+), 74 deletions(-) diff --git a/papaparse.js b/papaparse.js index 83e98f3..86adbb0 100644 --- a/papaparse.js +++ b/papaparse.js @@ -39,8 +39,7 @@ global.Papa.BYTE_ORDER_MARK = "\ufeff"; global.Papa.BAD_DELIMITERS = ["\r", "\n", "\"", global.Papa.BYTE_ORDER_MARK]; global.Papa.WORKERS_SUPPORTED = !!global.Worker; - // Must be set externally if using workers and Papa Parse is loaded asynchronously - global.Papa.SCRIPT_PATH = null; + global.Papa.SCRIPT_PATH = null; // Must be set manually if using workers and Papa Parse is loaded asynchronously // Configurable chunk sizes for local and remote files, respectively global.Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB @@ -156,11 +155,15 @@ else if (Papa.WORKERS_SUPPORTED) { AUTO_SCRIPT_PATH = getScriptPath(); + // Check if the script was loaded synchronously - if ( !document.body ) { + if (!document.body) + { // Body doesn't exist yet, must be synchronous LOADED_SYNC = true; - } else { + } + else + { document.addEventListener('DOMContentLoaded', function () { LOADED_SYNC = true; }, true); @@ -367,64 +370,39 @@ } } - function bindFunction(f, self) - { - return function() { - f.apply(self, arguments); - } - } - + // ChunkStreamer is the base prototype for various streamer implementations. function ChunkStreamer(config) { this._handle = null; this._paused = false; this._finished = false; - this._userComplete = null; this._input = null; this._baseIndex = 0; this._partialLine = ""; this._rowCount = 0; this._start = 0; - this._config = replaceConfig.call(this, config); - - this.resume = function() - { - this._paused = false; - this._nextChunk(); - }; - - this.finished = function() - { - return this._finished; - }; - - this.pause = function() - { - this._paused = true; - }; - - this.abort = function() - { - this._finished = true; - if (isFunction(this._userComplete)) - this._userComplete({ data: [], errors: [], meta: { aborted: true } }); - }; - this._nextChunk = null; + replaceConfig.call(this, config); - this.parseChunk = function(chunk) { + this.parseChunk = function(chunk) + { // Rejoin the line we likely just split in two by chunking the file var aggregate = this._partialLine + chunk; this._partialLine = ""; var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); - if (this._handle.paused()) return; + + if (this._handle.paused()) + return; + var lastIndex = results.meta.cursor; + if (!this._finished) { this._partialLine = aggregate.substring(lastIndex - this._baseIndex); this._baseIndex = lastIndex; } + if (results && results.data) this._rowCount += results.data.length; @@ -446,9 +424,9 @@ results = undefined; } - if (isFunction(this._userComplete) && finishedIncludingPreview) - this._userComplete(results); - + if (finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) + this._config.complete(results); + if (!finishedIncludingPreview && (!results || !results.meta.paused)) this._nextChunk(); @@ -471,17 +449,12 @@ function replaceConfig(config) { - // Deep-copy the config so we can edit it; we need - // to call the complete function if we are to ensure - // that the last chunk callback, if any, will be called - // BEFORE the complete function. + // Deep-copy the config so we can edit it var configCopy = copy(config); - this._userComplete = configCopy.complete; - configCopy.complete = undefined; configCopy.chunkSize = parseInt(configCopy.chunkSize); // VERY important so we don't concatenate strings! this._handle = new ParserHandle(configCopy); this._handle.streamer = this; - return configCopy; + this._config = configCopy; // persist the copy to the caller } } @@ -593,7 +566,6 @@ config.chunkSize = Papa.LocalChunkSize; ChunkStreamer.call(this, config); - var reader, slice; // FileReader is better than FileReaderSync (even in worker) - see http://stackoverflow.com/q/24708649/1048862 @@ -722,6 +694,9 @@ }; } + // Parses input. Most users won't need, and shouldn't mess with, the baseIndex + // and ignoreLastRow parameters. They are used by streamers (wrapper functions) + // when an input comes in multiple chunks, like from a file. this.parse = function(input, baseIndex, ignoreLastRow) { if (!_config.newline) @@ -1277,10 +1252,11 @@ var abort = function() { aborted = true; completeWorker(msg.workerId, { data: [], errors: [], meta: { aborted: true } }); - } + }; + var handle = { abort: abort, - pause: abort, + pause: notImplemented, resume: notImplemented }; @@ -1300,15 +1276,13 @@ } else if (isFunction(worker.userChunk)) { - worker.userChunk(msg.results, msg.file); + worker.userChunk(msg.results, handle, msg.file); delete msg.results; } } if (msg.finished && !aborted) - { completeWorker(msg.workerId, msg.results); - } } function completeWorker(workerId, results) { @@ -1362,6 +1336,13 @@ return cpy; } + function bindFunction(f, self) + { + return function() { + f.apply(self, arguments); + } + } + function isFunction(func) { return typeof func === 'function'; diff --git a/player/player.js b/player/player.js index d47d9ad..02438e6 100644 --- a/player/player.js +++ b/player/player.js @@ -149,13 +149,13 @@ function chunkFn(results, streamer, file) if (pauseChecked) { - console.log("Pausing; " + results.data.length + " rows in chunk", file); + console.log("Pausing; " + results.data.length + " rows in chunk; file:", file); streamer.pause(); return; } if (printStepChecked) - console.log(results.data.length + " rows in chunk", file); + console.log("Chunk data:", results.data.length, results); } function errorFn(error, file) diff --git a/tests/test-cases.js b/tests/test-cases.js index bfc0705..5e2ae14 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1066,7 +1066,8 @@ var CUSTOM_TESTS = [ Papa.parse('A,b,c\nd,E,f\nG,h,i', { step: function(response) { updates.push(response.meta.cursor); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1081,7 +1082,8 @@ var CUSTOM_TESTS = [ download: true, step: function(response) { updates.push(response.meta.cursor); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1097,7 +1099,8 @@ var CUSTOM_TESTS = [ chunkSize: 500, step: function(response) { updates.push(response.meta.cursor); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1114,7 +1117,8 @@ var CUSTOM_TESTS = [ worker: true, step: function(response) { updates.push(response.meta.cursor); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1130,7 +1134,8 @@ var CUSTOM_TESTS = [ chunkSize: 500, chunk: function(response) { updates.push(response.data.length); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1146,7 +1151,8 @@ var CUSTOM_TESTS = [ chunkSize: 500, chunk: function(response) { updates.push(response.meta.cursor); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1162,7 +1168,8 @@ var CUSTOM_TESTS = [ download: true, step: function(response) { updates.push(response.meta.cursor); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1178,7 +1185,8 @@ var CUSTOM_TESTS = [ chunkSize: 3, step: function(response) { updates.push(response.meta.cursor); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1194,7 +1202,8 @@ var CUSTOM_TESTS = [ chunkSize: 3, step: function(response) { updates.push(response.data[0]); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1209,7 +1218,8 @@ var CUSTOM_TESTS = [ step: function(response, handle) { updates.push(response.data[0]); handle.abort(); - }, complete: function() { + }, + complete: function() { callback(updates); } }); @@ -1225,7 +1235,8 @@ var CUSTOM_TESTS = [ updates.push(response.data[0]); handle.pause(); callback(updates); - }, complete: function() { + }, + complete: function() { callback('incorrect complete callback'); } }); @@ -1245,7 +1256,8 @@ var CUSTOM_TESTS = [ handle = h; handle.pause(); first = false; - }, complete: function() { + }, + complete: function() { callback(updates); } }); diff --git a/tests/test-runner.js b/tests/test-runner.js index 8d54c1f..8a3dd94 100644 --- a/tests/test-runner.js +++ b/tests/test-runner.js @@ -387,14 +387,13 @@ function runCustomTests(asyncDone) if (!results.passed) $('#test-' + testId + ' td.rvl').click(); - if (results.passed) { + if (results.passed) passCount++; - } else { + else failCount++; - } - if (--asyncRemaining === 0) { + + if (--asyncRemaining === 0) asyncDone(); - } }