From b038e4799e141e77628e7a213b0a6b84c43e9202 Mon Sep 17 00:00:00 2001 From: Braden Anderson Date: Thu, 12 Mar 2015 23:25:39 -0600 Subject: [PATCH] read files in a single chunk for complete results (#179) --- papaparse.js | 17 ++++++++++++----- tests/test-cases.js | 12 ++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/papaparse.js b/papaparse.js index e6a4d05..e30eb24 100644 --- a/papaparse.js +++ b/papaparse.js @@ -450,6 +450,8 @@ // Deep-copy the config so we can edit it var configCopy = copy(config); configCopy.chunkSize = parseInt(configCopy.chunkSize); // VERY important so we don't concatenate strings! + if (!config.step && !config.chunk) + configCopy.chunkSize = null; this._handle = new ParserHandle(configCopy); this._handle.streamer = this; this._config = configCopy; // persist the copy to the caller @@ -506,7 +508,7 @@ xhr.open("GET", this._input, !IS_WORKER); - if (this._config.step || this._config.chunk) + if (this._config.chunkSize) { var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive xhr.setRequestHeader("Range", "bytes="+this._start+"-"+end); @@ -537,7 +539,7 @@ return; } - this._finished = (!this._config.step && !this._config.chunk) || this._start > getFileSize(xhr); + this._finished = !this._config.chunkSize || this._start > getFileSize(xhr); this.parseChunk(xhr.responseText); } @@ -595,8 +597,13 @@ this._readChunk = function() { - var end = Math.min(this._start + this._config.chunkSize, this._input.size); - var txt = reader.readAsText(slice.call(this._input, this._start, end), this._config.encoding); + var input = this._input; + if (this._config.chunkSize) + { + var end = Math.min(this._start + this._config.chunkSize, this._input.size); + input = slice.call(input, this._start, end); + } + var txt = reader.readAsText(input, this._config.encoding); if (!usingAsyncReader) this._chunkLoaded({ target: { result: txt } }); // mimic the async signature } @@ -605,7 +612,7 @@ { // Very important to increment start each time before handling results this._start += this._config.chunkSize; - this._finished = this._start >= this._input.size; + this._finished = !this._config.chunkSize || this._start >= this._input.size; this.parseChunk(event.target.result); } diff --git a/tests/test-cases.js b/tests/test-cases.js index 524f991..256b028 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1041,6 +1041,18 @@ var UNPARSE_TESTS = [ var CUSTOM_TESTS = [ + { + description: "Complete is called with all results if neither step nor chunk is defined", + expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']], + run: function(callback) { + Papa.parse(new File(['A,b,c\nd,E,f\nG,h,i'], 'sample.csv'), { + chunkSize: 3, + complete: function(response) { + callback(response.data); + } + }); + } + }, { description: "Step is called for each row", expected: 2,