akira
/
PapaParse
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge branch 'master' of github.com:FlatFilers/PapaParse

pull/693/head
Monkey D Zeke 6 years ago
parent
791034bc79 a6e3a36585
commit
5c4b853b71
12 changed files with 524 additions and 286 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      .eslintrc.js
  2. 3
      .travis.yml
  3. 6
      Gruntfile.js
  4. 16
      docs/docs.html
  5. 6
      docs/faq.html
  6. 425
      docs/resources/js/papaparse.js
  7. 12
      package.json
  8. 138
      papaparse.js
  9. 12
      papaparse.min.js
  10. 139
      tests/test-cases.js
  11. 4
      tests/test.js
  12. 7
      tests/tests.html

2
.eslintrc.js
Unescape View File

@ -178,7 +178,7 @@ module.exports = {
"no-tabs": "off", "no-tabs": "off",
"no-template-curly-in-string": "error", "no-template-curly-in-string": "error",
"no-ternary": "off", "no-ternary": "off",
"no-throw-literal": "off", "no-throw-literal": "error",
"no-trailing-spaces": "error", "no-trailing-spaces": "error",
"no-undef-init": "error", "no-undef-init": "error",
"no-undefined": "off", "no-undefined": "off",

3
.travis.yml
Unescape View File

@ -1,7 +1,6 @@
language: node_js language: node_js
node_js: node_js:
- "6"
- "8" - "8"
- "9" - "9"
- "10" - "10"
- "11"

6
Gruntfile.js
Unescape View File

@ -2,6 +2,12 @@ module.exports = function(grunt) {
grunt.initConfig({ grunt.initConfig({
uglify: { uglify: {
options: { options: {
compress: {
global_defs: {
'PAPA_BROWSER_CONTEXT': true
},
dead_code: true
},
output: { output: {
comments: 'some', comments: 'some',
}, },

16
docs/docs.html
Unescape View File

@ -340,7 +340,7 @@ var csv = Papa.unparse({
quoteChar: '"', quoteChar: '"',
escapeChar: '"', escapeChar: '"',
header: false, header: false,
trimHeaders: false, transformHeader: undefined,
dynamicTyping: false, dynamicTyping: false,
preview: 0, preview: 0,
encoding: "", encoding: "",
@ -412,10 +412,10 @@ var csv = Papa.unparse({
</tr> </tr>
<tr> <tr>
<td> <td>
<code>trimHeaders</code> <code>transformHeader</code>
</td> </td>
<td> <td>
If true leading/trailing spaces will be trimed from headers. A function to apply on each header. Requires <code>header</code> to be <code>true</code>. The function receives the header as its first argument.
</td> </td>
</tr> </tr>
<tr> <tr>
@ -447,7 +447,7 @@ var csv = Papa.unparse({
<code>worker</code> <code>worker</code>
</td> </td>
<td> <td>
Whether or not to use a <a href="/faq#workers">worker thread</a>. Using a worker will keep your page reactive, but may be slightly slower. Web Workers also load the entire Javascript file, so be careful when <a href="/faq#combine">combining other libraries</a> in the same file as Papa Parse. Note that worker option is only available when parsing files and not when converting from JSON to CSV. Whether or not to use a <a href="/faq#workers">worker thread</a>. Using a worker will keep your page reactive, but may be slightly slower.
</td> </td>
</tr> </tr>
<tr> <tr>
@ -545,7 +545,7 @@ var csv = Papa.unparse({
<code>transform</code> <code>transform</code>
</td> </td>
<td> <td>
A function to apply on each value. The function receives the value as its first argument and the column number as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping. A function to apply on each value. The function receives the value as its first argument and the column number or header name when enabled as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping.
</td> </td>
</tr> </tr>
</table> </table>
@ -749,12 +749,6 @@ var csv = Papa.unparse({
Whether or not the browser supports HTML5 Web Workers. If false, <code>worker: true</code> will have no effect. Whether or not the browser supports HTML5 Web Workers. If false, <code>worker: true</code> will have no effect.
</td> </td>
</tr> </tr>
<tr>
<td><code>Papa.SCRIPT_PATH</code></td>
<td>
The relative path to Papa Parse. This is automatically detected when Papa Parse is loaded synchronously. However, if you load Papa Parse asynchronously (e.g. with RequireJS), you need to set this variable manually in order to use Web Workers. (In those cases, this variable is <i>not</i> read-only and you should set it!)
</td>
</tr>
</table> </table>
</div> </div>

6
docs/faq.html
Unescape View File

@ -84,7 +84,7 @@
<h6 id="combine">Can I put other libraries in the same file as Papa Parse?</h6> <h6 id="combine">Can I put other libraries in the same file as Papa Parse?</h6>
<p> <p>
Yes, but then don't use the Web Worker feature unless your other dependencies are battle-hardened for worker threads. A worker thread loads an entire file, not just a function, so all those dependencies would be executed in an environment without a DOM and other <code>window</code> features. If any of those dependencies crash (<code>Cannot read property "defaultView" of undefined</code> <a href="https://github.com/mholt/PapaParse/issues/114">is</a> <a href="https://github.com/mholt/PapaParse/issues/163">common</a>), the whole worker thread will crash and parsing will not succeed. Yes.
</p> </p>
@ -96,7 +96,7 @@
<h6 id="async">Can Papa Parse be loaded asynchronously (after the page loads)?</h6> <h6 id="async">Can Papa Parse be loaded asynchronously (after the page loads)?</h6>
<p> <p>
Yes. But if you want to use Web Workers, you'll need to specify the relative path to Papa Parse. To do this, set <a href="/docs#readonly">Papa.SCRIPT_PATH</a> to the relative path of the Papa Parse file. In synchronous loading, this is automatically detected. Yes.
</p> </p>
@ -209,7 +209,7 @@
<h6>Can I use a worker if I combine/concatenate my Javascript files?</h6> <h6>Can I use a worker if I combine/concatenate my Javascript files?</h6>
<p> <p>
Probably not. It's safest to concatenate the rest of your dependencies and include Papa Parse in a seperate file. Any library that expects to have access to the <code>window</code> or DOM will crash when executed in a worker thread. Only put <a href="/faq#combine">other libraries in the same file</a> if they are ready to be used in worker threads. Yes.
</p> </p>
<h6>When should I use a worker?</h6> <h6>When should I use a worker?</h6>

425
docs/resources/js/papaparse.js
Unescape View File

@ -1,11 +1,22 @@
/*! /* @license
Papa Parse Papa Parse
v4.3.7 v4.6.1
https://github.com/mholt/PapaParse https://github.com/mholt/PapaParse
License: MIT License: MIT
*/ */
// Polyfills
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill
if (!Array.isArray)
{
Array.isArray = function(arg) {
return Object.prototype.toString.call(arg) === '[object Array]';
};
}
(function(root, factory) (function(root, factory)
{ {
/* globals define */
if (typeof define === 'function' && define.amd) if (typeof define === 'function' && define.amd)
{ {
// AMD. Register as an anonymous module. // AMD. Register as an anonymous module.
@ -27,7 +38,7 @@
{ {
'use strict'; 'use strict';
var global = (function () { var global = (function() {
// alternative method, similar to `Function('return this')()` // alternative method, similar to `Function('return this')()`
// but without using `eval` (which is disabled when // but without using `eval` (which is disabled when
// using Content Security Policy). // using Content Security Policy).
@ -40,7 +51,6 @@
return {}; return {};
})(); })();
var IS_WORKER = !global.document && !!global.postMessage, var IS_WORKER = !global.document && !!global.postMessage,
IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search),
LOADED_SYNC = false, AUTO_SCRIPT_PATH; LOADED_SYNC = false, AUTO_SCRIPT_PATH;
@ -57,6 +67,7 @@
Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK];
Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker;
Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously
Papa.NODE_STREAM_INPUT = 1;
// Configurable chunk sizes for local and remote files, respectively // Configurable chunk sizes for local and remote files, respectively
Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB
@ -70,6 +81,7 @@
Papa.FileStreamer = FileStreamer; Papa.FileStreamer = FileStreamer;
Papa.StringStreamer = StringStreamer; Papa.StringStreamer = StringStreamer;
Papa.ReadableStreamStreamer = ReadableStreamStreamer; Papa.ReadableStreamStreamer = ReadableStreamStreamer;
Papa.DuplexStreamStreamer = DuplexStreamStreamer;
if (global.jQuery) if (global.jQuery)
{ {
@ -162,7 +174,7 @@
queue.splice(0, 1); queue.splice(0, 1);
parseNextFile(); parseNextFile();
} }
} };
} }
@ -182,7 +194,7 @@
} }
else else
{ {
document.addEventListener('DOMContentLoaded', function () { document.addEventListener('DOMContentLoaded', function() {
LOADED_SYNC = true; LOADED_SYNC = true;
}, true); }, true);
} }
@ -202,6 +214,8 @@
} }
_config.dynamicTyping = dynamicTyping; _config.dynamicTyping = dynamicTyping;
_config.transform = isFunction(_config.transform) ? _config.transform : false;
if (_config.worker && Papa.WORKERS_SUPPORTED) if (_config.worker && Papa.WORKERS_SUPPORTED)
{ {
var w = newWorker(); var w = newWorker();
@ -227,7 +241,14 @@
} }
var streamer = null; var streamer = null;
if (typeof _input === 'string') if (_input === Papa.NODE_STREAM_INPUT)
{
// create a node Duplex stream for use
// with .pipe
streamer = new DuplexStreamStreamer(_config);
return streamer.getStream();
}
else if (typeof _input === 'string')
{ {
if (_config.download) if (_config.download)
streamer = new NetworkStreamer(_config); streamer = new NetworkStreamer(_config);
@ -251,9 +272,6 @@
function JsonToCsv(_input, _config) function JsonToCsv(_input, _config)
{ {
var _output = '';
var _fields = [];
// Default configuration // Default configuration
/** whether to surround every datum with quotes */ /** whether to surround every datum with quotes */
@ -262,7 +280,7 @@
/** whether to write headers */ /** whether to write headers */
var _writeHeader = true; var _writeHeader = true;
/** delimiting character */ /** delimiting character(s) */
var _delimiter = ','; var _delimiter = ',';
/** newline character(s) */ /** newline character(s) */
@ -271,6 +289,9 @@
/** quote character */ /** quote character */
var _quoteChar = '"'; var _quoteChar = '"';
/** whether to skip empty lines */
var _skipEmptyLines = false;
unpackConfig(); unpackConfig();
var quoteCharRegex = new RegExp(_quoteChar, 'g'); var quoteCharRegex = new RegExp(_quoteChar, 'g');
@ -278,33 +299,33 @@
if (typeof _input === 'string') if (typeof _input === 'string')
_input = JSON.parse(_input); _input = JSON.parse(_input);
if (_input instanceof Array) if (Array.isArray(_input))
{ {
if (!_input.length || _input[0] instanceof Array) if (!_input.length || Array.isArray(_input[0]))
return serialize(null, _input); return serialize(null, _input, _skipEmptyLines);
else if (typeof _input[0] === 'object') else if (typeof _input[0] === 'object')
return serialize(objectKeys(_input[0]), _input); return serialize(objectKeys(_input[0]), _input, _skipEmptyLines);
} }
else if (typeof _input === 'object') else if (typeof _input === 'object')
{ {
if (typeof _input.data === 'string') if (typeof _input.data === 'string')
_input.data = JSON.parse(_input.data); _input.data = JSON.parse(_input.data);
if (_input.data instanceof Array) if (Array.isArray(_input.data))
{ {
if (!_input.fields) if (!_input.fields)
_input.fields = _input.meta && _input.meta.fields; _input.fields = _input.meta && _input.meta.fields;
if (!_input.fields) if (!_input.fields)
_input.fields = _input.data[0] instanceof Array _input.fields = Array.isArray(_input.data[0])
? _input.fields ? _input.fields
: objectKeys(_input.data[0]); : objectKeys(_input.data[0]);
if (!(_input.data[0] instanceof Array) && typeof _input.data[0] !== 'object') if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object')
_input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf']
} }
return serialize(_input.fields || [], _input.data || []); return serialize(_input.fields || [], _input.data || [], _skipEmptyLines);
} }
// Default (any valid paths should return before this) // Default (any valid paths should return before this)
@ -317,16 +338,19 @@
return; return;
if (typeof _config.delimiter === 'string' if (typeof _config.delimiter === 'string'
&& _config.delimiter.length === 1 && !Papa.BAD_DELIMITERS.filter(function(value) { return _config.delimiter.indexOf(value) !== -1; }).length)
&& Papa.BAD_DELIMITERS.indexOf(_config.delimiter) === -1)
{ {
_delimiter = _config.delimiter; _delimiter = _config.delimiter;
} }
if (typeof _config.quotes === 'boolean' if (typeof _config.quotes === 'boolean'
|| _config.quotes instanceof Array) || Array.isArray(_config.quotes))
_quotes = _config.quotes; _quotes = _config.quotes;
if (typeof _config.skipEmptyLines === 'boolean'
|| typeof _config.skipEmptyLines === 'string')
_skipEmptyLines = _config.skipEmptyLines;
if (typeof _config.newline === 'string') if (typeof _config.newline === 'string')
_newline = _config.newline; _newline = _config.newline;
@ -350,7 +374,7 @@
} }
/** The double for loop that iterates the data and writes out a CSV string including header row */ /** The double for loop that iterates the data and writes out a CSV string including header row */
function serialize(fields, data) function serialize(fields, data, skipEmptyLines)
{ {
var csv = ''; var csv = '';
@ -359,8 +383,8 @@
if (typeof data === 'string') if (typeof data === 'string')
data = JSON.parse(data); data = JSON.parse(data);
var hasHeader = fields instanceof Array && fields.length > 0; var hasHeader = Array.isArray(fields) && fields.length > 0;
var dataKeyedByField = !(data[0] instanceof Array); var dataKeyedByField = !(Array.isArray(data[0]));
// If there a header row, write it first // If there a header row, write it first
if (hasHeader && _writeHeader) if (hasHeader && _writeHeader)
@ -379,7 +403,10 @@
for (var row = 0; row < data.length; row++) for (var row = 0; row < data.length; row++)
{ {
var maxCol = hasHeader ? fields.length : data[row].length; var maxCol = hasHeader ? fields.length : data[row].length;
var r = hasHeader ? fields : data[row];
if (skipEmptyLines !== 'greedy' || r.join('').trim() !== '')
{
for (var col = 0; col < maxCol; col++) for (var col = 0; col < maxCol; col++)
{ {
if (col > 0) if (col > 0)
@ -387,11 +414,10 @@
var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; var colIdx = hasHeader && dataKeyedByField ? fields[col] : col;
csv += safe(data[row][colIdx], col); csv += safe(data[row][colIdx], col);
} }
if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0))
if (row < data.length - 1)
csv += _newline; csv += _newline;
} }
}
return csv; return csv;
} }
@ -401,10 +427,13 @@
if (typeof str === 'undefined' || str === null) if (typeof str === 'undefined' || str === null)
return ''; return '';
str = str.toString().replace(quoteCharRegex, _quoteChar+_quoteChar); if (str.constructor === Date)
return JSON.stringify(str).slice(1, 25);
str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar);
var needsQuotes = (typeof _quotes === 'boolean' && _quotes) var needsQuotes = (typeof _quotes === 'boolean' && _quotes)
|| (_quotes instanceof Array && _quotes[col]) || (Array.isArray(_quotes) && _quotes[col])
|| hasAny(str, Papa.BAD_DELIMITERS) || hasAny(str, Papa.BAD_DELIMITERS)
|| str.indexOf(_delimiter) > -1 || str.indexOf(_delimiter) > -1
|| str.charAt(0) === ' ' || str.charAt(0) === ' '
@ -426,8 +455,8 @@
function ChunkStreamer(config) function ChunkStreamer(config)
{ {
this._handle = null; this._handle = null;
this._paused = false;
this._finished = false; this._finished = false;
this._completed = false;
this._input = null; this._input = null;
this._baseIndex = 0; this._baseIndex = 0;
this._partialLine = ''; this._partialLine = '';
@ -442,7 +471,7 @@
}; };
replaceConfig.call(this, config); replaceConfig.call(this, config);
this.parseChunk = function(chunk) this.parseChunk = function(chunk, isFakeChunk)
{ {
// First chunk pre-processing // First chunk pre-processing
if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk))
@ -483,10 +512,10 @@
finished: finishedIncludingPreview finished: finishedIncludingPreview
}); });
} }
else if (isFunction(this._config.chunk)) else if (isFunction(this._config.chunk) && !isFakeChunk)
{ {
this._config.chunk(results, this._handle); this._config.chunk(results, this._handle);
if (this._paused) if (this._handle.paused() || this._handle.aborted())
return; return;
results = undefined; results = undefined;
this._completeResults = undefined; this._completeResults = undefined;
@ -498,8 +527,10 @@
this._completeResults.meta = results.meta; this._completeResults.meta = results.meta;
} }
if (finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) {
this._config.complete(this._completeResults, this._input); this._config.complete(this._completeResults, this._input);
this._completed = true;
}
if (!finishedIncludingPreview && (!results || !results.meta.paused)) if (!finishedIncludingPreview && (!results || !results.meta.paused))
this._nextChunk(); this._nextChunk();
@ -602,7 +633,7 @@
if (this._config.chunkSize) if (this._config.chunkSize)
{ {
var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive
xhr.setRequestHeader('Range', 'bytes='+this._start+'-'+end); xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end);
xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672 xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672
} }
@ -617,11 +648,11 @@
this._chunkError(); this._chunkError();
else else
this._start += this._config.chunkSize; this._start += this._config.chunkSize;
} };
this._chunkLoaded = function() this._chunkLoaded = function()
{ {
if (xhr.readyState != 4) if (xhr.readyState !== 4)
return; return;
if (xhr.status < 200 || xhr.status >= 400) if (xhr.status < 200 || xhr.status >= 400)
@ -632,13 +663,13 @@
this._finished = !this._config.chunkSize || this._start > getFileSize(xhr); this._finished = !this._config.chunkSize || this._start > getFileSize(xhr);
this.parseChunk(xhr.responseText); this.parseChunk(xhr.responseText);
} };
this._chunkError = function(errorMessage) this._chunkError = function(errorMessage)
{ {
var errorText = xhr.statusText || errorMessage; var errorText = xhr.statusText || errorMessage;
this._sendError(errorText); this._sendError(new Error(errorText));
} };
function getFileSize(xhr) function getFileSize(xhr)
{ {
@ -687,7 +718,7 @@
{ {
if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview)) if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview))
this._readChunk(); this._readChunk();
} };
this._readChunk = function() this._readChunk = function()
{ {
@ -700,7 +731,7 @@
var txt = reader.readAsText(input, this._config.encoding); var txt = reader.readAsText(input, this._config.encoding);
if (!usingAsyncReader) if (!usingAsyncReader)
this._chunkLoaded({ target: { result: txt } }); // mimic the async signature this._chunkLoaded({ target: { result: txt } }); // mimic the async signature
} };
this._chunkLoaded = function(event) this._chunkLoaded = function(event)
{ {
@ -708,12 +739,12 @@
this._start += this._config.chunkSize; this._start += this._config.chunkSize;
this._finished = !this._config.chunkSize || this._start >= this._input.size; this._finished = !this._config.chunkSize || this._start >= this._input.size;
this.parseChunk(event.target.result); this.parseChunk(event.target.result);
} };
this._chunkError = function() this._chunkError = function()
{ {
this._sendError(reader.error.message); this._sendError(reader.error);
} };
} }
FileStreamer.prototype = Object.create(ChunkStreamer.prototype); FileStreamer.prototype = Object.create(ChunkStreamer.prototype);
@ -725,14 +756,12 @@
config = config || {}; config = config || {};
ChunkStreamer.call(this, config); ChunkStreamer.call(this, config);
var string;
var remaining; var remaining;
this.stream = function(s) this.stream = function(s)
{ {
string = s;
remaining = s; remaining = s;
return this._nextChunk(); return this._nextChunk();
} };
this._nextChunk = function() this._nextChunk = function()
{ {
if (this._finished) return; if (this._finished) return;
@ -741,7 +770,7 @@
remaining = size ? remaining.substr(size) : ''; remaining = size ? remaining.substr(size) : '';
this._finished = !remaining; this._finished = !remaining;
return this.parseChunk(chunk); return this.parseChunk(chunk);
} };
} }
StringStreamer.prototype = Object.create(StringStreamer.prototype); StringStreamer.prototype = Object.create(StringStreamer.prototype);
StringStreamer.prototype.constructor = StringStreamer; StringStreamer.prototype.constructor = StringStreamer;
@ -755,6 +784,19 @@
var queue = []; var queue = [];
var parseOnData = true; var parseOnData = true;
var streamHasEnded = false;
this.pause = function()
{
ChunkStreamer.prototype.pause.apply(this, arguments);
this._input.pause();
};
this.resume = function()
{
ChunkStreamer.prototype.resume.apply(this, arguments);
this._input.resume();
};
this.stream = function(stream) this.stream = function(stream)
{ {
@ -763,10 +805,18 @@
this._input.on('data', this._streamData); this._input.on('data', this._streamData);
this._input.on('end', this._streamEnd); this._input.on('end', this._streamEnd);
this._input.on('error', this._streamError); this._input.on('error', this._streamError);
};
this._checkIsFinished = function()
{
if (streamHasEnded && queue.length === 1) {
this._finished = true;
} }
};
this._nextChunk = function() this._nextChunk = function()
{ {
this._checkIsFinished();
if (queue.length) if (queue.length)
{ {
this.parseChunk(queue.shift()); this.parseChunk(queue.shift());
@ -775,7 +825,7 @@
{ {
parseOnData = true; parseOnData = true;
} }
} };
this._streamData = bindFunction(function(chunk) this._streamData = bindFunction(function(chunk)
{ {
@ -786,6 +836,7 @@
if (parseOnData) if (parseOnData)
{ {
parseOnData = false; parseOnData = false;
this._checkIsFinished();
this.parseChunk(queue.shift()); this.parseChunk(queue.shift());
} }
} }
@ -798,13 +849,13 @@
this._streamError = bindFunction(function(error) this._streamError = bindFunction(function(error)
{ {
this._streamCleanUp(); this._streamCleanUp();
this._sendError(error.message); this._sendError(error);
}, this); }, this);
this._streamEnd = bindFunction(function() this._streamEnd = bindFunction(function()
{ {
this._streamCleanUp(); this._streamCleanUp();
this._finished = true; streamHasEnded = true;
this._streamData(''); this._streamData('');
}, this); }, this);
@ -819,14 +870,117 @@
ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer; ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer;
function DuplexStreamStreamer(_config) {
var Duplex = require('stream').Duplex;
var config = copy(_config);
var parseOnWrite = true;
var writeStreamHasFinished = false;
var parseCallbackQueue = [];
var stream = null;
this._onCsvData = function(results)
{
var data = results.data;
for (var i = 0; i < data.length; i++) {
if (!stream.push(data[i]) && !this._handle.paused()) {
// the writeable consumer buffer has filled up
// so we need to pause until more items
// can be processed
this._handle.pause();
}
}
};
this._onCsvComplete = function()
{
// node will finish the read stream when
// null is pushed
stream.push(null);
};
config.step = bindFunction(this._onCsvData, this);
config.complete = bindFunction(this._onCsvComplete, this);
ChunkStreamer.call(this, config);
this._nextChunk = function()
{
if (writeStreamHasFinished && parseCallbackQueue.length === 1) {
this._finished = true;
}
if (parseCallbackQueue.length) {
parseCallbackQueue.shift()();
} else {
parseOnWrite = true;
}
};
this._addToParseQueue = function(chunk, callback)
{
// add to queue so that we can indicate
// completion via callback
// node will automatically pause the incoming stream
// when too many items have been added without their
// callback being invoked
parseCallbackQueue.push(bindFunction(function() {
this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding));
if (isFunction(callback)) {
return callback();
}
}, this));
if (parseOnWrite) {
parseOnWrite = false;
this._nextChunk();
}
};
this._onRead = function()
{
if (this._handle.paused()) {
// the writeable consumer can handle more data
// so resume the chunk parsing
this._handle.resume();
}
};
this._onWrite = function(chunk, encoding, callback)
{
this._addToParseQueue(chunk, callback);
};
this._onWriteComplete = function()
{
writeStreamHasFinished = true;
// have to write empty string
// so parser knows its done
this._addToParseQueue('');
};
this.getStream = function()
{
return stream;
};
stream = new Duplex({
readableObjectMode: true,
decodeStrings: false,
read: bindFunction(this._onRead, this),
write: bindFunction(this._onWrite, this)
});
stream.once('finish', bindFunction(this._onWriteComplete, this));
}
DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype);
DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer;
// Use one ParserHandle per entire CSV file or string // Use one ParserHandle per entire CSV file or string
function ParserHandle(_config) function ParserHandle(_config)
{ {
// One goal is to minimize the use of regular expressions... // One goal is to minimize the use of regular expressions...
var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
var self = this; var self = this;
var _stepCounter = 0; // Number of times step was called (number of rows parsed) var _stepCounter = 0; // Number of times step was called (number of rows parsed)
var _rowCounter = 0; // Number of rows that have been parsed so far
var _input; // The input being parsed var _input; // The input being parsed
var _parser; // The core parser being used var _parser; // The core parser being used
var _paused = false; // Whether we are paused or not var _paused = false; // Whether we are paused or not
@ -872,13 +1026,14 @@
*/ */
this.parse = function(input, baseIndex, ignoreLastRow) this.parse = function(input, baseIndex, ignoreLastRow)
{ {
var quoteChar = _config.quoteChar || '"';
if (!_config.newline) if (!_config.newline)
_config.newline = guessLineEndings(input); _config.newline = guessLineEndings(input, quoteChar);
_delimiterError = false; _delimiterError = false;
if (!_config.delimiter) if (!_config.delimiter)
{ {
var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines); var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments);
if (delimGuess.successful) if (delimGuess.successful)
_config.delimiter = delimGuess.bestDelimiter; _config.delimiter = delimGuess.bestDelimiter;
else else
@ -920,10 +1075,10 @@
this.resume = function() this.resume = function()
{ {
_paused = false; _paused = false;
self.streamer.parseChunk(_input); self.streamer.parseChunk(_input, true);
}; };
this.aborted = function () this.aborted = function()
{ {
return _aborted; return _aborted;
}; };
@ -938,25 +1093,29 @@
_input = ''; _input = '';
}; };
function testEmptyLine(s) {
return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0;
}
function processResults() function processResults()
{ {
if (_results && _delimiterError) if (_results && _delimiterError)
{ {
addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \''+Papa.DefaultDelimiter+'\''); addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \'' + Papa.DefaultDelimiter + '\'');
_delimiterError = false; _delimiterError = false;
} }
if (_config.skipEmptyLines) if (_config.skipEmptyLines)
{ {
for (var i = 0; i < _results.data.length; i++) for (var i = 0; i < _results.data.length; i++)
if (_results.data[i].length === 1 && _results.data[i][0] === '') if (testEmptyLine(_results.data[i]))
_results.data.splice(i--, 1); _results.data.splice(i--, 1);
} }
if (needsHeaderRow()) if (needsHeaderRow())
fillHeaderFields(); fillHeaderFields();
return applyHeaderAndDynamicTyping(); return applyHeaderAndDynamicTypingAndTransformation();
} }
function needsHeaderRow() function needsHeaderRow()
@ -970,7 +1129,15 @@
return; return;
for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
for (var j = 0; j < _results.data[i].length; j++) for (var j = 0; j < _results.data[i].length; j++)
_fields.push(_results.data[i][j]); {
var header = _results.data[i][j];
if (_config.trimHeaders) {
header = header.trim();
}
_fields.push(header);
}
_results.data.splice(0, 1); _results.data.splice(0, 1);
} }
@ -979,7 +1146,7 @@
if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) { if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) {
_config.dynamicTyping[field] = _config.dynamicTypingFunction(field); _config.dynamicTyping[field] = _config.dynamicTypingFunction(field);
} }
return (_config.dynamicTyping[field] || _config.dynamicTyping) === true return (_config.dynamicTyping[field] || _config.dynamicTyping) === true;
} }
function parseDynamic(field, value) function parseDynamic(field, value)
@ -990,22 +1157,27 @@
return true; return true;
else if (value === 'false' || value === 'FALSE') else if (value === 'false' || value === 'FALSE')
return false; return false;
else if (FLOAT.test(value))
return parseFloat(value);
else if (ISO_DATE.test(value))
return new Date(value);
else else
return tryParseFloat(value); return (value === '' ? null : value);
} }
return value; return value;
} }
function applyHeaderAndDynamicTyping() function applyHeaderAndDynamicTypingAndTransformation()
{ {
if (!_results || (!_config.header && !_config.dynamicTyping)) if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform))
return _results; return _results;
for (var i = 0; i < _results.data.length; i++) for (var i = 0; i < _results.data.length; i++)
{ {
var row = _config.header ? {} : []; var row = _config.header ? {} : [];
for (var j = 0; j < _results.data[i].length; j++) var j;
for (j = 0; j < _results.data[i].length; j++)
{ {
var field = j; var field = j;
var value = _results.data[i][j]; var value = _results.data[i][j];
@ -1013,6 +1185,9 @@
if (_config.header) if (_config.header)
field = j >= _fields.length ? '__parsed_extra' : _fields[j]; field = j >= _fields.length ? '__parsed_extra' : _fields[j];
if (_config.transform)
value = _config.transform(value,field);
value = parseDynamic(field, value); value = parseDynamic(field, value);
if (field === '__parsed_extra') if (field === '__parsed_extra')
@ -1029,18 +1204,20 @@
if (_config.header) if (_config.header)
{ {
if (j > _fields.length) if (j > _fields.length)
addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, i); addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i);
else if (j < _fields.length) else if (j < _fields.length)
addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, i); addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i);
} }
} }
if (_config.header && _results.meta) if (_config.header && _results.meta)
_results.meta.fields = _fields; _results.meta.fields = _fields;
_rowCounter += _results.data.length;
return _results; return _results;
} }
function guessDelimiter(input, newline, skipEmptyLines) function guessDelimiter(input, newline, skipEmptyLines, comments)
{ {
var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP];
var bestDelim, bestDelta, fieldCountPrevRow; var bestDelim, bestDelta, fieldCountPrevRow;
@ -1052,6 +1229,7 @@
fieldCountPrevRow = undefined; fieldCountPrevRow = undefined;
var preview = new Parser({ var preview = new Parser({
comments: comments,
delimiter: delim, delimiter: delim,
newline: newline, newline: newline,
preview: 10 preview: 10
@ -1059,9 +1237,10 @@
for (var j = 0; j < preview.data.length; j++) for (var j = 0; j < preview.data.length; j++)
{ {
if (skipEmptyLines && preview.data[j].length === 1 && preview.data[j][0].length === 0) { if (skipEmptyLines && testEmptyLine(preview.data[j]))
emptyLinesCount++ {
continue emptyLinesCount++;
continue;
} }
var fieldCount = preview.data[j].length; var fieldCount = preview.data[j].length;
avgFieldCount += fieldCount; avgFieldCount += fieldCount;
@ -1094,12 +1273,15 @@
return { return {
successful: !!bestDelim, successful: !!bestDelim,
bestDelimiter: bestDelim bestDelimiter: bestDelim
} };
} }
function guessLineEndings(input) function guessLineEndings(input, quoteChar)
{ {
input = input.substr(0, 1024*1024); // max length 1 MB input = input.substr(0, 1024 * 1024); // max length 1 MB
// Replace all the text inside quotes
var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm');
input = input.replace(re, '');
var r = input.split('\r'); var r = input.split('\r');
@ -1120,12 +1302,6 @@
return numWithN >= r.length / 2 ? '\r\n' : '\r'; return numWithN >= r.length / 2 ? '\r\n' : '\r';
} }
function tryParseFloat(val)
{
var isNumber = FLOAT.test(val);
return isNumber ? parseFloat(val) : val;
}
function addError(type, code, msg, row) function addError(type, code, msg, row)
{ {
_results.errors.push({ _results.errors.push({
@ -1137,9 +1313,11 @@
} }
} }
/** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */
function escapeRegExp(string)
{
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
}
/** The core parser implements speedy and correct CSV parsing */ /** The core parser implements speedy and correct CSV parsing */
function Parser(config) function Parser(config)
@ -1152,11 +1330,16 @@
var step = config.step; var step = config.step;
var preview = config.preview; var preview = config.preview;
var fastMode = config.fastMode; var fastMode = config.fastMode;
var quoteChar;
/** Allows for no quoteChar by setting quoteChar to undefined in config */ /** Allows for no quoteChar by setting quoteChar to undefined in config */
if (config.quoteChar === undefined){ if (config.quoteChar === undefined) {
var quoteChar = '"'; quoteChar = '"';
} else { } else {
var quoteChar = config.quoteChar; quoteChar = config.quoteChar;
}
var escapeChar = quoteChar;
if (config.escapeChar !== undefined) {
escapeChar = config.escapeChar;
} }
// Delimiter must be valid // Delimiter must be valid
@ -1174,7 +1357,7 @@
comments = false; comments = false;
// Newline must be valid: \r, \n, or \r\n // Newline must be valid: \r, \n, or \r\n
if (newline != '\n' && newline != '\r' && newline != '\r\n') if (newline !== '\n' && newline !== '\r' && newline !== '\r\n')
newline = '\n'; newline = '\n';
// We're gonna need these at the Parser scope // We're gonna need these at the Parser scope
@ -1207,7 +1390,7 @@
var rows = input.split(newline); var rows = input.split(newline);
for (var i = 0; i < rows.length; i++) for (var i = 0; i < rows.length; i++)
{ {
var row = rows[i]; row = rows[i];
cursor += row.length; cursor += row.length;
if (i !== rows.length - 1) if (i !== rows.length - 1)
cursor += newline.length; cursor += newline.length;
@ -1236,7 +1419,8 @@
var nextDelim = input.indexOf(delim, cursor); var nextDelim = input.indexOf(delim, cursor);
var nextNewline = input.indexOf(newline, cursor); var nextNewline = input.indexOf(newline, cursor);
var quoteCharRegex = new RegExp(quoteChar+quoteChar, 'g'); var quoteCharRegex = new RegExp(escapeChar.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&') + quoteChar, 'g');
var quoteSearch;
// Parser loop // Parser loop
for (;;) for (;;)
@ -1245,7 +1429,7 @@
if (input[cursor] === quoteChar) if (input[cursor] === quoteChar)
{ {
// Start our search for the closing quote where the cursor is // Start our search for the closing quote where the cursor is
var quoteSearch = cursor; quoteSearch = cursor;
// Skip the opening quote // Skip the opening quote
cursor++; cursor++;
@ -1253,7 +1437,7 @@
for (;;) for (;;)
{ {
// Find closing quote // Find closing quote
var quoteSearch = input.indexOf(quoteChar, quoteSearch+1); quoteSearch = input.indexOf(quoteChar, quoteSearch + 1);
//No other quotes are found - no other delimiters //No other quotes are found - no other delimiters
if (quoteSearch === -1) if (quoteSearch === -1)
@ -1272,34 +1456,47 @@
} }
// Closing quote at EOF // Closing quote at EOF
if (quoteSearch === inputLen-1) if (quoteSearch === inputLen - 1)
{ {
var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar); var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar);
return finish(value); return finish(value);
} }
// If this quote is escaped, it's part of the data; skip it // If this quote is escaped, it's part of the data; skip it
if (input[quoteSearch+1] === quoteChar) // If the quote character is the escape character, then check if the next character is the escape character
if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar)
{ {
quoteSearch++; quoteSearch++;
continue; continue;
} }
// Closing quote followed by delimiter // If the quote character is not the escape character, then check if the previous character was the escape character
if (input[quoteSearch+1] === delim) if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar)
{
continue;
}
// Check up to nextDelim or nextNewline, whichever is closest
var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline);
var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo);
// Closing quote followed by delimiter or 'unnecessary spaces + delimiter'
if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter] === delim)
{ {
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
cursor = quoteSearch + 1 + delimLen; cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen;
nextDelim = input.indexOf(delim, cursor); nextDelim = input.indexOf(delim, cursor);
nextNewline = input.indexOf(newline, cursor); nextNewline = input.indexOf(newline, cursor);
break; break;
} }
// Closing quote followed by newline var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline);
if (input.substr(quoteSearch+1, newlineLen) === newline)
// Closing quote followed by newline or 'unnecessary spaces + newLine'
if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, newlineLen) === newline)
{ {
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
saveRow(quoteSearch + 1 + newlineLen); saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen);
nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field
if (stepIsFunction) if (stepIsFunction)
@ -1385,6 +1582,21 @@
lastCursor = cursor; lastCursor = cursor;
} }
/**
* checks if there are extra spaces after closing quote and given index without any text
* if Yes, returns the number of spaces
*/
function extraSpaces(index) {
var spaceLength = 0;
if (index !== -1) {
var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index);
if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === '') {
spaceLength = textBetweenClosingQuoteAndIndex.length;
}
}
return spaceLength;
}
/** /**
* Appends the remaining input from cursor to the end into * Appends the remaining input from cursor to the end into
* row, saves the row, calls step, and returns the results. * row, saves the row, calls step, and returns the results.
@ -1437,7 +1649,8 @@
function doStep() function doStep()
{ {
step(returnable()); step(returnable());
data = [], errors = []; data = [];
errors = [];
} }
}; };
@ -1572,9 +1785,9 @@
/** Makes a deep copy of an array or object (mostly) */ /** Makes a deep copy of an array or object (mostly) */
function copy(obj) function copy(obj)
{ {
if (typeof obj !== 'object') if (typeof obj !== 'object' || obj === null)
return obj; return obj;
var cpy = obj instanceof Array ? [] : {}; var cpy = Array.isArray(obj) ? [] : {};
for (var key in obj) for (var key in obj)
cpy[key] = copy(obj[key]); cpy[key] = copy(obj[key]);
return cpy; return cpy;

12
package.json
Unescape View File

@ -1,6 +1,6 @@
{ {
"name": "papaparse", "name": "papaparse",
"version": "4.6.0", "version": "5.0.0-beta.0",
"description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.",
"keywords": [ "keywords": [
"csv", "csv",
@ -35,23 +35,23 @@
}, },
"license": "MIT", "license": "MIT",
"main": "papaparse.js", "main": "papaparse.js",
"browser": "papaparse.min.js",
"devDependencies": { "devDependencies": {
"chai": "^4.2.0", "chai": "^4.2.0",
"connect": "^3.3.3", "connect": "^3.3.3",
"eslint": "^4.19.1", "eslint": "^4.19.1",
"grunt": "^1.0.2", "grunt": "^1.0.2",
"grunt-contrib-uglify": "^3.3.0", "grunt-contrib-uglify": "^3.3.0",
"mocha": "^3.5.0", "mocha": "^5.2.0",
"mocha-phantomjs": "^4.1.0", "mocha-headless-chrome": "^2.0.1",
"open": "0.0.5", "open": "0.0.5",
"phantomjs-prebuilt": "^2.1.16",
"serve-static": "^1.7.1" "serve-static": "^1.7.1"
}, },
"scripts": { "scripts": {
"lint": "eslint --no-ignore papaparse.js Gruntfile.js .eslintrc.js 'tests/**/*.js'", "lint": "eslint --no-ignore papaparse.js Gruntfile.js .eslintrc.js 'tests/**/*.js'",
"test-browser": "node tests/test.js", "test-browser": "node tests/test.js",
"test-phantomjs": "node tests/test.js --phantomjs", "test-mocha-headless-chrome": "node tests/test.js --mocha-headless-chrome",
"test-node": "mocha tests/node-tests.js tests/test-cases.js", "test-node": "mocha tests/node-tests.js tests/test-cases.js",
"test": "npm run lint && npm run test-node && npm run test-phantomjs" "test": "npm run lint && npm run test-node && npm run test-mocha-headless-chrome"
} }
} }

138
papaparse.js
Unescape View File

@ -1,9 +1,10 @@
/*@license /* @license
Papa Parse Papa Parse
v4.6.0 v5.0.0-beta.0
https://github.com/mholt/PapaParse https://github.com/mholt/PapaParse
License: MIT License: MIT
*/ */
(function(root, factory) (function(root, factory)
{ {
/* globals define */ /* globals define */
@ -24,7 +25,10 @@
// Browser globals (root is window) // Browser globals (root is window)
root.Papa = factory(); root.Papa = factory();
} }
}(this, function() // in strict mode we cannot access arguments.callee, so we need a named reference to
// stringify the factory method for the blob worker
// eslint-disable-next-line func-name
}(this, function moduleFactory()
{ {
'use strict'; 'use strict';
@ -42,9 +46,14 @@
})(); })();
function getWorkerBlob() {
var URL = global.URL || global.webkitURL || null;
var code = moduleFactory.toString();
return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(['(', code, ')();'], {type: 'text/javascript'})));
}
var IS_WORKER = !global.document && !!global.postMessage, var IS_WORKER = !global.document && !!global.postMessage,
IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), IS_PAPA_WORKER = IS_WORKER && /blob:/i.test((global.location || {}).protocol);
LOADED_SYNC = false, AUTO_SCRIPT_PATH;
var workers = {}, workerIdCounter = 0; var workers = {}, workerIdCounter = 0;
var Papa = {}; var Papa = {};
@ -57,7 +66,6 @@
Papa.BYTE_ORDER_MARK = '\ufeff'; Papa.BYTE_ORDER_MARK = '\ufeff';
Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK];
Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker;
Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously
Papa.NODE_STREAM_INPUT = 1; Papa.NODE_STREAM_INPUT = 1;
// Configurable chunk sizes for local and remote files, respectively // Configurable chunk sizes for local and remote files, respectively
@ -72,7 +80,9 @@
Papa.FileStreamer = FileStreamer; Papa.FileStreamer = FileStreamer;
Papa.StringStreamer = StringStreamer; Papa.StringStreamer = StringStreamer;
Papa.ReadableStreamStreamer = ReadableStreamStreamer; Papa.ReadableStreamStreamer = ReadableStreamStreamer;
if (typeof PAPA_BROWSER_CONTEXT === 'undefined') {
Papa.DuplexStreamStreamer = DuplexStreamStreamer; Papa.DuplexStreamStreamer = DuplexStreamStreamer;
}
if (global.jQuery) if (global.jQuery)
{ {
@ -173,23 +183,6 @@
{ {
global.onmessage = workerThreadReceivedMessage; global.onmessage = workerThreadReceivedMessage;
} }
else if (Papa.WORKERS_SUPPORTED)
{
AUTO_SCRIPT_PATH = getScriptPath();
// Check if the script was loaded synchronously
if (!document.body)
{
// Body doesn't exist yet, must be synchronous
LOADED_SYNC = true;
}
else
{
document.addEventListener('DOMContentLoaded', function() {
LOADED_SYNC = true;
}, true);
}
}
@ -232,7 +225,7 @@
} }
var streamer = null; var streamer = null;
if (_input === Papa.NODE_STREAM_INPUT) if (_input === Papa.NODE_STREAM_INPUT && typeof PAPA_BROWSER_CONTEXT === 'undefined')
{ {
// create a node Duplex stream for use // create a node Duplex stream for use
// with .pipe // with .pipe
@ -285,14 +278,14 @@
unpackConfig(); unpackConfig();
var quoteCharRegex = new RegExp(_quoteChar, 'g'); var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g');
if (typeof _input === 'string') if (typeof _input === 'string')
_input = JSON.parse(_input); _input = JSON.parse(_input);
if (_input instanceof Array) if (Array.isArray(_input))
{ {
if (!_input.length || _input[0] instanceof Array) if (!_input.length || Array.isArray(_input[0]))
return serialize(null, _input, _skipEmptyLines); return serialize(null, _input, _skipEmptyLines);
else if (typeof _input[0] === 'object') else if (typeof _input[0] === 'object')
return serialize(objectKeys(_input[0]), _input, _skipEmptyLines); return serialize(objectKeys(_input[0]), _input, _skipEmptyLines);
@ -302,17 +295,17 @@
if (typeof _input.data === 'string') if (typeof _input.data === 'string')
_input.data = JSON.parse(_input.data); _input.data = JSON.parse(_input.data);
if (_input.data instanceof Array) if (Array.isArray(_input.data))
{ {
if (!_input.fields) if (!_input.fields)
_input.fields = _input.meta && _input.meta.fields; _input.fields = _input.meta && _input.meta.fields;
if (!_input.fields) if (!_input.fields)
_input.fields = _input.data[0] instanceof Array _input.fields = Array.isArray(_input.data[0])
? _input.fields ? _input.fields
: objectKeys(_input.data[0]); : objectKeys(_input.data[0]);
if (!(_input.data[0] instanceof Array) && typeof _input.data[0] !== 'object') if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object')
_input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf']
} }
@ -320,7 +313,7 @@
} }
// Default (any valid paths should return before this) // Default (any valid paths should return before this)
throw 'exception: Unable to serialize unrecognized input'; throw new Error('Unable to serialize unrecognized input');
function unpackConfig() function unpackConfig()
@ -335,7 +328,7 @@
} }
if (typeof _config.quotes === 'boolean' if (typeof _config.quotes === 'boolean'
|| _config.quotes instanceof Array) || Array.isArray(_config.quotes))
_quotes = _config.quotes; _quotes = _config.quotes;
if (typeof _config.skipEmptyLines === 'boolean' if (typeof _config.skipEmptyLines === 'boolean'
@ -374,8 +367,8 @@
if (typeof data === 'string') if (typeof data === 'string')
data = JSON.parse(data); data = JSON.parse(data);
var hasHeader = fields instanceof Array && fields.length > 0; var hasHeader = Array.isArray(fields) && fields.length > 0;
var dataKeyedByField = !(data[0] instanceof Array); var dataKeyedByField = !(Array.isArray(data[0]));
// If there a header row, write it first // If there a header row, write it first
if (hasHeader && _writeHeader) if (hasHeader && _writeHeader)
@ -394,21 +387,36 @@
for (var row = 0; row < data.length; row++) for (var row = 0; row < data.length; row++)
{ {
var maxCol = hasHeader ? fields.length : data[row].length; var maxCol = hasHeader ? fields.length : data[row].length;
var r = hasHeader ? fields : data[row];
if (skipEmptyLines !== 'greedy' || r.join('').trim() !== '') var emptyLine = false;
var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0;
if (skipEmptyLines && !hasHeader)
{
emptyLine = skipEmptyLines === 'greedy' ? data[row].join('').trim() === '' : data[row].length === 1 && data[row][0].length === 0;
}
if (skipEmptyLines === 'greedy' && hasHeader) {
var line = [];
for (var c = 0; c < maxCol; c++) {
var cx = dataKeyedByField ? fields[c] : c;
line.push(data[row][cx]);
}
emptyLine = line.join('').trim() === '';
}
if (!emptyLine)
{ {
for (var col = 0; col < maxCol; col++) for (var col = 0; col < maxCol; col++)
{ {
if (col > 0) if (col > 0 && !nullLine)
csv += _delimiter; csv += _delimiter;
var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; var colIdx = hasHeader && dataKeyedByField ? fields[col] : col;
csv += safe(data[row][colIdx], col); csv += safe(data[row][colIdx], col);
} }
if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0)) if (row < data.length - 1 && (!skipEmptyLines || (maxCol > 0 && !nullLine)))
{
csv += _newline; csv += _newline;
} }
} }
}
return csv; return csv;
} }
@ -424,7 +432,7 @@
str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar); str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar);
var needsQuotes = (typeof _quotes === 'boolean' && _quotes) var needsQuotes = (typeof _quotes === 'boolean' && _quotes)
|| (_quotes instanceof Array && _quotes[col]) || (Array.isArray(_quotes) && _quotes[col])
|| hasAny(str, Papa.BAD_DELIMITERS) || hasAny(str, Papa.BAD_DELIMITERS)
|| str.indexOf(_delimiter) > -1 || str.indexOf(_delimiter) > -1
|| str.charAt(0) === ' ' || str.charAt(0) === ' '
@ -625,7 +633,6 @@
{ {
var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive
xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end); xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end);
xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672
} }
try { try {
@ -872,14 +879,12 @@
this._onCsvData = function(results) this._onCsvData = function(results)
{ {
var data = results.data; var data = results.data;
for (var i = 0; i < data.length; i++) { if (!stream.push(data) && !this._handle.paused()) {
if (!stream.push(data[i]) && !this._handle.paused()) {
// the writeable consumer buffer has filled up // the writeable consumer buffer has filled up
// so we need to pause until more items // so we need to pause until more items
// can be processed // can be processed
this._handle.pause(); this._handle.pause();
} }
}
}; };
this._onCsvComplete = function() this._onCsvComplete = function()
@ -958,8 +963,10 @@
}); });
stream.once('finish', bindFunction(this._onWriteComplete, this)); stream.once('finish', bindFunction(this._onWriteComplete, this));
} }
if (typeof PAPA_BROWSER_CONTEXT === 'undefined') {
DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype);
DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer;
}
// Use one ParserHandle per entire CSV file or string // Use one ParserHandle per entire CSV file or string
@ -1123,8 +1130,8 @@
{ {
var header = _results.data[i][j]; var header = _results.data[i][j];
if (_config.trimHeaders) { if (isFunction(_config.transformHeader)) {
header = header.trim(); header = _config.transformHeader(header);
} }
_fields.push(header); _fields.push(header);
@ -1340,7 +1347,7 @@
// Comment character must be valid // Comment character must be valid
if (comments === delim) if (comments === delim)
throw 'Comment character same as delimiter'; throw new Error('Comment character same as delimiter');
else if (comments === true) else if (comments === true)
comments = '#'; comments = '#';
else if (typeof comments !== 'string' else if (typeof comments !== 'string'
@ -1359,7 +1366,7 @@
{ {
// For some reason, in Chrome, this speeds things up (!?) // For some reason, in Chrome, this speeds things up (!?)
if (typeof input !== 'string') if (typeof input !== 'string')
throw 'Input must be a string'; throw new Error('Input must be a string');
// We don't need to compute some of these every time parse() is called, // We don't need to compute some of these every time parse() is called,
// but having them in a more local scope seems to perform better // but having them in a more local scope seems to perform better
@ -1410,7 +1417,7 @@
var nextDelim = input.indexOf(delim, cursor); var nextDelim = input.indexOf(delim, cursor);
var nextNewline = input.indexOf(newline, cursor); var nextNewline = input.indexOf(newline, cursor);
var quoteCharRegex = new RegExp(escapeChar.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&') + quoteChar, 'g'); var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), 'g');
var quoteSearch; var quoteSearch;
// Parser loop // Parser loop
@ -1621,10 +1628,11 @@
} }
/** Returns an object with the results, errors, and meta. */ /** Returns an object with the results, errors, and meta. */
function returnable(stopped) function returnable(stopped, step)
{ {
var isStep = step || false;
return { return {
data: data, data: isStep ? data[0] : data,
errors: errors, errors: errors,
meta: { meta: {
delimiter: delim, delimiter: delim,
@ -1639,7 +1647,7 @@
/** Executes the user's step function and resets data & errors. */ /** Executes the user's step function and resets data & errors. */
function doStep() function doStep()
{ {
step(returnable()); step(returnable(undefined, true));
data = []; data = [];
errors = []; errors = [];
} }
@ -1659,26 +1667,12 @@
} }
// If you need to load Papa Parse asynchronously and you also need worker threads, hard-code
// the script path here. See: https://github.com/mholt/PapaParse/issues/87#issuecomment-57885358
function getScriptPath()
{
var scripts = document.getElementsByTagName('script');
return scripts.length ? scripts[scripts.length - 1].src : '';
}
function newWorker() function newWorker()
{ {
if (!Papa.WORKERS_SUPPORTED) if (!Papa.WORKERS_SUPPORTED)
return false; return false;
if (!LOADED_SYNC && Papa.SCRIPT_PATH === null)
throw new Error( var workerUrl = getWorkerBlob();
'Script path cannot be determined automatically when Papa Parse is loaded asynchronously. ' +
'You need to set Papa.SCRIPT_PATH manually.'
);
var workerUrl = Papa.SCRIPT_PATH || AUTO_SCRIPT_PATH;
// Append 'papaworker' to the search string to tell papaparse that this is our worker.
workerUrl += (workerUrl.indexOf('?') !== -1 ? '&' : '?') + 'papaworker';
var w = new global.Worker(workerUrl); var w = new global.Worker(workerUrl);
w.onmessage = mainThreadReceivedMessage; w.onmessage = mainThreadReceivedMessage;
w.id = workerIdCounter++; w.id = workerIdCounter++;
@ -1742,7 +1736,7 @@
} }
function notImplemented() { function notImplemented() {
throw 'Not implemented.'; throw new Error('Not implemented.');
} }
/** Callback when worker thread receives a message */ /** Callback when worker thread receives a message */
@ -1778,7 +1772,7 @@
{ {
if (typeof obj !== 'object' || obj === null) if (typeof obj !== 'object' || obj === null)
return obj; return obj;
var cpy = obj instanceof Array ? [] : {}; var cpy = Array.isArray(obj) ? [] : {};
for (var key in obj) for (var key in obj)
cpy[key] = copy(obj[key]); cpy[key] = copy(obj[key]);
return cpy; return cpy;

12
papaparse.min.js vendored
View File

File diff suppressed because one or more lines are too long

139
tests/test-cases.js
Unescape View File

@ -7,6 +7,7 @@ if (typeof module !== 'undefined' && module.exports) {
var assert = chai.assert; var assert = chai.assert;
var BASE_PATH = (typeof document === 'undefined') ? './' : document.getElementById('test-cases').src.replace(/test-cases\.js$/, '');
var RECORD_SEP = String.fromCharCode(30); var RECORD_SEP = String.fromCharCode(30);
var UNIT_SEP = String.fromCharCode(31); var UNIT_SEP = String.fromCharCode(31);
var FILES_ENABLED = false; var FILES_ENABLED = false;
@ -725,11 +726,11 @@ var PARSE_TESTS = [
} }
}, },
{ {
description: "Header rows are trimmed when trimHeaders is set", description: "Header rows are transformed when transformHeader function is provided",
input: ' A , B , C \r\na,b ,c', input: 'A,B,C\r\na,b,c',
config: { header: true, trimHeaders: true }, config: { header: true, transformHeader: function(header) { return header.toLowerCase(); } },
expected: { expected: {
data: [{"A": "a", "B": "b ", "C": "c"}], data: [{"a": "a", "b": "b", "c": "c"}],
errors: [] errors: []
} }
}, },
@ -912,6 +913,39 @@ var PARSE_TESTS = [
errors: [] errors: []
} }
}, },
{
description: "Custom transform accepts column number also",
input: 'A,B,C\r\nd,e,f',
config: {
transform: function(value, column) {
if (column % 2) {
value = value.toLowerCase();
}
return value;
}
},
expected: {
data: [["A","b","C"], ["d","e","f"]],
errors: []
}
},
{
description: "Custom transform accepts header name when using header",
input: 'A,B,C\r\nd,e,f',
config: {
header: true,
transform: function(value, name) {
if (name === 'B') {
value = value.toUpperCase();
}
return value;
}
},
expected: {
data: [{'A': "d", 'B': "E", 'C': "f"}],
errors: []
}
},
{ {
description: "Dynamic typing converts ISO date strings to Dates", description: "Dynamic typing converts ISO date strings to Dates",
input: 'ISO date,long date\r\n2018-05-04T21:08:03.269Z,Fri May 04 2018 14:08:03 GMT-0700 (PDT)\r\n2018-05-08T15:20:22.642Z,Tue May 08 2018 08:20:22 GMT-0700 (PDT)', input: 'ISO date,long date\r\n2018-05-04T21:08:03.269Z,Fri May 04 2018 14:08:03 GMT-0700 (PDT)\r\n2018-05-08T15:20:22.642Z,Tue May 08 2018 08:20:22 GMT-0700 (PDT)',
@ -1324,7 +1358,7 @@ var PARSE_TESTS = [
} }
}, },
{ {
description: "Using reserved regex characters as quote characters", description: "Using reserved regex character . as quote character",
input: '.a\na.,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', input: '.a\na.,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j',
config: { quoteChar: '.' }, config: { quoteChar: '.' },
expected: { expected: {
@ -1339,6 +1373,22 @@ var PARSE_TESTS = [
} }
} }
}, },
{
description: "Using reserved regex character | as quote character",
input: '|a\na|,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j',
config: { quoteChar: '|' },
expected: {
data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']],
errors: [],
meta: {
linebreak: '\r\n',
delimiter: ',',
cursor: 27,
aborted: false,
truncated: false
}
}
},
{ {
description: "Parsing with skipEmptyLines set to 'greedy'", description: "Parsing with skipEmptyLines set to 'greedy'",
notes: "Must parse correctly without lines with no content", notes: "Must parse correctly without lines with no content",
@ -1396,7 +1446,7 @@ var PARSE_ASYNC_TESTS = [
}, },
{ {
description: "Simple download", description: "Simple download",
input: "sample.csv", input: BASE_PATH + "sample.csv",
config: { config: {
download: true download: true
}, },
@ -1408,7 +1458,7 @@ var PARSE_ASYNC_TESTS = [
}, },
{ {
description: "Simple download + worker", description: "Simple download + worker",
input: "tests/sample.csv", input: BASE_PATH + "sample.csv",
config: { config: {
worker: true, worker: true,
download: true download: true
@ -1654,6 +1704,24 @@ var UNPARSE_TESTS = [
input: [[null, ' '], [], ['1', '2']], input: [[null, ' '], [], ['1', '2']],
config: {skipEmptyLines: 'greedy'}, config: {skipEmptyLines: 'greedy'},
expected: '1,2' expected: '1,2'
},
{
description: "Returns empty rows when empty rows are passed and skipEmptyLines is false with headers",
input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}],
config: {skipEmptyLines: false, header: true},
expected: 'a,b\r\n," "\r\n\r\n1,2'
},
{
description: "Returns without empty rows when skipEmptyLines is true with headers",
input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}],
config: {skipEmptyLines: true, header: true},
expected: 'a,b\r\n," "\r\n1,2'
},
{
description: "Returns without rows with no content when skipEmptyLines is 'greedy' with headers",
input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}],
config: {skipEmptyLines: 'greedy', header: true},
expected: 'a,b\r\n1,2'
} }
]; ];
@ -1717,7 +1785,7 @@ var CUSTOM_TESTS = [
run: function(callback) { run: function(callback) {
Papa.parse('A,b,c', { Papa.parse('A,b,c', {
step: function(response) { step: function(response) {
callback(response.data[0]); callback(response.data);
} }
}); });
} }
@ -1743,7 +1811,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = []; var updates = [];
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
download: true, download: true,
step: function(response) { step: function(response) {
updates.push(response.meta.cursor); updates.push(response.meta.cursor);
@ -1760,7 +1828,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = []; var updates = [];
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
download: true, download: true,
chunkSize: 500, chunkSize: 500,
step: function(response) { step: function(response) {
@ -1778,7 +1846,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = []; var updates = [];
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
download: true, download: true,
chunkSize: 500, chunkSize: 500,
worker: true, worker: true,
@ -1797,7 +1865,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = []; var updates = [];
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
download: true, download: true,
chunkSize: 500, chunkSize: 500,
chunk: function(response) { chunk: function(response) {
@ -1815,7 +1883,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = []; var updates = [];
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
download: true, download: true,
chunkSize: 500, chunkSize: 500,
chunk: function(response) { chunk: function(response) {
@ -1939,7 +2007,7 @@ var CUSTOM_TESTS = [
Papa.parse(new File(['A,B,C\nX,"Y\n1\n2\n3",Z'], 'sample.csv'), { Papa.parse(new File(['A,B,C\nX,"Y\n1\n2\n3",Z'], 'sample.csv'), {
chunkSize: 3, chunkSize: 3,
step: function(response) { step: function(response) {
updates.push(response.data[0]); updates.push(response.data);
}, },
complete: function() { complete: function() {
callback(updates); callback(updates);
@ -1954,7 +2022,7 @@ var CUSTOM_TESTS = [
var updates = []; var updates = [];
Papa.parse('A,b,c\nd,E,f\nG,h,i', { Papa.parse('A,b,c\nd,E,f\nG,h,i', {
step: function(response, handle) { step: function(response, handle) {
updates.push(response.data[0]); updates.push(response.data);
handle.abort(); handle.abort();
callback(updates); callback(updates);
}, },
@ -1984,7 +2052,7 @@ var CUSTOM_TESTS = [
var updates = []; var updates = [];
Papa.parse('A,b,c\nd,E,f\nG,h,i', { Papa.parse('A,b,c\nd,E,f\nG,h,i', {
step: function(response, handle) { step: function(response, handle) {
updates.push(response.data[0]); updates.push(response.data);
handle.pause(); handle.pause();
callback(updates); callback(updates);
}, },
@ -2003,7 +2071,7 @@ var CUSTOM_TESTS = [
var first = true; var first = true;
Papa.parse('A,b,c\nd,E,f\nG,h,i', { Papa.parse('A,b,c\nd,E,f\nG,h,i', {
step: function(response, h) { step: function(response, h) {
updates.push(response.data[0]); updates.push(response.data);
if (!first) return; if (!first) return;
handle = h; handle = h;
handle.pause(); handle.pause();
@ -2024,7 +2092,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = 0; var updates = 0;
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
worker: true, worker: true,
download: true, download: true,
chunkSize: 500, chunkSize: 500,
@ -2044,7 +2112,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = 0; var updates = 0;
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
download: true, download: true,
chunkSize: 500, chunkSize: 500,
beforeFirstChunk: function(chunk) { beforeFirstChunk: function(chunk) {
@ -2065,7 +2133,7 @@ var CUSTOM_TESTS = [
disabled: !XHR_ENABLED, disabled: !XHR_ENABLED,
run: function(callback) { run: function(callback) {
var updates = 0; var updates = 0;
Papa.parse("/tests/long-sample.csv", { Papa.parse(BASE_PATH + "long-sample.csv", {
download: true, download: true,
chunkSize: 500, chunkSize: 500,
beforeFirstChunk: function(chunk) { beforeFirstChunk: function(chunk) {
@ -2078,37 +2146,6 @@ var CUSTOM_TESTS = [
} }
}); });
} }
},
{
description: "Should not assume we own the worker unless papaworker is in the search string",
disabled: typeof Worker === 'undefined',
expected: [false, true, true, true, true],
run: function(callback) {
var searchStrings = [
'',
'?papaworker',
'?x=1&papaworker',
'?x=1&papaworker&y=1',
'?x=1&papaworker=1'
];
var results = searchStrings.map(function() { return false; });
var workers = [];
// Give it .5s to do something
setTimeout(function() {
workers.forEach(function(w) { w.terminate(); });
callback(results);
}, 500);
searchStrings.forEach(function(searchString, idx) {
var w = new Worker('../papaparse.js' + searchString);
workers.push(w);
w.addEventListener('message', function() {
results[idx] = true;
});
w.postMessage({input: 'a,b,c\n1,2,3'});
});
}
} }
]; ];

4
tests/test.js
Unescape View File

@ -5,8 +5,8 @@ var path = require('path');
var childProcess = require('child_process'); var childProcess = require('child_process');
var server = connect().use(serveStatic(path.join(__dirname, '/..'))).listen(8071, function() { var server = connect().use(serveStatic(path.join(__dirname, '/..'))).listen(8071, function() {
if (process.argv.indexOf('--phantomjs') !== -1) { if (process.argv.indexOf('--mocha-headless-chrome') !== -1) {
childProcess.spawn('node_modules/.bin/mocha-phantomjs', ['http://localhost:8071/tests/tests.html'], { childProcess.spawn('node_modules/.bin/mocha-headless-chrome', ['-f', 'http://localhost:8071/tests/tests.html'], {
stdio: 'inherit' stdio: 'inherit'
}).on('exit', function(code) { }).on('exit', function(code) {
server.close(); server.close();

7
tests/tests.html
Unescape View File

@ -9,19 +9,14 @@
<script src="../node_modules/chai/chai.js"></script> <script src="../node_modules/chai/chai.js"></script>
<script>mocha.setup('bdd')</script> <script>mocha.setup('bdd')</script>
<script src="test-cases.js"></script> <script src="test-cases.js" id="test-cases"></script>
</head> </head>
<body> <body>
<div id="mocha"></div> <div id="mocha"></div>
<script> <script>
if (window.mochaPhantomJS) {
mochaPhantomJS.run();
} else {
mocha.checkLeaks(); mocha.checkLeaks();
mocha.run(); mocha.run();
}
</script> </script>
</body> </body>
</html> </html>

Write Preview
Loading…
Cancel
Save