Browse Source

Use latest version on demo website

pull/580/head
Sergi Almacellas Abellana 6 years ago
parent
commit
d61d43f8d9
  1. 447
      docs/resources/js/papaparse.js

447
docs/resources/js/papaparse.js

@ -1,11 +1,22 @@
/*! /* @license
Papa Parse Papa Parse
v4.3.7 v4.6.1
https://github.com/mholt/PapaParse https://github.com/mholt/PapaParse
License: MIT License: MIT
*/ */
// Polyfills
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill
if (!Array.isArray)
{
Array.isArray = function(arg) {
return Object.prototype.toString.call(arg) === '[object Array]';
};
}
(function(root, factory) (function(root, factory)
{ {
/* globals define */
if (typeof define === 'function' && define.amd) if (typeof define === 'function' && define.amd)
{ {
// AMD. Register as an anonymous module. // AMD. Register as an anonymous module.
@ -27,7 +38,7 @@
{ {
'use strict'; 'use strict';
var global = (function () { var global = (function() {
// alternative method, similar to `Function('return this')()` // alternative method, similar to `Function('return this')()`
// but without using `eval` (which is disabled when // but without using `eval` (which is disabled when
// using Content Security Policy). // using Content Security Policy).
@ -40,7 +51,6 @@
return {}; return {};
})(); })();
var IS_WORKER = !global.document && !!global.postMessage, var IS_WORKER = !global.document && !!global.postMessage,
IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search),
LOADED_SYNC = false, AUTO_SCRIPT_PATH; LOADED_SYNC = false, AUTO_SCRIPT_PATH;
@ -57,6 +67,7 @@
Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK];
Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker;
Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously
Papa.NODE_STREAM_INPUT = 1;
// Configurable chunk sizes for local and remote files, respectively // Configurable chunk sizes for local and remote files, respectively
Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB
@ -70,6 +81,7 @@
Papa.FileStreamer = FileStreamer; Papa.FileStreamer = FileStreamer;
Papa.StringStreamer = StringStreamer; Papa.StringStreamer = StringStreamer;
Papa.ReadableStreamStreamer = ReadableStreamStreamer; Papa.ReadableStreamStreamer = ReadableStreamStreamer;
Papa.DuplexStreamStreamer = DuplexStreamStreamer;
if (global.jQuery) if (global.jQuery)
{ {
@ -162,7 +174,7 @@
queue.splice(0, 1); queue.splice(0, 1);
parseNextFile(); parseNextFile();
} }
} };
} }
@ -182,7 +194,7 @@
} }
else else
{ {
document.addEventListener('DOMContentLoaded', function () { document.addEventListener('DOMContentLoaded', function() {
LOADED_SYNC = true; LOADED_SYNC = true;
}, true); }, true);
} }
@ -202,6 +214,8 @@
} }
_config.dynamicTyping = dynamicTyping; _config.dynamicTyping = dynamicTyping;
_config.transform = isFunction(_config.transform) ? _config.transform : false;
if (_config.worker && Papa.WORKERS_SUPPORTED) if (_config.worker && Papa.WORKERS_SUPPORTED)
{ {
var w = newWorker(); var w = newWorker();
@ -227,7 +241,14 @@
} }
var streamer = null; var streamer = null;
if (typeof _input === 'string') if (_input === Papa.NODE_STREAM_INPUT)
{
// create a node Duplex stream for use
// with .pipe
streamer = new DuplexStreamStreamer(_config);
return streamer.getStream();
}
else if (typeof _input === 'string')
{ {
if (_config.download) if (_config.download)
streamer = new NetworkStreamer(_config); streamer = new NetworkStreamer(_config);
@ -251,9 +272,6 @@
function JsonToCsv(_input, _config) function JsonToCsv(_input, _config)
{ {
var _output = '';
var _fields = [];
// Default configuration // Default configuration
/** whether to surround every datum with quotes */ /** whether to surround every datum with quotes */
@ -262,7 +280,7 @@
/** whether to write headers */ /** whether to write headers */
var _writeHeader = true; var _writeHeader = true;
/** delimiting character */ /** delimiting character(s) */
var _delimiter = ','; var _delimiter = ',';
/** newline character(s) */ /** newline character(s) */
@ -271,6 +289,9 @@
/** quote character */ /** quote character */
var _quoteChar = '"'; var _quoteChar = '"';
/** whether to skip empty lines */
var _skipEmptyLines = false;
unpackConfig(); unpackConfig();
var quoteCharRegex = new RegExp(_quoteChar, 'g'); var quoteCharRegex = new RegExp(_quoteChar, 'g');
@ -278,33 +299,33 @@
if (typeof _input === 'string') if (typeof _input === 'string')
_input = JSON.parse(_input); _input = JSON.parse(_input);
if (_input instanceof Array) if (Array.isArray(_input))
{ {
if (!_input.length || _input[0] instanceof Array) if (!_input.length || Array.isArray(_input[0]))
return serialize(null, _input); return serialize(null, _input, _skipEmptyLines);
else if (typeof _input[0] === 'object') else if (typeof _input[0] === 'object')
return serialize(objectKeys(_input[0]), _input); return serialize(objectKeys(_input[0]), _input, _skipEmptyLines);
} }
else if (typeof _input === 'object') else if (typeof _input === 'object')
{ {
if (typeof _input.data === 'string') if (typeof _input.data === 'string')
_input.data = JSON.parse(_input.data); _input.data = JSON.parse(_input.data);
if (_input.data instanceof Array) if (Array.isArray(_input.data))
{ {
if (!_input.fields) if (!_input.fields)
_input.fields = _input.meta && _input.meta.fields; _input.fields = _input.meta && _input.meta.fields;
if (!_input.fields) if (!_input.fields)
_input.fields = _input.data[0] instanceof Array _input.fields = Array.isArray(_input.data[0])
? _input.fields ? _input.fields
: objectKeys(_input.data[0]); : objectKeys(_input.data[0]);
if (!(_input.data[0] instanceof Array) && typeof _input.data[0] !== 'object') if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object')
_input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf']
} }
return serialize(_input.fields || [], _input.data || []); return serialize(_input.fields || [], _input.data || [], _skipEmptyLines);
} }
// Default (any valid paths should return before this) // Default (any valid paths should return before this)
@ -317,16 +338,19 @@
return; return;
if (typeof _config.delimiter === 'string' if (typeof _config.delimiter === 'string'
&& _config.delimiter.length === 1 && !Papa.BAD_DELIMITERS.filter(function(value) { return _config.delimiter.indexOf(value) !== -1; }).length)
&& Papa.BAD_DELIMITERS.indexOf(_config.delimiter) === -1)
{ {
_delimiter = _config.delimiter; _delimiter = _config.delimiter;
} }
if (typeof _config.quotes === 'boolean' if (typeof _config.quotes === 'boolean'
|| _config.quotes instanceof Array) || Array.isArray(_config.quotes))
_quotes = _config.quotes; _quotes = _config.quotes;
if (typeof _config.skipEmptyLines === 'boolean'
|| typeof _config.skipEmptyLines === 'string')
_skipEmptyLines = _config.skipEmptyLines;
if (typeof _config.newline === 'string') if (typeof _config.newline === 'string')
_newline = _config.newline; _newline = _config.newline;
@ -350,7 +374,7 @@
} }
/** The double for loop that iterates the data and writes out a CSV string including header row */ /** The double for loop that iterates the data and writes out a CSV string including header row */
function serialize(fields, data) function serialize(fields, data, skipEmptyLines)
{ {
var csv = ''; var csv = '';
@ -359,8 +383,8 @@
if (typeof data === 'string') if (typeof data === 'string')
data = JSON.parse(data); data = JSON.parse(data);
var hasHeader = fields instanceof Array && fields.length > 0; var hasHeader = Array.isArray(fields) && fields.length > 0;
var dataKeyedByField = !(data[0] instanceof Array); var dataKeyedByField = !(Array.isArray(data[0]));
// If there a header row, write it first // If there a header row, write it first
if (hasHeader && _writeHeader) if (hasHeader && _writeHeader)
@ -379,19 +403,21 @@
for (var row = 0; row < data.length; row++) for (var row = 0; row < data.length; row++)
{ {
var maxCol = hasHeader ? fields.length : data[row].length; var maxCol = hasHeader ? fields.length : data[row].length;
var r = hasHeader ? fields : data[row];
for (var col = 0; col < maxCol; col++) if (skipEmptyLines !== 'greedy' || r.join('').trim() !== '')
{ {
if (col > 0) for (var col = 0; col < maxCol; col++)
csv += _delimiter; {
var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; if (col > 0)
csv += safe(data[row][colIdx], col); csv += _delimiter;
var colIdx = hasHeader && dataKeyedByField ? fields[col] : col;
csv += safe(data[row][colIdx], col);
}
if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0))
csv += _newline;
} }
if (row < data.length - 1)
csv += _newline;
} }
return csv; return csv;
} }
@ -401,10 +427,13 @@
if (typeof str === 'undefined' || str === null) if (typeof str === 'undefined' || str === null)
return ''; return '';
str = str.toString().replace(quoteCharRegex, _quoteChar+_quoteChar); if (str.constructor === Date)
return JSON.stringify(str).slice(1, 25);
str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar);
var needsQuotes = (typeof _quotes === 'boolean' && _quotes) var needsQuotes = (typeof _quotes === 'boolean' && _quotes)
|| (_quotes instanceof Array && _quotes[col]) || (Array.isArray(_quotes) && _quotes[col])
|| hasAny(str, Papa.BAD_DELIMITERS) || hasAny(str, Papa.BAD_DELIMITERS)
|| str.indexOf(_delimiter) > -1 || str.indexOf(_delimiter) > -1
|| str.charAt(0) === ' ' || str.charAt(0) === ' '
@ -426,8 +455,8 @@
function ChunkStreamer(config) function ChunkStreamer(config)
{ {
this._handle = null; this._handle = null;
this._paused = false;
this._finished = false; this._finished = false;
this._completed = false;
this._input = null; this._input = null;
this._baseIndex = 0; this._baseIndex = 0;
this._partialLine = ''; this._partialLine = '';
@ -442,7 +471,7 @@
}; };
replaceConfig.call(this, config); replaceConfig.call(this, config);
this.parseChunk = function(chunk) this.parseChunk = function(chunk, isFakeChunk)
{ {
// First chunk pre-processing // First chunk pre-processing
if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk))
@ -483,10 +512,10 @@
finished: finishedIncludingPreview finished: finishedIncludingPreview
}); });
} }
else if (isFunction(this._config.chunk)) else if (isFunction(this._config.chunk) && !isFakeChunk)
{ {
this._config.chunk(results, this._handle); this._config.chunk(results, this._handle);
if (this._paused) if (this._handle.paused() || this._handle.aborted())
return; return;
results = undefined; results = undefined;
this._completeResults = undefined; this._completeResults = undefined;
@ -498,8 +527,10 @@
this._completeResults.meta = results.meta; this._completeResults.meta = results.meta;
} }
if (finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) {
this._config.complete(this._completeResults, this._input); this._config.complete(this._completeResults, this._input);
this._completed = true;
}
if (!finishedIncludingPreview && (!results || !results.meta.paused)) if (!finishedIncludingPreview && (!results || !results.meta.paused))
this._nextChunk(); this._nextChunk();
@ -602,7 +633,7 @@
if (this._config.chunkSize) if (this._config.chunkSize)
{ {
var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive
xhr.setRequestHeader('Range', 'bytes='+this._start+'-'+end); xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end);
xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672 xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672
} }
@ -617,11 +648,11 @@
this._chunkError(); this._chunkError();
else else
this._start += this._config.chunkSize; this._start += this._config.chunkSize;
} };
this._chunkLoaded = function() this._chunkLoaded = function()
{ {
if (xhr.readyState != 4) if (xhr.readyState !== 4)
return; return;
if (xhr.status < 200 || xhr.status >= 400) if (xhr.status < 200 || xhr.status >= 400)
@ -632,20 +663,20 @@
this._finished = !this._config.chunkSize || this._start > getFileSize(xhr); this._finished = !this._config.chunkSize || this._start > getFileSize(xhr);
this.parseChunk(xhr.responseText); this.parseChunk(xhr.responseText);
} };
this._chunkError = function(errorMessage) this._chunkError = function(errorMessage)
{ {
var errorText = xhr.statusText || errorMessage; var errorText = xhr.statusText || errorMessage;
this._sendError(errorText); this._sendError(new Error(errorText));
} };
function getFileSize(xhr) function getFileSize(xhr)
{ {
var contentRange = xhr.getResponseHeader('Content-Range'); var contentRange = xhr.getResponseHeader('Content-Range');
if (contentRange === null) { // no content range, then finish! if (contentRange === null) { // no content range, then finish!
return -1; return -1;
} }
return parseInt(contentRange.substr(contentRange.lastIndexOf('/') + 1)); return parseInt(contentRange.substr(contentRange.lastIndexOf('/') + 1));
} }
} }
@ -687,7 +718,7 @@
{ {
if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview)) if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview))
this._readChunk(); this._readChunk();
} };
this._readChunk = function() this._readChunk = function()
{ {
@ -700,7 +731,7 @@
var txt = reader.readAsText(input, this._config.encoding); var txt = reader.readAsText(input, this._config.encoding);
if (!usingAsyncReader) if (!usingAsyncReader)
this._chunkLoaded({ target: { result: txt } }); // mimic the async signature this._chunkLoaded({ target: { result: txt } }); // mimic the async signature
} };
this._chunkLoaded = function(event) this._chunkLoaded = function(event)
{ {
@ -708,12 +739,12 @@
this._start += this._config.chunkSize; this._start += this._config.chunkSize;
this._finished = !this._config.chunkSize || this._start >= this._input.size; this._finished = !this._config.chunkSize || this._start >= this._input.size;
this.parseChunk(event.target.result); this.parseChunk(event.target.result);
} };
this._chunkError = function() this._chunkError = function()
{ {
this._sendError(reader.error.message); this._sendError(reader.error);
} };
} }
FileStreamer.prototype = Object.create(ChunkStreamer.prototype); FileStreamer.prototype = Object.create(ChunkStreamer.prototype);
@ -725,14 +756,12 @@
config = config || {}; config = config || {};
ChunkStreamer.call(this, config); ChunkStreamer.call(this, config);
var string;
var remaining; var remaining;
this.stream = function(s) this.stream = function(s)
{ {
string = s;
remaining = s; remaining = s;
return this._nextChunk(); return this._nextChunk();
} };
this._nextChunk = function() this._nextChunk = function()
{ {
if (this._finished) return; if (this._finished) return;
@ -741,7 +770,7 @@
remaining = size ? remaining.substr(size) : ''; remaining = size ? remaining.substr(size) : '';
this._finished = !remaining; this._finished = !remaining;
return this.parseChunk(chunk); return this.parseChunk(chunk);
} };
} }
StringStreamer.prototype = Object.create(StringStreamer.prototype); StringStreamer.prototype = Object.create(StringStreamer.prototype);
StringStreamer.prototype.constructor = StringStreamer; StringStreamer.prototype.constructor = StringStreamer;
@ -755,6 +784,19 @@
var queue = []; var queue = [];
var parseOnData = true; var parseOnData = true;
var streamHasEnded = false;
this.pause = function()
{
ChunkStreamer.prototype.pause.apply(this, arguments);
this._input.pause();
};
this.resume = function()
{
ChunkStreamer.prototype.resume.apply(this, arguments);
this._input.resume();
};
this.stream = function(stream) this.stream = function(stream)
{ {
@ -763,10 +805,18 @@
this._input.on('data', this._streamData); this._input.on('data', this._streamData);
this._input.on('end', this._streamEnd); this._input.on('end', this._streamEnd);
this._input.on('error', this._streamError); this._input.on('error', this._streamError);
} };
this._checkIsFinished = function()
{
if (streamHasEnded && queue.length === 1) {
this._finished = true;
}
};
this._nextChunk = function() this._nextChunk = function()
{ {
this._checkIsFinished();
if (queue.length) if (queue.length)
{ {
this.parseChunk(queue.shift()); this.parseChunk(queue.shift());
@ -775,7 +825,7 @@
{ {
parseOnData = true; parseOnData = true;
} }
} };
this._streamData = bindFunction(function(chunk) this._streamData = bindFunction(function(chunk)
{ {
@ -786,6 +836,7 @@
if (parseOnData) if (parseOnData)
{ {
parseOnData = false; parseOnData = false;
this._checkIsFinished();
this.parseChunk(queue.shift()); this.parseChunk(queue.shift());
} }
} }
@ -798,13 +849,13 @@
this._streamError = bindFunction(function(error) this._streamError = bindFunction(function(error)
{ {
this._streamCleanUp(); this._streamCleanUp();
this._sendError(error.message); this._sendError(error);
}, this); }, this);
this._streamEnd = bindFunction(function() this._streamEnd = bindFunction(function()
{ {
this._streamCleanUp(); this._streamCleanUp();
this._finished = true; streamHasEnded = true;
this._streamData(''); this._streamData('');
}, this); }, this);
@ -819,14 +870,117 @@
ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer; ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer;
function DuplexStreamStreamer(_config) {
var Duplex = require('stream').Duplex;
var config = copy(_config);
var parseOnWrite = true;
var writeStreamHasFinished = false;
var parseCallbackQueue = [];
var stream = null;
this._onCsvData = function(results)
{
var data = results.data;
for (var i = 0; i < data.length; i++) {
if (!stream.push(data[i]) && !this._handle.paused()) {
// the writeable consumer buffer has filled up
// so we need to pause until more items
// can be processed
this._handle.pause();
}
}
};
this._onCsvComplete = function()
{
// node will finish the read stream when
// null is pushed
stream.push(null);
};
config.step = bindFunction(this._onCsvData, this);
config.complete = bindFunction(this._onCsvComplete, this);
ChunkStreamer.call(this, config);
this._nextChunk = function()
{
if (writeStreamHasFinished && parseCallbackQueue.length === 1) {
this._finished = true;
}
if (parseCallbackQueue.length) {
parseCallbackQueue.shift()();
} else {
parseOnWrite = true;
}
};
this._addToParseQueue = function(chunk, callback)
{
// add to queue so that we can indicate
// completion via callback
// node will automatically pause the incoming stream
// when too many items have been added without their
// callback being invoked
parseCallbackQueue.push(bindFunction(function() {
this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding));
if (isFunction(callback)) {
return callback();
}
}, this));
if (parseOnWrite) {
parseOnWrite = false;
this._nextChunk();
}
};
this._onRead = function()
{
if (this._handle.paused()) {
// the writeable consumer can handle more data
// so resume the chunk parsing
this._handle.resume();
}
};
this._onWrite = function(chunk, encoding, callback)
{
this._addToParseQueue(chunk, callback);
};
this._onWriteComplete = function()
{
writeStreamHasFinished = true;
// have to write empty string
// so parser knows its done
this._addToParseQueue('');
};
this.getStream = function()
{
return stream;
};
stream = new Duplex({
readableObjectMode: true,
decodeStrings: false,
read: bindFunction(this._onRead, this),
write: bindFunction(this._onWrite, this)
});
stream.once('finish', bindFunction(this._onWriteComplete, this));
}
DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype);
DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer;
// Use one ParserHandle per entire CSV file or string // Use one ParserHandle per entire CSV file or string
function ParserHandle(_config) function ParserHandle(_config)
{ {
// One goal is to minimize the use of regular expressions... // One goal is to minimize the use of regular expressions...
var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
var self = this; var self = this;
var _stepCounter = 0; // Number of times step was called (number of rows parsed) var _stepCounter = 0; // Number of times step was called (number of rows parsed)
var _rowCounter = 0; // Number of rows that have been parsed so far
var _input; // The input being parsed var _input; // The input being parsed
var _parser; // The core parser being used var _parser; // The core parser being used
var _paused = false; // Whether we are paused or not var _paused = false; // Whether we are paused or not
@ -872,13 +1026,14 @@
*/ */
this.parse = function(input, baseIndex, ignoreLastRow) this.parse = function(input, baseIndex, ignoreLastRow)
{ {
var quoteChar = _config.quoteChar || '"';
if (!_config.newline) if (!_config.newline)
_config.newline = guessLineEndings(input); _config.newline = guessLineEndings(input, quoteChar);
_delimiterError = false; _delimiterError = false;
if (!_config.delimiter) if (!_config.delimiter)
{ {
var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines); var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments);
if (delimGuess.successful) if (delimGuess.successful)
_config.delimiter = delimGuess.bestDelimiter; _config.delimiter = delimGuess.bestDelimiter;
else else
@ -920,10 +1075,10 @@
this.resume = function() this.resume = function()
{ {
_paused = false; _paused = false;
self.streamer.parseChunk(_input); self.streamer.parseChunk(_input, true);
}; };
this.aborted = function () this.aborted = function()
{ {
return _aborted; return _aborted;
}; };
@ -938,25 +1093,29 @@
_input = ''; _input = '';
}; };
function testEmptyLine(s) {
return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0;
}
function processResults() function processResults()
{ {
if (_results && _delimiterError) if (_results && _delimiterError)
{ {
addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \''+Papa.DefaultDelimiter+'\''); addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \'' + Papa.DefaultDelimiter + '\'');
_delimiterError = false; _delimiterError = false;
} }
if (_config.skipEmptyLines) if (_config.skipEmptyLines)
{ {
for (var i = 0; i < _results.data.length; i++) for (var i = 0; i < _results.data.length; i++)
if (_results.data[i].length === 1 && _results.data[i][0] === '') if (testEmptyLine(_results.data[i]))
_results.data.splice(i--, 1); _results.data.splice(i--, 1);
} }
if (needsHeaderRow()) if (needsHeaderRow())
fillHeaderFields(); fillHeaderFields();
return applyHeaderAndDynamicTyping(); return applyHeaderAndDynamicTypingAndTransformation();
} }
function needsHeaderRow() function needsHeaderRow()
@ -970,7 +1129,15 @@
return; return;
for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
for (var j = 0; j < _results.data[i].length; j++) for (var j = 0; j < _results.data[i].length; j++)
_fields.push(_results.data[i][j]); {
var header = _results.data[i][j];
if (_config.trimHeaders) {
header = header.trim();
}
_fields.push(header);
}
_results.data.splice(0, 1); _results.data.splice(0, 1);
} }
@ -979,7 +1146,7 @@
if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) { if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) {
_config.dynamicTyping[field] = _config.dynamicTypingFunction(field); _config.dynamicTyping[field] = _config.dynamicTypingFunction(field);
} }
return (_config.dynamicTyping[field] || _config.dynamicTyping) === true return (_config.dynamicTyping[field] || _config.dynamicTyping) === true;
} }
function parseDynamic(field, value) function parseDynamic(field, value)
@ -990,22 +1157,27 @@
return true; return true;
else if (value === 'false' || value === 'FALSE') else if (value === 'false' || value === 'FALSE')
return false; return false;
else if (FLOAT.test(value))
return parseFloat(value);
else if (ISO_DATE.test(value))
return new Date(value);
else else
return tryParseFloat(value); return (value === '' ? null : value);
} }
return value; return value;
} }
function applyHeaderAndDynamicTyping() function applyHeaderAndDynamicTypingAndTransformation()
{ {
if (!_results || (!_config.header && !_config.dynamicTyping)) if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform))
return _results; return _results;
for (var i = 0; i < _results.data.length; i++) for (var i = 0; i < _results.data.length; i++)
{ {
var row = _config.header ? {} : []; var row = _config.header ? {} : [];
for (var j = 0; j < _results.data[i].length; j++) var j;
for (j = 0; j < _results.data[i].length; j++)
{ {
var field = j; var field = j;
var value = _results.data[i][j]; var value = _results.data[i][j];
@ -1013,6 +1185,9 @@
if (_config.header) if (_config.header)
field = j >= _fields.length ? '__parsed_extra' : _fields[j]; field = j >= _fields.length ? '__parsed_extra' : _fields[j];
if (_config.transform)
value = _config.transform(value,field);
value = parseDynamic(field, value); value = parseDynamic(field, value);
if (field === '__parsed_extra') if (field === '__parsed_extra')
@ -1029,18 +1204,20 @@
if (_config.header) if (_config.header)
{ {
if (j > _fields.length) if (j > _fields.length)
addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, i); addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i);
else if (j < _fields.length) else if (j < _fields.length)
addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, i); addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i);
} }
} }
if (_config.header && _results.meta) if (_config.header && _results.meta)
_results.meta.fields = _fields; _results.meta.fields = _fields;
_rowCounter += _results.data.length;
return _results; return _results;
} }
function guessDelimiter(input, newline, skipEmptyLines) function guessDelimiter(input, newline, skipEmptyLines, comments)
{ {
var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP];
var bestDelim, bestDelta, fieldCountPrevRow; var bestDelim, bestDelta, fieldCountPrevRow;
@ -1052,6 +1229,7 @@
fieldCountPrevRow = undefined; fieldCountPrevRow = undefined;
var preview = new Parser({ var preview = new Parser({
comments: comments,
delimiter: delim, delimiter: delim,
newline: newline, newline: newline,
preview: 10 preview: 10
@ -1059,9 +1237,10 @@
for (var j = 0; j < preview.data.length; j++) for (var j = 0; j < preview.data.length; j++)
{ {
if (skipEmptyLines && preview.data[j].length === 1 && preview.data[j][0].length === 0) { if (skipEmptyLines && testEmptyLine(preview.data[j]))
emptyLinesCount++ {
continue emptyLinesCount++;
continue;
} }
var fieldCount = preview.data[j].length; var fieldCount = preview.data[j].length;
avgFieldCount += fieldCount; avgFieldCount += fieldCount;
@ -1094,12 +1273,15 @@
return { return {
successful: !!bestDelim, successful: !!bestDelim,
bestDelimiter: bestDelim bestDelimiter: bestDelim
} };
} }
function guessLineEndings(input) function guessLineEndings(input, quoteChar)
{ {
input = input.substr(0, 1024*1024); // max length 1 MB input = input.substr(0, 1024 * 1024); // max length 1 MB
// Replace all the text inside quotes
var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm');
input = input.replace(re, '');
var r = input.split('\r'); var r = input.split('\r');
@ -1120,12 +1302,6 @@
return numWithN >= r.length / 2 ? '\r\n' : '\r'; return numWithN >= r.length / 2 ? '\r\n' : '\r';
} }
function tryParseFloat(val)
{
var isNumber = FLOAT.test(val);
return isNumber ? parseFloat(val) : val;
}
function addError(type, code, msg, row) function addError(type, code, msg, row)
{ {
_results.errors.push({ _results.errors.push({
@ -1137,9 +1313,11 @@
} }
} }
/** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */
function escapeRegExp(string)
{
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
}
/** The core parser implements speedy and correct CSV parsing */ /** The core parser implements speedy and correct CSV parsing */
function Parser(config) function Parser(config)
@ -1152,11 +1330,16 @@
var step = config.step; var step = config.step;
var preview = config.preview; var preview = config.preview;
var fastMode = config.fastMode; var fastMode = config.fastMode;
var quoteChar;
/** Allows for no quoteChar by setting quoteChar to undefined in config */ /** Allows for no quoteChar by setting quoteChar to undefined in config */
if (config.quoteChar === undefined){ if (config.quoteChar === undefined) {
var quoteChar = '"'; quoteChar = '"';
} else { } else {
var quoteChar = config.quoteChar; quoteChar = config.quoteChar;
}
var escapeChar = quoteChar;
if (config.escapeChar !== undefined) {
escapeChar = config.escapeChar;
} }
// Delimiter must be valid // Delimiter must be valid
@ -1174,7 +1357,7 @@
comments = false; comments = false;
// Newline must be valid: \r, \n, or \r\n // Newline must be valid: \r, \n, or \r\n
if (newline != '\n' && newline != '\r' && newline != '\r\n') if (newline !== '\n' && newline !== '\r' && newline !== '\r\n')
newline = '\n'; newline = '\n';
// We're gonna need these at the Parser scope // We're gonna need these at the Parser scope
@ -1207,7 +1390,7 @@
var rows = input.split(newline); var rows = input.split(newline);
for (var i = 0; i < rows.length; i++) for (var i = 0; i < rows.length; i++)
{ {
var row = rows[i]; row = rows[i];
cursor += row.length; cursor += row.length;
if (i !== rows.length - 1) if (i !== rows.length - 1)
cursor += newline.length; cursor += newline.length;
@ -1236,7 +1419,8 @@
var nextDelim = input.indexOf(delim, cursor); var nextDelim = input.indexOf(delim, cursor);
var nextNewline = input.indexOf(newline, cursor); var nextNewline = input.indexOf(newline, cursor);
var quoteCharRegex = new RegExp(quoteChar+quoteChar, 'g'); var quoteCharRegex = new RegExp(escapeChar.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&') + quoteChar, 'g');
var quoteSearch;
// Parser loop // Parser loop
for (;;) for (;;)
@ -1245,7 +1429,7 @@
if (input[cursor] === quoteChar) if (input[cursor] === quoteChar)
{ {
// Start our search for the closing quote where the cursor is // Start our search for the closing quote where the cursor is
var quoteSearch = cursor; quoteSearch = cursor;
// Skip the opening quote // Skip the opening quote
cursor++; cursor++;
@ -1253,7 +1437,7 @@
for (;;) for (;;)
{ {
// Find closing quote // Find closing quote
var quoteSearch = input.indexOf(quoteChar, quoteSearch+1); quoteSearch = input.indexOf(quoteChar, quoteSearch + 1);
//No other quotes are found - no other delimiters //No other quotes are found - no other delimiters
if (quoteSearch === -1) if (quoteSearch === -1)
@ -1272,34 +1456,47 @@
} }
// Closing quote at EOF // Closing quote at EOF
if (quoteSearch === inputLen-1) if (quoteSearch === inputLen - 1)
{ {
var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar); var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar);
return finish(value); return finish(value);
} }
// If this quote is escaped, it's part of the data; skip it // If this quote is escaped, it's part of the data; skip it
if (input[quoteSearch+1] === quoteChar) // If the quote character is the escape character, then check if the next character is the escape character
if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar)
{ {
quoteSearch++; quoteSearch++;
continue; continue;
} }
// Closing quote followed by delimiter // If the quote character is not the escape character, then check if the previous character was the escape character
if (input[quoteSearch+1] === delim) if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar)
{
continue;
}
// Check up to nextDelim or nextNewline, whichever is closest
var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline);
var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo);
// Closing quote followed by delimiter or 'unnecessary spaces + delimiter'
if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter] === delim)
{ {
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
cursor = quoteSearch + 1 + delimLen; cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen;
nextDelim = input.indexOf(delim, cursor); nextDelim = input.indexOf(delim, cursor);
nextNewline = input.indexOf(newline, cursor); nextNewline = input.indexOf(newline, cursor);
break; break;
} }
// Closing quote followed by newline var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline);
if (input.substr(quoteSearch+1, newlineLen) === newline)
// Closing quote followed by newline or 'unnecessary spaces + newLine'
if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, newlineLen) === newline)
{ {
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
saveRow(quoteSearch + 1 + newlineLen); saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen);
nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field
if (stepIsFunction) if (stepIsFunction)
@ -1385,6 +1582,21 @@
lastCursor = cursor; lastCursor = cursor;
} }
/**
* checks if there are extra spaces after closing quote and given index without any text
* if Yes, returns the number of spaces
*/
function extraSpaces(index) {
var spaceLength = 0;
if (index !== -1) {
var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index);
if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === '') {
spaceLength = textBetweenClosingQuoteAndIndex.length;
}
}
return spaceLength;
}
/** /**
* Appends the remaining input from cursor to the end into * Appends the remaining input from cursor to the end into
* row, saves the row, calls step, and returns the results. * row, saves the row, calls step, and returns the results.
@ -1437,7 +1649,8 @@
function doStep() function doStep()
{ {
step(returnable()); step(returnable());
data = [], errors = []; data = [];
errors = [];
} }
}; };
@ -1572,9 +1785,9 @@
/** Makes a deep copy of an array or object (mostly) */ /** Makes a deep copy of an array or object (mostly) */
function copy(obj) function copy(obj)
{ {
if (typeof obj !== 'object') if (typeof obj !== 'object' || obj === null)
return obj; return obj;
var cpy = obj instanceof Array ? [] : {}; var cpy = Array.isArray(obj) ? [] : {};
for (var key in obj) for (var key in obj)
cpy[key] = copy(obj[key]); cpy[key] = copy(obj[key]);
return cpy; return cpy;

Loading…
Cancel
Save