Browse Source

merging in updates

pull/693/head
Monkey D Zeke 6 years ago
parent
commit
0882c55e07
  1. 7
      README.md
  2. 143
      docs/docs.html
  3. 10
      docs/index.html
  4. 8
      docs/resources/js/lovers.js
  5. 107
      papaparse.js
  6. 36
      tests/node-tests.js
  7. 158
      tests/test-cases.js

7
README.md

@ -26,7 +26,7 @@ can be installed with the following command: @@ -26,7 +26,7 @@ can be installed with the following command:
npm install papaparse
If you don't want to use npm, [papaparse.min.js](https://github.com/mholt/PapaParse/blob/master/papaparse.min.js) can be downloaded to your project source.
If you don't want to use npm, [papaparse.min.js](https://unpkg.com/papaparse@latest/papaparse.min.js) can be downloaded to your project source.
Homepage & Demo
@ -39,9 +39,8 @@ To learn how to use Papa Parse: @@ -39,9 +39,8 @@ To learn how to use Papa Parse:
- [Documentation](http://papaparse.com/docs)
The website is hosted on on [Github Pages](https://pages.github.com/). If
you want to contribute just clone the gh-branch of this repository and
open a pull request.
The website is hosted on on [Github Pages](https://pages.github.com/). It's content is also inclued on the docs folder of this repository. If
you want to contribute on it just clone the master of this repository and open a pull request.
Papa Parse for Node

143
docs/docs.html

@ -99,7 +99,7 @@ @@ -99,7 +99,7 @@
</div>
<div class="grid-50">
<pre><code class="language-javascript">Papa.parse(csvString<i>[, <a href="#config">config</a>]</i>)</pre></code>
<pre><code class="language-javascript">Papa.parse(csvString<i>[, <a href="#config">config</a>]</i>)</code></pre>
</div>
<div class="grid-50">
@ -188,7 +188,7 @@ @@ -188,7 +188,7 @@
reason: "Some reason",
config: <span class="comment">// altered config...</span>
}</code></pre>
to alter the flow of parsing. Actions can be <code>"abort"</code> to skip this and all other files in the queue, <code>"skip"</code> to skip just this file, or <code>"continue"</code> to carry on (equivalent to returning nothing). <code>reason</code> can be a reason for aborting. <code>config</code> can be a modified <a href="#config">configuration</a> for parsing just this file.</li>
to alter the flow of parsing. Actions can be <code>"abort"</code> to skip this and all other files in the queue, <code>"skip"</code> to skip just this file, or <code>"continue"</code> to carry on (equivalent to returning nothing). <code>reason</code> can be a reason for aborting. <code>config</code> can be a modified <a href="#config">configuration</a> for parsing just this file.
</li>
<li>The <code>complete</code> callback shown here is executed after <i>all</i> files are finished and does not receive any data. Use the complete callback in <a href="#config">config</a> for per-file results.</li>
</ul>
@ -228,7 +228,7 @@ @@ -228,7 +228,7 @@
<div class="grid-50">
<pre><code class="language-javascript">Papa.unparse(data<i>[, config]</i>)</code></pre>
<pre><code class="language-javascript">Papa.unparse(data<i>[, <a href="#unparse-config-default">config</a>]</i>)</code></pre>
</div>
<div class="grid-50">
@ -243,19 +243,104 @@ @@ -243,19 +243,104 @@
</ul>
</li>
<li>
<code>config</code> is an optional object with any of these properties:
<pre><code class="language-javascript">// defaults shown
<code>config</code> is an optional <a href="#unparse-config-default">config object</a>
</li>
</ul>
</div>
<div class="clear"></div>
<div class="grid-100">
<h5 id="unparse-config-default">Default Unparse Config with all options</h5>
</div>
<div class="prefix-25 grid-50 suffix-25">
<pre><code class="language-javascript">
{
quotes: false,
quotes: false, //or array of booleans
quoteChar: '"',
escapeChar: '"',
delimiter: ",",
header: true,
newline: "\r\n"
}</code></pre>
Set <code>quotes</code> to <code>true</code> to always enclose each field in quotes, or an array of true/false values correlating to specific to columns to force-quote. The character used to quote can be customized using <code>quoteChar</code>. The character used to escape the <code>quoteChar</code> within a field can be customized using <code>escapeChar</code>. The <code>delimiter</code> can be any valid delimiting character. The <code>newline</code> character(s) may also be customized. Setting <code>header</code> to <code>false</code> will omit the header row.
</li>
</ul>
newline: "\r\n",
skipEmptyLines: false, //or 'greedy',
columns: null //or array of strings
}
</code></pre>
</div>
<div class="clear"></div>
<div class="grid-100">
<h5>Unparse Config Options</h5>
</div>
<div class="grid-100" style="overflow-x: auto;">
<table>
<tr>
<th style="width: 20%;">Option</th>
<th style="width: 80%;">Explanation</th>
</tr>
<tr>
<td>
<code>quotes</code>
</td>
<td>
If <code>true</code>, forces all fields to be enclosed in quotes. If an array of <code>true/false</code> values, specifies which fields should be force-quoted (first boolean is for the first column, second boolean for the second column, ...).
</td>
</tr>
<tr>
<td><code>quoteChar</code></td>
<td>
The character used to quote fields.
</td>
</tr>
<tr>
<td><code>escapeChar</code></td>
<td>
The character used to escape <code>quoteChar</code> inside field values.
</td>
</tr>
<tr>
<td>
<code>delimiter</code>
</td>
<td>
The delimiting character. It must not be found in <a href="#readonly">Papa.BAD_DELIMITERS</a>.
</td>
</tr>
<tr>
<td>
<code>header</code>
</td>
<td>
If <code>false</code>, will omit the header row. If <code>data</code> is an array of arrays this option is ignored. If <code>data</code> is an array of objects the keys of the first object are the header row. If <code>data</code> is an object with the keys <code>fields</code> and <code>data</code> the <code>fields</code> are the header row.
</td>
</tr>
<tr>
<td>
<code>newline</code>
</td>
<td>
The newline sequence. Must be one of <code>"\r"</code>, <code>"\n"</code>, or <code>"\r\n"</code>.
</td>
</tr>
<tr>
<td>
<code>skipEmptyLines</code>
</td>
<td>
If <code>true</code>, lines that are completely empty (those which evaluate to an empty string) will be skipped. If set to <code>'greedy'</code>, lines that don't have any content (those which have only whitespace after parsing) will also be skipped.
</td>
</tr>
<tr>
<td>
<code>columns</code>
</td>
<td>
If <code>data</code> is an array of objects this option can be used to manually specify the keys (columns) you expect in the objects. If not set the keys of the first objects are used as column.
</td>
</tr>
</table>
</div>
<div class="clear"></div>
@ -289,8 +374,8 @@ var csv = Papa.unparse([ @@ -289,8 +374,8 @@ var csv = Papa.unparse([
<div class="grid-33">
<pre><code class="language-javascript">// Specifying fields and data explicitly
var csv = Papa.unparse({
fields: ["Column 1", "Column 2"],
data: [
"fields": ["Column 1", "Column 2"],
"data": [
["foo", "bar"],
["abc", "def"]
]
@ -350,12 +435,14 @@ var csv = Papa.unparse({ @@ -350,12 +435,14 @@ var csv = Papa.unparse({
complete: undefined,
error: undefined,
download: false,
downloadRequestHeaders: undefined,
skipEmptyLines: false,
chunk: undefined,
fastMode: undefined,
beforeFirstChunk: undefined,
withCredentials: undefined,
transform: undefined
transform: undefined,
delimitersToGuess: [',', '\t', '|', ';', <a href="#readonly">Papa.RECORD_SEP</a>, <a href="#readonly">Papa.UNIT_SEP</a>]
}</code></pre>
</div>
<div class="clear"></div>
@ -375,7 +462,7 @@ var csv = Papa.unparse({ @@ -375,7 +462,7 @@ var csv = Papa.unparse({
<code>delimiter</code>
</td>
<td>
The delimiting character. Leave blank to auto-detect from a list of most common delimiters. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in <a href="#readonly">Papa.BAD_DELIMITERS</a>.
The delimiting character. Leave blank to auto-detect from a list of most common delimiters, or any values passed in through <code>delimitersToGuess</code>. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in <a href="#readonly">Papa.BAD_DELIMITERS</a>.
</td>
</tr>
<tr>
@ -415,7 +502,8 @@ var csv = Papa.unparse({ @@ -415,7 +502,8 @@ var csv = Papa.unparse({
<code>transformHeader</code>
</td>
<td>
A function to apply on each header. Requires <code>header</code> to be <code>true</code>. The function receives the header as its first argument.
A function to apply on each header. Requires <code>header</code> to be <code>true</code>. The function receives the header as its first argument.<br>
Only available starting with version 5.0.
</td>
</tr>
<tr>
@ -431,7 +519,7 @@ var csv = Papa.unparse({ @@ -431,7 +519,7 @@ var csv = Papa.unparse({
<code>preview</code>
</td>
<td>
If > 0, only that many rows will be parsed.
If &gt; 0, only that many rows will be parsed.
</td>
</tr>
<tr>
@ -500,6 +588,19 @@ var csv = Papa.unparse({ @@ -500,6 +588,19 @@ var csv = Papa.unparse({
If true, this indicates that the string you passed as the first argument to <code>parse()</code> is actually a URL from which to download a file and parse its contents.
</td>
</tr>
<tr>
<td>
<code>downloadRequestHeaders</code>
</td>
<td>
If defined, should be an object that describes the headers, example:
<pre>
<code class="language-javascript">downloadRequestHeaders: {
'Authorization': 'token 123345678901234567890',
}</code>
</pre>
</tr>
<tr>
<td>
<code>skipEmptyLines</code>
@ -548,6 +649,14 @@ var csv = Papa.unparse({ @@ -548,6 +649,14 @@ var csv = Papa.unparse({
A function to apply on each value. The function receives the value as its first argument and the column number or header name when enabled as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping.
</td>
</tr>
<tr>
<td>
<code>delimitersToGuess</code>
</td>
<td>
An array of delimiters to guess from if the <code>delimiter</code> option is not set.
</td>
</tr>
</table>
</div>
</div>

10
docs/index.html

@ -91,9 +91,6 @@ Papa.parse(bigFile, { @@ -91,9 +91,6 @@ Papa.parse(bigFile, {
<a href="http://stackoverflow.com/questions/tagged/papaparse">
<i class="fa fa-stack-overflow fa-lg"></i> Help
</a>
<!-- <a href="https://matt.life/pay" class="donate">
<i class="fa fa-heart fa-lg"></i> Donate
</a> -->
</div>
</div>
</div>
@ -506,6 +503,7 @@ var csv = Papa.unparse(yourData);</code></pre> @@ -506,6 +503,7 @@ var csv = Papa.unparse(yourData);</code></pre>
<i class="fa fa-book"></i>&nbsp; Documentation
</a>
</div>
</div>
</section>
@ -534,7 +532,7 @@ var csv = Papa.unparse(yourData);</code></pre> @@ -534,7 +532,7 @@ var csv = Papa.unparse(yourData);</code></pre>
</div>
<div class="grid-15 mobile-grid-50 links">
<h5>Project</h5>
<a href="https://gratipay.com/mholt">Donate</a>
<a href="https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=S6VTL9FQ6L8EN&item_name=PapaParse&currency_code=EUR&source=url">Donate</a>
<a href="https://github.com/mholt/PapaParse">GitHub</a>
<a href="https://twitter.com/search?q=%23PapaParse">Share</a>
</div>
@ -543,8 +541,8 @@ var csv = Papa.unparse(yourData);</code></pre> @@ -543,8 +541,8 @@ var csv = Papa.unparse(yourData);</code></pre>
<h5>Download</h5>
<a href="https://github.com/mholt/PapaParse/archive/master.zip">Latest (master)</a>
<hr>
<a href="https://github.com/mholt/PapaParse/blob/master/papaparse.min.js">Lil' Papa</a>
<a href="https://github.com/mholt/PapaParse/blob/master/papaparse.js">Fat Papa</a>
<a href="https://unpkg.com/papaparse@latest/papaparse.min.js">Lil' Papa</a>
<a href="https://unpkg.com/papaparse@latest/papaparse.js">Fat Papa</a>
</div>
<div class="grid-15 mobile-grid-50 links">
<h5>Community</h5>

8
docs/resources/js/lovers.js

@ -61,7 +61,7 @@ var peopleLovePapa = [ @@ -61,7 +61,7 @@ var peopleLovePapa = [
description: "created a video showing how to use Papa Parse and FileDrop.js to create a drag-and-drop CSV-JSON converter.",
quote: "It's often easy to convert data to CSV. With Papa, it's easy to turn that CSV into JSON."
},
{
{
link: "http://www.yolpo.com/social/gist.github?1dbd4556e748bdb830b3&autoplay=1&interimresults=0&failfast=1",
name: "Yolpo",
description: "created a simple regression test for Papa Parse.",
@ -78,5 +78,11 @@ var peopleLovePapa = [ @@ -78,5 +78,11 @@ var peopleLovePapa = [
name: "Novel.js",
description: "is a text adventure framework that uses Papa Parse to enable user-friendly translations.",
quote: "Papa saves countless hours of work and makes reading large CSV files so easy!"
},
{
link: "https://mailcheck.co",
name: "Mailcheck.co",
description: "Mailcheck is email validation service. All emails usually stored in CSV's. We use Papa Parse to process data from our customers in browser",
quote: "Papa Parser allowed our customers to preview and process csv's in browser, without uploading them to server. It saves lots of time and space :)"
}
];

107
papaparse.js

@ -273,9 +273,15 @@ License: MIT @@ -273,9 +273,15 @@ License: MIT
/** quote character */
var _quoteChar = '"';
/** escaped quote character, either "" or <config.escapeChar>" */
var _escapedQuote = _quoteChar + _quoteChar;
/** whether to skip empty lines */
var _skipEmptyLines = false;
/** the columns (keys) we expect when we unparse objects */
var _columns = null;
unpackConfig();
var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g');
@ -288,7 +294,7 @@ License: MIT @@ -288,7 +294,7 @@ License: MIT
if (!_input.length || Array.isArray(_input[0]))
return serialize(null, _input, _skipEmptyLines);
else if (typeof _input[0] === 'object')
return serialize(objectKeys(_input[0]), _input, _skipEmptyLines);
return serialize(_columns || objectKeys(_input[0]), _input, _skipEmptyLines);
}
else if (typeof _input === 'object')
{
@ -343,6 +349,17 @@ License: MIT @@ -343,6 +349,17 @@ License: MIT
if (typeof _config.header === 'boolean')
_writeHeader = _config.header;
if (Array.isArray(_config.columns)) {
if (_config.columns.length === 0) throw new Error('Option columns is empty');
_columns = _config.columns;
}
if (_config.escapeChar !== undefined) {
_escapedQuote = _config.escapeChar + _quoteChar;
}
}
@ -429,7 +446,7 @@ License: MIT @@ -429,7 +446,7 @@ License: MIT
if (str.constructor === Date)
return JSON.stringify(str).slice(1, 25);
str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar);
str = str.toString().replace(quoteCharRegex, _escapedQuote);
var needsQuotes = (typeof _quotes === 'boolean' && _quotes)
|| (Array.isArray(_quotes) && _quotes[col])
@ -456,6 +473,7 @@ License: MIT @@ -456,6 +473,7 @@ License: MIT
this._handle = null;
this._finished = false;
this._completed = false;
this._halted = false;
this._input = null;
this._baseIndex = 0;
this._partialLine = '';
@ -480,6 +498,7 @@ License: MIT @@ -480,6 +498,7 @@ License: MIT
chunk = modifiedChunk;
}
this.isFirstChunk = false;
this._halted = false;
// Rejoin the line we likely just split in two by chunking the file
var aggregate = this._partialLine + chunk;
@ -487,8 +506,10 @@ License: MIT @@ -487,8 +506,10 @@ License: MIT
var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
if (this._handle.paused() || this._handle.aborted())
if (this._handle.paused() || this._handle.aborted()) {
this._halted = true;
return;
}
var lastIndex = results.meta.cursor;
@ -514,8 +535,10 @@ License: MIT @@ -514,8 +535,10 @@ License: MIT
else if (isFunction(this._config.chunk) && !isFakeChunk)
{
this._config.chunk(results, this._handle);
if (this._handle.paused() || this._handle.aborted())
if (this._handle.paused() || this._handle.aborted()) {
this._halted = true;
return;
}
results = undefined;
this._completeResults = undefined;
}
@ -975,7 +998,6 @@ License: MIT @@ -975,7 +998,6 @@ License: MIT
// One goal is to minimize the use of regular expressions...
var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
var self = this;
var _stepCounter = 0; // Number of times step was called (number of rows parsed)
var _rowCounter = 0; // Number of rows that have been parsed so far
@ -1031,7 +1053,7 @@ License: MIT @@ -1031,7 +1053,7 @@ License: MIT
_delimiterError = false;
if (!_config.delimiter)
{
var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments);
var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess);
if (delimGuess.successful)
_config.delimiter = delimGuess.bestDelimiter;
else
@ -1072,8 +1094,14 @@ License: MIT @@ -1072,8 +1094,14 @@ License: MIT
this.resume = function()
{
_paused = false;
self.streamer.parseChunk(_input, true);
if(self.streamer._halted) {
_paused = false;
self.streamer.parseChunk(_input, true);
} else {
// Bugfix: #636 In case the processing hasn't halted yet
// wait for it to halt in order to resume
setTimeout(this.resume, 3);
}
};
this.aborted = function()
@ -1125,18 +1153,25 @@ License: MIT @@ -1125,18 +1153,25 @@ License: MIT
{
if (!_results)
return;
for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
for (var j = 0; j < _results.data[i].length; j++)
{
var header = _results.data[i][j];
if (isFunction(_config.transformHeader)) {
header = _config.transformHeader(header);
}
function addHeder(header)
{
if (isFunction(_config.transformHeader))
header = _config.transformHeader(header);
_fields.push(header);
}
_results.data.splice(0, 1);
_fields.push(header);
}
if (Array.isArray(_results.data[0]))
{
for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
_results.data[i].forEach(addHeder);
_results.data.splice(0, 1);
}
// if _results.data[0] is not an array, we are in a step where _results.data is the row.
else
_results.data.forEach(addHeder);
}
function shouldApplyDynamicTyping(field) {
@ -1170,15 +1205,15 @@ License: MIT @@ -1170,15 +1205,15 @@ License: MIT
if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform))
return _results;
for (var i = 0; i < _results.data.length; i++)
function processRow(rowSource, i)
{
var row = _config.header ? {} : [];
var j;
for (j = 0; j < _results.data[i].length; j++)
for (j = 0; j < rowSource.length; j++)
{
var field = j;
var value = _results.data[i][j];
var value = rowSource[j];
if (_config.header)
field = j >= _fields.length ? '__parsed_extra' : _fields[j];
@ -1197,7 +1232,6 @@ License: MIT @@ -1197,7 +1232,6 @@ License: MIT
row[field] = value;
}
_results.data[i] = row;
if (_config.header)
{
@ -1206,23 +1240,36 @@ License: MIT @@ -1206,23 +1240,36 @@ License: MIT
else if (j < _fields.length)
addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i);
}
return row;
}
var incrementBy = 1;
if (!_results.data[0] || Array.isArray(_results.data[0]))
{
_results.data = _results.data.map(processRow);
incrementBy = _results.data.length;
}
else
_results.data = processRow(_results.data, 0);
if (_config.header && _results.meta)
_results.meta.fields = _fields;
_rowCounter += _results.data.length;
_rowCounter += incrementBy;
return _results;
}
function guessDelimiter(input, newline, skipEmptyLines, comments)
function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess)
{
var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP];
var bestDelim, bestDelta, fieldCountPrevRow;
for (var i = 0; i < delimChoices.length; i++)
delimitersToGuess = delimitersToGuess || [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP];
for (var i = 0; i < delimitersToGuess.length; i++)
{
var delim = delimChoices[i];
var delim = delimitersToGuess[i];
var delta = 0, avgFieldCount = 0, emptyLinesCount = 0;
fieldCountPrevRow = undefined;
@ -1245,7 +1292,7 @@ License: MIT @@ -1245,7 +1292,7 @@ License: MIT
if (typeof fieldCountPrevRow === 'undefined')
{
fieldCountPrevRow = fieldCount;
fieldCountPrevRow = 0;
continue;
}
else if (fieldCount > 1)
@ -1258,7 +1305,7 @@ License: MIT @@ -1258,7 +1305,7 @@ License: MIT
if (preview.data.length > 0)
avgFieldCount /= (preview.data.length - emptyLinesCount);
if ((typeof bestDelta === 'undefined' || delta < bestDelta)
if ((typeof bestDelta === 'undefined' || delta > bestDelta)
&& avgFieldCount > 1.99)
{
bestDelta = delta;
@ -1707,7 +1754,7 @@ License: MIT @@ -1707,7 +1754,7 @@ License: MIT
for (var i = 0; i < msg.results.data.length; i++)
{
worker.userStep({
data: [msg.results.data[i]],
data: msg.results.data[i],
errors: msg.results.errors,
meta: msg.results.meta
}, handle);

36
tests/node-tests.js

@ -41,6 +41,42 @@ describe('PapaParse', function() { @@ -41,6 +41,42 @@ describe('PapaParse', function() {
assertLongSampleParsedCorrectly(Papa.parse(longSampleRawCsv));
});
it('Pause and resume works (Regression Test for Bug #636)', function(done) {
this.timeout(30000);
var mod200Rows = [
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Lorem ipsum dolor sit","42","ABC"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Etiam a dolor vitae est vestibulum","84"],
["Lorem ipsum dolor sit","42","ABC"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Lorem ipsum dolor sit","42","ABC"],
["Lorem ipsum dolor sit","42"]
];
var stepped = 0;
var dataRows = [];
Papa.parse(fs.createReadStream(__dirname + '/verylong-sample.csv'), {
step: function(results, parser) {
stepped++;
if (results)
{
parser.pause();
parser.resume();
if (results.data && stepped % 200 === 0) {
dataRows.push(results.data);
}
}
},
complete: function() {
assert.strictEqual(2001, stepped);
assert.deepEqual(mod200Rows, dataRows);
done();
}
});
});
it('asynchronously parsed CSV should be correctly parsed', function(done) {
Papa.parse(longSampleRawCsv, {
complete: function(parsedCsv) {

158
tests/test-cases.js

@ -1193,6 +1193,16 @@ var PARSE_TESTS = [ @@ -1193,6 +1193,16 @@ var PARSE_TESTS = [
errors: []
}
},
{
description: "Pipe delimiter is guessed correctly when mixed with comas",
notes: "Guessing the delimiter should work even if there are many lines of comments at the start of the file",
input: 'one|two,two|three\nfour|five,five|six',
config: {},
expected: {
data: [['one','two,two','three'],['four','five,five','six']],
errors: []
}
},
{
description: "Single quote as quote character",
notes: "Must parse correctly when single quote is specified as a quote character",
@ -1722,6 +1732,25 @@ var UNPARSE_TESTS = [ @@ -1722,6 +1732,25 @@ var UNPARSE_TESTS = [
input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}],
config: {skipEmptyLines: 'greedy', header: true},
expected: 'a,b\r\n1,2'
},
{
description: "Column option used to manually specify keys",
notes: "Should not throw any error when attempting to serialize key not present in object. Columns are different than keys of the first object. When an object is missing a key then the serialized value should be an empty string.",
input: [{a: 1, b: '2'}, {}, {a: 3, d: 'd', c: 4,}],
config: {columns: ['a', 'b', 'c']},
expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4'
},
{
description: "Use different escapeChar",
input: [{a: 'foo', b: '"quoted"'}],
config: {header: false, escapeChar: '\\'},
expected: 'foo,"\\"quoted\\""'
},
{
description: "test defeault escapeChar",
input: [{a: 'foo', b: '"quoted"'}],
config: {header: false},
expected: 'foo,"""quoted"""'
}
];
@ -1751,6 +1780,49 @@ describe('Unparse Tests', function() { @@ -1751,6 +1780,49 @@ describe('Unparse Tests', function() {
var CUSTOM_TESTS = [
{
description: "Pause and resume works (Regression Test for Bug #636)",
disabled: !XHR_ENABLED,
timeout: 30000,
expected: [2001, [
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Lorem ipsum dolor sit","42","ABC"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Etiam a dolor vitae est vestibulum","84"],
["Lorem ipsum dolor sit","42","ABC"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Etiam a dolor vitae est vestibulum","84","DEF"],
["Lorem ipsum dolor sit","42","ABC"],
["Lorem ipsum dolor sit","42"]
], 0],
run: function(callback) {
var stepped = 0;
var dataRows = [];
var errorCount = 0;
var output = [];
Papa.parse(BASE_PATH + "verylong-sample.csv", {
download: true,
step: function(results, parser) {
stepped++;
if (results)
{
parser.pause();
parser.resume();
if (results.data && stepped % 200 === 0) {
dataRows.push(results.data);
}
}
},
complete: function() {
output.push(stepped);
output.push(dataRows);
output.push(errorCount);
callback(output);
}
});
}
},
{
description: "Complete is called with all results if neither step nor chunk is defined",
expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']],
@ -1779,6 +1851,70 @@ var CUSTOM_TESTS = [ @@ -1779,6 +1851,70 @@ var CUSTOM_TESTS = [
});
}
},
{
description: "Data is correctly parsed with steps",
expected: [['A', 'b', 'c'], ['d', 'E', 'f']],
run: function(callback) {
var data = [];
Papa.parse('A,b,c\nd,E,f', {
step: function(results) {
data.push(results.data);
},
complete: function() {
callback(data);
}
});
}
},
{
description: "Data is correctly parsed with steps (headers)",
expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}],
run: function(callback) {
var data = [];
Papa.parse('One,Two,Three\nA,b,c\nd,E,f', {
header: true,
step: function(results) {
data.push(results.data);
},
complete: function() {
callback(data);
}
});
}
},
{
description: "Data is correctly parsed with steps and worker (headers)",
expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}],
run: function(callback) {
var data = [];
Papa.parse('One,Two,Three\nA,b,c\nd,E,f', {
header: true,
worker: true,
step: function(results) {
data.push(results.data);
},
complete: function() {
callback(data);
}
});
}
},
{
description: "Data is correctly parsed with steps and worker",
expected: [['A', 'b', 'c'], ['d', 'E', 'f']],
run: function(callback) {
var data = [];
Papa.parse('A,b,c\nd,E,f', {
worker: true,
step: function(results) {
data.push(results.data);
},
complete: function() {
callback(data);
}
});
}
},
{
description: "Step is called with the contents of the row",
expected: ['A', 'b', 'c'],
@ -2146,13 +2282,33 @@ var CUSTOM_TESTS = [ @@ -2146,13 +2282,33 @@ var CUSTOM_TESTS = [
}
});
}
},
{
description: "Should correctly guess custom delimiter when passed delimiters to guess.",
expected: "~",
run: function(callback) {
var results = Papa.parse('"A"~"B"~"C"~"D"', {
delimitersToGuess: ['~', '@', '%']
});
callback(results.meta.delimiter);
}
},
{
description: "Should still correctly guess default delimiters when delimiters to guess are not given.",
expected: ",",
run: function(callback) {
var results = Papa.parse('"A","B","C","D"');
callback(results.meta.delimiter);
}
}
];
describe('Custom Tests', function() {
function generateTest(test) {
(test.disabled ? it.skip : it)(test.description, function(done) {
if(test.timeout) {
this.timeout(test.timeout);
}
test.run(function(actual) {
assert.deepEqual(JSON.stringify(actual), JSON.stringify(test.expected));
done();

Loading…
Cancel
Save