diff --git a/README.md b/README.md index 9e00358..eca0188 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ can be installed with the following command: npm install papaparse -If you don't want to use npm, [papaparse.min.js](https://github.com/mholt/PapaParse/blob/master/papaparse.min.js) can be downloaded to your project source. +If you don't want to use npm, [papaparse.min.js](https://unpkg.com/papaparse@latest/papaparse.min.js) can be downloaded to your project source. Homepage & Demo @@ -39,9 +39,8 @@ To learn how to use Papa Parse: - [Documentation](http://papaparse.com/docs) -The website is hosted on on [Github Pages](https://pages.github.com/). If -you want to contribute just clone the gh-branch of this repository and -open a pull request. +The website is hosted on on [Github Pages](https://pages.github.com/). It's content is also inclued on the docs folder of this repository. If +you want to contribute on it just clone the master of this repository and open a pull request. Papa Parse for Node diff --git a/docs/docs.html b/docs/docs.html index 1da2126..261caa2 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -99,7 +99,7 @@
- 
Papa.parse(csvString[, config])
+ 
Papa.parse(csvString[, config])
@@ -188,7 +188,7 @@ reason: "Some reason", config: // altered config... } - to alter the flow of parsing. Actions can be "abort" to skip this and all other files in the queue, "skip" to skip just this file, or "continue" to carry on (equivalent to returning nothing). reason can be a reason for aborting. config can be a modified configuration for parsing just this file. + to alter the flow of parsing. Actions can be "abort" to skip this and all other files in the queue, "skip" to skip just this file, or "continue" to carry on (equivalent to returning nothing). reason can be a reason for aborting. config can be a modified configuration for parsing just this file.
  • The complete callback shown here is executed after all files are finished and does not receive any data. Use the complete callback in config for per-file results.
    • @@ -228,7 +228,7 @@
    - 
    Papa.unparse(data[, config])
    + 
    Papa.unparse(data[, config])
    @@ -243,19 +243,104 @@
  • - config is an optional object with any of these properties: - 
    // defaults shown
+								config is an optional config object
+
    • + +
    +
    + +
    +
    Default Unparse Config with all options
    +
    + +
    + 
    
 {
-	quotes: false,
+	quotes: false, //or array of booleans
 	quoteChar: '"',
 	escapeChar: '"',
 	delimiter: ",",
 	header: true,
-	newline: "\r\n"
-}
    - Set quotes to true to always enclose each field in quotes, or an array of true/false values correlating to specific to columns to force-quote. The character used to quote can be customized using quoteChar. The character used to escape the quoteChar within a field can be customized using escapeChar. The delimiter can be any valid delimiting character. The newline character(s) may also be customized. Setting header to false will omit the header row. - - + newline: "\r\n", + skipEmptyLines: false, //or 'greedy', + columns: null //or array of strings +} +     +
    + +
    + +
    +
    Unparse Config Options
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionExplanation
    + quotes + + If true, forces all fields to be enclosed in quotes. If an array of true/false values, specifies which fields should be force-quoted (first boolean is for the first column, second boolean for the second column, ...). +
    quoteChar + The character used to quote fields. +
    escapeChar + The character used to escape quoteChar inside field values. +
    + delimiter + + The delimiting character. It must not be found in Papa.BAD_DELIMITERS. +
    + header + + If false, will omit the header row. If data is an array of arrays this option is ignored. If data is an array of objects the keys of the first object are the header row. If data is an object with the keys fields and data the fields are the header row. +
    + newline + + The newline sequence. Must be one of "\r", "\n", or "\r\n". +
    + skipEmptyLines + + If true, lines that are completely empty (those which evaluate to an empty string) will be skipped. If set to 'greedy', lines that don't have any content (those which have only whitespace after parsing) will also be skipped. +
    + columns + + If data is an array of objects this option can be used to manually specify the keys (columns) you expect in the objects. If not set the keys of the first objects are used as column. +
    @@ -289,8 +374,8 @@ var csv = Papa.unparse([ 
    // Specifying fields and data explicitly
 var csv = Papa.unparse({
-	fields: ["Column 1", "Column 2"],
-	data: [
+	"fields": ["Column 1", "Column 2"],
+	"data": [
 		["foo", "bar"],
 		["abc", "def"]
 	]
@@ -350,12 +435,14 @@ var csv = Papa.unparse({
 	complete: undefined,
 	error: undefined,
 	download: false,
+	downloadRequestHeaders: undefined,
 	skipEmptyLines: false,
 	chunk: undefined,
 	fastMode: undefined,
 	beforeFirstChunk: undefined,
 	withCredentials: undefined,
-	transform: undefined
+	transform: undefined,
+	delimitersToGuess: [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]
 }
    @@ -375,7 +462,7 @@ var csv = Papa.unparse({ delimiter - The delimiting character. Leave blank to auto-detect from a list of most common delimiters. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in Papa.BAD_DELIMITERS. + The delimiting character. Leave blank to auto-detect from a list of most common delimiters, or any values passed in through delimitersToGuess. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in Papa.BAD_DELIMITERS. @@ -415,7 +502,8 @@ var csv = Papa.unparse({ transformHeader - A function to apply on each header. Requires header to be true. The function receives the header as its first argument. + A function to apply on each header. Requires header to be true. The function receives the header as its first argument.
    + Only available starting with version 5.0. @@ -431,7 +519,7 @@ var csv = Papa.unparse({ preview - If > 0, only that many rows will be parsed. + If > 0, only that many rows will be parsed. @@ -500,6 +588,19 @@ var csv = Papa.unparse({ If true, this indicates that the string you passed as the first argument to parse() is actually a URL from which to download a file and parse its contents. + + + downloadRequestHeaders + + + If defined, should be an object that describes the headers, example: + + 
    +										downloadRequestHeaders: {
+'Authorization': 'token 123345678901234567890',
+}
+
    + skipEmptyLines @@ -548,6 +649,14 @@ var csv = Papa.unparse({ A function to apply on each value. The function receives the value as its first argument and the column number or header name when enabled as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping. + + + delimitersToGuess + + + An array of delimiters to guess from if the delimiter option is not set. + +
    diff --git a/docs/index.html b/docs/index.html index 5190b1d..ab427f4 100644 --- a/docs/index.html +++ b/docs/index.html @@ -91,9 +91,6 @@ Papa.parse(bigFile, { Help - @@ -506,6 +503,7 @@ var csv = Papa.unparse(yourData);   Documentation + @@ -534,7 +532,7 @@ var csv = Papa.unparse(yourData);
    Project
    - Donate + Donate GitHub Share
    @@ -543,8 +541,8 @@ var csv = Papa.unparse(yourData);
    Download
    Latest (master)
    - Lil' Papa - Fat Papa + Lil' Papa + Fat Papa
    Community
    diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 2d02965..8e8a24d 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -61,7 +61,7 @@ var peopleLovePapa = [ description: "created a video showing how to use Papa Parse and FileDrop.js to create a drag-and-drop CSV-JSON converter.", quote: "It's often easy to convert data to CSV. With Papa, it's easy to turn that CSV into JSON." }, - { + { link: "http://www.yolpo.com/social/gist.github?1dbd4556e748bdb830b3&autoplay=1&interimresults=0&failfast=1", name: "Yolpo", description: "created a simple regression test for Papa Parse.", @@ -78,5 +78,11 @@ var peopleLovePapa = [ name: "Novel.js", description: "is a text adventure framework that uses Papa Parse to enable user-friendly translations.", quote: "Papa saves countless hours of work and makes reading large CSV files so easy!" + }, + { + link: "https://mailcheck.co", + name: "Mailcheck.co", + description: "Mailcheck is email validation service. All emails usually stored in CSV's. We use Papa Parse to process data from our customers in browser", + quote: "Papa Parser allowed our customers to preview and process csv's in browser, without uploading them to server. It saves lots of time and space :)" } ]; diff --git a/papaparse.js b/papaparse.js index 7febbaf..6e68a52 100755 --- a/papaparse.js +++ b/papaparse.js @@ -273,9 +273,15 @@ License: MIT /** quote character */ var _quoteChar = '"'; + /** escaped quote character, either "" or " */ + var _escapedQuote = _quoteChar + _quoteChar; + /** whether to skip empty lines */ var _skipEmptyLines = false; + /** the columns (keys) we expect when we unparse objects */ + var _columns = null; + unpackConfig(); var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g'); @@ -288,7 +294,7 @@ License: MIT if (!_input.length || Array.isArray(_input[0])) return serialize(null, _input, _skipEmptyLines); else if (typeof _input[0] === 'object') - return serialize(objectKeys(_input[0]), _input, _skipEmptyLines); + return serialize(_columns || objectKeys(_input[0]), _input, _skipEmptyLines); } else if (typeof _input === 'object') { @@ -343,6 +349,17 @@ License: MIT if (typeof _config.header === 'boolean') _writeHeader = _config.header; + + if (Array.isArray(_config.columns)) { + + if (_config.columns.length === 0) throw new Error('Option columns is empty'); + + _columns = _config.columns; + } + + if (_config.escapeChar !== undefined) { + _escapedQuote = _config.escapeChar + _quoteChar; + } } @@ -429,7 +446,7 @@ License: MIT if (str.constructor === Date) return JSON.stringify(str).slice(1, 25); - str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar); + str = str.toString().replace(quoteCharRegex, _escapedQuote); var needsQuotes = (typeof _quotes === 'boolean' && _quotes) || (Array.isArray(_quotes) && _quotes[col]) @@ -456,6 +473,7 @@ License: MIT this._handle = null; this._finished = false; this._completed = false; + this._halted = false; this._input = null; this._baseIndex = 0; this._partialLine = ''; @@ -480,6 +498,7 @@ License: MIT chunk = modifiedChunk; } this.isFirstChunk = false; + this._halted = false; // Rejoin the line we likely just split in two by chunking the file var aggregate = this._partialLine + chunk; @@ -487,8 +506,10 @@ License: MIT var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); - if (this._handle.paused() || this._handle.aborted()) + if (this._handle.paused() || this._handle.aborted()) { + this._halted = true; return; + } var lastIndex = results.meta.cursor; @@ -514,8 +535,10 @@ License: MIT else if (isFunction(this._config.chunk) && !isFakeChunk) { this._config.chunk(results, this._handle); - if (this._handle.paused() || this._handle.aborted()) + if (this._handle.paused() || this._handle.aborted()) { + this._halted = true; return; + } results = undefined; this._completeResults = undefined; } @@ -975,7 +998,6 @@ License: MIT // One goal is to minimize the use of regular expressions... var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/; - var self = this; var _stepCounter = 0; // Number of times step was called (number of rows parsed) var _rowCounter = 0; // Number of rows that have been parsed so far @@ -1031,7 +1053,7 @@ License: MIT _delimiterError = false; if (!_config.delimiter) { - var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments); + var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess); if (delimGuess.successful) _config.delimiter = delimGuess.bestDelimiter; else @@ -1072,8 +1094,14 @@ License: MIT this.resume = function() { - _paused = false; - self.streamer.parseChunk(_input, true); + if(self.streamer._halted) { + _paused = false; + self.streamer.parseChunk(_input, true); + } else { + // Bugfix: #636 In case the processing hasn't halted yet + // wait for it to halt in order to resume + setTimeout(this.resume, 3); + } }; this.aborted = function() @@ -1125,18 +1153,25 @@ License: MIT { if (!_results) return; - for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) - for (var j = 0; j < _results.data[i].length; j++) - { - var header = _results.data[i][j]; - if (isFunction(_config.transformHeader)) { - header = _config.transformHeader(header); - } + function addHeder(header) + { + if (isFunction(_config.transformHeader)) + header = _config.transformHeader(header); - _fields.push(header); - } - _results.data.splice(0, 1); + _fields.push(header); + } + + if (Array.isArray(_results.data[0])) + { + for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) + _results.data[i].forEach(addHeder); + + _results.data.splice(0, 1); + } + // if _results.data[0] is not an array, we are in a step where _results.data is the row. + else + _results.data.forEach(addHeder); } function shouldApplyDynamicTyping(field) { @@ -1170,15 +1205,15 @@ License: MIT if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform)) return _results; - for (var i = 0; i < _results.data.length; i++) + function processRow(rowSource, i) { var row = _config.header ? {} : []; var j; - for (j = 0; j < _results.data[i].length; j++) + for (j = 0; j < rowSource.length; j++) { var field = j; - var value = _results.data[i][j]; + var value = rowSource[j]; if (_config.header) field = j >= _fields.length ? '__parsed_extra' : _fields[j]; @@ -1197,7 +1232,6 @@ License: MIT row[field] = value; } - _results.data[i] = row; if (_config.header) { @@ -1206,23 +1240,36 @@ License: MIT else if (j < _fields.length) addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); } + + return row; } + var incrementBy = 1; + if (!_results.data[0] || Array.isArray(_results.data[0])) + { + _results.data = _results.data.map(processRow); + incrementBy = _results.data.length; + } + else + _results.data = processRow(_results.data, 0); + + if (_config.header && _results.meta) _results.meta.fields = _fields; - _rowCounter += _results.data.length; + _rowCounter += incrementBy; return _results; } - function guessDelimiter(input, newline, skipEmptyLines, comments) + function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) { - var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; var bestDelim, bestDelta, fieldCountPrevRow; - for (var i = 0; i < delimChoices.length; i++) + delimitersToGuess = delimitersToGuess || [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; + + for (var i = 0; i < delimitersToGuess.length; i++) { - var delim = delimChoices[i]; + var delim = delimitersToGuess[i]; var delta = 0, avgFieldCount = 0, emptyLinesCount = 0; fieldCountPrevRow = undefined; @@ -1245,7 +1292,7 @@ License: MIT if (typeof fieldCountPrevRow === 'undefined') { - fieldCountPrevRow = fieldCount; + fieldCountPrevRow = 0; continue; } else if (fieldCount > 1) @@ -1258,7 +1305,7 @@ License: MIT if (preview.data.length > 0) avgFieldCount /= (preview.data.length - emptyLinesCount); - if ((typeof bestDelta === 'undefined' || delta < bestDelta) + if ((typeof bestDelta === 'undefined' || delta > bestDelta) && avgFieldCount > 1.99) { bestDelta = delta; @@ -1707,7 +1754,7 @@ License: MIT for (var i = 0; i < msg.results.data.length; i++) { worker.userStep({ - data: [msg.results.data[i]], + data: msg.results.data[i], errors: msg.results.errors, meta: msg.results.meta }, handle); diff --git a/tests/node-tests.js b/tests/node-tests.js index fe671df..a067bfe 100644 --- a/tests/node-tests.js +++ b/tests/node-tests.js @@ -41,6 +41,42 @@ describe('PapaParse', function() { assertLongSampleParsedCorrectly(Papa.parse(longSampleRawCsv)); }); + it('Pause and resume works (Regression Test for Bug #636)', function(done) { + this.timeout(30000); + var mod200Rows = [ + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Lorem ipsum dolor sit","42"] + ]; + var stepped = 0; + var dataRows = []; + Papa.parse(fs.createReadStream(__dirname + '/verylong-sample.csv'), { + step: function(results, parser) { + stepped++; + if (results) + { + parser.pause(); + parser.resume(); + if (results.data && stepped % 200 === 0) { + dataRows.push(results.data); + } + } + }, + complete: function() { + assert.strictEqual(2001, stepped); + assert.deepEqual(mod200Rows, dataRows); + done(); + } + }); + }); + it('asynchronously parsed CSV should be correctly parsed', function(done) { Papa.parse(longSampleRawCsv, { complete: function(parsedCsv) { diff --git a/tests/test-cases.js b/tests/test-cases.js index 2e19d68..d9dd459 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1193,6 +1193,16 @@ var PARSE_TESTS = [ errors: [] } }, + { + description: "Pipe delimiter is guessed correctly when mixed with comas", + notes: "Guessing the delimiter should work even if there are many lines of comments at the start of the file", + input: 'one|two,two|three\nfour|five,five|six', + config: {}, + expected: { + data: [['one','two,two','three'],['four','five,five','six']], + errors: [] + } + }, { description: "Single quote as quote character", notes: "Must parse correctly when single quote is specified as a quote character", @@ -1722,6 +1732,25 @@ var UNPARSE_TESTS = [ input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], config: {skipEmptyLines: 'greedy', header: true}, expected: 'a,b\r\n1,2' + }, + { + description: "Column option used to manually specify keys", + notes: "Should not throw any error when attempting to serialize key not present in object. Columns are different than keys of the first object. When an object is missing a key then the serialized value should be an empty string.", + input: [{a: 1, b: '2'}, {}, {a: 3, d: 'd', c: 4,}], + config: {columns: ['a', 'b', 'c']}, + expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4' + }, + { + description: "Use different escapeChar", + input: [{a: 'foo', b: '"quoted"'}], + config: {header: false, escapeChar: '\\'}, + expected: 'foo,"\\"quoted\\""' + }, + { + description: "test defeault escapeChar", + input: [{a: 'foo', b: '"quoted"'}], + config: {header: false}, + expected: 'foo,"""quoted"""' } ]; @@ -1751,6 +1780,49 @@ describe('Unparse Tests', function() { var CUSTOM_TESTS = [ + { + description: "Pause and resume works (Regression Test for Bug #636)", + disabled: !XHR_ENABLED, + timeout: 30000, + expected: [2001, [ + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Lorem ipsum dolor sit","42"] + ], 0], + run: function(callback) { + var stepped = 0; + var dataRows = []; + var errorCount = 0; + var output = []; + Papa.parse(BASE_PATH + "verylong-sample.csv", { + download: true, + step: function(results, parser) { + stepped++; + if (results) + { + parser.pause(); + parser.resume(); + if (results.data && stepped % 200 === 0) { + dataRows.push(results.data); + } + } + }, + complete: function() { + output.push(stepped); + output.push(dataRows); + output.push(errorCount); + callback(output); + } + }); + } + }, { description: "Complete is called with all results if neither step nor chunk is defined", expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']], @@ -1779,6 +1851,70 @@ var CUSTOM_TESTS = [ }); } }, + { + description: "Data is correctly parsed with steps", + expected: [['A', 'b', 'c'], ['d', 'E', 'f']], + run: function(callback) { + var data = []; + Papa.parse('A,b,c\nd,E,f', { + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps (headers)", + expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}], + run: function(callback) { + var data = []; + Papa.parse('One,Two,Three\nA,b,c\nd,E,f', { + header: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps and worker (headers)", + expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}], + run: function(callback) { + var data = []; + Papa.parse('One,Two,Three\nA,b,c\nd,E,f', { + header: true, + worker: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps and worker", + expected: [['A', 'b', 'c'], ['d', 'E', 'f']], + run: function(callback) { + var data = []; + Papa.parse('A,b,c\nd,E,f', { + worker: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, { description: "Step is called with the contents of the row", expected: ['A', 'b', 'c'], @@ -2146,13 +2282,33 @@ var CUSTOM_TESTS = [ } }); } + }, + { + description: "Should correctly guess custom delimiter when passed delimiters to guess.", + expected: "~", + run: function(callback) { + var results = Papa.parse('"A"~"B"~"C"~"D"', { + delimitersToGuess: ['~', '@', '%'] + }); + callback(results.meta.delimiter); + } + }, + { + description: "Should still correctly guess default delimiters when delimiters to guess are not given.", + expected: ",", + run: function(callback) { + var results = Papa.parse('"A","B","C","D"'); + callback(results.meta.delimiter); + } } - ]; describe('Custom Tests', function() { function generateTest(test) { (test.disabled ? it.skip : it)(test.description, function(done) { + if(test.timeout) { + this.timeout(test.timeout); + } test.run(function(actual) { assert.deepEqual(JSON.stringify(actual), JSON.stringify(test.expected)); done();