From 884c0bf123019c7dd8e5765d15fad9ea675bb2d9 Mon Sep 17 00:00:00 2001 From: Jay L Date: Tue, 2 Oct 2018 00:31:31 -0700 Subject: [PATCH 01/25] adding chai as an npm devDependency (#577) Resolves #576 --- package.json | 2 +- tests/tests.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 7340311..2adb9e1 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "license": "MIT", "main": "papaparse.js", "devDependencies": { - "chai": "^4.1.2", + "chai": "^4.2.0", "connect": "^3.3.3", "eslint": "^4.19.1", "grunt": "^1.0.2", diff --git a/tests/tests.html b/tests/tests.html index 625c1cb..421e96b 100644 --- a/tests/tests.html +++ b/tests/tests.html @@ -5,8 +5,8 @@ - - + + From 18ef015e1906802ca4479da5206eb282e32d5aa9 Mon Sep 17 00:00:00 2001 From: Ole Ersoy Date: Tue, 2 Oct 2018 02:46:19 -0500 Subject: [PATCH 02/25] Bumping font size and darkening color on h5, footer h4 (#575) Fixes #574 --- docs/resources/css/common.css | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/resources/css/common.css b/docs/resources/css/common.css index e8e0c4f..813113b 100644 --- a/docs/resources/css/common.css +++ b/docs/resources/css/common.css @@ -77,14 +77,13 @@ h4 { line-height: 1.25em; } -h5, -footer h4 { - font-size: 12px; - letter-spacing: 2px; - text-transform: uppercase; - color: #A1B2C2; /* darker alternative: #698EB0 */ - line-height: 1em; - margin: 50px auto; +h5, footer h4 { + font-size: 20px; + letter-spacing: 2px; + text-transform: uppercase; + color: #242627; + line-height: 1em; + margin: 50px auto; } h6 { @@ -511,4 +510,4 @@ footer hr { footer h5 { margin-top: 40px; } -} \ No newline at end of file +} From 40ab90aee33c3ce80c450d0a92b673d14cc71bda Mon Sep 17 00:00:00 2001 From: Jay L Date: Wed, 3 Oct 2018 06:05:41 -0700 Subject: [PATCH 03/25] Use Array.isArray for determining arraynes (#570) Also add polyfill in case it's not supported by browser version --- papaparse.js | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/papaparse.js b/papaparse.js index 059ccbb..31b58f5 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,9 +1,19 @@ /*@license - Papa Parse - v4.6.0 - https://github.com/mholt/PapaParse - License: MIT +Papa Parse +v4.6.0 +https://github.com/mholt/PapaParse +License: MIT */ + +// Polyfills +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill +if (!Array.isArray) +{ + Array.isArray = function(arg) { + return Object.prototype.toString.call(arg) === '[object Array]'; + }; +} + (function(root, factory) { /* globals define */ @@ -41,7 +51,6 @@ return {}; })(); - var IS_WORKER = !global.document && !!global.postMessage, IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), LOADED_SYNC = false, AUTO_SCRIPT_PATH; @@ -290,9 +299,9 @@ if (typeof _input === 'string') _input = JSON.parse(_input); - if (_input instanceof Array) + if (Array.isArray(_input)) { - if (!_input.length || _input[0] instanceof Array) + if (!_input.length || Array.isArray(_input[0])) return serialize(null, _input, _skipEmptyLines); else if (typeof _input[0] === 'object') return serialize(objectKeys(_input[0]), _input, _skipEmptyLines); @@ -302,17 +311,17 @@ if (typeof _input.data === 'string') _input.data = JSON.parse(_input.data); - if (_input.data instanceof Array) + if (Array.isArray(_input.data)) { if (!_input.fields) _input.fields = _input.meta && _input.meta.fields; if (!_input.fields) - _input.fields = _input.data[0] instanceof Array + _input.fields = Array.isArray(_input.data[0]) ? _input.fields : objectKeys(_input.data[0]); - if (!(_input.data[0] instanceof Array) && typeof _input.data[0] !== 'object') + if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object') _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] } @@ -335,7 +344,7 @@ } if (typeof _config.quotes === 'boolean' - || _config.quotes instanceof Array) + || Array.isArray(_config.quotes)) _quotes = _config.quotes; if (typeof _config.skipEmptyLines === 'boolean' @@ -374,8 +383,8 @@ if (typeof data === 'string') data = JSON.parse(data); - var hasHeader = fields instanceof Array && fields.length > 0; - var dataKeyedByField = !(data[0] instanceof Array); + var hasHeader = Array.isArray(fields) && fields.length > 0; + var dataKeyedByField = !(Array.isArray(data[0])); // If there a header row, write it first if (hasHeader && _writeHeader) @@ -424,7 +433,7 @@ str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar); var needsQuotes = (typeof _quotes === 'boolean' && _quotes) - || (_quotes instanceof Array && _quotes[col]) + || (Array.isArray(_quotes) && _quotes[col]) || hasAny(str, Papa.BAD_DELIMITERS) || str.indexOf(_delimiter) > -1 || str.charAt(0) === ' ' @@ -1778,7 +1787,7 @@ { if (typeof obj !== 'object' || obj === null) return obj; - var cpy = obj instanceof Array ? [] : {}; + var cpy = Array.isArray(obj) ? [] : {}; for (var key in obj) cpy[key] = copy(obj[key]); return cpy; From 55d2c48bae5a51941ad4d310607fe7bad57366ef Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 4 Oct 2018 12:59:25 +0200 Subject: [PATCH 04/25] Add space between comment and license clasifier Fixes #565 --- papaparse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 31b58f5..e96ef32 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,4 +1,4 @@ -/*@license +/* @license Papa Parse v4.6.0 https://github.com/mholt/PapaParse From c8ca35904af1e99af1d604255060d4bffc014578 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 4 Oct 2018 13:01:38 +0200 Subject: [PATCH 05/25] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 2adb9e1..7b9e671 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "4.6.0", + "version": "4.6.1", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index e96ef32..94c81b6 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v4.6.0 +v4.6.1 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 68fb77d..5828b1d 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ -/*@license - Papa Parse - v4.6.0 - https://github.com/mholt/PapaParse - License: MIT +/* @license +Papa Parse +v4.6.1 +https://github.com/mholt/PapaParse +License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},r=!f.document&&!!f.postMessage,o=r&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,k={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&k.WORKERS_SUPPORTED){var n=function(){if(!k.WORKERS_SUPPORTED)return!1;if(!a&&null===k.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=k.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=v,t.id=u++,h[t.id]=t}();return n.userStep=t.step,n.userChunk=t.chunk,n.userComplete=t.complete,n.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void n.postMessage({input:e,config:t,workerId:n.id})}var r=null;{if(e===k.NODE_STREAM_INPUT)return(r=new g(t)).getStream();"string"==typeof e?r=t.download?new l(t):new _(t):!0===e.readable&&M(e.read)&&M(e.on)?r=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(r=new p(t))}return r.stream(e)},unparse:function(e,t){var n=!1,f=!0,d=",",c="\r\n",r='"';!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||k.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(d=t.delimiter);("boolean"==typeof t.quotes||t.quotes instanceof Array)&&(n=t.quotes);"string"==typeof t.newline&&(c=t.newline);"string"==typeof t.quoteChar&&(r=t.quoteChar);"boolean"==typeof t.header&&(f=t.header)}();var s=new RegExp(r,"g");"string"==typeof e&&(e=JSON.parse(e));if(e instanceof Array){if(!e.length||e[0]instanceof Array)return a(null,e);if("object"==typeof e[0])return a(i(e[0]),e)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),e.data instanceof Array&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=e.data[0]instanceof Array?e.fields:i(e.data[0])),e.data[0]instanceof Array||"object"==typeof e.data[0]||(e.data=[e.data])),a(e.fields||[],e.data||[]);throw"exception: Unable to serialize unrecognized input";function i(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function a(e,t){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=e instanceof Array&&0=this._config.preview;if(o)f.postMessage({results:r,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return;r=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!a||!M(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||r&&r.meta.paused||this._nextChunk(),r}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function l(e){var n;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),c.call(this,e),this._nextChunk=r?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(n=new XMLHttpRequest,this._config.withCredentials&&(n.withCredentials=this._config.withCredentials),r||(n.onload=R(this._chunkLoaded,this),n.onerror=R(this._chunkError,this)),n.open("GET",this._input,!r),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)n.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;n.setRequestHeader("Range","bytes="+this._start+"-"+i),n.setRequestHeader("If-None-Match","webkit-no-cache")}try{n.send()}catch(e){this._chunkError(e.message)}r&&0===n.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===n.readyState&&(n.status<200||400<=n.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(n),this.parseChunk(n.responseText)))},this._chunkError=function(e){var t=n.statusText||e;this._sendError(new Error(t))}}function p(e){var n,r;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),c.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,s?((n=new FileReader).onload=R(this._chunkLoaded,this),n.onerror=R(this._chunkError,this)):n=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(n.error)}}function _(e){var i;c.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){c.call(this,e=e||{});var t=[],i=!0,n=!1;this.pause=function(){c.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){c.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){n&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),n=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),n=!0,r=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():c(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function l(){if(d&&h&&(v("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(r=m.transform(r,n)),r=_(n,r),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(r)):i[n]=r}d.data[e]=i,m.header&&(t>f.length?v("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=n.length/2?"\r\n":"\r"}(e,n)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var r=function(e,t,i,n){for(var r,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=A,A++;;){if(-1===(p=n.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:A}),b();if(p===r-1)return b(n.substring(A,p).replace(g,S));if(S!==L||n[p+1]!==L){if(S===L||0===p||n[p-1]!==L){var v=C(-1===m?_:Math.min(_,m));if(n[p+1+v]===x){f.push(n.substring(A,p).replace(g,S)),A=p+1+v+e,_=n.indexOf(x,A),m=n.indexOf(T,A);break}var k=C(m);if(n.substr(p+1+k,s)===T){if(f.push(n.substring(A,p).replace(g,S)),w(p+1+k+s),_=n.indexOf(x,A),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:A}),p++}}else p++}return b();function y(e){h.push(e),d=A}function C(e){var t=0;if(-1!==e){var i=n.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=n.substr(A)),f.push(e),A=r,y(f),o&&R()),E()}function w(e){A=e,y(f),f=[],m=n.indexOf(T,A)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:T,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return A}}function v(e){var t=e.data,i=h[t.workerId],n=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var r={abort:function(){n=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s Date: Thu, 4 Oct 2018 13:03:47 +0200 Subject: [PATCH 06/25] Use latest version on demo website --- docs/resources/js/papaparse.js | 447 ++++++++++++++++++++++++--------- 1 file changed, 330 insertions(+), 117 deletions(-) diff --git a/docs/resources/js/papaparse.js b/docs/resources/js/papaparse.js index 5bef3cc..94c81b6 100644 --- a/docs/resources/js/papaparse.js +++ b/docs/resources/js/papaparse.js @@ -1,11 +1,22 @@ -/*! - Papa Parse - v4.3.7 - https://github.com/mholt/PapaParse - License: MIT +/* @license +Papa Parse +v4.6.1 +https://github.com/mholt/PapaParse +License: MIT */ + +// Polyfills +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill +if (!Array.isArray) +{ + Array.isArray = function(arg) { + return Object.prototype.toString.call(arg) === '[object Array]'; + }; +} + (function(root, factory) { + /* globals define */ if (typeof define === 'function' && define.amd) { // AMD. Register as an anonymous module. @@ -27,7 +38,7 @@ { 'use strict'; - var global = (function () { + var global = (function() { // alternative method, similar to `Function('return this')()` // but without using `eval` (which is disabled when // using Content Security Policy). @@ -40,7 +51,6 @@ return {}; })(); - var IS_WORKER = !global.document && !!global.postMessage, IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), LOADED_SYNC = false, AUTO_SCRIPT_PATH; @@ -57,6 +67,7 @@ Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously + Papa.NODE_STREAM_INPUT = 1; // Configurable chunk sizes for local and remote files, respectively Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB @@ -70,6 +81,7 @@ Papa.FileStreamer = FileStreamer; Papa.StringStreamer = StringStreamer; Papa.ReadableStreamStreamer = ReadableStreamStreamer; + Papa.DuplexStreamStreamer = DuplexStreamStreamer; if (global.jQuery) { @@ -162,7 +174,7 @@ queue.splice(0, 1); parseNextFile(); } - } + }; } @@ -182,7 +194,7 @@ } else { - document.addEventListener('DOMContentLoaded', function () { + document.addEventListener('DOMContentLoaded', function() { LOADED_SYNC = true; }, true); } @@ -202,6 +214,8 @@ } _config.dynamicTyping = dynamicTyping; + _config.transform = isFunction(_config.transform) ? _config.transform : false; + if (_config.worker && Papa.WORKERS_SUPPORTED) { var w = newWorker(); @@ -227,7 +241,14 @@ } var streamer = null; - if (typeof _input === 'string') + if (_input === Papa.NODE_STREAM_INPUT) + { + // create a node Duplex stream for use + // with .pipe + streamer = new DuplexStreamStreamer(_config); + return streamer.getStream(); + } + else if (typeof _input === 'string') { if (_config.download) streamer = new NetworkStreamer(_config); @@ -251,9 +272,6 @@ function JsonToCsv(_input, _config) { - var _output = ''; - var _fields = []; - // Default configuration /** whether to surround every datum with quotes */ @@ -262,7 +280,7 @@ /** whether to write headers */ var _writeHeader = true; - /** delimiting character */ + /** delimiting character(s) */ var _delimiter = ','; /** newline character(s) */ @@ -271,6 +289,9 @@ /** quote character */ var _quoteChar = '"'; + /** whether to skip empty lines */ + var _skipEmptyLines = false; + unpackConfig(); var quoteCharRegex = new RegExp(_quoteChar, 'g'); @@ -278,33 +299,33 @@ if (typeof _input === 'string') _input = JSON.parse(_input); - if (_input instanceof Array) + if (Array.isArray(_input)) { - if (!_input.length || _input[0] instanceof Array) - return serialize(null, _input); + if (!_input.length || Array.isArray(_input[0])) + return serialize(null, _input, _skipEmptyLines); else if (typeof _input[0] === 'object') - return serialize(objectKeys(_input[0]), _input); + return serialize(objectKeys(_input[0]), _input, _skipEmptyLines); } else if (typeof _input === 'object') { if (typeof _input.data === 'string') _input.data = JSON.parse(_input.data); - if (_input.data instanceof Array) + if (Array.isArray(_input.data)) { if (!_input.fields) _input.fields = _input.meta && _input.meta.fields; if (!_input.fields) - _input.fields = _input.data[0] instanceof Array - ? _input.fields - : objectKeys(_input.data[0]); + _input.fields = Array.isArray(_input.data[0]) + ? _input.fields + : objectKeys(_input.data[0]); - if (!(_input.data[0] instanceof Array) && typeof _input.data[0] !== 'object') + if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object') _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] } - return serialize(_input.fields || [], _input.data || []); + return serialize(_input.fields || [], _input.data || [], _skipEmptyLines); } // Default (any valid paths should return before this) @@ -317,16 +338,19 @@ return; if (typeof _config.delimiter === 'string' - && _config.delimiter.length === 1 - && Papa.BAD_DELIMITERS.indexOf(_config.delimiter) === -1) + && !Papa.BAD_DELIMITERS.filter(function(value) { return _config.delimiter.indexOf(value) !== -1; }).length) { _delimiter = _config.delimiter; } if (typeof _config.quotes === 'boolean' - || _config.quotes instanceof Array) + || Array.isArray(_config.quotes)) _quotes = _config.quotes; + if (typeof _config.skipEmptyLines === 'boolean' + || typeof _config.skipEmptyLines === 'string') + _skipEmptyLines = _config.skipEmptyLines; + if (typeof _config.newline === 'string') _newline = _config.newline; @@ -350,7 +374,7 @@ } /** The double for loop that iterates the data and writes out a CSV string including header row */ - function serialize(fields, data) + function serialize(fields, data, skipEmptyLines) { var csv = ''; @@ -359,8 +383,8 @@ if (typeof data === 'string') data = JSON.parse(data); - var hasHeader = fields instanceof Array && fields.length > 0; - var dataKeyedByField = !(data[0] instanceof Array); + var hasHeader = Array.isArray(fields) && fields.length > 0; + var dataKeyedByField = !(Array.isArray(data[0])); // If there a header row, write it first if (hasHeader && _writeHeader) @@ -379,19 +403,21 @@ for (var row = 0; row < data.length; row++) { var maxCol = hasHeader ? fields.length : data[row].length; + var r = hasHeader ? fields : data[row]; - for (var col = 0; col < maxCol; col++) + if (skipEmptyLines !== 'greedy' || r.join('').trim() !== '') { - if (col > 0) - csv += _delimiter; - var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; - csv += safe(data[row][colIdx], col); + for (var col = 0; col < maxCol; col++) + { + if (col > 0) + csv += _delimiter; + var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; + csv += safe(data[row][colIdx], col); + } + if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0)) + csv += _newline; } - - if (row < data.length - 1) - csv += _newline; } - return csv; } @@ -401,10 +427,13 @@ if (typeof str === 'undefined' || str === null) return ''; - str = str.toString().replace(quoteCharRegex, _quoteChar+_quoteChar); + if (str.constructor === Date) + return JSON.stringify(str).slice(1, 25); + + str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar); var needsQuotes = (typeof _quotes === 'boolean' && _quotes) - || (_quotes instanceof Array && _quotes[col]) + || (Array.isArray(_quotes) && _quotes[col]) || hasAny(str, Papa.BAD_DELIMITERS) || str.indexOf(_delimiter) > -1 || str.charAt(0) === ' ' @@ -426,8 +455,8 @@ function ChunkStreamer(config) { this._handle = null; - this._paused = false; this._finished = false; + this._completed = false; this._input = null; this._baseIndex = 0; this._partialLine = ''; @@ -442,7 +471,7 @@ }; replaceConfig.call(this, config); - this.parseChunk = function(chunk) + this.parseChunk = function(chunk, isFakeChunk) { // First chunk pre-processing if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) @@ -483,10 +512,10 @@ finished: finishedIncludingPreview }); } - else if (isFunction(this._config.chunk)) + else if (isFunction(this._config.chunk) && !isFakeChunk) { this._config.chunk(results, this._handle); - if (this._paused) + if (this._handle.paused() || this._handle.aborted()) return; results = undefined; this._completeResults = undefined; @@ -498,8 +527,10 @@ this._completeResults.meta = results.meta; } - if (finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) + if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) { this._config.complete(this._completeResults, this._input); + this._completed = true; + } if (!finishedIncludingPreview && (!results || !results.meta.paused)) this._nextChunk(); @@ -602,7 +633,7 @@ if (this._config.chunkSize) { var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive - xhr.setRequestHeader('Range', 'bytes='+this._start+'-'+end); + xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end); xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672 } @@ -617,11 +648,11 @@ this._chunkError(); else this._start += this._config.chunkSize; - } + }; this._chunkLoaded = function() { - if (xhr.readyState != 4) + if (xhr.readyState !== 4) return; if (xhr.status < 200 || xhr.status >= 400) @@ -632,20 +663,20 @@ this._finished = !this._config.chunkSize || this._start > getFileSize(xhr); this.parseChunk(xhr.responseText); - } + }; this._chunkError = function(errorMessage) { var errorText = xhr.statusText || errorMessage; - this._sendError(errorText); - } + this._sendError(new Error(errorText)); + }; function getFileSize(xhr) { var contentRange = xhr.getResponseHeader('Content-Range'); if (contentRange === null) { // no content range, then finish! - return -1; - } + return -1; + } return parseInt(contentRange.substr(contentRange.lastIndexOf('/') + 1)); } } @@ -687,7 +718,7 @@ { if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview)) this._readChunk(); - } + }; this._readChunk = function() { @@ -700,7 +731,7 @@ var txt = reader.readAsText(input, this._config.encoding); if (!usingAsyncReader) this._chunkLoaded({ target: { result: txt } }); // mimic the async signature - } + }; this._chunkLoaded = function(event) { @@ -708,12 +739,12 @@ this._start += this._config.chunkSize; this._finished = !this._config.chunkSize || this._start >= this._input.size; this.parseChunk(event.target.result); - } + }; this._chunkError = function() { - this._sendError(reader.error.message); - } + this._sendError(reader.error); + }; } FileStreamer.prototype = Object.create(ChunkStreamer.prototype); @@ -725,14 +756,12 @@ config = config || {}; ChunkStreamer.call(this, config); - var string; var remaining; this.stream = function(s) { - string = s; remaining = s; return this._nextChunk(); - } + }; this._nextChunk = function() { if (this._finished) return; @@ -741,7 +770,7 @@ remaining = size ? remaining.substr(size) : ''; this._finished = !remaining; return this.parseChunk(chunk); - } + }; } StringStreamer.prototype = Object.create(StringStreamer.prototype); StringStreamer.prototype.constructor = StringStreamer; @@ -755,6 +784,19 @@ var queue = []; var parseOnData = true; + var streamHasEnded = false; + + this.pause = function() + { + ChunkStreamer.prototype.pause.apply(this, arguments); + this._input.pause(); + }; + + this.resume = function() + { + ChunkStreamer.prototype.resume.apply(this, arguments); + this._input.resume(); + }; this.stream = function(stream) { @@ -763,10 +805,18 @@ this._input.on('data', this._streamData); this._input.on('end', this._streamEnd); this._input.on('error', this._streamError); - } + }; + + this._checkIsFinished = function() + { + if (streamHasEnded && queue.length === 1) { + this._finished = true; + } + }; this._nextChunk = function() { + this._checkIsFinished(); if (queue.length) { this.parseChunk(queue.shift()); @@ -775,7 +825,7 @@ { parseOnData = true; } - } + }; this._streamData = bindFunction(function(chunk) { @@ -786,6 +836,7 @@ if (parseOnData) { parseOnData = false; + this._checkIsFinished(); this.parseChunk(queue.shift()); } } @@ -798,13 +849,13 @@ this._streamError = bindFunction(function(error) { this._streamCleanUp(); - this._sendError(error.message); + this._sendError(error); }, this); this._streamEnd = bindFunction(function() { this._streamCleanUp(); - this._finished = true; + streamHasEnded = true; this._streamData(''); }, this); @@ -819,14 +870,117 @@ ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer; + function DuplexStreamStreamer(_config) { + var Duplex = require('stream').Duplex; + var config = copy(_config); + var parseOnWrite = true; + var writeStreamHasFinished = false; + var parseCallbackQueue = []; + var stream = null; + + this._onCsvData = function(results) + { + var data = results.data; + for (var i = 0; i < data.length; i++) { + if (!stream.push(data[i]) && !this._handle.paused()) { + // the writeable consumer buffer has filled up + // so we need to pause until more items + // can be processed + this._handle.pause(); + } + } + }; + + this._onCsvComplete = function() + { + // node will finish the read stream when + // null is pushed + stream.push(null); + }; + + config.step = bindFunction(this._onCsvData, this); + config.complete = bindFunction(this._onCsvComplete, this); + ChunkStreamer.call(this, config); + + this._nextChunk = function() + { + if (writeStreamHasFinished && parseCallbackQueue.length === 1) { + this._finished = true; + } + if (parseCallbackQueue.length) { + parseCallbackQueue.shift()(); + } else { + parseOnWrite = true; + } + }; + + this._addToParseQueue = function(chunk, callback) + { + // add to queue so that we can indicate + // completion via callback + // node will automatically pause the incoming stream + // when too many items have been added without their + // callback being invoked + parseCallbackQueue.push(bindFunction(function() { + this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding)); + if (isFunction(callback)) { + return callback(); + } + }, this)); + if (parseOnWrite) { + parseOnWrite = false; + this._nextChunk(); + } + }; + + this._onRead = function() + { + if (this._handle.paused()) { + // the writeable consumer can handle more data + // so resume the chunk parsing + this._handle.resume(); + } + }; + + this._onWrite = function(chunk, encoding, callback) + { + this._addToParseQueue(chunk, callback); + }; + + this._onWriteComplete = function() + { + writeStreamHasFinished = true; + // have to write empty string + // so parser knows its done + this._addToParseQueue(''); + }; + + this.getStream = function() + { + return stream; + }; + stream = new Duplex({ + readableObjectMode: true, + decodeStrings: false, + read: bindFunction(this._onRead, this), + write: bindFunction(this._onWrite, this) + }); + stream.once('finish', bindFunction(this._onWriteComplete, this)); + } + DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); + DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; + + // Use one ParserHandle per entire CSV file or string function ParserHandle(_config) { // One goal is to minimize the use of regular expressions... var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; + var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/; var self = this; var _stepCounter = 0; // Number of times step was called (number of rows parsed) + var _rowCounter = 0; // Number of rows that have been parsed so far var _input; // The input being parsed var _parser; // The core parser being used var _paused = false; // Whether we are paused or not @@ -872,13 +1026,14 @@ */ this.parse = function(input, baseIndex, ignoreLastRow) { + var quoteChar = _config.quoteChar || '"'; if (!_config.newline) - _config.newline = guessLineEndings(input); + _config.newline = guessLineEndings(input, quoteChar); _delimiterError = false; if (!_config.delimiter) { - var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines); + var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments); if (delimGuess.successful) _config.delimiter = delimGuess.bestDelimiter; else @@ -920,10 +1075,10 @@ this.resume = function() { _paused = false; - self.streamer.parseChunk(_input); + self.streamer.parseChunk(_input, true); }; - this.aborted = function () + this.aborted = function() { return _aborted; }; @@ -938,25 +1093,29 @@ _input = ''; }; + function testEmptyLine(s) { + return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0; + } + function processResults() { if (_results && _delimiterError) { - addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \''+Papa.DefaultDelimiter+'\''); + addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \'' + Papa.DefaultDelimiter + '\''); _delimiterError = false; } if (_config.skipEmptyLines) { for (var i = 0; i < _results.data.length; i++) - if (_results.data[i].length === 1 && _results.data[i][0] === '') + if (testEmptyLine(_results.data[i])) _results.data.splice(i--, 1); } if (needsHeaderRow()) fillHeaderFields(); - return applyHeaderAndDynamicTyping(); + return applyHeaderAndDynamicTypingAndTransformation(); } function needsHeaderRow() @@ -970,7 +1129,15 @@ return; for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) for (var j = 0; j < _results.data[i].length; j++) - _fields.push(_results.data[i][j]); + { + var header = _results.data[i][j]; + + if (_config.trimHeaders) { + header = header.trim(); + } + + _fields.push(header); + } _results.data.splice(0, 1); } @@ -979,7 +1146,7 @@ if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) { _config.dynamicTyping[field] = _config.dynamicTypingFunction(field); } - return (_config.dynamicTyping[field] || _config.dynamicTyping) === true + return (_config.dynamicTyping[field] || _config.dynamicTyping) === true; } function parseDynamic(field, value) @@ -990,22 +1157,27 @@ return true; else if (value === 'false' || value === 'FALSE') return false; + else if (FLOAT.test(value)) + return parseFloat(value); + else if (ISO_DATE.test(value)) + return new Date(value); else - return tryParseFloat(value); + return (value === '' ? null : value); } return value; } - function applyHeaderAndDynamicTyping() + function applyHeaderAndDynamicTypingAndTransformation() { - if (!_results || (!_config.header && !_config.dynamicTyping)) + if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform)) return _results; for (var i = 0; i < _results.data.length; i++) { var row = _config.header ? {} : []; - for (var j = 0; j < _results.data[i].length; j++) + var j; + for (j = 0; j < _results.data[i].length; j++) { var field = j; var value = _results.data[i][j]; @@ -1013,6 +1185,9 @@ if (_config.header) field = j >= _fields.length ? '__parsed_extra' : _fields[j]; + if (_config.transform) + value = _config.transform(value,field); + value = parseDynamic(field, value); if (field === '__parsed_extra') @@ -1029,18 +1204,20 @@ if (_config.header) { if (j > _fields.length) - addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, i); + addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); else if (j < _fields.length) - addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, i); + addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); } } if (_config.header && _results.meta) _results.meta.fields = _fields; + + _rowCounter += _results.data.length; return _results; } - function guessDelimiter(input, newline, skipEmptyLines) + function guessDelimiter(input, newline, skipEmptyLines, comments) { var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; var bestDelim, bestDelta, fieldCountPrevRow; @@ -1052,6 +1229,7 @@ fieldCountPrevRow = undefined; var preview = new Parser({ + comments: comments, delimiter: delim, newline: newline, preview: 10 @@ -1059,9 +1237,10 @@ for (var j = 0; j < preview.data.length; j++) { - if (skipEmptyLines && preview.data[j].length === 1 && preview.data[j][0].length === 0) { - emptyLinesCount++ - continue + if (skipEmptyLines && testEmptyLine(preview.data[j])) + { + emptyLinesCount++; + continue; } var fieldCount = preview.data[j].length; avgFieldCount += fieldCount; @@ -1094,12 +1273,15 @@ return { successful: !!bestDelim, bestDelimiter: bestDelim - } + }; } - function guessLineEndings(input) + function guessLineEndings(input, quoteChar) { - input = input.substr(0, 1024*1024); // max length 1 MB + input = input.substr(0, 1024 * 1024); // max length 1 MB + // Replace all the text inside quotes + var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); + input = input.replace(re, ''); var r = input.split('\r'); @@ -1120,12 +1302,6 @@ return numWithN >= r.length / 2 ? '\r\n' : '\r'; } - function tryParseFloat(val) - { - var isNumber = FLOAT.test(val); - return isNumber ? parseFloat(val) : val; - } - function addError(type, code, msg, row) { _results.errors.push({ @@ -1137,9 +1313,11 @@ } } - - - + /** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */ + function escapeRegExp(string) + { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string + } /** The core parser implements speedy and correct CSV parsing */ function Parser(config) @@ -1152,11 +1330,16 @@ var step = config.step; var preview = config.preview; var fastMode = config.fastMode; + var quoteChar; /** Allows for no quoteChar by setting quoteChar to undefined in config */ - if (config.quoteChar === undefined){ - var quoteChar = '"'; + if (config.quoteChar === undefined) { + quoteChar = '"'; } else { - var quoteChar = config.quoteChar; + quoteChar = config.quoteChar; + } + var escapeChar = quoteChar; + if (config.escapeChar !== undefined) { + escapeChar = config.escapeChar; } // Delimiter must be valid @@ -1174,7 +1357,7 @@ comments = false; // Newline must be valid: \r, \n, or \r\n - if (newline != '\n' && newline != '\r' && newline != '\r\n') + if (newline !== '\n' && newline !== '\r' && newline !== '\r\n') newline = '\n'; // We're gonna need these at the Parser scope @@ -1207,7 +1390,7 @@ var rows = input.split(newline); for (var i = 0; i < rows.length; i++) { - var row = rows[i]; + row = rows[i]; cursor += row.length; if (i !== rows.length - 1) cursor += newline.length; @@ -1236,7 +1419,8 @@ var nextDelim = input.indexOf(delim, cursor); var nextNewline = input.indexOf(newline, cursor); - var quoteCharRegex = new RegExp(quoteChar+quoteChar, 'g'); + var quoteCharRegex = new RegExp(escapeChar.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&') + quoteChar, 'g'); + var quoteSearch; // Parser loop for (;;) @@ -1245,7 +1429,7 @@ if (input[cursor] === quoteChar) { // Start our search for the closing quote where the cursor is - var quoteSearch = cursor; + quoteSearch = cursor; // Skip the opening quote cursor++; @@ -1253,7 +1437,7 @@ for (;;) { // Find closing quote - var quoteSearch = input.indexOf(quoteChar, quoteSearch+1); + quoteSearch = input.indexOf(quoteChar, quoteSearch + 1); //No other quotes are found - no other delimiters if (quoteSearch === -1) @@ -1272,34 +1456,47 @@ } // Closing quote at EOF - if (quoteSearch === inputLen-1) + if (quoteSearch === inputLen - 1) { var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar); return finish(value); } // If this quote is escaped, it's part of the data; skip it - if (input[quoteSearch+1] === quoteChar) + // If the quote character is the escape character, then check if the next character is the escape character + if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) { quoteSearch++; continue; } - // Closing quote followed by delimiter - if (input[quoteSearch+1] === delim) + // If the quote character is not the escape character, then check if the previous character was the escape character + if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar) + { + continue; + } + + // Check up to nextDelim or nextNewline, whichever is closest + var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline); + var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo); + + // Closing quote followed by delimiter or 'unnecessary spaces + delimiter' + if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter] === delim) { row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); - cursor = quoteSearch + 1 + delimLen; + cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen; nextDelim = input.indexOf(delim, cursor); nextNewline = input.indexOf(newline, cursor); break; } - // Closing quote followed by newline - if (input.substr(quoteSearch+1, newlineLen) === newline) + var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline); + + // Closing quote followed by newline or 'unnecessary spaces + newLine' + if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, newlineLen) === newline) { row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); - saveRow(quoteSearch + 1 + newlineLen); + saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen); nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field if (stepIsFunction) @@ -1385,6 +1582,21 @@ lastCursor = cursor; } + /** + * checks if there are extra spaces after closing quote and given index without any text + * if Yes, returns the number of spaces + */ + function extraSpaces(index) { + var spaceLength = 0; + if (index !== -1) { + var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index); + if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === '') { + spaceLength = textBetweenClosingQuoteAndIndex.length; + } + } + return spaceLength; + } + /** * Appends the remaining input from cursor to the end into * row, saves the row, calls step, and returns the results. @@ -1437,7 +1649,8 @@ function doStep() { step(returnable()); - data = [], errors = []; + data = []; + errors = []; } }; @@ -1572,9 +1785,9 @@ /** Makes a deep copy of an array or object (mostly) */ function copy(obj) { - if (typeof obj !== 'object') + if (typeof obj !== 'object' || obj === null) return obj; - var cpy = obj instanceof Array ? [] : {}; + var cpy = Array.isArray(obj) ? [] : {}; for (var key in obj) cpy[key] = copy(obj[key]); return cpy; From c6c3876e861245c2a31e3c5eba3b5048bb690f8c Mon Sep 17 00:00:00 2001 From: Jay L Date: Tue, 9 Oct 2018 01:34:28 -0700 Subject: [PATCH 07/25] Fixes unparsing using 'skipEmptyLines' = 'greedy' when using headers (#569) --- papaparse.js | 23 +++++++++++++++++++---- tests/test-cases.js | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/papaparse.js b/papaparse.js index 94c81b6..6a0a115 100755 --- a/papaparse.js +++ b/papaparse.js @@ -403,19 +403,34 @@ if (!Array.isArray) for (var row = 0; row < data.length; row++) { var maxCol = hasHeader ? fields.length : data[row].length; - var r = hasHeader ? fields : data[row]; - if (skipEmptyLines !== 'greedy' || r.join('').trim() !== '') + var emptyLine = false; + var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0; + if (skipEmptyLines && !hasHeader) + { + emptyLine = skipEmptyLines === 'greedy' ? data[row].join('').trim() === '' : data[row].length === 1 && data[row][0].length === 0; + } + if (skipEmptyLines === 'greedy' && hasHeader) { + var line = []; + for (var c = 0; c < maxCol; c++) { + var cx = dataKeyedByField ? fields[c] : c; + line.push(data[row][cx]); + } + emptyLine = line.join('').trim() === ''; + } + if (!emptyLine) { for (var col = 0; col < maxCol; col++) { - if (col > 0) + if (col > 0 && !nullLine) csv += _delimiter; var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; csv += safe(data[row][colIdx], col); } - if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0)) + if (row < data.length - 1 && (!skipEmptyLines || (maxCol > 0 && !nullLine))) + { csv += _newline; + } } } return csv; diff --git a/tests/test-cases.js b/tests/test-cases.js index deb2e65..98509de 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1654,6 +1654,24 @@ var UNPARSE_TESTS = [ input: [[null, ' '], [], ['1', '2']], config: {skipEmptyLines: 'greedy'}, expected: '1,2' + }, + { + description: "Returns empty rows when empty rows are passed and skipEmptyLines is false with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: false, header: true}, + expected: 'a,b\r\n," "\r\n\r\n1,2' + }, + { + description: "Returns without empty rows when skipEmptyLines is true with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: true, header: true}, + expected: 'a,b\r\n," "\r\n1,2' + }, + { + description: "Returns without rows with no content when skipEmptyLines is 'greedy' with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: 'greedy', header: true}, + expected: 'a,b\r\n1,2' } ]; From 857d4558e17f8953102c9f2b56459980238a0ef7 Mon Sep 17 00:00:00 2001 From: Jeff Barnes Date: Mon, 8 Oct 2018 10:48:17 -0800 Subject: [PATCH 08/25] Avoid use of node stream in browser context. --- Gruntfile.js | 6 ++++++ package.json | 1 + papaparse.js | 12 ++++++++---- papaparse.min.js | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index e645b52..93b1256 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -2,6 +2,12 @@ module.exports = function(grunt) { grunt.initConfig({ uglify: { options: { + compress: { + global_defs: { + 'PAPA_BROWSER_CONTEXT': true + }, + dead_code: true + }, output: { comments: 'some', }, diff --git a/package.json b/package.json index 7b9e671..ab6e6fc 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ }, "license": "MIT", "main": "papaparse.js", + "browser": "papaparse.min.js", "devDependencies": { "chai": "^4.2.0", "connect": "^3.3.3", diff --git a/papaparse.js b/papaparse.js index 6a0a115..bcf3139 100755 --- a/papaparse.js +++ b/papaparse.js @@ -81,7 +81,9 @@ if (!Array.isArray) Papa.FileStreamer = FileStreamer; Papa.StringStreamer = StringStreamer; Papa.ReadableStreamStreamer = ReadableStreamStreamer; - Papa.DuplexStreamStreamer = DuplexStreamStreamer; + if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { + Papa.DuplexStreamStreamer = DuplexStreamStreamer; + } if (global.jQuery) { @@ -241,7 +243,7 @@ if (!Array.isArray) } var streamer = null; - if (_input === Papa.NODE_STREAM_INPUT) + if (_input === Papa.NODE_STREAM_INPUT && typeof PAPA_BROWSER_CONTEXT === 'undefined') { // create a node Duplex stream for use // with .pipe @@ -982,8 +984,10 @@ if (!Array.isArray) }); stream.once('finish', bindFunction(this._onWriteComplete, this)); } - DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); - DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; + if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { + DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); + DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; + } // Use one ParserHandle per entire CSV file or string diff --git a/papaparse.min.js b/papaparse.min.js index 5828b1d..32ebc35 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -4,4 +4,4 @@ v4.6.1 https://github.com/mholt/PapaParse License: MIT */ -Array.isArray||(Array.isArray=function(e){return"[object Array]"===Object.prototype.toString.call(e)}),function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},n=!f.document&&!!f.postMessage,o=n&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,v={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&v.WORKERS_SUPPORTED){var r=function(){if(!v.WORKERS_SUPPORTED)return!1;if(!a&&null===v.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=v.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=y,t.id=u++,h[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;{if(e===v.NODE_STREAM_INPUT)return(n=new g(t)).getStream();"string"==typeof e?n=t.download?new c(t):new _(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new p(t))}return n.stream(e)},unparse:function(e,t){var r=!1,l=!0,c=",",p="\r\n",n='"',i=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(c=t.delimiter);("boolean"==typeof t.quotes||Array.isArray(t.quotes))&&(r=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(p=t.newline);"string"==typeof t.quoteChar&&(n=t.quoteChar);"boolean"==typeof t.header&&(l=t.header)}();var s=new RegExp(n,"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return o(null,e,i);if("object"==typeof e[0])return o(a(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:a(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),o(e.fields||[],e.data||[],i);throw"exception: Unable to serialize unrecognized input";function a(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function o(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s Date: Wed, 14 Nov 2018 10:35:09 +0100 Subject: [PATCH 09/25] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index ab6e6fc..90e8ca8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "4.6.1", + "version": "4.6.2", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index bcf3139..f39a108 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v4.6.1 +v4.6.2 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 32ebc35..95d7b63 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v4.6.1 +v4.6.2 https://github.com/mholt/PapaParse License: MIT */ -Array.isArray||(Array.isArray=function(e){return"[object Array]"===Object.prototype.toString.call(e)}),function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},n=!f.document&&!!f.postMessage,o=n&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,v={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&v.WORKERS_SUPPORTED){var r=function(){if(!v.WORKERS_SUPPORTED)return!1;if(!a&&null===v.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=v.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=y,t.id=u++,h[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;{if(e===v.NODE_STREAM_INPUT)return(n=new g(t)).getStream();"string"==typeof e?n=t.download?new c(t):new _(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new p(t))}return n.stream(e)},unparse:function(e,t){var r=!1,l=!0,c=",",p="\r\n",n='"',i=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(c=t.delimiter);("boolean"==typeof t.quotes||Array.isArray(t.quotes))&&(r=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(p=t.newline);"string"==typeof t.quoteChar&&(n=t.quoteChar);"boolean"==typeof t.header&&(l=t.header)}();var s=new RegExp(n,"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return o(null,e,i);if("object"==typeof e[0])return o(a(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:a(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),o(e.fields||[],e.data||[],i);throw"exception: Unable to serialize unrecognized input";function a(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function o(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function c(e){var i;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r),i.setRequestHeader("If-None-Match","webkit-no-cache")}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function p(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function g(e){var r;l.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function _(e){l.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=C(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=C(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=C(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=C(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(_){var a,o,h,i=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,n=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,s=0,u=!1,e=!1,f=[],d={data:[],errors:[],meta:{}};if(M(_.step)){var l=_.step;_.step=function(e){if(d=e,p())c();else{if(c(),0===d.data.length)return;r+=e.data.length,_.preview&&r>_.preview?o.abort():l(d,t)}}}function m(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),_.transform&&(n=_.transform(n,i)),n=g(i,n),"__parsed_extra"===i?(r[i]=r[i]||[],r[i].push(n)):r[i]=n}d.data[e]=r,_.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=i.length/2?"\r\n":"\r"}(e,i)),h=!1,_.delimiter)M(_.delimiter)&&(_.delimiter=_.delimiter(e),d.meta.delimiter=_.delimiter);else{var n=function(e,t,r,i){for(var n,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=A)return C(!0)}else for(p=P,P++;;){if(-1===(p=i.indexOf(S,p+1)))return r||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),E();if(p===n-1)return E(i.substring(P,p).replace(m,S));if(S!==L||i[p+1]!==L){if(S===L||0===p||i[p-1]!==L){var y=b(-1===_?g:Math.min(g,_));if(i[p+1+y]===O){f.push(i.substring(P,p).replace(m,S)),P=p+1+y+e,g=i.indexOf(O,P),_=i.indexOf(x,P);break}var v=b(_);if(i.substr(p+1+v,s)===x){if(f.push(i.substring(P,p).replace(m,S)),w(p+1+v+s),g=i.indexOf(O,P),o&&(R(),F))return C();if(A&&h.length>=A)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return E();function k(e){h.push(e),d=P}function b(e){var t=0;if(-1!==e){var r=i.substring(p+1,e);r&&""===r.trim()&&(t=r.length)}return t}function E(e){return r||(void 0===e&&(e=i.substr(P)),f.push(e),P=n,k(f),o&&R()),C()}function w(e){P=e,k(f),f=[],_=i.indexOf(x,P)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(C()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function m(e){var t=e.data,r=h[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,y(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:E,resume:E};if(M(r.userStep)){for(var s=0;s Date: Wed, 14 Nov 2018 11:23:05 +0100 Subject: [PATCH 10/25] Use 5.0.0-alpha as version number Closes #582 --- package.json | 2 +- papaparse.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 90e8ca8..ae43241 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "4.6.2", + "version": "5.0.0-alpha", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index f39a108..8703ec8 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v4.6.2 +v5.0.0-alpha https://github.com/mholt/PapaParse License: MIT */ From 16ba9bb4a516c4c3e8757d1535e8967733651691 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 14 Nov 2018 11:26:05 +0100 Subject: [PATCH 11/25] Remove support for node6 and add support for node11 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4ef1c4..f00e0fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: node_js node_js: - - "6" - "8" - "9" - "10" - + - "11" From 8880a53401285092c8cf5e0b9e5233103d9de30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Sardinha?= Date: Wed, 14 Nov 2018 10:29:12 +0000 Subject: [PATCH 12/25] Allow transforming header columns (#589) * Deprecate trimHeaders: The same result can be achieved by using transformHeader parameter. --- docs/docs.html | 6 +++--- papaparse.js | 4 ++-- tests/test-cases.js | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index 68f1132..b424c7a 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -340,7 +340,7 @@ var csv = Papa.unparse({ quoteChar: '"', escapeChar: '"', header: false, - trimHeaders: false, + transformHeader: undefined, dynamicTyping: false, preview: 0, encoding: "", @@ -412,10 +412,10 @@ var csv = Papa.unparse({ - trimHeaders + transformHeader - If true leading/trailing spaces will be trimed from headers. + A function to apply on each header. Requires header to be true. The function receives the header as its first argument. diff --git a/papaparse.js b/papaparse.js index 8703ec8..ababc70 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1151,8 +1151,8 @@ if (!Array.isArray) { var header = _results.data[i][j]; - if (_config.trimHeaders) { - header = header.trim(); + if (isFunction(_config.transformHeader)) { + header = _config.transformHeader(header); } _fields.push(header); diff --git a/tests/test-cases.js b/tests/test-cases.js index 98509de..bfc1d04 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -725,11 +725,11 @@ var PARSE_TESTS = [ } }, { - description: "Header rows are trimmed when trimHeaders is set", - input: ' A , B , C \r\na,b ,c', - config: { header: true, trimHeaders: true }, + description: "Header rows are transformed when transformHeader function is provided", + input: 'A,B,C\r\na,b,c', + config: { header: true, transformHeader: function(header) { return header.toLowerCase(); } }, expected: { - data: [{"A": "a", "B": "b ", "C": "c"}], + data: [{"a": "a", "b": "b", "c": "c"}], errors: [] } }, From 491c8fa8cf978297242af65d4c4a749d0cc6d43a Mon Sep 17 00:00:00 2001 From: Gabe Gorelick Date: Wed, 14 Nov 2018 04:36:35 -0600 Subject: [PATCH 13/25] Throw Error objects instead of strings (#497) Unlike strings, Error objects allow you to get stack traces. Also, a lot of libraries will complain if they encounter a thrown non-Error object. This is in line with standard Javascript best practices. Per https://eslint.org/docs/rules/no-throw-literal: --- .eslintrc.js | 2 +- papaparse.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index f63bc09..c3ed7c5 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -178,7 +178,7 @@ module.exports = { "no-tabs": "off", "no-template-curly-in-string": "error", "no-ternary": "off", - "no-throw-literal": "off", + "no-throw-literal": "error", "no-trailing-spaces": "error", "no-undef-init": "error", "no-undefined": "off", diff --git a/papaparse.js b/papaparse.js index ababc70..0edec76 100755 --- a/papaparse.js +++ b/papaparse.js @@ -331,7 +331,7 @@ if (!Array.isArray) } // Default (any valid paths should return before this) - throw 'exception: Unable to serialize unrecognized input'; + throw new Error('Unable to serialize unrecognized input'); function unpackConfig() @@ -1368,7 +1368,7 @@ if (!Array.isArray) // Comment character must be valid if (comments === delim) - throw 'Comment character same as delimiter'; + throw new Error('Comment character same as delimiter'); else if (comments === true) comments = '#'; else if (typeof comments !== 'string' @@ -1387,7 +1387,7 @@ if (!Array.isArray) { // For some reason, in Chrome, this speeds things up (!?) if (typeof input !== 'string') - throw 'Input must be a string'; + throw new Error('Input must be a string'); // We don't need to compute some of these every time parse() is called, // but having them in a more local scope seems to perform better @@ -1770,7 +1770,7 @@ if (!Array.isArray) } function notImplemented() { - throw 'Not implemented.'; + throw new Error('Not implemented.'); } /** Callback when worker thread receives a message */ From 803452f2b5fe35b6881fbfa6efec005f4257b894 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 14 Nov 2018 11:37:57 +0100 Subject: [PATCH 14/25] Return unique row as data on step function (#500) --- papaparse.js | 7 ++++--- tests/test-cases.js | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/papaparse.js b/papaparse.js index 0edec76..8676056 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1649,10 +1649,11 @@ if (!Array.isArray) } /** Returns an object with the results, errors, and meta. */ - function returnable(stopped) + function returnable(stopped, step) { + var isStep = step || false; return { - data: data, + data: isStep ? data[0] : data, errors: errors, meta: { delimiter: delim, @@ -1667,7 +1668,7 @@ if (!Array.isArray) /** Executes the user's step function and resets data & errors. */ function doStep() { - step(returnable()); + step(returnable(undefined, true)); data = []; errors = []; } diff --git a/tests/test-cases.js b/tests/test-cases.js index bfc1d04..44521e5 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1735,7 +1735,7 @@ var CUSTOM_TESTS = [ run: function(callback) { Papa.parse('A,b,c', { step: function(response) { - callback(response.data[0]); + callback(response.data); } }); } @@ -1972,7 +1972,7 @@ var CUSTOM_TESTS = [ var updates = []; Papa.parse('A,b,c\nd,E,f\nG,h,i', { step: function(response, handle) { - updates.push(response.data[0]); + updates.push(response.data); handle.abort(); callback(updates); }, @@ -2002,7 +2002,7 @@ var CUSTOM_TESTS = [ var updates = []; Papa.parse('A,b,c\nd,E,f\nG,h,i', { step: function(response, handle) { - updates.push(response.data[0]); + updates.push(response.data); handle.pause(); callback(updates); }, @@ -2021,7 +2021,7 @@ var CUSTOM_TESTS = [ var first = true; Papa.parse('A,b,c\nd,E,f\nG,h,i', { step: function(response, h) { - updates.push(response.data[0]); + updates.push(response.data); if (!first) return; handle = h; handle.pause(); From e6a27ad79326c3684e334008b5ed5d2241def6eb Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 14 Nov 2018 11:59:02 +0100 Subject: [PATCH 15/25] Return data directly on NodeStream as step function now returns a single row --- papaparse.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/papaparse.js b/papaparse.js index 8676056..9d1fcc6 100755 --- a/papaparse.js +++ b/papaparse.js @@ -898,13 +898,11 @@ if (!Array.isArray) this._onCsvData = function(results) { var data = results.data; - for (var i = 0; i < data.length; i++) { - if (!stream.push(data[i]) && !this._handle.paused()) { - // the writeable consumer buffer has filled up - // so we need to pause until more items - // can be processed - this._handle.pause(); - } + if (!stream.push(data) && !this._handle.paused()) { + // the writeable consumer buffer has filled up + // so we need to pause until more items + // can be processed + this._handle.pause(); } }; From f39ba6faf17eaf6a7ae237a42bb68be52b8c4d3a Mon Sep 17 00:00:00 2001 From: Leo Anthias Date: Wed, 14 Nov 2018 11:03:18 +0000 Subject: [PATCH 16/25] Remove If-None-Match header. Fixes #595 (#596) --- papaparse.js | 1 - 1 file changed, 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 9d1fcc6..81c772c 100755 --- a/papaparse.js +++ b/papaparse.js @@ -651,7 +651,6 @@ if (!Array.isArray) { var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end); - xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672 } try { From 054303b91526b7b7474a9fa2fa34bdb6f4de792e Mon Sep 17 00:00:00 2001 From: Jacopo Farina Date: Thu, 11 Oct 2018 14:26:20 +0200 Subject: [PATCH 17/25] Move to mocha-headless-chrome for running tests Additionally, upgrade mocha. --- package.json | 8 ++++---- tests/test.js | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index ae43241..0072ac1 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "eslint": "^4.19.1", "grunt": "^1.0.2", "grunt-contrib-uglify": "^3.3.0", - "mocha": "^3.5.0", - "mocha-phantomjs": "^4.1.0", + "mocha": "^5.2.0", + "mocha-headless-chrome": "^2.0.1", "open": "0.0.5", "phantomjs-prebuilt": "^2.1.16", "serve-static": "^1.7.1" @@ -51,8 +51,8 @@ "scripts": { "lint": "eslint --no-ignore papaparse.js Gruntfile.js .eslintrc.js 'tests/**/*.js'", "test-browser": "node tests/test.js", - "test-phantomjs": "node tests/test.js --phantomjs", + "test-mocha-headless-chrome": "node tests/test.js --mocha-headless-chrome", "test-node": "mocha tests/node-tests.js tests/test-cases.js", - "test": "npm run lint && npm run test-node && npm run test-phantomjs" + "test": "npm run lint && npm run test-node && npm run test-mocha-headless-chrome" } } diff --git a/tests/test.js b/tests/test.js index f4fed5f..2741831 100644 --- a/tests/test.js +++ b/tests/test.js @@ -5,8 +5,8 @@ var path = require('path'); var childProcess = require('child_process'); var server = connect().use(serveStatic(path.join(__dirname, '/..'))).listen(8071, function() { - if (process.argv.indexOf('--phantomjs') !== -1) { - childProcess.spawn('node_modules/.bin/mocha-phantomjs', ['http://localhost:8071/tests/tests.html'], { + if (process.argv.indexOf('--mocha-headless-chrome') !== -1) { + childProcess.spawn('node_modules/.bin/mocha-headless-chrome', ['-f', 'http://localhost:8071/tests/tests.html'], { stdio: 'inherit' }).on('exit', function(code) { server.close(); From 25cbfae1616c7eaaabc40f135307710bb413d38d Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 20 Nov 2018 12:58:35 +0100 Subject: [PATCH 18/25] Update file test case to do not filter response data Now a single row is returned on the step function --- tests/test-cases.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-cases.js b/tests/test-cases.js index 44521e5..9477892 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1957,7 +1957,7 @@ var CUSTOM_TESTS = [ Papa.parse(new File(['A,B,C\nX,"Y\n1\n2\n3",Z'], 'sample.csv'), { chunkSize: 3, step: function(response) { - updates.push(response.data[0]); + updates.push(response.data); }, complete: function() { callback(updates); From 7fbb6d62432e17117e02fe7e9b95aba709c85eb7 Mon Sep 17 00:00:00 2001 From: Jonathan Grimes Date: Mon, 19 Nov 2018 13:00:36 -0600 Subject: [PATCH 19/25] Support workers from inline-blobs Simplify worker to just use inline-blob --- docs/docs.html | 8 +------ docs/faq.html | 6 +++--- papaparse.js | 51 ++++++++++++-------------------------------- tests/test-cases.js | 52 ++++++++++----------------------------------- tests/tests.html | 2 +- 5 files changed, 30 insertions(+), 89 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index b424c7a..bd842f1 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -447,7 +447,7 @@ var csv = Papa.unparse({ worker - Whether or not to use a worker thread. Using a worker will keep your page reactive, but may be slightly slower. Web Workers also load the entire Javascript file, so be careful when combining other libraries in the same file as Papa Parse. Note that worker option is only available when parsing files and not when converting from JSON to CSV. + Whether or not to use a worker thread. Using a worker will keep your page reactive, but may be slightly slower. @@ -749,12 +749,6 @@ var csv = Papa.unparse({ Whether or not the browser supports HTML5 Web Workers. If false, worker: true will have no effect. - - Papa.SCRIPT_PATH - - The relative path to Papa Parse. This is automatically detected when Papa Parse is loaded synchronously. However, if you load Papa Parse asynchronously (e.g. with RequireJS), you need to set this variable manually in order to use Web Workers. (In those cases, this variable is not read-only and you should set it!) - - diff --git a/docs/faq.html b/docs/faq.html index 9af0863..eacb409 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -84,7 +84,7 @@
Can I put other libraries in the same file as Papa Parse?

- Yes, but then don't use the Web Worker feature unless your other dependencies are battle-hardened for worker threads. A worker thread loads an entire file, not just a function, so all those dependencies would be executed in an environment without a DOM and other window features. If any of those dependencies crash (Cannot read property "defaultView" of undefined is common), the whole worker thread will crash and parsing will not succeed. + Yes.

@@ -96,7 +96,7 @@
Can Papa Parse be loaded asynchronously (after the page loads)?

- Yes. But if you want to use Web Workers, you'll need to specify the relative path to Papa Parse. To do this, set Papa.SCRIPT_PATH to the relative path of the Papa Parse file. In synchronous loading, this is automatically detected. + Yes.

@@ -209,7 +209,7 @@
Can I use a worker if I combine/concatenate my Javascript files?

- Probably not. It's safest to concatenate the rest of your dependencies and include Papa Parse in a seperate file. Any library that expects to have access to the window or DOM will crash when executed in a worker thread. Only put other libraries in the same file if they are ready to be used in worker threads. + Yes.

When should I use a worker?
diff --git a/papaparse.js b/papaparse.js index 81c772c..14edf60 100755 --- a/papaparse.js +++ b/papaparse.js @@ -34,7 +34,10 @@ if (!Array.isArray) // Browser globals (root is window) root.Papa = factory(); } -}(this, function() + // in strict mode we cannot access arguments.callee, so we need a named reference to + // stringify the factory method for the blob worker + // eslint-disable-next-line func-name +}(this, function moduleFactory() { 'use strict'; @@ -51,9 +54,15 @@ if (!Array.isArray) return {}; })(); + + function getWorkerBlob() { + var URL = global.URL || global.webkitURL || null; + var code = moduleFactory.toString(); + return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(['(', code, ')();'], {type: 'text/javascript'}))); + } + var IS_WORKER = !global.document && !!global.postMessage, - IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), - LOADED_SYNC = false, AUTO_SCRIPT_PATH; + IS_PAPA_WORKER = IS_WORKER && /blob:/i.test((global.location || {}).protocol); var workers = {}, workerIdCounter = 0; var Papa = {}; @@ -66,7 +75,6 @@ if (!Array.isArray) Papa.BYTE_ORDER_MARK = '\ufeff'; Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; - Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously Papa.NODE_STREAM_INPUT = 1; // Configurable chunk sizes for local and remote files, respectively @@ -184,23 +192,6 @@ if (!Array.isArray) { global.onmessage = workerThreadReceivedMessage; } - else if (Papa.WORKERS_SUPPORTED) - { - AUTO_SCRIPT_PATH = getScriptPath(); - - // Check if the script was loaded synchronously - if (!document.body) - { - // Body doesn't exist yet, must be synchronous - LOADED_SYNC = true; - } - else - { - document.addEventListener('DOMContentLoaded', function() { - LOADED_SYNC = true; - }, true); - } - } @@ -1685,26 +1676,12 @@ if (!Array.isArray) } - // If you need to load Papa Parse asynchronously and you also need worker threads, hard-code - // the script path here. See: https://github.com/mholt/PapaParse/issues/87#issuecomment-57885358 - function getScriptPath() - { - var scripts = document.getElementsByTagName('script'); - return scripts.length ? scripts[scripts.length - 1].src : ''; - } - function newWorker() { if (!Papa.WORKERS_SUPPORTED) return false; - if (!LOADED_SYNC && Papa.SCRIPT_PATH === null) - throw new Error( - 'Script path cannot be determined automatically when Papa Parse is loaded asynchronously. ' + - 'You need to set Papa.SCRIPT_PATH manually.' - ); - var workerUrl = Papa.SCRIPT_PATH || AUTO_SCRIPT_PATH; - // Append 'papaworker' to the search string to tell papaparse that this is our worker. - workerUrl += (workerUrl.indexOf('?') !== -1 ? '&' : '?') + 'papaworker'; + + var workerUrl = getWorkerBlob(); var w = new global.Worker(workerUrl); w.onmessage = mainThreadReceivedMessage; w.id = workerIdCounter++; diff --git a/tests/test-cases.js b/tests/test-cases.js index 9477892..7e4986c 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -7,6 +7,7 @@ if (typeof module !== 'undefined' && module.exports) { var assert = chai.assert; +var BASE_PATH = (typeof document === 'undefined') ? './' : document.getElementById('test-cases').src.replace(/test-cases\.js$/, ''); var RECORD_SEP = String.fromCharCode(30); var UNIT_SEP = String.fromCharCode(31); var FILES_ENABLED = false; @@ -1396,7 +1397,7 @@ var PARSE_ASYNC_TESTS = [ }, { description: "Simple download", - input: "sample.csv", + input: BASE_PATH + "sample.csv", config: { download: true }, @@ -1408,7 +1409,7 @@ var PARSE_ASYNC_TESTS = [ }, { description: "Simple download + worker", - input: "tests/sample.csv", + input: BASE_PATH + "sample.csv", config: { worker: true, download: true @@ -1761,7 +1762,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, step: function(response) { updates.push(response.meta.cursor); @@ -1778,7 +1779,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, step: function(response) { @@ -1796,7 +1797,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, worker: true, @@ -1815,7 +1816,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, chunk: function(response) { @@ -1833,7 +1834,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, chunk: function(response) { @@ -2042,7 +2043,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = 0; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { worker: true, download: true, chunkSize: 500, @@ -2062,7 +2063,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = 0; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, beforeFirstChunk: function(chunk) { @@ -2083,7 +2084,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = 0; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, beforeFirstChunk: function(chunk) { @@ -2096,37 +2097,6 @@ var CUSTOM_TESTS = [ } }); } - }, - { - description: "Should not assume we own the worker unless papaworker is in the search string", - disabled: typeof Worker === 'undefined', - expected: [false, true, true, true, true], - run: function(callback) { - var searchStrings = [ - '', - '?papaworker', - '?x=1&papaworker', - '?x=1&papaworker&y=1', - '?x=1&papaworker=1' - ]; - var results = searchStrings.map(function() { return false; }); - var workers = []; - - // Give it .5s to do something - setTimeout(function() { - workers.forEach(function(w) { w.terminate(); }); - callback(results); - }, 500); - - searchStrings.forEach(function(searchString, idx) { - var w = new Worker('../papaparse.js' + searchString); - workers.push(w); - w.addEventListener('message', function() { - results[idx] = true; - }); - w.postMessage({input: 'a,b,c\n1,2,3'}); - }); - } } ]; diff --git a/tests/tests.html b/tests/tests.html index 421e96b..3d719d3 100644 --- a/tests/tests.html +++ b/tests/tests.html @@ -9,7 +9,7 @@ - +
From 998a74422fa567e6d555bae7c756aaad9595ce5d Mon Sep 17 00:00:00 2001 From: Jonathan Grimes Date: Tue, 20 Nov 2018 09:28:54 -0600 Subject: [PATCH 20/25] cleanup remnants of phantomjs --- package.json | 1 - tests/tests.html | 9 ++------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 0072ac1..8318774 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,6 @@ "mocha": "^5.2.0", "mocha-headless-chrome": "^2.0.1", "open": "0.0.5", - "phantomjs-prebuilt": "^2.1.16", "serve-static": "^1.7.1" }, "scripts": { diff --git a/tests/tests.html b/tests/tests.html index 3d719d3..a3ce51e 100644 --- a/tests/tests.html +++ b/tests/tests.html @@ -15,13 +15,8 @@
From e16b71b344b8f4c60eaa047d3ef9e3f1bc6046e5 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 20 Nov 2018 18:36:20 +0100 Subject: [PATCH 21/25] Remove trailing space on faqs --- docs/faq.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.html b/docs/faq.html index eacb409..edced65 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -96,7 +96,7 @@
Can Papa Parse be loaded asynchronously (after the page loads)?

- Yes. + Yes.

From 983b3c3db6727a119614724f1a1445e7d83f0c77 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 21 Nov 2018 09:45:08 +0100 Subject: [PATCH 22/25] Remove Array.isArray() polyfill Fixes #598 --- papaparse.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/papaparse.js b/papaparse.js index 14edf60..d905513 100755 --- a/papaparse.js +++ b/papaparse.js @@ -5,15 +5,6 @@ https://github.com/mholt/PapaParse License: MIT */ -// Polyfills -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill -if (!Array.isArray) -{ - Array.isArray = function(arg) { - return Object.prototype.toString.call(arg) === '[object Array]'; - }; -} - (function(root, factory) { /* globals define */ From 46ff5c7a66e8acb18c194b2e9a6235c4eb993e0e Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 21 Nov 2018 10:01:50 +0100 Subject: [PATCH 23/25] Beta version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 8318774..d83361f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.0.0-alpha", + "version": "5.0.0-beta.0", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index d905513..66dba9d 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.0.0-alpha +v5.0.0-beta.0 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 95d7b63..c960d68 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v4.6.2 +v5.0.0-beta.0 https://github.com/mholt/PapaParse License: MIT */ -Array.isArray||(Array.isArray=function(e){return"[object Array]"===Object.prototype.toString.call(e)}),function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},n=!f.document&&!!f.postMessage,o=n&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,k={parse:function(e,t){var r=(t=t||{}).dynamicTyping||!1;M(r)&&(t.dynamicTypingFunction=r,r={});if(t.dynamicTyping=r,t.transform=!!M(t.transform)&&t.transform,t.worker&&k.WORKERS_SUPPORTED){var i=function(){if(!k.WORKERS_SUPPORTED)return!1;if(!a&&null===k.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=k.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=m,t.id=u++,h[t.id]=t}();return i.userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void i.postMessage({input:e,config:t,workerId:i.id})}var n=null;k.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new c(t):new g(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new _(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new p(t));return n.stream(e)},unparse:function(e,t){var i=!1,_=!0,m=",",y="\r\n",n='"',r=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||k.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||Array.isArray(t.quotes))&&(i=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(r=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(n=t.quoteChar);"boolean"==typeof t.header&&(_=t.header)}();var s=new RegExp(n,"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return o(null,e,r);if("object"==typeof e[0])return o(a(e[0]),e,r)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:a(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),o(e.fields||[],e.data||[],r);throw"exception: Unable to serialize unrecognized input";function a(e){if("object"!=typeof e)return[];var t=[];for(var r in e)t.push(r);return t}function o(e,t,r){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function c(e){var i;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r),i.setRequestHeader("If-None-Match","webkit-no-cache")}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function p(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function g(e){var r;l.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function _(e){l.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=C(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=C(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=C(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=C(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(_){var a,o,h,i=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,n=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,s=0,u=!1,e=!1,f=[],d={data:[],errors:[],meta:{}};if(M(_.step)){var l=_.step;_.step=function(e){if(d=e,p())c();else{if(c(),0===d.data.length)return;r+=e.data.length,_.preview&&r>_.preview?o.abort():l(d,t)}}}function m(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),_.transform&&(n=_.transform(n,i)),n=g(i,n),"__parsed_extra"===i?(r[i]=r[i]||[],r[i].push(n)):r[i]=n}d.data[e]=r,_.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=i.length/2?"\r\n":"\r"}(e,i)),h=!1,_.delimiter)M(_.delimiter)&&(_.delimiter=_.delimiter(e),d.meta.delimiter=_.delimiter);else{var n=function(e,t,r,i){for(var n,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=A)return C(!0)}else for(p=P,P++;;){if(-1===(p=i.indexOf(S,p+1)))return r||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),E();if(p===n-1)return E(i.substring(P,p).replace(m,S));if(S!==L||i[p+1]!==L){if(S===L||0===p||i[p-1]!==L){var y=b(-1===_?g:Math.min(g,_));if(i[p+1+y]===O){f.push(i.substring(P,p).replace(m,S)),P=p+1+y+e,g=i.indexOf(O,P),_=i.indexOf(x,P);break}var v=b(_);if(i.substr(p+1+v,s)===x){if(f.push(i.substring(P,p).replace(m,S)),w(p+1+v+s),g=i.indexOf(O,P),o&&(R(),F))return C();if(A&&h.length>=A)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return E();function k(e){h.push(e),d=P}function b(e){var t=0;if(-1!==e){var r=i.substring(p+1,e);r&&""===r.trim()&&(t=r.length)}return t}function E(e){return r||(void 0===e&&(e=i.substr(P)),f.push(e),P=n,k(f),o&&R()),C()}function w(e){P=e,k(f),f=[],_=i.indexOf(x,P)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(C()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function m(e){var t=e.data,r=h[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,y(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:E,resume:E};if(M(r.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=w(this._chunkLoaded,this),i.onerror=w(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=w(this._chunkLoaded,this),i.onerror=w(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;u.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function _(e){u.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=w(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=w(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=w(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=w(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(g){var a,o,h,i=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,n=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,s=0,u=!1,e=!1,f=[],d={data:[],errors:[],meta:{}};if(M(g.step)){var l=g.step;g.step=function(e){if(d=e,p())c();else{if(c(),0===d.data.length)return;r+=e.data.length,g.preview&&r>g.preview?o.abort():l(d,t)}}}function m(e){return"greedy"===g.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(v("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),g.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),g.transform&&(n=g.transform(n,i)),n=_(i,n),"__parsed_extra"===i?(r[i]=r[i]||[],r[i].push(n)):r[i]=n}d.data[e]=r,g.header&&(t>f.length?v("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=i.length/2?"\r\n":"\r"}(e,i)),h=!1,g.delimiter)M(g.delimiter)&&(g.delimiter=g.delimiter(e),d.meta.delimiter=g.delimiter);else{var n=function(e,t,r,i){for(var n,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=T)return C(!0)}else for(p=F,F++;;){if(-1===(p=i.indexOf(S,p+1)))return t||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:F}),E();if(p===n-1)return E(i.substring(F,p).replace(m,S));if(S!==A||i[p+1]!==A){if(S===A||0===p||i[p-1]!==A){var v=b(-1===g?_:Math.min(_,g));if(i[p+1+v]===O){f.push(i.substring(F,p).replace(m,S)),F=p+1+v+e,_=i.indexOf(O,F),g=i.indexOf(x,F);break}var y=b(g);if(i.substr(p+1+y,s)===x){if(f.push(i.substring(F,p).replace(m,S)),w(p+1+y+s),_=i.indexOf(O,F),o&&(R(),z))return C();if(T&&h.length>=T)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:F}),p++}}else p++}return E();function k(e){h.push(e),d=F}function b(e){var t=0;if(-1!==e){var r=i.substring(p+1,e);r&&""===r.trim()&&(t=r.length)}return t}function E(e){return t||(void 0===e&&(e=i.substr(F)),f.push(e),F=n,k(f),o&&R()),C()}function w(e){F=e,k(f),f=[],g=i.indexOf(x,F)}function C(e,t){return{data:t||!1?h[0]:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:z,truncated:!!e,cursor:d+(r||0)}}}function R(){D(C(void 0,!0)),h=[],u=[]}},this.abort=function(){z=!0},this.getCharIndex=function(){return F}}function g(e){var t=e.data,r=a[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:v,resume:v};if(M(r.userStep)){for(var s=0;s Date: Wed, 28 Nov 2018 17:38:57 +0100 Subject: [PATCH 24/25] Improve transform function docs and tests. Fixes #601 --- docs/docs.html | 2 +- tests/test-cases.js | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index bd842f1..1da2126 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -545,7 +545,7 @@ var csv = Papa.unparse({ transform - A function to apply on each value. The function receives the value as its first argument and the column number as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping. + A function to apply on each value. The function receives the value as its first argument and the column number or header name when enabled as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping. diff --git a/tests/test-cases.js b/tests/test-cases.js index 7e4986c..69b9b25 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -913,6 +913,39 @@ var PARSE_TESTS = [ errors: [] } }, + { + description: "Custom transform accepts column number also", + input: 'A,B,C\r\nd,e,f', + config: { + transform: function(value, column) { + if (column % 2) { + value = value.toLowerCase(); + } + return value; + } + }, + expected: { + data: [["A","b","C"], ["d","e","f"]], + errors: [] + } + }, + { + description: "Custom transform accepts header name when using header", + input: 'A,B,C\r\nd,e,f', + config: { + header: true, + transform: function(value, name) { + if (name === 'B') { + value = value.toUpperCase(); + } + return value; + } + }, + expected: { + data: [{'A': "d", 'B': "E", 'C': "f"}], + errors: [] + } + }, { description: "Dynamic typing converts ISO date strings to Dates", input: 'ISO date,long date\r\n2018-05-04T21:08:03.269Z,Fri May 04 2018 14:08:03 GMT-0700 (PDT)\r\n2018-05-08T15:20:22.642Z,Tue May 08 2018 08:20:22 GMT-0700 (PDT)', From a6e3a36585db737f96f0425be08e7ae1492c0347 Mon Sep 17 00:00:00 2001 From: Chris Zubak-Skees Date: Sun, 2 Dec 2018 14:16:23 -0500 Subject: [PATCH 25/25] Consistently apply regex escaping to quoteChar (#602) --- papaparse.js | 4 ++-- tests/test-cases.js | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/papaparse.js b/papaparse.js index 66dba9d..7febbaf 100755 --- a/papaparse.js +++ b/papaparse.js @@ -278,7 +278,7 @@ License: MIT unpackConfig(); - var quoteCharRegex = new RegExp(_quoteChar, 'g'); + var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g'); if (typeof _input === 'string') _input = JSON.parse(_input); @@ -1417,7 +1417,7 @@ License: MIT var nextDelim = input.indexOf(delim, cursor); var nextNewline = input.indexOf(newline, cursor); - var quoteCharRegex = new RegExp(escapeChar.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&') + quoteChar, 'g'); + var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), 'g'); var quoteSearch; // Parser loop diff --git a/tests/test-cases.js b/tests/test-cases.js index 69b9b25..2e19d68 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1358,7 +1358,7 @@ var PARSE_TESTS = [ } }, { - description: "Using reserved regex characters as quote characters", + description: "Using reserved regex character . as quote character", input: '.a\na.,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', config: { quoteChar: '.' }, expected: { @@ -1373,6 +1373,22 @@ var PARSE_TESTS = [ } } }, + { + description: "Using reserved regex character | as quote character", + input: '|a\na|,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', + config: { quoteChar: '|' }, + expected: { + data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 27, + aborted: false, + truncated: false + } + } + }, { description: "Parsing with skipEmptyLines set to 'greedy'", notes: "Must parse correctly without lines with no content",