From 4e0a80370f4afb5038c56a026fdb701f4dda7bc0 Mon Sep 17 00:00:00 2001 From: Jay L Date: Wed, 3 Oct 2018 06:05:41 -0700 Subject: [PATCH 01/40] Use Array.isArray for determining arraynes (#570) Also add polyfill in case it's not supported by browser version --- papaparse.js | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/papaparse.js b/papaparse.js index 059ccbb..31b58f5 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,9 +1,19 @@ /*@license - Papa Parse - v4.6.0 - https://github.com/mholt/PapaParse - License: MIT +Papa Parse +v4.6.0 +https://github.com/mholt/PapaParse +License: MIT */ + +// Polyfills +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill +if (!Array.isArray) +{ + Array.isArray = function(arg) { + return Object.prototype.toString.call(arg) === '[object Array]'; + }; +} + (function(root, factory) { /* globals define */ @@ -41,7 +51,6 @@ return {}; })(); - var IS_WORKER = !global.document && !!global.postMessage, IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), LOADED_SYNC = false, AUTO_SCRIPT_PATH; @@ -290,9 +299,9 @@ if (typeof _input === 'string') _input = JSON.parse(_input); - if (_input instanceof Array) + if (Array.isArray(_input)) { - if (!_input.length || _input[0] instanceof Array) + if (!_input.length || Array.isArray(_input[0])) return serialize(null, _input, _skipEmptyLines); else if (typeof _input[0] === 'object') return serialize(objectKeys(_input[0]), _input, _skipEmptyLines); @@ -302,17 +311,17 @@ if (typeof _input.data === 'string') _input.data = JSON.parse(_input.data); - if (_input.data instanceof Array) + if (Array.isArray(_input.data)) { if (!_input.fields) _input.fields = _input.meta && _input.meta.fields; if (!_input.fields) - _input.fields = _input.data[0] instanceof Array + _input.fields = Array.isArray(_input.data[0]) ? _input.fields : objectKeys(_input.data[0]); - if (!(_input.data[0] instanceof Array) && typeof _input.data[0] !== 'object') + if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object') _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] } @@ -335,7 +344,7 @@ } if (typeof _config.quotes === 'boolean' - || _config.quotes instanceof Array) + || Array.isArray(_config.quotes)) _quotes = _config.quotes; if (typeof _config.skipEmptyLines === 'boolean' @@ -374,8 +383,8 @@ if (typeof data === 'string') data = JSON.parse(data); - var hasHeader = fields instanceof Array && fields.length > 0; - var dataKeyedByField = !(data[0] instanceof Array); + var hasHeader = Array.isArray(fields) && fields.length > 0; + var dataKeyedByField = !(Array.isArray(data[0])); // If there a header row, write it first if (hasHeader && _writeHeader) @@ -424,7 +433,7 @@ str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar); var needsQuotes = (typeof _quotes === 'boolean' && _quotes) - || (_quotes instanceof Array && _quotes[col]) + || (Array.isArray(_quotes) && _quotes[col]) || hasAny(str, Papa.BAD_DELIMITERS) || str.indexOf(_delimiter) > -1 || str.charAt(0) === ' ' @@ -1778,7 +1787,7 @@ { if (typeof obj !== 'object' || obj === null) return obj; - var cpy = obj instanceof Array ? [] : {}; + var cpy = Array.isArray(obj) ? [] : {}; for (var key in obj) cpy[key] = copy(obj[key]); return cpy; From 2cf39cef65bc17a9a7b07db912c11d0ee9d35e41 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 4 Oct 2018 12:59:25 +0200 Subject: [PATCH 02/40] Add space between comment and license clasifier Fixes #565 --- papaparse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 31b58f5..e96ef32 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,4 +1,4 @@ -/*@license +/* @license Papa Parse v4.6.0 https://github.com/mholt/PapaParse From 4bb68a5365d277db9dd290914cf64ac3c461a59b Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 4 Oct 2018 13:01:38 +0200 Subject: [PATCH 03/40] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 2adb9e1..7b9e671 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "4.6.0", + "version": "4.6.1", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index e96ef32..94c81b6 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v4.6.0 +v4.6.1 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 68fb77d..5828b1d 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ -/*@license - Papa Parse - v4.6.0 - https://github.com/mholt/PapaParse - License: MIT +/* @license +Papa Parse +v4.6.1 +https://github.com/mholt/PapaParse +License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},r=!f.document&&!!f.postMessage,o=r&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,k={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&k.WORKERS_SUPPORTED){var n=function(){if(!k.WORKERS_SUPPORTED)return!1;if(!a&&null===k.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=k.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=v,t.id=u++,h[t.id]=t}();return n.userStep=t.step,n.userChunk=t.chunk,n.userComplete=t.complete,n.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void n.postMessage({input:e,config:t,workerId:n.id})}var r=null;{if(e===k.NODE_STREAM_INPUT)return(r=new g(t)).getStream();"string"==typeof e?r=t.download?new l(t):new _(t):!0===e.readable&&M(e.read)&&M(e.on)?r=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(r=new p(t))}return r.stream(e)},unparse:function(e,t){var n=!1,f=!0,d=",",c="\r\n",r='"';!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||k.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(d=t.delimiter);("boolean"==typeof t.quotes||t.quotes instanceof Array)&&(n=t.quotes);"string"==typeof t.newline&&(c=t.newline);"string"==typeof t.quoteChar&&(r=t.quoteChar);"boolean"==typeof t.header&&(f=t.header)}();var s=new RegExp(r,"g");"string"==typeof e&&(e=JSON.parse(e));if(e instanceof Array){if(!e.length||e[0]instanceof Array)return a(null,e);if("object"==typeof e[0])return a(i(e[0]),e)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),e.data instanceof Array&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=e.data[0]instanceof Array?e.fields:i(e.data[0])),e.data[0]instanceof Array||"object"==typeof e.data[0]||(e.data=[e.data])),a(e.fields||[],e.data||[]);throw"exception: Unable to serialize unrecognized input";function i(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function a(e,t){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=e instanceof Array&&0=this._config.preview;if(o)f.postMessage({results:r,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return;r=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!a||!M(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||r&&r.meta.paused||this._nextChunk(),r}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function l(e){var n;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),c.call(this,e),this._nextChunk=r?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(n=new XMLHttpRequest,this._config.withCredentials&&(n.withCredentials=this._config.withCredentials),r||(n.onload=R(this._chunkLoaded,this),n.onerror=R(this._chunkError,this)),n.open("GET",this._input,!r),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)n.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;n.setRequestHeader("Range","bytes="+this._start+"-"+i),n.setRequestHeader("If-None-Match","webkit-no-cache")}try{n.send()}catch(e){this._chunkError(e.message)}r&&0===n.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===n.readyState&&(n.status<200||400<=n.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(n),this.parseChunk(n.responseText)))},this._chunkError=function(e){var t=n.statusText||e;this._sendError(new Error(t))}}function p(e){var n,r;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),c.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,s?((n=new FileReader).onload=R(this._chunkLoaded,this),n.onerror=R(this._chunkError,this)):n=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(n.error)}}function _(e){var i;c.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){c.call(this,e=e||{});var t=[],i=!0,n=!1;this.pause=function(){c.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){c.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){n&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),n=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),n=!0,r=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():c(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function l(){if(d&&h&&(v("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(r=m.transform(r,n)),r=_(n,r),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(r)):i[n]=r}d.data[e]=i,m.header&&(t>f.length?v("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=n.length/2?"\r\n":"\r"}(e,n)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var r=function(e,t,i,n){for(var r,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=A,A++;;){if(-1===(p=n.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:A}),b();if(p===r-1)return b(n.substring(A,p).replace(g,S));if(S!==L||n[p+1]!==L){if(S===L||0===p||n[p-1]!==L){var v=C(-1===m?_:Math.min(_,m));if(n[p+1+v]===x){f.push(n.substring(A,p).replace(g,S)),A=p+1+v+e,_=n.indexOf(x,A),m=n.indexOf(T,A);break}var k=C(m);if(n.substr(p+1+k,s)===T){if(f.push(n.substring(A,p).replace(g,S)),w(p+1+k+s),_=n.indexOf(x,A),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:A}),p++}}else p++}return b();function y(e){h.push(e),d=A}function C(e){var t=0;if(-1!==e){var i=n.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=n.substr(A)),f.push(e),A=r,y(f),o&&R()),E()}function w(e){A=e,y(f),f=[],m=n.indexOf(T,A)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:T,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return A}}function v(e){var t=e.data,i=h[t.workerId],n=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var r={abort:function(){n=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s Date: Thu, 4 Oct 2018 13:03:47 +0200 Subject: [PATCH 04/40] Use latest version on demo website --- docs/resources/js/papaparse.js | 447 ++++++++++++++++++++++++--------- 1 file changed, 330 insertions(+), 117 deletions(-) diff --git a/docs/resources/js/papaparse.js b/docs/resources/js/papaparse.js index 5bef3cc..94c81b6 100644 --- a/docs/resources/js/papaparse.js +++ b/docs/resources/js/papaparse.js @@ -1,11 +1,22 @@ -/*! - Papa Parse - v4.3.7 - https://github.com/mholt/PapaParse - License: MIT +/* @license +Papa Parse +v4.6.1 +https://github.com/mholt/PapaParse +License: MIT */ + +// Polyfills +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill +if (!Array.isArray) +{ + Array.isArray = function(arg) { + return Object.prototype.toString.call(arg) === '[object Array]'; + }; +} + (function(root, factory) { + /* globals define */ if (typeof define === 'function' && define.amd) { // AMD. Register as an anonymous module. @@ -27,7 +38,7 @@ { 'use strict'; - var global = (function () { + var global = (function() { // alternative method, similar to `Function('return this')()` // but without using `eval` (which is disabled when // using Content Security Policy). @@ -40,7 +51,6 @@ return {}; })(); - var IS_WORKER = !global.document && !!global.postMessage, IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), LOADED_SYNC = false, AUTO_SCRIPT_PATH; @@ -57,6 +67,7 @@ Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously + Papa.NODE_STREAM_INPUT = 1; // Configurable chunk sizes for local and remote files, respectively Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB @@ -70,6 +81,7 @@ Papa.FileStreamer = FileStreamer; Papa.StringStreamer = StringStreamer; Papa.ReadableStreamStreamer = ReadableStreamStreamer; + Papa.DuplexStreamStreamer = DuplexStreamStreamer; if (global.jQuery) { @@ -162,7 +174,7 @@ queue.splice(0, 1); parseNextFile(); } - } + }; } @@ -182,7 +194,7 @@ } else { - document.addEventListener('DOMContentLoaded', function () { + document.addEventListener('DOMContentLoaded', function() { LOADED_SYNC = true; }, true); } @@ -202,6 +214,8 @@ } _config.dynamicTyping = dynamicTyping; + _config.transform = isFunction(_config.transform) ? _config.transform : false; + if (_config.worker && Papa.WORKERS_SUPPORTED) { var w = newWorker(); @@ -227,7 +241,14 @@ } var streamer = null; - if (typeof _input === 'string') + if (_input === Papa.NODE_STREAM_INPUT) + { + // create a node Duplex stream for use + // with .pipe + streamer = new DuplexStreamStreamer(_config); + return streamer.getStream(); + } + else if (typeof _input === 'string') { if (_config.download) streamer = new NetworkStreamer(_config); @@ -251,9 +272,6 @@ function JsonToCsv(_input, _config) { - var _output = ''; - var _fields = []; - // Default configuration /** whether to surround every datum with quotes */ @@ -262,7 +280,7 @@ /** whether to write headers */ var _writeHeader = true; - /** delimiting character */ + /** delimiting character(s) */ var _delimiter = ','; /** newline character(s) */ @@ -271,6 +289,9 @@ /** quote character */ var _quoteChar = '"'; + /** whether to skip empty lines */ + var _skipEmptyLines = false; + unpackConfig(); var quoteCharRegex = new RegExp(_quoteChar, 'g'); @@ -278,33 +299,33 @@ if (typeof _input === 'string') _input = JSON.parse(_input); - if (_input instanceof Array) + if (Array.isArray(_input)) { - if (!_input.length || _input[0] instanceof Array) - return serialize(null, _input); + if (!_input.length || Array.isArray(_input[0])) + return serialize(null, _input, _skipEmptyLines); else if (typeof _input[0] === 'object') - return serialize(objectKeys(_input[0]), _input); + return serialize(objectKeys(_input[0]), _input, _skipEmptyLines); } else if (typeof _input === 'object') { if (typeof _input.data === 'string') _input.data = JSON.parse(_input.data); - if (_input.data instanceof Array) + if (Array.isArray(_input.data)) { if (!_input.fields) _input.fields = _input.meta && _input.meta.fields; if (!_input.fields) - _input.fields = _input.data[0] instanceof Array - ? _input.fields - : objectKeys(_input.data[0]); + _input.fields = Array.isArray(_input.data[0]) + ? _input.fields + : objectKeys(_input.data[0]); - if (!(_input.data[0] instanceof Array) && typeof _input.data[0] !== 'object') + if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object') _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] } - return serialize(_input.fields || [], _input.data || []); + return serialize(_input.fields || [], _input.data || [], _skipEmptyLines); } // Default (any valid paths should return before this) @@ -317,16 +338,19 @@ return; if (typeof _config.delimiter === 'string' - && _config.delimiter.length === 1 - && Papa.BAD_DELIMITERS.indexOf(_config.delimiter) === -1) + && !Papa.BAD_DELIMITERS.filter(function(value) { return _config.delimiter.indexOf(value) !== -1; }).length) { _delimiter = _config.delimiter; } if (typeof _config.quotes === 'boolean' - || _config.quotes instanceof Array) + || Array.isArray(_config.quotes)) _quotes = _config.quotes; + if (typeof _config.skipEmptyLines === 'boolean' + || typeof _config.skipEmptyLines === 'string') + _skipEmptyLines = _config.skipEmptyLines; + if (typeof _config.newline === 'string') _newline = _config.newline; @@ -350,7 +374,7 @@ } /** The double for loop that iterates the data and writes out a CSV string including header row */ - function serialize(fields, data) + function serialize(fields, data, skipEmptyLines) { var csv = ''; @@ -359,8 +383,8 @@ if (typeof data === 'string') data = JSON.parse(data); - var hasHeader = fields instanceof Array && fields.length > 0; - var dataKeyedByField = !(data[0] instanceof Array); + var hasHeader = Array.isArray(fields) && fields.length > 0; + var dataKeyedByField = !(Array.isArray(data[0])); // If there a header row, write it first if (hasHeader && _writeHeader) @@ -379,19 +403,21 @@ for (var row = 0; row < data.length; row++) { var maxCol = hasHeader ? fields.length : data[row].length; + var r = hasHeader ? fields : data[row]; - for (var col = 0; col < maxCol; col++) + if (skipEmptyLines !== 'greedy' || r.join('').trim() !== '') { - if (col > 0) - csv += _delimiter; - var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; - csv += safe(data[row][colIdx], col); + for (var col = 0; col < maxCol; col++) + { + if (col > 0) + csv += _delimiter; + var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; + csv += safe(data[row][colIdx], col); + } + if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0)) + csv += _newline; } - - if (row < data.length - 1) - csv += _newline; } - return csv; } @@ -401,10 +427,13 @@ if (typeof str === 'undefined' || str === null) return ''; - str = str.toString().replace(quoteCharRegex, _quoteChar+_quoteChar); + if (str.constructor === Date) + return JSON.stringify(str).slice(1, 25); + + str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar); var needsQuotes = (typeof _quotes === 'boolean' && _quotes) - || (_quotes instanceof Array && _quotes[col]) + || (Array.isArray(_quotes) && _quotes[col]) || hasAny(str, Papa.BAD_DELIMITERS) || str.indexOf(_delimiter) > -1 || str.charAt(0) === ' ' @@ -426,8 +455,8 @@ function ChunkStreamer(config) { this._handle = null; - this._paused = false; this._finished = false; + this._completed = false; this._input = null; this._baseIndex = 0; this._partialLine = ''; @@ -442,7 +471,7 @@ }; replaceConfig.call(this, config); - this.parseChunk = function(chunk) + this.parseChunk = function(chunk, isFakeChunk) { // First chunk pre-processing if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) @@ -483,10 +512,10 @@ finished: finishedIncludingPreview }); } - else if (isFunction(this._config.chunk)) + else if (isFunction(this._config.chunk) && !isFakeChunk) { this._config.chunk(results, this._handle); - if (this._paused) + if (this._handle.paused() || this._handle.aborted()) return; results = undefined; this._completeResults = undefined; @@ -498,8 +527,10 @@ this._completeResults.meta = results.meta; } - if (finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) + if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) { this._config.complete(this._completeResults, this._input); + this._completed = true; + } if (!finishedIncludingPreview && (!results || !results.meta.paused)) this._nextChunk(); @@ -602,7 +633,7 @@ if (this._config.chunkSize) { var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive - xhr.setRequestHeader('Range', 'bytes='+this._start+'-'+end); + xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end); xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672 } @@ -617,11 +648,11 @@ this._chunkError(); else this._start += this._config.chunkSize; - } + }; this._chunkLoaded = function() { - if (xhr.readyState != 4) + if (xhr.readyState !== 4) return; if (xhr.status < 200 || xhr.status >= 400) @@ -632,20 +663,20 @@ this._finished = !this._config.chunkSize || this._start > getFileSize(xhr); this.parseChunk(xhr.responseText); - } + }; this._chunkError = function(errorMessage) { var errorText = xhr.statusText || errorMessage; - this._sendError(errorText); - } + this._sendError(new Error(errorText)); + }; function getFileSize(xhr) { var contentRange = xhr.getResponseHeader('Content-Range'); if (contentRange === null) { // no content range, then finish! - return -1; - } + return -1; + } return parseInt(contentRange.substr(contentRange.lastIndexOf('/') + 1)); } } @@ -687,7 +718,7 @@ { if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview)) this._readChunk(); - } + }; this._readChunk = function() { @@ -700,7 +731,7 @@ var txt = reader.readAsText(input, this._config.encoding); if (!usingAsyncReader) this._chunkLoaded({ target: { result: txt } }); // mimic the async signature - } + }; this._chunkLoaded = function(event) { @@ -708,12 +739,12 @@ this._start += this._config.chunkSize; this._finished = !this._config.chunkSize || this._start >= this._input.size; this.parseChunk(event.target.result); - } + }; this._chunkError = function() { - this._sendError(reader.error.message); - } + this._sendError(reader.error); + }; } FileStreamer.prototype = Object.create(ChunkStreamer.prototype); @@ -725,14 +756,12 @@ config = config || {}; ChunkStreamer.call(this, config); - var string; var remaining; this.stream = function(s) { - string = s; remaining = s; return this._nextChunk(); - } + }; this._nextChunk = function() { if (this._finished) return; @@ -741,7 +770,7 @@ remaining = size ? remaining.substr(size) : ''; this._finished = !remaining; return this.parseChunk(chunk); - } + }; } StringStreamer.prototype = Object.create(StringStreamer.prototype); StringStreamer.prototype.constructor = StringStreamer; @@ -755,6 +784,19 @@ var queue = []; var parseOnData = true; + var streamHasEnded = false; + + this.pause = function() + { + ChunkStreamer.prototype.pause.apply(this, arguments); + this._input.pause(); + }; + + this.resume = function() + { + ChunkStreamer.prototype.resume.apply(this, arguments); + this._input.resume(); + }; this.stream = function(stream) { @@ -763,10 +805,18 @@ this._input.on('data', this._streamData); this._input.on('end', this._streamEnd); this._input.on('error', this._streamError); - } + }; + + this._checkIsFinished = function() + { + if (streamHasEnded && queue.length === 1) { + this._finished = true; + } + }; this._nextChunk = function() { + this._checkIsFinished(); if (queue.length) { this.parseChunk(queue.shift()); @@ -775,7 +825,7 @@ { parseOnData = true; } - } + }; this._streamData = bindFunction(function(chunk) { @@ -786,6 +836,7 @@ if (parseOnData) { parseOnData = false; + this._checkIsFinished(); this.parseChunk(queue.shift()); } } @@ -798,13 +849,13 @@ this._streamError = bindFunction(function(error) { this._streamCleanUp(); - this._sendError(error.message); + this._sendError(error); }, this); this._streamEnd = bindFunction(function() { this._streamCleanUp(); - this._finished = true; + streamHasEnded = true; this._streamData(''); }, this); @@ -819,14 +870,117 @@ ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer; + function DuplexStreamStreamer(_config) { + var Duplex = require('stream').Duplex; + var config = copy(_config); + var parseOnWrite = true; + var writeStreamHasFinished = false; + var parseCallbackQueue = []; + var stream = null; + + this._onCsvData = function(results) + { + var data = results.data; + for (var i = 0; i < data.length; i++) { + if (!stream.push(data[i]) && !this._handle.paused()) { + // the writeable consumer buffer has filled up + // so we need to pause until more items + // can be processed + this._handle.pause(); + } + } + }; + + this._onCsvComplete = function() + { + // node will finish the read stream when + // null is pushed + stream.push(null); + }; + + config.step = bindFunction(this._onCsvData, this); + config.complete = bindFunction(this._onCsvComplete, this); + ChunkStreamer.call(this, config); + + this._nextChunk = function() + { + if (writeStreamHasFinished && parseCallbackQueue.length === 1) { + this._finished = true; + } + if (parseCallbackQueue.length) { + parseCallbackQueue.shift()(); + } else { + parseOnWrite = true; + } + }; + + this._addToParseQueue = function(chunk, callback) + { + // add to queue so that we can indicate + // completion via callback + // node will automatically pause the incoming stream + // when too many items have been added without their + // callback being invoked + parseCallbackQueue.push(bindFunction(function() { + this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding)); + if (isFunction(callback)) { + return callback(); + } + }, this)); + if (parseOnWrite) { + parseOnWrite = false; + this._nextChunk(); + } + }; + + this._onRead = function() + { + if (this._handle.paused()) { + // the writeable consumer can handle more data + // so resume the chunk parsing + this._handle.resume(); + } + }; + + this._onWrite = function(chunk, encoding, callback) + { + this._addToParseQueue(chunk, callback); + }; + + this._onWriteComplete = function() + { + writeStreamHasFinished = true; + // have to write empty string + // so parser knows its done + this._addToParseQueue(''); + }; + + this.getStream = function() + { + return stream; + }; + stream = new Duplex({ + readableObjectMode: true, + decodeStrings: false, + read: bindFunction(this._onRead, this), + write: bindFunction(this._onWrite, this) + }); + stream.once('finish', bindFunction(this._onWriteComplete, this)); + } + DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); + DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; + + // Use one ParserHandle per entire CSV file or string function ParserHandle(_config) { // One goal is to minimize the use of regular expressions... var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; + var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/; var self = this; var _stepCounter = 0; // Number of times step was called (number of rows parsed) + var _rowCounter = 0; // Number of rows that have been parsed so far var _input; // The input being parsed var _parser; // The core parser being used var _paused = false; // Whether we are paused or not @@ -872,13 +1026,14 @@ */ this.parse = function(input, baseIndex, ignoreLastRow) { + var quoteChar = _config.quoteChar || '"'; if (!_config.newline) - _config.newline = guessLineEndings(input); + _config.newline = guessLineEndings(input, quoteChar); _delimiterError = false; if (!_config.delimiter) { - var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines); + var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments); if (delimGuess.successful) _config.delimiter = delimGuess.bestDelimiter; else @@ -920,10 +1075,10 @@ this.resume = function() { _paused = false; - self.streamer.parseChunk(_input); + self.streamer.parseChunk(_input, true); }; - this.aborted = function () + this.aborted = function() { return _aborted; }; @@ -938,25 +1093,29 @@ _input = ''; }; + function testEmptyLine(s) { + return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0; + } + function processResults() { if (_results && _delimiterError) { - addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \''+Papa.DefaultDelimiter+'\''); + addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \'' + Papa.DefaultDelimiter + '\''); _delimiterError = false; } if (_config.skipEmptyLines) { for (var i = 0; i < _results.data.length; i++) - if (_results.data[i].length === 1 && _results.data[i][0] === '') + if (testEmptyLine(_results.data[i])) _results.data.splice(i--, 1); } if (needsHeaderRow()) fillHeaderFields(); - return applyHeaderAndDynamicTyping(); + return applyHeaderAndDynamicTypingAndTransformation(); } function needsHeaderRow() @@ -970,7 +1129,15 @@ return; for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) for (var j = 0; j < _results.data[i].length; j++) - _fields.push(_results.data[i][j]); + { + var header = _results.data[i][j]; + + if (_config.trimHeaders) { + header = header.trim(); + } + + _fields.push(header); + } _results.data.splice(0, 1); } @@ -979,7 +1146,7 @@ if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) { _config.dynamicTyping[field] = _config.dynamicTypingFunction(field); } - return (_config.dynamicTyping[field] || _config.dynamicTyping) === true + return (_config.dynamicTyping[field] || _config.dynamicTyping) === true; } function parseDynamic(field, value) @@ -990,22 +1157,27 @@ return true; else if (value === 'false' || value === 'FALSE') return false; + else if (FLOAT.test(value)) + return parseFloat(value); + else if (ISO_DATE.test(value)) + return new Date(value); else - return tryParseFloat(value); + return (value === '' ? null : value); } return value; } - function applyHeaderAndDynamicTyping() + function applyHeaderAndDynamicTypingAndTransformation() { - if (!_results || (!_config.header && !_config.dynamicTyping)) + if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform)) return _results; for (var i = 0; i < _results.data.length; i++) { var row = _config.header ? {} : []; - for (var j = 0; j < _results.data[i].length; j++) + var j; + for (j = 0; j < _results.data[i].length; j++) { var field = j; var value = _results.data[i][j]; @@ -1013,6 +1185,9 @@ if (_config.header) field = j >= _fields.length ? '__parsed_extra' : _fields[j]; + if (_config.transform) + value = _config.transform(value,field); + value = parseDynamic(field, value); if (field === '__parsed_extra') @@ -1029,18 +1204,20 @@ if (_config.header) { if (j > _fields.length) - addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, i); + addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); else if (j < _fields.length) - addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, i); + addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); } } if (_config.header && _results.meta) _results.meta.fields = _fields; + + _rowCounter += _results.data.length; return _results; } - function guessDelimiter(input, newline, skipEmptyLines) + function guessDelimiter(input, newline, skipEmptyLines, comments) { var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; var bestDelim, bestDelta, fieldCountPrevRow; @@ -1052,6 +1229,7 @@ fieldCountPrevRow = undefined; var preview = new Parser({ + comments: comments, delimiter: delim, newline: newline, preview: 10 @@ -1059,9 +1237,10 @@ for (var j = 0; j < preview.data.length; j++) { - if (skipEmptyLines && preview.data[j].length === 1 && preview.data[j][0].length === 0) { - emptyLinesCount++ - continue + if (skipEmptyLines && testEmptyLine(preview.data[j])) + { + emptyLinesCount++; + continue; } var fieldCount = preview.data[j].length; avgFieldCount += fieldCount; @@ -1094,12 +1273,15 @@ return { successful: !!bestDelim, bestDelimiter: bestDelim - } + }; } - function guessLineEndings(input) + function guessLineEndings(input, quoteChar) { - input = input.substr(0, 1024*1024); // max length 1 MB + input = input.substr(0, 1024 * 1024); // max length 1 MB + // Replace all the text inside quotes + var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); + input = input.replace(re, ''); var r = input.split('\r'); @@ -1120,12 +1302,6 @@ return numWithN >= r.length / 2 ? '\r\n' : '\r'; } - function tryParseFloat(val) - { - var isNumber = FLOAT.test(val); - return isNumber ? parseFloat(val) : val; - } - function addError(type, code, msg, row) { _results.errors.push({ @@ -1137,9 +1313,11 @@ } } - - - + /** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */ + function escapeRegExp(string) + { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string + } /** The core parser implements speedy and correct CSV parsing */ function Parser(config) @@ -1152,11 +1330,16 @@ var step = config.step; var preview = config.preview; var fastMode = config.fastMode; + var quoteChar; /** Allows for no quoteChar by setting quoteChar to undefined in config */ - if (config.quoteChar === undefined){ - var quoteChar = '"'; + if (config.quoteChar === undefined) { + quoteChar = '"'; } else { - var quoteChar = config.quoteChar; + quoteChar = config.quoteChar; + } + var escapeChar = quoteChar; + if (config.escapeChar !== undefined) { + escapeChar = config.escapeChar; } // Delimiter must be valid @@ -1174,7 +1357,7 @@ comments = false; // Newline must be valid: \r, \n, or \r\n - if (newline != '\n' && newline != '\r' && newline != '\r\n') + if (newline !== '\n' && newline !== '\r' && newline !== '\r\n') newline = '\n'; // We're gonna need these at the Parser scope @@ -1207,7 +1390,7 @@ var rows = input.split(newline); for (var i = 0; i < rows.length; i++) { - var row = rows[i]; + row = rows[i]; cursor += row.length; if (i !== rows.length - 1) cursor += newline.length; @@ -1236,7 +1419,8 @@ var nextDelim = input.indexOf(delim, cursor); var nextNewline = input.indexOf(newline, cursor); - var quoteCharRegex = new RegExp(quoteChar+quoteChar, 'g'); + var quoteCharRegex = new RegExp(escapeChar.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&') + quoteChar, 'g'); + var quoteSearch; // Parser loop for (;;) @@ -1245,7 +1429,7 @@ if (input[cursor] === quoteChar) { // Start our search for the closing quote where the cursor is - var quoteSearch = cursor; + quoteSearch = cursor; // Skip the opening quote cursor++; @@ -1253,7 +1437,7 @@ for (;;) { // Find closing quote - var quoteSearch = input.indexOf(quoteChar, quoteSearch+1); + quoteSearch = input.indexOf(quoteChar, quoteSearch + 1); //No other quotes are found - no other delimiters if (quoteSearch === -1) @@ -1272,34 +1456,47 @@ } // Closing quote at EOF - if (quoteSearch === inputLen-1) + if (quoteSearch === inputLen - 1) { var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar); return finish(value); } // If this quote is escaped, it's part of the data; skip it - if (input[quoteSearch+1] === quoteChar) + // If the quote character is the escape character, then check if the next character is the escape character + if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) { quoteSearch++; continue; } - // Closing quote followed by delimiter - if (input[quoteSearch+1] === delim) + // If the quote character is not the escape character, then check if the previous character was the escape character + if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar) + { + continue; + } + + // Check up to nextDelim or nextNewline, whichever is closest + var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline); + var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo); + + // Closing quote followed by delimiter or 'unnecessary spaces + delimiter' + if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter] === delim) { row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); - cursor = quoteSearch + 1 + delimLen; + cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen; nextDelim = input.indexOf(delim, cursor); nextNewline = input.indexOf(newline, cursor); break; } - // Closing quote followed by newline - if (input.substr(quoteSearch+1, newlineLen) === newline) + var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline); + + // Closing quote followed by newline or 'unnecessary spaces + newLine' + if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, newlineLen) === newline) { row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); - saveRow(quoteSearch + 1 + newlineLen); + saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen); nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field if (stepIsFunction) @@ -1385,6 +1582,21 @@ lastCursor = cursor; } + /** + * checks if there are extra spaces after closing quote and given index without any text + * if Yes, returns the number of spaces + */ + function extraSpaces(index) { + var spaceLength = 0; + if (index !== -1) { + var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index); + if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === '') { + spaceLength = textBetweenClosingQuoteAndIndex.length; + } + } + return spaceLength; + } + /** * Appends the remaining input from cursor to the end into * row, saves the row, calls step, and returns the results. @@ -1437,7 +1649,8 @@ function doStep() { step(returnable()); - data = [], errors = []; + data = []; + errors = []; } }; @@ -1572,9 +1785,9 @@ /** Makes a deep copy of an array or object (mostly) */ function copy(obj) { - if (typeof obj !== 'object') + if (typeof obj !== 'object' || obj === null) return obj; - var cpy = obj instanceof Array ? [] : {}; + var cpy = Array.isArray(obj) ? [] : {}; for (var key in obj) cpy[key] = copy(obj[key]); return cpy; From fff1ee1be640510c17041951e1a5a5e8bac5dcb7 Mon Sep 17 00:00:00 2001 From: Jay L Date: Tue, 9 Oct 2018 01:34:28 -0700 Subject: [PATCH 05/40] Fixes unparsing using 'skipEmptyLines' = 'greedy' when using headers (#569) --- papaparse.js | 23 +++++++++++++++++++---- tests/test-cases.js | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/papaparse.js b/papaparse.js index 94c81b6..6a0a115 100755 --- a/papaparse.js +++ b/papaparse.js @@ -403,19 +403,34 @@ if (!Array.isArray) for (var row = 0; row < data.length; row++) { var maxCol = hasHeader ? fields.length : data[row].length; - var r = hasHeader ? fields : data[row]; - if (skipEmptyLines !== 'greedy' || r.join('').trim() !== '') + var emptyLine = false; + var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0; + if (skipEmptyLines && !hasHeader) + { + emptyLine = skipEmptyLines === 'greedy' ? data[row].join('').trim() === '' : data[row].length === 1 && data[row][0].length === 0; + } + if (skipEmptyLines === 'greedy' && hasHeader) { + var line = []; + for (var c = 0; c < maxCol; c++) { + var cx = dataKeyedByField ? fields[c] : c; + line.push(data[row][cx]); + } + emptyLine = line.join('').trim() === ''; + } + if (!emptyLine) { for (var col = 0; col < maxCol; col++) { - if (col > 0) + if (col > 0 && !nullLine) csv += _delimiter; var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; csv += safe(data[row][colIdx], col); } - if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0)) + if (row < data.length - 1 && (!skipEmptyLines || (maxCol > 0 && !nullLine))) + { csv += _newline; + } } } return csv; diff --git a/tests/test-cases.js b/tests/test-cases.js index deb2e65..98509de 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1654,6 +1654,24 @@ var UNPARSE_TESTS = [ input: [[null, ' '], [], ['1', '2']], config: {skipEmptyLines: 'greedy'}, expected: '1,2' + }, + { + description: "Returns empty rows when empty rows are passed and skipEmptyLines is false with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: false, header: true}, + expected: 'a,b\r\n," "\r\n\r\n1,2' + }, + { + description: "Returns without empty rows when skipEmptyLines is true with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: true, header: true}, + expected: 'a,b\r\n," "\r\n1,2' + }, + { + description: "Returns without rows with no content when skipEmptyLines is 'greedy' with headers", + input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], + config: {skipEmptyLines: 'greedy', header: true}, + expected: 'a,b\r\n1,2' } ]; From 0e392e2f2468aff246969a9102ff9762c37c2875 Mon Sep 17 00:00:00 2001 From: Jeff Barnes Date: Mon, 8 Oct 2018 10:48:17 -0800 Subject: [PATCH 06/40] Avoid use of node stream in browser context. --- Gruntfile.js | 6 ++++++ package.json | 1 + papaparse.js | 12 ++++++++---- papaparse.min.js | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index e645b52..93b1256 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -2,6 +2,12 @@ module.exports = function(grunt) { grunt.initConfig({ uglify: { options: { + compress: { + global_defs: { + 'PAPA_BROWSER_CONTEXT': true + }, + dead_code: true + }, output: { comments: 'some', }, diff --git a/package.json b/package.json index 7b9e671..ab6e6fc 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ }, "license": "MIT", "main": "papaparse.js", + "browser": "papaparse.min.js", "devDependencies": { "chai": "^4.2.0", "connect": "^3.3.3", diff --git a/papaparse.js b/papaparse.js index 6a0a115..bcf3139 100755 --- a/papaparse.js +++ b/papaparse.js @@ -81,7 +81,9 @@ if (!Array.isArray) Papa.FileStreamer = FileStreamer; Papa.StringStreamer = StringStreamer; Papa.ReadableStreamStreamer = ReadableStreamStreamer; - Papa.DuplexStreamStreamer = DuplexStreamStreamer; + if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { + Papa.DuplexStreamStreamer = DuplexStreamStreamer; + } if (global.jQuery) { @@ -241,7 +243,7 @@ if (!Array.isArray) } var streamer = null; - if (_input === Papa.NODE_STREAM_INPUT) + if (_input === Papa.NODE_STREAM_INPUT && typeof PAPA_BROWSER_CONTEXT === 'undefined') { // create a node Duplex stream for use // with .pipe @@ -982,8 +984,10 @@ if (!Array.isArray) }); stream.once('finish', bindFunction(this._onWriteComplete, this)); } - DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); - DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; + if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { + DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); + DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; + } // Use one ParserHandle per entire CSV file or string diff --git a/papaparse.min.js b/papaparse.min.js index 5828b1d..32ebc35 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -4,4 +4,4 @@ v4.6.1 https://github.com/mholt/PapaParse License: MIT */ -Array.isArray||(Array.isArray=function(e){return"[object Array]"===Object.prototype.toString.call(e)}),function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},n=!f.document&&!!f.postMessage,o=n&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,v={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&v.WORKERS_SUPPORTED){var r=function(){if(!v.WORKERS_SUPPORTED)return!1;if(!a&&null===v.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=v.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=y,t.id=u++,h[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;{if(e===v.NODE_STREAM_INPUT)return(n=new g(t)).getStream();"string"==typeof e?n=t.download?new c(t):new _(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new p(t))}return n.stream(e)},unparse:function(e,t){var r=!1,l=!0,c=",",p="\r\n",n='"',i=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(c=t.delimiter);("boolean"==typeof t.quotes||Array.isArray(t.quotes))&&(r=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(p=t.newline);"string"==typeof t.quoteChar&&(n=t.quoteChar);"boolean"==typeof t.header&&(l=t.header)}();var s=new RegExp(n,"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return o(null,e,i);if("object"==typeof e[0])return o(a(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:a(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),o(e.fields||[],e.data||[],i);throw"exception: Unable to serialize unrecognized input";function a(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function o(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s Date: Wed, 14 Nov 2018 10:35:09 +0100 Subject: [PATCH 07/40] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index ab6e6fc..90e8ca8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "4.6.1", + "version": "4.6.2", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index bcf3139..f39a108 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v4.6.1 +v4.6.2 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 32ebc35..95d7b63 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v4.6.1 +v4.6.2 https://github.com/mholt/PapaParse License: MIT */ -Array.isArray||(Array.isArray=function(e){return"[object Array]"===Object.prototype.toString.call(e)}),function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},n=!f.document&&!!f.postMessage,o=n&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,v={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&v.WORKERS_SUPPORTED){var r=function(){if(!v.WORKERS_SUPPORTED)return!1;if(!a&&null===v.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=v.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=y,t.id=u++,h[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;{if(e===v.NODE_STREAM_INPUT)return(n=new g(t)).getStream();"string"==typeof e?n=t.download?new c(t):new _(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new p(t))}return n.stream(e)},unparse:function(e,t){var r=!1,l=!0,c=",",p="\r\n",n='"',i=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(c=t.delimiter);("boolean"==typeof t.quotes||Array.isArray(t.quotes))&&(r=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(p=t.newline);"string"==typeof t.quoteChar&&(n=t.quoteChar);"boolean"==typeof t.header&&(l=t.header)}();var s=new RegExp(n,"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return o(null,e,i);if("object"==typeof e[0])return o(a(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:a(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),o(e.fields||[],e.data||[],i);throw"exception: Unable to serialize unrecognized input";function a(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function o(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:v.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function c(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i),r.setRequestHeader("If-None-Match","webkit-no-cache")}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function p(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=R(this._chunkLoaded,this),r.onerror=R(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function _(e){var i;l.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?i.substr(0,e):i;return i=e?i.substr(e):"",this._finished=!i,this.parseChunk(t)}}}function m(e){l.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=R(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=R(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=R(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=R(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function g(e){var t=require("stream").Duplex,i=E(e),r=!0,n=!1,s=[],a=null;this._onCsvData=function(e){for(var t=e.data,i=0;im.preview?o.abort():l(d,t)}}}function g(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),m.transform&&(n=m.transform(n,r)),n=_(r,n),"__parsed_extra"===r?(i[r]=i[r]||[],i[r].push(n)):i[r]=n}d.data[e]=i,m.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),d.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r){for(var n,s,a,o=[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP],h=0;h=D)return E(!0)}else for(p=P,P++;;){if(-1===(p=r.indexOf(S,p+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),b();if(p===n-1)return b(r.substring(P,p).replace(g,S));if(S!==L||r[p+1]!==L){if(S===L||0===p||r[p-1]!==L){var y=C(-1===m?_:Math.min(_,m));if(r[p+1+y]===x){f.push(r.substring(P,p).replace(g,S)),P=p+1+y+e,_=r.indexOf(x,P),m=r.indexOf(O,P);break}var v=C(m);if(r.substr(p+1+v,s)===O){if(f.push(r.substring(P,p).replace(g,S)),w(p+1+v+s),_=r.indexOf(x,P),o&&(R(),F))return E();if(D&&h.length>=D)return E(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return b();function k(e){h.push(e),d=P}function C(e){var t=0;if(-1!==e){var i=r.substring(p+1,e);i&&""===i.trim()&&(t=i.length)}return t}function b(e){return i||(void 0===e&&(e=r.substr(P)),f.push(e),P=n,k(f),o&&R()),E()}function w(e){P=e,k(f),f=[],m=r.indexOf(O,P)}function E(e){return{data:h,errors:u,meta:{delimiter:x,linebreak:O,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(E()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function y(e){var t=e.data,i=h[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,b(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:w,resume:w};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function c(e){var i;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r),i.setRequestHeader("If-None-Match","webkit-no-cache")}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function p(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function g(e){var r;l.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function _(e){l.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=C(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=C(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=C(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=C(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(_){var a,o,h,i=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,n=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,s=0,u=!1,e=!1,f=[],d={data:[],errors:[],meta:{}};if(M(_.step)){var l=_.step;_.step=function(e){if(d=e,p())c();else{if(c(),0===d.data.length)return;r+=e.data.length,_.preview&&r>_.preview?o.abort():l(d,t)}}}function m(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),_.transform&&(n=_.transform(n,i)),n=g(i,n),"__parsed_extra"===i?(r[i]=r[i]||[],r[i].push(n)):r[i]=n}d.data[e]=r,_.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=i.length/2?"\r\n":"\r"}(e,i)),h=!1,_.delimiter)M(_.delimiter)&&(_.delimiter=_.delimiter(e),d.meta.delimiter=_.delimiter);else{var n=function(e,t,r,i){for(var n,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=A)return C(!0)}else for(p=P,P++;;){if(-1===(p=i.indexOf(S,p+1)))return r||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),E();if(p===n-1)return E(i.substring(P,p).replace(m,S));if(S!==L||i[p+1]!==L){if(S===L||0===p||i[p-1]!==L){var y=b(-1===_?g:Math.min(g,_));if(i[p+1+y]===O){f.push(i.substring(P,p).replace(m,S)),P=p+1+y+e,g=i.indexOf(O,P),_=i.indexOf(x,P);break}var v=b(_);if(i.substr(p+1+v,s)===x){if(f.push(i.substring(P,p).replace(m,S)),w(p+1+v+s),g=i.indexOf(O,P),o&&(R(),F))return C();if(A&&h.length>=A)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return E();function k(e){h.push(e),d=P}function b(e){var t=0;if(-1!==e){var r=i.substring(p+1,e);r&&""===r.trim()&&(t=r.length)}return t}function E(e){return r||(void 0===e&&(e=i.substr(P)),f.push(e),P=n,k(f),o&&R()),C()}function w(e){P=e,k(f),f=[],_=i.indexOf(x,P)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(C()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function m(e){var t=e.data,r=h[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,y(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:E,resume:E};if(M(r.userStep)){for(var s=0;s Date: Wed, 14 Nov 2018 11:23:05 +0100 Subject: [PATCH 08/40] Use 5.0.0-alpha as version number Closes #582 --- package.json | 2 +- papaparse.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 90e8ca8..ae43241 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "4.6.2", + "version": "5.0.0-alpha", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index f39a108..8703ec8 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v4.6.2 +v5.0.0-alpha https://github.com/mholt/PapaParse License: MIT */ From 30cf3247058564534e88421f70fbe58e5b5f0305 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 14 Nov 2018 11:26:05 +0100 Subject: [PATCH 09/40] Remove support for node6 and add support for node11 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4ef1c4..f00e0fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: node_js node_js: - - "6" - "8" - "9" - "10" - + - "11" From c95db1f64c95ef1e0c33fa50fd0e0b7a92facc75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Sardinha?= Date: Wed, 14 Nov 2018 10:29:12 +0000 Subject: [PATCH 10/40] Allow transforming header columns (#589) * Deprecate trimHeaders: The same result can be achieved by using transformHeader parameter. --- docs/docs.html | 6 +++--- papaparse.js | 4 ++-- tests/test-cases.js | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index 68f1132..b424c7a 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -340,7 +340,7 @@ var csv = Papa.unparse({ quoteChar: '"', escapeChar: '"', header: false, - trimHeaders: false, + transformHeader: undefined, dynamicTyping: false, preview: 0, encoding: "", @@ -412,10 +412,10 @@ var csv = Papa.unparse({ - trimHeaders + transformHeader - If true leading/trailing spaces will be trimed from headers. + A function to apply on each header. Requires header to be true. The function receives the header as its first argument. diff --git a/papaparse.js b/papaparse.js index 8703ec8..ababc70 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1151,8 +1151,8 @@ if (!Array.isArray) { var header = _results.data[i][j]; - if (_config.trimHeaders) { - header = header.trim(); + if (isFunction(_config.transformHeader)) { + header = _config.transformHeader(header); } _fields.push(header); diff --git a/tests/test-cases.js b/tests/test-cases.js index 98509de..bfc1d04 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -725,11 +725,11 @@ var PARSE_TESTS = [ } }, { - description: "Header rows are trimmed when trimHeaders is set", - input: ' A , B , C \r\na,b ,c', - config: { header: true, trimHeaders: true }, + description: "Header rows are transformed when transformHeader function is provided", + input: 'A,B,C\r\na,b,c', + config: { header: true, transformHeader: function(header) { return header.toLowerCase(); } }, expected: { - data: [{"A": "a", "B": "b ", "C": "c"}], + data: [{"a": "a", "b": "b", "c": "c"}], errors: [] } }, From 2151cf1b906f6e168ce6d2198bb7237c4ef4a3b2 Mon Sep 17 00:00:00 2001 From: Gabe Gorelick Date: Wed, 14 Nov 2018 04:36:35 -0600 Subject: [PATCH 11/40] Throw Error objects instead of strings (#497) Unlike strings, Error objects allow you to get stack traces. Also, a lot of libraries will complain if they encounter a thrown non-Error object. This is in line with standard Javascript best practices. Per https://eslint.org/docs/rules/no-throw-literal: --- .eslintrc.js | 2 +- papaparse.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index f63bc09..c3ed7c5 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -178,7 +178,7 @@ module.exports = { "no-tabs": "off", "no-template-curly-in-string": "error", "no-ternary": "off", - "no-throw-literal": "off", + "no-throw-literal": "error", "no-trailing-spaces": "error", "no-undef-init": "error", "no-undefined": "off", diff --git a/papaparse.js b/papaparse.js index ababc70..0edec76 100755 --- a/papaparse.js +++ b/papaparse.js @@ -331,7 +331,7 @@ if (!Array.isArray) } // Default (any valid paths should return before this) - throw 'exception: Unable to serialize unrecognized input'; + throw new Error('Unable to serialize unrecognized input'); function unpackConfig() @@ -1368,7 +1368,7 @@ if (!Array.isArray) // Comment character must be valid if (comments === delim) - throw 'Comment character same as delimiter'; + throw new Error('Comment character same as delimiter'); else if (comments === true) comments = '#'; else if (typeof comments !== 'string' @@ -1387,7 +1387,7 @@ if (!Array.isArray) { // For some reason, in Chrome, this speeds things up (!?) if (typeof input !== 'string') - throw 'Input must be a string'; + throw new Error('Input must be a string'); // We don't need to compute some of these every time parse() is called, // but having them in a more local scope seems to perform better @@ -1770,7 +1770,7 @@ if (!Array.isArray) } function notImplemented() { - throw 'Not implemented.'; + throw new Error('Not implemented.'); } /** Callback when worker thread receives a message */ From a7eb6343be9b9e06b9c28c908899c52b1a7d4ee5 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 14 Nov 2018 11:37:57 +0100 Subject: [PATCH 12/40] Return unique row as data on step function (#500) --- papaparse.js | 7 ++++--- tests/test-cases.js | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/papaparse.js b/papaparse.js index 0edec76..8676056 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1649,10 +1649,11 @@ if (!Array.isArray) } /** Returns an object with the results, errors, and meta. */ - function returnable(stopped) + function returnable(stopped, step) { + var isStep = step || false; return { - data: data, + data: isStep ? data[0] : data, errors: errors, meta: { delimiter: delim, @@ -1667,7 +1668,7 @@ if (!Array.isArray) /** Executes the user's step function and resets data & errors. */ function doStep() { - step(returnable()); + step(returnable(undefined, true)); data = []; errors = []; } diff --git a/tests/test-cases.js b/tests/test-cases.js index bfc1d04..44521e5 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1735,7 +1735,7 @@ var CUSTOM_TESTS = [ run: function(callback) { Papa.parse('A,b,c', { step: function(response) { - callback(response.data[0]); + callback(response.data); } }); } @@ -1972,7 +1972,7 @@ var CUSTOM_TESTS = [ var updates = []; Papa.parse('A,b,c\nd,E,f\nG,h,i', { step: function(response, handle) { - updates.push(response.data[0]); + updates.push(response.data); handle.abort(); callback(updates); }, @@ -2002,7 +2002,7 @@ var CUSTOM_TESTS = [ var updates = []; Papa.parse('A,b,c\nd,E,f\nG,h,i', { step: function(response, handle) { - updates.push(response.data[0]); + updates.push(response.data); handle.pause(); callback(updates); }, @@ -2021,7 +2021,7 @@ var CUSTOM_TESTS = [ var first = true; Papa.parse('A,b,c\nd,E,f\nG,h,i', { step: function(response, h) { - updates.push(response.data[0]); + updates.push(response.data); if (!first) return; handle = h; handle.pause(); From a0026683f7b8ebeda9e54da9b9825b2d41058480 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 14 Nov 2018 11:59:02 +0100 Subject: [PATCH 13/40] Return data directly on NodeStream as step function now returns a single row --- papaparse.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/papaparse.js b/papaparse.js index 8676056..9d1fcc6 100755 --- a/papaparse.js +++ b/papaparse.js @@ -898,13 +898,11 @@ if (!Array.isArray) this._onCsvData = function(results) { var data = results.data; - for (var i = 0; i < data.length; i++) { - if (!stream.push(data[i]) && !this._handle.paused()) { - // the writeable consumer buffer has filled up - // so we need to pause until more items - // can be processed - this._handle.pause(); - } + if (!stream.push(data) && !this._handle.paused()) { + // the writeable consumer buffer has filled up + // so we need to pause until more items + // can be processed + this._handle.pause(); } }; From 66727899059c3b1593bdfe1404cac75103a68d16 Mon Sep 17 00:00:00 2001 From: Leo Anthias Date: Wed, 14 Nov 2018 11:03:18 +0000 Subject: [PATCH 14/40] Remove If-None-Match header. Fixes #595 (#596) --- papaparse.js | 1 - 1 file changed, 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 9d1fcc6..81c772c 100755 --- a/papaparse.js +++ b/papaparse.js @@ -651,7 +651,6 @@ if (!Array.isArray) { var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end); - xhr.setRequestHeader('If-None-Match', 'webkit-no-cache'); // https://bugs.webkit.org/show_bug.cgi?id=82672 } try { From 9a4a00a1ee8753eb2a106ef2de86d3a8a8bba698 Mon Sep 17 00:00:00 2001 From: Jacopo Farina Date: Thu, 11 Oct 2018 14:26:20 +0200 Subject: [PATCH 15/40] Move to mocha-headless-chrome for running tests Additionally, upgrade mocha. --- package.json | 8 ++++---- tests/test.js | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index ae43241..0072ac1 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "eslint": "^4.19.1", "grunt": "^1.0.2", "grunt-contrib-uglify": "^3.3.0", - "mocha": "^3.5.0", - "mocha-phantomjs": "^4.1.0", + "mocha": "^5.2.0", + "mocha-headless-chrome": "^2.0.1", "open": "0.0.5", "phantomjs-prebuilt": "^2.1.16", "serve-static": "^1.7.1" @@ -51,8 +51,8 @@ "scripts": { "lint": "eslint --no-ignore papaparse.js Gruntfile.js .eslintrc.js 'tests/**/*.js'", "test-browser": "node tests/test.js", - "test-phantomjs": "node tests/test.js --phantomjs", + "test-mocha-headless-chrome": "node tests/test.js --mocha-headless-chrome", "test-node": "mocha tests/node-tests.js tests/test-cases.js", - "test": "npm run lint && npm run test-node && npm run test-phantomjs" + "test": "npm run lint && npm run test-node && npm run test-mocha-headless-chrome" } } diff --git a/tests/test.js b/tests/test.js index f4fed5f..2741831 100644 --- a/tests/test.js +++ b/tests/test.js @@ -5,8 +5,8 @@ var path = require('path'); var childProcess = require('child_process'); var server = connect().use(serveStatic(path.join(__dirname, '/..'))).listen(8071, function() { - if (process.argv.indexOf('--phantomjs') !== -1) { - childProcess.spawn('node_modules/.bin/mocha-phantomjs', ['http://localhost:8071/tests/tests.html'], { + if (process.argv.indexOf('--mocha-headless-chrome') !== -1) { + childProcess.spawn('node_modules/.bin/mocha-headless-chrome', ['-f', 'http://localhost:8071/tests/tests.html'], { stdio: 'inherit' }).on('exit', function(code) { server.close(); From ded2feb4f201f9f127d226dce3cf710c7bbe1f04 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 20 Nov 2018 12:58:35 +0100 Subject: [PATCH 16/40] Update file test case to do not filter response data Now a single row is returned on the step function --- tests/test-cases.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-cases.js b/tests/test-cases.js index 44521e5..9477892 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1957,7 +1957,7 @@ var CUSTOM_TESTS = [ Papa.parse(new File(['A,B,C\nX,"Y\n1\n2\n3",Z'], 'sample.csv'), { chunkSize: 3, step: function(response) { - updates.push(response.data[0]); + updates.push(response.data); }, complete: function() { callback(updates); From 4ecae0b5bbcc76a16abecded59792ed9cea750cb Mon Sep 17 00:00:00 2001 From: Jonathan Grimes Date: Mon, 19 Nov 2018 13:00:36 -0600 Subject: [PATCH 17/40] Support workers from inline-blobs Simplify worker to just use inline-blob --- docs/docs.html | 8 +------ docs/faq.html | 6 +++--- papaparse.js | 51 ++++++++++++-------------------------------- tests/test-cases.js | 52 ++++++++++----------------------------------- tests/tests.html | 2 +- 5 files changed, 30 insertions(+), 89 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index b424c7a..bd842f1 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -447,7 +447,7 @@ var csv = Papa.unparse({ worker - Whether or not to use a worker thread. Using a worker will keep your page reactive, but may be slightly slower. Web Workers also load the entire Javascript file, so be careful when combining other libraries in the same file as Papa Parse. Note that worker option is only available when parsing files and not when converting from JSON to CSV. + Whether or not to use a worker thread. Using a worker will keep your page reactive, but may be slightly slower. @@ -749,12 +749,6 @@ var csv = Papa.unparse({ Whether or not the browser supports HTML5 Web Workers. If false, worker: true will have no effect. - - Papa.SCRIPT_PATH - - The relative path to Papa Parse. This is automatically detected when Papa Parse is loaded synchronously. However, if you load Papa Parse asynchronously (e.g. with RequireJS), you need to set this variable manually in order to use Web Workers. (In those cases, this variable is not read-only and you should set it!) - - diff --git a/docs/faq.html b/docs/faq.html index 9af0863..eacb409 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -84,7 +84,7 @@
Can I put other libraries in the same file as Papa Parse?

- Yes, but then don't use the Web Worker feature unless your other dependencies are battle-hardened for worker threads. A worker thread loads an entire file, not just a function, so all those dependencies would be executed in an environment without a DOM and other window features. If any of those dependencies crash (Cannot read property "defaultView" of undefined is common), the whole worker thread will crash and parsing will not succeed. + Yes.

@@ -96,7 +96,7 @@
Can Papa Parse be loaded asynchronously (after the page loads)?

- Yes. But if you want to use Web Workers, you'll need to specify the relative path to Papa Parse. To do this, set Papa.SCRIPT_PATH to the relative path of the Papa Parse file. In synchronous loading, this is automatically detected. + Yes.

@@ -209,7 +209,7 @@
Can I use a worker if I combine/concatenate my Javascript files?

- Probably not. It's safest to concatenate the rest of your dependencies and include Papa Parse in a seperate file. Any library that expects to have access to the window or DOM will crash when executed in a worker thread. Only put other libraries in the same file if they are ready to be used in worker threads. + Yes.

When should I use a worker?
diff --git a/papaparse.js b/papaparse.js index 81c772c..14edf60 100755 --- a/papaparse.js +++ b/papaparse.js @@ -34,7 +34,10 @@ if (!Array.isArray) // Browser globals (root is window) root.Papa = factory(); } -}(this, function() + // in strict mode we cannot access arguments.callee, so we need a named reference to + // stringify the factory method for the blob worker + // eslint-disable-next-line func-name +}(this, function moduleFactory() { 'use strict'; @@ -51,9 +54,15 @@ if (!Array.isArray) return {}; })(); + + function getWorkerBlob() { + var URL = global.URL || global.webkitURL || null; + var code = moduleFactory.toString(); + return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(['(', code, ')();'], {type: 'text/javascript'}))); + } + var IS_WORKER = !global.document && !!global.postMessage, - IS_PAPA_WORKER = IS_WORKER && /(\?|&)papaworker(=|&|$)/.test(global.location.search), - LOADED_SYNC = false, AUTO_SCRIPT_PATH; + IS_PAPA_WORKER = IS_WORKER && /blob:/i.test((global.location || {}).protocol); var workers = {}, workerIdCounter = 0; var Papa = {}; @@ -66,7 +75,6 @@ if (!Array.isArray) Papa.BYTE_ORDER_MARK = '\ufeff'; Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; - Papa.SCRIPT_PATH = null; // Must be set by your code if you use workers and this lib is loaded asynchronously Papa.NODE_STREAM_INPUT = 1; // Configurable chunk sizes for local and remote files, respectively @@ -184,23 +192,6 @@ if (!Array.isArray) { global.onmessage = workerThreadReceivedMessage; } - else if (Papa.WORKERS_SUPPORTED) - { - AUTO_SCRIPT_PATH = getScriptPath(); - - // Check if the script was loaded synchronously - if (!document.body) - { - // Body doesn't exist yet, must be synchronous - LOADED_SYNC = true; - } - else - { - document.addEventListener('DOMContentLoaded', function() { - LOADED_SYNC = true; - }, true); - } - } @@ -1685,26 +1676,12 @@ if (!Array.isArray) } - // If you need to load Papa Parse asynchronously and you also need worker threads, hard-code - // the script path here. See: https://github.com/mholt/PapaParse/issues/87#issuecomment-57885358 - function getScriptPath() - { - var scripts = document.getElementsByTagName('script'); - return scripts.length ? scripts[scripts.length - 1].src : ''; - } - function newWorker() { if (!Papa.WORKERS_SUPPORTED) return false; - if (!LOADED_SYNC && Papa.SCRIPT_PATH === null) - throw new Error( - 'Script path cannot be determined automatically when Papa Parse is loaded asynchronously. ' + - 'You need to set Papa.SCRIPT_PATH manually.' - ); - var workerUrl = Papa.SCRIPT_PATH || AUTO_SCRIPT_PATH; - // Append 'papaworker' to the search string to tell papaparse that this is our worker. - workerUrl += (workerUrl.indexOf('?') !== -1 ? '&' : '?') + 'papaworker'; + + var workerUrl = getWorkerBlob(); var w = new global.Worker(workerUrl); w.onmessage = mainThreadReceivedMessage; w.id = workerIdCounter++; diff --git a/tests/test-cases.js b/tests/test-cases.js index 9477892..7e4986c 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -7,6 +7,7 @@ if (typeof module !== 'undefined' && module.exports) { var assert = chai.assert; +var BASE_PATH = (typeof document === 'undefined') ? './' : document.getElementById('test-cases').src.replace(/test-cases\.js$/, ''); var RECORD_SEP = String.fromCharCode(30); var UNIT_SEP = String.fromCharCode(31); var FILES_ENABLED = false; @@ -1396,7 +1397,7 @@ var PARSE_ASYNC_TESTS = [ }, { description: "Simple download", - input: "sample.csv", + input: BASE_PATH + "sample.csv", config: { download: true }, @@ -1408,7 +1409,7 @@ var PARSE_ASYNC_TESTS = [ }, { description: "Simple download + worker", - input: "tests/sample.csv", + input: BASE_PATH + "sample.csv", config: { worker: true, download: true @@ -1761,7 +1762,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, step: function(response) { updates.push(response.meta.cursor); @@ -1778,7 +1779,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, step: function(response) { @@ -1796,7 +1797,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, worker: true, @@ -1815,7 +1816,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, chunk: function(response) { @@ -1833,7 +1834,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = []; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, chunk: function(response) { @@ -2042,7 +2043,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = 0; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { worker: true, download: true, chunkSize: 500, @@ -2062,7 +2063,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = 0; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, beforeFirstChunk: function(chunk) { @@ -2083,7 +2084,7 @@ var CUSTOM_TESTS = [ disabled: !XHR_ENABLED, run: function(callback) { var updates = 0; - Papa.parse("/tests/long-sample.csv", { + Papa.parse(BASE_PATH + "long-sample.csv", { download: true, chunkSize: 500, beforeFirstChunk: function(chunk) { @@ -2096,37 +2097,6 @@ var CUSTOM_TESTS = [ } }); } - }, - { - description: "Should not assume we own the worker unless papaworker is in the search string", - disabled: typeof Worker === 'undefined', - expected: [false, true, true, true, true], - run: function(callback) { - var searchStrings = [ - '', - '?papaworker', - '?x=1&papaworker', - '?x=1&papaworker&y=1', - '?x=1&papaworker=1' - ]; - var results = searchStrings.map(function() { return false; }); - var workers = []; - - // Give it .5s to do something - setTimeout(function() { - workers.forEach(function(w) { w.terminate(); }); - callback(results); - }, 500); - - searchStrings.forEach(function(searchString, idx) { - var w = new Worker('../papaparse.js' + searchString); - workers.push(w); - w.addEventListener('message', function() { - results[idx] = true; - }); - w.postMessage({input: 'a,b,c\n1,2,3'}); - }); - } } ]; diff --git a/tests/tests.html b/tests/tests.html index 421e96b..3d719d3 100644 --- a/tests/tests.html +++ b/tests/tests.html @@ -9,7 +9,7 @@ - +
From 296c89049bcb5f6ab0dc7008b326d641e1d6a531 Mon Sep 17 00:00:00 2001 From: Jonathan Grimes Date: Tue, 20 Nov 2018 09:28:54 -0600 Subject: [PATCH 18/40] cleanup remnants of phantomjs --- package.json | 1 - tests/tests.html | 9 ++------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 0072ac1..8318774 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,6 @@ "mocha": "^5.2.0", "mocha-headless-chrome": "^2.0.1", "open": "0.0.5", - "phantomjs-prebuilt": "^2.1.16", "serve-static": "^1.7.1" }, "scripts": { diff --git a/tests/tests.html b/tests/tests.html index 3d719d3..a3ce51e 100644 --- a/tests/tests.html +++ b/tests/tests.html @@ -15,13 +15,8 @@
From 2fe54b7a81360fb69af705238fcc25db0c86a7e2 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 20 Nov 2018 18:36:20 +0100 Subject: [PATCH 19/40] Remove trailing space on faqs --- docs/faq.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.html b/docs/faq.html index eacb409..edced65 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -96,7 +96,7 @@
Can Papa Parse be loaded asynchronously (after the page loads)?

- Yes. + Yes.

From 46160eed95bf39fa8a6ed0ec7291fe5054827446 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 21 Nov 2018 09:45:08 +0100 Subject: [PATCH 20/40] Remove Array.isArray() polyfill Fixes #598 --- papaparse.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/papaparse.js b/papaparse.js index 14edf60..d905513 100755 --- a/papaparse.js +++ b/papaparse.js @@ -5,15 +5,6 @@ https://github.com/mholt/PapaParse License: MIT */ -// Polyfills -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#Polyfill -if (!Array.isArray) -{ - Array.isArray = function(arg) { - return Object.prototype.toString.call(arg) === '[object Array]'; - }; -} - (function(root, factory) { /* globals define */ From bac638610d777719e97c2813afaad693caabd2c8 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 21 Nov 2018 10:01:50 +0100 Subject: [PATCH 21/40] Beta version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 8318774..d83361f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.0.0-alpha", + "version": "5.0.0-beta.0", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index d905513..66dba9d 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.0.0-alpha +v5.0.0-beta.0 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 95d7b63..c960d68 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v4.6.2 +v5.0.0-beta.0 https://github.com/mholt/PapaParse License: MIT */ -Array.isArray||(Array.isArray=function(e){return"[object Array]"===Object.prototype.toString.call(e)}),function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function(){"use strict";var s,e,f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{},n=!f.document&&!!f.postMessage,o=n&&/(\?|&)papaworker(=|&|$)/.test(f.location.search),a=!1,h={},u=0,k={parse:function(e,t){var r=(t=t||{}).dynamicTyping||!1;M(r)&&(t.dynamicTypingFunction=r,r={});if(t.dynamicTyping=r,t.transform=!!M(t.transform)&&t.transform,t.worker&&k.WORKERS_SUPPORTED){var i=function(){if(!k.WORKERS_SUPPORTED)return!1;if(!a&&null===k.SCRIPT_PATH)throw new Error("Script path cannot be determined automatically when Papa Parse is loaded asynchronously. You need to set Papa.SCRIPT_PATH manually.");var e=k.SCRIPT_PATH||s;e+=(-1!==e.indexOf("?")?"&":"?")+"papaworker";var t=new f.Worker(e);return t.onmessage=m,t.id=u++,h[t.id]=t}();return i.userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void i.postMessage({input:e,config:t,workerId:i.id})}var n=null;k.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new c(t):new g(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new _(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new p(t));return n.stream(e)},unparse:function(e,t){var i=!1,_=!0,m=",",y="\r\n",n='"',r=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||k.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||Array.isArray(t.quotes))&&(i=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(r=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(n=t.quoteChar);"boolean"==typeof t.header&&(_=t.header)}();var s=new RegExp(n,"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return o(null,e,r);if("object"==typeof e[0])return o(a(e[0]),e,r)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:a(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),o(e.fields||[],e.data||[],r);throw"exception: Unable to serialize unrecognized input";function a(e){if("object"!=typeof e)return[];var t=[];for(var r in e)t.push(r);return t}function o(e,t,r){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function c(e){var i;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),l.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r),i.setRequestHeader("If-None-Match","webkit-no-cache")}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function p(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),l.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=C(this._chunkLoaded,this),i.onerror=C(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function g(e){var r;l.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function _(e){l.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){l.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){l.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=C(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=C(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=C(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=C(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(_){var a,o,h,i=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,n=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,s=0,u=!1,e=!1,f=[],d={data:[],errors:[],meta:{}};if(M(_.step)){var l=_.step;_.step=function(e){if(d=e,p())c();else{if(c(),0===d.data.length)return;r+=e.data.length,_.preview&&r>_.preview?o.abort():l(d,t)}}}function m(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(y("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),_.transform&&(n=_.transform(n,i)),n=g(i,n),"__parsed_extra"===i?(r[i]=r[i]||[],r[i].push(n)):r[i]=n}d.data[e]=r,_.header&&(t>f.length?y("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=i.length/2?"\r\n":"\r"}(e,i)),h=!1,_.delimiter)M(_.delimiter)&&(_.delimiter=_.delimiter(e),d.meta.delimiter=_.delimiter);else{var n=function(e,t,r,i){for(var n,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=A)return C(!0)}else for(p=P,P++;;){if(-1===(p=i.indexOf(S,p+1)))return r||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:P}),E();if(p===n-1)return E(i.substring(P,p).replace(m,S));if(S!==L||i[p+1]!==L){if(S===L||0===p||i[p-1]!==L){var y=b(-1===_?g:Math.min(g,_));if(i[p+1+y]===O){f.push(i.substring(P,p).replace(m,S)),P=p+1+y+e,g=i.indexOf(O,P),_=i.indexOf(x,P);break}var v=b(_);if(i.substr(p+1+v,s)===x){if(f.push(i.substring(P,p).replace(m,S)),w(p+1+v+s),g=i.indexOf(O,P),o&&(R(),F))return C();if(A&&h.length>=A)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:P}),p++}}else p++}return E();function k(e){h.push(e),d=P}function b(e){var t=0;if(-1!==e){var r=i.substring(p+1,e);r&&""===r.trim()&&(t=r.length)}return t}function E(e){return r||(void 0===e&&(e=i.substr(P)),f.push(e),P=n,k(f),o&&R()),C()}function w(e){P=e,k(f),f=[],_=i.indexOf(x,P)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:F,truncated:!!e,cursor:d+(t||0)}}}function R(){I(C()),h=[],u=[]}},this.abort=function(){F=!0},this.getCharIndex=function(){return P}}function m(e){var t=e.data,r=h[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,y(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:E,resume:E};if(M(r.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:k.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return;n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:k.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=k.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=w(this._chunkLoaded,this),i.onerror=w(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=k.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=w(this._chunkLoaded,this),i.onerror=w(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;u.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function _(e){u.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=w(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=w(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=w(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=w(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(g){var a,o,h,i=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,n=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,s=0,u=!1,e=!1,f=[],d={data:[],errors:[],meta:{}};if(M(g.step)){var l=g.step;g.step=function(e){if(d=e,p())c();else{if(c(),0===d.data.length)return;r+=e.data.length,g.preview&&r>g.preview?o.abort():l(d,t)}}}function m(e){return"greedy"===g.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function c(){if(d&&h&&(v("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+k.DefaultDelimiter+"'"),h=!1),g.skipEmptyLines)for(var e=0;e=f.length?"__parsed_extra":f[t]),g.transform&&(n=g.transform(n,i)),n=_(i,n),"__parsed_extra"===i?(r[i]=r[i]||[],r[i].push(n)):r[i]=n}d.data[e]=r,g.header&&(t>f.length?v("FieldMismatch","TooManyFields","Too many fields: expected "+f.length+" fields but parsed "+t,s+e):t=i.length/2?"\r\n":"\r"}(e,i)),h=!1,g.delimiter)M(g.delimiter)&&(g.delimiter=g.delimiter(e),d.meta.delimiter=g.delimiter);else{var n=function(e,t,r,i){for(var n,s,a,o=[",","\t","|",";",k.RECORD_SEP,k.UNIT_SEP],h=0;h=T)return C(!0)}else for(p=F,F++;;){if(-1===(p=i.indexOf(S,p+1)))return t||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:F}),E();if(p===n-1)return E(i.substring(F,p).replace(m,S));if(S!==A||i[p+1]!==A){if(S===A||0===p||i[p-1]!==A){var v=b(-1===g?_:Math.min(_,g));if(i[p+1+v]===O){f.push(i.substring(F,p).replace(m,S)),F=p+1+v+e,_=i.indexOf(O,F),g=i.indexOf(x,F);break}var y=b(g);if(i.substr(p+1+y,s)===x){if(f.push(i.substring(F,p).replace(m,S)),w(p+1+y+s),_=i.indexOf(O,F),o&&(R(),z))return C();if(T&&h.length>=T)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:F}),p++}}else p++}return E();function k(e){h.push(e),d=F}function b(e){var t=0;if(-1!==e){var r=i.substring(p+1,e);r&&""===r.trim()&&(t=r.length)}return t}function E(e){return t||(void 0===e&&(e=i.substr(F)),f.push(e),F=n,k(f),o&&R()),C()}function w(e){F=e,k(f),f=[],g=i.indexOf(x,F)}function C(e,t){return{data:t||!1?h[0]:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:z,truncated:!!e,cursor:d+(r||0)}}}function R(){D(C(void 0,!0)),h=[],u=[]}},this.abort=function(){z=!0},this.getCharIndex=function(){return F}}function g(e){var t=e.data,r=a[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:v,resume:v};if(M(r.userStep)){for(var s=0;s Date: Wed, 28 Nov 2018 17:38:57 +0100 Subject: [PATCH 22/40] Improve transform function docs and tests. Fixes #601 --- docs/docs.html | 2 +- tests/test-cases.js | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index bd842f1..1da2126 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -545,7 +545,7 @@ var csv = Papa.unparse({ transform - A function to apply on each value. The function receives the value as its first argument and the column number as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping. + A function to apply on each value. The function receives the value as its first argument and the column number or header name when enabled as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping. diff --git a/tests/test-cases.js b/tests/test-cases.js index 7e4986c..69b9b25 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -913,6 +913,39 @@ var PARSE_TESTS = [ errors: [] } }, + { + description: "Custom transform accepts column number also", + input: 'A,B,C\r\nd,e,f', + config: { + transform: function(value, column) { + if (column % 2) { + value = value.toLowerCase(); + } + return value; + } + }, + expected: { + data: [["A","b","C"], ["d","e","f"]], + errors: [] + } + }, + { + description: "Custom transform accepts header name when using header", + input: 'A,B,C\r\nd,e,f', + config: { + header: true, + transform: function(value, name) { + if (name === 'B') { + value = value.toUpperCase(); + } + return value; + } + }, + expected: { + data: [{'A': "d", 'B': "E", 'C': "f"}], + errors: [] + } + }, { description: "Dynamic typing converts ISO date strings to Dates", input: 'ISO date,long date\r\n2018-05-04T21:08:03.269Z,Fri May 04 2018 14:08:03 GMT-0700 (PDT)\r\n2018-05-08T15:20:22.642Z,Tue May 08 2018 08:20:22 GMT-0700 (PDT)', From 9a541b1e56de1e5da7c41c9f0170438530436fba Mon Sep 17 00:00:00 2001 From: Chris Zubak-Skees Date: Sun, 2 Dec 2018 14:16:23 -0500 Subject: [PATCH 23/40] Consistently apply regex escaping to quoteChar (#602) --- papaparse.js | 4 ++-- tests/test-cases.js | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/papaparse.js b/papaparse.js index 66dba9d..7febbaf 100755 --- a/papaparse.js +++ b/papaparse.js @@ -278,7 +278,7 @@ License: MIT unpackConfig(); - var quoteCharRegex = new RegExp(_quoteChar, 'g'); + var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g'); if (typeof _input === 'string') _input = JSON.parse(_input); @@ -1417,7 +1417,7 @@ License: MIT var nextDelim = input.indexOf(delim, cursor); var nextNewline = input.indexOf(newline, cursor); - var quoteCharRegex = new RegExp(escapeChar.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&') + quoteChar, 'g'); + var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), 'g'); var quoteSearch; // Parser loop diff --git a/tests/test-cases.js b/tests/test-cases.js index 69b9b25..2e19d68 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1358,7 +1358,7 @@ var PARSE_TESTS = [ } }, { - description: "Using reserved regex characters as quote characters", + description: "Using reserved regex character . as quote character", input: '.a\na.,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', config: { quoteChar: '.' }, expected: { @@ -1373,6 +1373,22 @@ var PARSE_TESTS = [ } } }, + { + description: "Using reserved regex character | as quote character", + input: '|a\na|,b\r\nc,d\r\ne,f\r\ng,h\r\ni,j', + config: { quoteChar: '|' }, + expected: { + data: [['a\na', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j']], + errors: [], + meta: { + linebreak: '\r\n', + delimiter: ',', + cursor: 27, + aborted: false, + truncated: false + } + } + }, { description: "Parsing with skipEmptyLines set to 'greedy'", notes: "Must parse correctly without lines with no content", From a5ba84600d423eeb44d47a6a7ba343c6f527edf2 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 9 Jan 2019 09:52:07 +0100 Subject: [PATCH 24/40] Update donate link Fixes #614 --- docs/index.html | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/index.html b/docs/index.html index 5190b1d..e1507c2 100644 --- a/docs/index.html +++ b/docs/index.html @@ -91,9 +91,6 @@ Papa.parse(bigFile, { Help - @@ -534,7 +531,7 @@ var csv = Papa.unparse(yourData); From f6b1b36c3f8faf5ab8e436709925e1563b0f75de Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 9 Jan 2019 09:58:24 +0100 Subject: [PATCH 25/40] Correctly guess deliminter when mixed with commas Closes #592 --- papaparse.js | 4 ++-- tests/test-cases.js | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/papaparse.js b/papaparse.js index 7febbaf..0b15bfc 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1245,7 +1245,7 @@ License: MIT if (typeof fieldCountPrevRow === 'undefined') { - fieldCountPrevRow = fieldCount; + fieldCountPrevRow = 0; continue; } else if (fieldCount > 1) @@ -1258,7 +1258,7 @@ License: MIT if (preview.data.length > 0) avgFieldCount /= (preview.data.length - emptyLinesCount); - if ((typeof bestDelta === 'undefined' || delta < bestDelta) + if ((typeof bestDelta === 'undefined' || delta > bestDelta) && avgFieldCount > 1.99) { bestDelta = delta; diff --git a/tests/test-cases.js b/tests/test-cases.js index 2e19d68..1dceaa3 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1193,6 +1193,16 @@ var PARSE_TESTS = [ errors: [] } }, + { + description: "Pipe delimiter is guessed correctly when mixed with comas", + notes: "Guessing the delimiter should work even if there are many lines of comments at the start of the file", + input: 'one|two,two|three\nfour|five,five|six', + config: {}, + expected: { + data: [['one','two,two','three'],['four','five,five','six']], + errors: [] + } + }, { description: "Single quote as quote character", notes: "Must parse correctly when single quote is specified as a quote character", From 71c2eee63c7249460571be04630e1f18c003fe2c Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Fri, 11 Jan 2019 18:43:35 +0100 Subject: [PATCH 26/40] Use latest stable version for file download links Closes #615 --- README.md | 2 +- docs/index.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9e00358..3270ee6 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ can be installed with the following command: npm install papaparse -If you don't want to use npm, [papaparse.min.js](https://github.com/mholt/PapaParse/blob/master/papaparse.min.js) can be downloaded to your project source. +If you don't want to use npm, [papaparse.min.js](https://unpkg.com/papaparse@latest/papaparse.min.js) can be downloaded to your project source. Homepage & Demo diff --git a/docs/index.html b/docs/index.html index e1507c2..09e3292 100644 --- a/docs/index.html +++ b/docs/index.html @@ -540,8 +540,8 @@ var csv = Papa.unparse(yourData);
Download
Latest (master)
- Lil' Papa - Fat Papa + Lil' Papa + Fat Papa From b7529303e33f3a22c1a4773bc7ab245f935e18d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Beli=C3=ABn?= Date: Fri, 1 Feb 2019 15:34:54 +0100 Subject: [PATCH 29/40] =?UTF-8?q?Ex=C3=B1.a(#625)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See #597 --- docs/docs.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index 7680ef1..2c31312 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -415,7 +415,8 @@ var csv = Papa.unparse({ transformHeader - A function to apply on each header. Requires header to be true. The function receives the header as its first argument. + A function to apply on each header. Requires header to be true. The function receives the header as its first argument.
+ Only available starting with version 5.0. From 757b1bf6e04353b7e82a960e4fd9936495cd4e3c Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Mon, 4 Feb 2019 09:45:35 +0100 Subject: [PATCH 30/40] Add DelimitersToGuess config option (#555) --- docs/docs.html | 13 +++++++++++-- papaparse.js | 11 ++++++----- tests/test-cases.js | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index 2c31312..59c87d5 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -355,7 +355,8 @@ var csv = Papa.unparse({ fastMode: undefined, beforeFirstChunk: undefined, withCredentials: undefined, - transform: undefined + transform: undefined, + delimitersToGuess: [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP] }
@@ -375,7 +376,7 @@ var csv = Papa.unparse({ delimiter - The delimiting character. Leave blank to auto-detect from a list of most common delimiters. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in Papa.BAD_DELIMITERS. + The delimiting character. Leave blank to auto-detect from a list of most common delimiters, or any values passed in through delimitersToGuess. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in Papa.BAD_DELIMITERS. @@ -549,6 +550,14 @@ var csv = Papa.unparse({ A function to apply on each value. The function receives the value as its first argument and the column number or header name when enabled as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping. + + + delimitersToGuess + + + An array of delimiters to guess from if the delimiter option is not set. + + diff --git a/papaparse.js b/papaparse.js index 6f18efe..e0ddca9 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1031,7 +1031,7 @@ License: MIT _delimiterError = false; if (!_config.delimiter) { - var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments); + var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess); if (delimGuess.successful) _config.delimiter = delimGuess.bestDelimiter; else @@ -1215,14 +1215,15 @@ License: MIT return _results; } - function guessDelimiter(input, newline, skipEmptyLines, comments) + function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) { - var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; var bestDelim, bestDelta, fieldCountPrevRow; - for (var i = 0; i < delimChoices.length; i++) + delimitersToGuess = delimitersToGuess || [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; + + for (var i = 0; i < delimitersToGuess.length; i++) { - var delim = delimChoices[i]; + var delim = delimitersToGuess[i]; var delta = 0, avgFieldCount = 0, emptyLinesCount = 0; fieldCountPrevRow = undefined; diff --git a/tests/test-cases.js b/tests/test-cases.js index e8b424f..72338f1 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -2188,7 +2188,6 @@ var CUSTOM_TESTS = [ }); } } - ]; describe('Custom Tests', function() { @@ -2205,3 +2204,40 @@ describe('Custom Tests', function() { generateTest(CUSTOM_TESTS[i]); } }); + + +var DELIMITERS_TO_GUESS_TESTS = [ + { + description: "Should correctly guess custom delimiter when passed delimiters to guess.", + expected: "~", + run: function(callback) { + var results = Papa.parse('"A"~"B"~"C"~"D"', { + delimitersToGuess: ['~', '@', '%'] + }); + callback(results.meta.delimiter); + } + }, + { + description: "Should still correctly guess default delimiters when delimiters to guess are not given.", + expected: ",", + run: function(callback) { + var results = Papa.parse('"A","B","C","D"'); + callback(results.meta.delimiter); + } + } +]; + +describe('Delimiters to Guess Tests', function() { + function generateTest(test) { + (test.disabled ? it.skip : it)(test.description, function(done) { + test.run(function(actual) { + assert.deepEqual(JSON.stringify(actual), JSON.stringify(test.expected)); + done(); + }); + }); + } + + for (var i = 0; i < DELIMITERS_TO_GUESS_TESTS.length; i++) { + generateTest(DELIMITERS_TO_GUESS_TESTS[i]); + } +}); From 0e7f50be0ad1e404e14783dd5d6d127e6423684a Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Mon, 4 Feb 2019 09:49:31 +0100 Subject: [PATCH 31/40] Include delimiter to guess tests on custom tests --- tests/test-cases.js | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/tests/test-cases.js b/tests/test-cases.js index 72338f1..0b34f0d 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -2187,26 +2187,7 @@ var CUSTOM_TESTS = [ } }); } - } -]; - -describe('Custom Tests', function() { - function generateTest(test) { - (test.disabled ? it.skip : it)(test.description, function(done) { - test.run(function(actual) { - assert.deepEqual(JSON.stringify(actual), JSON.stringify(test.expected)); - done(); - }); - }); - } - - for (var i = 0; i < CUSTOM_TESTS.length; i++) { - generateTest(CUSTOM_TESTS[i]); - } -}); - - -var DELIMITERS_TO_GUESS_TESTS = [ + }, { description: "Should correctly guess custom delimiter when passed delimiters to guess.", expected: "~", @@ -2227,7 +2208,7 @@ var DELIMITERS_TO_GUESS_TESTS = [ } ]; -describe('Delimiters to Guess Tests', function() { +describe('Custom Tests', function() { function generateTest(test) { (test.disabled ? it.skip : it)(test.description, function(done) { test.run(function(actual) { @@ -2237,7 +2218,7 @@ describe('Delimiters to Guess Tests', function() { }); } - for (var i = 0; i < DELIMITERS_TO_GUESS_TESTS.length; i++) { - generateTest(DELIMITERS_TO_GUESS_TESTS[i]); + for (var i = 0; i < CUSTOM_TESTS.length; i++) { + generateTest(CUSTOM_TESTS[i]); } }); From 265e09c67a16bea4e21ae9565defc6e913a2a0ea Mon Sep 17 00:00:00 2001 From: Jonathan Grimes Date: Mon, 4 Feb 2019 02:50:56 -0600 Subject: [PATCH 32/40] Ensure data is correctly parsed with header: true (#621) Fixes #619 --- papaparse.js | 48 +++++++++++++++++++++++++++++++-------------- tests/test-cases.js | 33 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/papaparse.js b/papaparse.js index e0ddca9..5ed703e 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1125,18 +1125,25 @@ License: MIT { if (!_results) return; - for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) - for (var j = 0; j < _results.data[i].length; j++) - { - var header = _results.data[i][j]; - if (isFunction(_config.transformHeader)) { - header = _config.transformHeader(header); - } + function addHeder(header) + { + if (isFunction(_config.transformHeader)) + header = _config.transformHeader(header); - _fields.push(header); - } - _results.data.splice(0, 1); + _fields.push(header); + } + + if (Array.isArray(_results.data[0])) + { + for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) + _results.data[i].forEach(addHeder); + + _results.data.splice(0, 1); + } + // if _results.data[0] is not an array, we are in a step where _results.data is the row. + else + _results.data.forEach(addHeder); } function shouldApplyDynamicTyping(field) { @@ -1170,15 +1177,15 @@ License: MIT if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform)) return _results; - for (var i = 0; i < _results.data.length; i++) + function processRow(rowSource, i) { var row = _config.header ? {} : []; var j; - for (j = 0; j < _results.data[i].length; j++) + for (j = 0; j < rowSource.length; j++) { var field = j; - var value = _results.data[i][j]; + var value = rowSource[j]; if (_config.header) field = j >= _fields.length ? '__parsed_extra' : _fields[j]; @@ -1197,7 +1204,6 @@ License: MIT row[field] = value; } - _results.data[i] = row; if (_config.header) { @@ -1206,12 +1212,24 @@ License: MIT else if (j < _fields.length) addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); } + + return row; + } + + var incrementBy = 1; + if (!_results.data[0] || Array.isArray(_results.data[0])) + { + _results.data = _results.data.map(processRow); + incrementBy = _results.data.length; } + else + _results.data = processRow(_results.data, 0); + if (_config.header && _results.meta) _results.meta.fields = _fields; - _rowCounter += _results.data.length; + _rowCounter += incrementBy; return _results; } diff --git a/tests/test-cases.js b/tests/test-cases.js index 0b34f0d..b0610dd 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1804,6 +1804,39 @@ var CUSTOM_TESTS = [ }); } }, + { + description: "Data is correctly parsed with steps (headers)", + expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}], + run: function(callback) { + var data = []; + Papa.parse('One,Two,Three\nA,b,c\nd,E,f', { + header: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, + { + description: "Data is correctly parsed with steps and worker (headers)", + expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}], + run: function(callback) { + var data = []; + Papa.parse('One,Two,Three\nA,b,c\nd,E,f', { + header: true, + worker: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, { description: "Data is correctly parsed with steps and worker", expected: [['A', 'b', 'c'], ['d', 'E', 'f']], From 108d91ceccba3570272fa2a297aa3cbc728b0911 Mon Sep 17 00:00:00 2001 From: janisdd Date: Thu, 7 Feb 2019 19:29:57 +0100 Subject: [PATCH 33/40] - added table for unparse config options (#628) --- docs/docs.html | 85 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index 59c87d5..f84d0ad 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -228,7 +228,7 @@
-
Papa.unparse(data[, config])
+
Papa.unparse(data[, config])
@@ -243,19 +243,88 @@
  • - config is an optional object with any of these properties: -
    // defaults shown
    +								config is an optional config object
    +							
  • + +
    +
    + +
    +
    Default Unparse Config with all options
    +
    + +
    +
    
     {
    -	quotes: false,
    +	quotes: false, //or array of booleans
     	quoteChar: '"',
     	delimiter: ",",
     	header: true,
     	newline: "\r\n",
     	skipEmptyLines: false, //or 'greedy'
    -}
    - Set quotes to true to always enclose each field in quotes, or an array of true/false values correlating to specific to columns to force-quote. The character used to quote can be customized using quoteChar. The delimiter can be any valid delimiting character. The newline character(s) may also be customized. Setting header to false will omit the header row. Setting skipEmptyLines to true will skip completely empty rows (those which evaluate to an empty string). If set to 'greedy', rows that don't have any content (those which have only whitespace after unparsing) will also be skipped.
    - - +} + +
    + +
    + +
    +
    Unparse Config Options
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionExplanation
    + quotes + + If true, forces all fields to be enclosed in quotes. If an array of true/false values, specifies which fields should be force-quoted (first boolean is for the first column, second boolean for the second column, ...). +
    quoteChar + The character used to quote fields. +
    + delimiter + + The delimiting character. It must not be found in Papa.BAD_DELIMITERS. +
    + header + + If false, will omit the header row. If data is an array of arrays this option is ignored. If data is an array of objects the keys of the first object are the header row. If data is an object with the keys fields and data the fields are the header row. +
    + newline + + The newline sequence. Must be one of "\r", "\n", or "\r\n". +
    + skipEmptyLines + + If true, lines that are completely empty (those which evaluate to an empty string) will be skipped. If set to 'greedy', lines that don't have any content (those which have only whitespace after parsing) will also be skipped. +
    From b9f1ebae32a01ef250d4751c0724f4a3924bf08e Mon Sep 17 00:00:00 2001 From: Konstantin Nosov Date: Fri, 8 Feb 2019 01:32:54 +0200 Subject: [PATCH 34/40] Add mailcheck.co to lovers.js (#629) Papa Parser is a great help for us. Such a function like guessing delimiters saved lots of our clients time. --- docs/resources/js/lovers.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 2d02965..80c4c27 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -78,5 +78,11 @@ var peopleLovePapa = [ name: "Novel.js", description: "is a text adventure framework that uses Papa Parse to enable user-friendly translations.", quote: "Papa saves countless hours of work and makes reading large CSV files so easy!" + }, + { + link: "https://mailcheck.co", + name: "Mailcheck.co", + description: "Mailcheck is email validation service. All emails usually stored in CSV's. We use Papa Parse to process data from our customers in browser", + quote: "Papa Parser allowed our customers to preview and process csv's in browser, without uploading them to server. It saves lots of time and space :)" } ]; From 6107789c6be98ec6dfbd6e0be1fa1e87da6ff46e Mon Sep 17 00:00:00 2001 From: janisdd Date: Fri, 8 Feb 2019 16:11:10 +0100 Subject: [PATCH 35/40] - some minor doc fixes --- docs/docs.html | 10 +++++----- docs/index.html | 1 + docs/resources/js/lovers.js | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index f84d0ad..dd37e50 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -99,7 +99,7 @@
    -
    Papa.parse(csvString[, config])
    +
    Papa.parse(csvString[, config])
    @@ -188,7 +188,7 @@ reason: "Some reason", config: // altered config... } - to alter the flow of parsing. Actions can be "abort" to skip this and all other files in the queue, "skip" to skip just this file, or "continue" to carry on (equivalent to returning nothing). reason can be a reason for aborting. config can be a modified configuration for parsing just this file. + to alter the flow of parsing. Actions can be "abort" to skip this and all other files in the queue, "skip" to skip just this file, or "continue" to carry on (equivalent to returning nothing). reason can be a reason for aborting. config can be a modified configuration for parsing just this file.
  • The complete callback shown here is executed after all files are finished and does not receive any data. Use the complete callback in config for per-file results.
  • @@ -358,8 +358,8 @@ var csv = Papa.unparse([
    // Specifying fields and data explicitly
     var csv = Papa.unparse({
    -	fields: ["Column 1", "Column 2"],
    -	data: [
    +	"fields": ["Column 1", "Column 2"],
    +	"data": [
     		["foo", "bar"],
     		["abc", "def"]
     	]
    @@ -502,7 +502,7 @@ var csv = Papa.unparse({
     									preview
     								
     								
    -									If > 0, only that many rows will be parsed.
    +									If > 0, only that many rows will be parsed.
     								
     							
     							
    diff --git a/docs/index.html b/docs/index.html
    index 09e3292..ab427f4 100644
    --- a/docs/index.html
    +++ b/docs/index.html
    @@ -503,6 +503,7 @@ var csv = Papa.unparse(yourData);
      Documentation
    +
    diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 80c4c27..8e8a24d 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -61,7 +61,7 @@ var peopleLovePapa = [ description: "created a video showing how to use Papa Parse and FileDrop.js to create a drag-and-drop CSV-JSON converter.", quote: "It's often easy to convert data to CSV. With Papa, it's easy to turn that CSV into JSON." }, - { + { link: "http://www.yolpo.com/social/gist.github?1dbd4556e748bdb830b3&autoplay=1&interimresults=0&failfast=1", name: "Yolpo", description: "created a simple regression test for Papa Parse.", From 941033094aa8e2f36b4f1de6a902b6905ff859d6 Mon Sep 17 00:00:00 2001 From: janisdd Date: Mon, 11 Feb 2019 13:48:08 +0100 Subject: [PATCH 36/40] Allow to specify the columns used for unparse (#632) --- docs/docs.html | 11 ++++++++++- papaparse.js | 12 +++++++++++- tests/test-cases.js | 7 +++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index dd37e50..8777802 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -261,7 +261,8 @@ delimiter: ",", header: true, newline: "\r\n", - skipEmptyLines: false, //or 'greedy' + skipEmptyLines: false, //or 'greedy', + columns: null //or array of strings } @@ -324,6 +325,14 @@ If true, lines that are completely empty (those which evaluate to an empty string) will be skipped. If set to 'greedy', lines that don't have any content (those which have only whitespace after parsing) will also be skipped. + + + columns + + + If data is an array of objects this option can be used to manually specify the keys (columns) you expect in the objects. If not set the keys of the first objects are used as column. + +
    diff --git a/papaparse.js b/papaparse.js index 5ed703e..5c7b5ed 100755 --- a/papaparse.js +++ b/papaparse.js @@ -276,6 +276,9 @@ License: MIT /** whether to skip empty lines */ var _skipEmptyLines = false; + /** the columns (keys) we expect when we unparse objects */ + var _columns = null; + unpackConfig(); var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g'); @@ -288,7 +291,7 @@ License: MIT if (!_input.length || Array.isArray(_input[0])) return serialize(null, _input, _skipEmptyLines); else if (typeof _input[0] === 'object') - return serialize(objectKeys(_input[0]), _input, _skipEmptyLines); + return serialize(_columns || objectKeys(_input[0]), _input, _skipEmptyLines); } else if (typeof _input === 'object') { @@ -343,6 +346,13 @@ License: MIT if (typeof _config.header === 'boolean') _writeHeader = _config.header; + + if (Array.isArray(_config.columns)) { + + if (_config.columns.length === 0) throw new Error('Option columns is empty'); + + _columns = _config.columns; + } } diff --git a/tests/test-cases.js b/tests/test-cases.js index b0610dd..5ea71e0 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1732,6 +1732,13 @@ var UNPARSE_TESTS = [ input: [{a: null, b: ' '}, {}, {a: '1', b: '2'}], config: {skipEmptyLines: 'greedy', header: true}, expected: 'a,b\r\n1,2' + }, + { + description: "Column option used to manually specify keys", + notes: "Should not throw any error when attempting to serialize key not present in object. Columns are different than keys of the first object. When an object is missing a key then the serialized value should be an empty string.", + input: [{a: 1, b: '2'}, {}, {a: 3, d: 'd', c: 4,}], + config: {columns: ['a', 'b', 'c']}, + expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4' } ]; From a627547d87b6c155ccb46d61ee100c7a20581cf8 Mon Sep 17 00:00:00 2001 From: Ledion Bitincka Date: Wed, 13 Feb 2019 00:37:09 -0800 Subject: [PATCH 37/40] Add ability to support escapeChar on unparse (#631) --- docs/docs.html | 7 +++++++ papaparse.js | 9 ++++++++- tests/test-cases.js | 12 ++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index 8777802..df87a72 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -258,6 +258,7 @@ { quotes: false, //or array of booleans quoteChar: '"', + escapeChar: '"', delimiter: ",", header: true, newline: "\r\n", @@ -293,6 +294,12 @@ The character used to quote fields. + + escapeChar + + The character used to escape quoteChar inside field values. + + delimiter diff --git a/papaparse.js b/papaparse.js index 5c7b5ed..70c4edc 100755 --- a/papaparse.js +++ b/papaparse.js @@ -273,6 +273,9 @@ License: MIT /** quote character */ var _quoteChar = '"'; + /** escaped quote character, either "" or " */ + var _escapedQuote = _quoteChar + _quoteChar; + /** whether to skip empty lines */ var _skipEmptyLines = false; @@ -353,6 +356,10 @@ License: MIT _columns = _config.columns; } + + if (_config.escapeChar !== undefined) { + _escapedQuote = _config.escapeChar + _quoteChar; + } } @@ -439,7 +446,7 @@ License: MIT if (str.constructor === Date) return JSON.stringify(str).slice(1, 25); - str = str.toString().replace(quoteCharRegex, _quoteChar + _quoteChar); + str = str.toString().replace(quoteCharRegex, _escapedQuote); var needsQuotes = (typeof _quotes === 'boolean' && _quotes) || (Array.isArray(_quotes) && _quotes[col]) diff --git a/tests/test-cases.js b/tests/test-cases.js index 5ea71e0..07b8a53 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1739,6 +1739,18 @@ var UNPARSE_TESTS = [ input: [{a: 1, b: '2'}, {}, {a: 3, d: 'd', c: 4,}], config: {columns: ['a', 'b', 'c']}, expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4' + }, + { + description: "Use different escapeChar", + input: [{a: 'foo', b: '"quoted"'}], + config: {header: false, escapeChar: '\\'}, + expected: 'foo,"\\"quoted\\""' + }, + { + description: "test defeault escapeChar", + input: [{a: 'foo', b: '"quoted"'}], + config: {header: false}, + expected: 'foo,"""quoted"""' } ]; From f2930716d0c743179708d6db0dfa76b5c1224d62 Mon Sep 17 00:00:00 2001 From: Varun Sharma Date: Tue, 19 Mar 2019 19:29:03 +0000 Subject: [PATCH 38/40] BugFix #636 Pause and resume (In a quick succession) gets you lot of exceptions and an infinite loop (#637) --- papaparse.js | 21 ++++++++++++++++----- tests/node-tests.js | 36 +++++++++++++++++++++++++++++++++++ tests/test-cases.js | 46 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/papaparse.js b/papaparse.js index 70c4edc..6e68a52 100755 --- a/papaparse.js +++ b/papaparse.js @@ -473,6 +473,7 @@ License: MIT this._handle = null; this._finished = false; this._completed = false; + this._halted = false; this._input = null; this._baseIndex = 0; this._partialLine = ''; @@ -497,6 +498,7 @@ License: MIT chunk = modifiedChunk; } this.isFirstChunk = false; + this._halted = false; // Rejoin the line we likely just split in two by chunking the file var aggregate = this._partialLine + chunk; @@ -504,8 +506,10 @@ License: MIT var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); - if (this._handle.paused() || this._handle.aborted()) + if (this._handle.paused() || this._handle.aborted()) { + this._halted = true; return; + } var lastIndex = results.meta.cursor; @@ -531,8 +535,10 @@ License: MIT else if (isFunction(this._config.chunk) && !isFakeChunk) { this._config.chunk(results, this._handle); - if (this._handle.paused() || this._handle.aborted()) + if (this._handle.paused() || this._handle.aborted()) { + this._halted = true; return; + } results = undefined; this._completeResults = undefined; } @@ -992,7 +998,6 @@ License: MIT // One goal is to minimize the use of regular expressions... var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/; - var self = this; var _stepCounter = 0; // Number of times step was called (number of rows parsed) var _rowCounter = 0; // Number of rows that have been parsed so far @@ -1089,8 +1094,14 @@ License: MIT this.resume = function() { - _paused = false; - self.streamer.parseChunk(_input, true); + if(self.streamer._halted) { + _paused = false; + self.streamer.parseChunk(_input, true); + } else { + // Bugfix: #636 In case the processing hasn't halted yet + // wait for it to halt in order to resume + setTimeout(this.resume, 3); + } }; this.aborted = function() diff --git a/tests/node-tests.js b/tests/node-tests.js index fe671df..a067bfe 100644 --- a/tests/node-tests.js +++ b/tests/node-tests.js @@ -41,6 +41,42 @@ describe('PapaParse', function() { assertLongSampleParsedCorrectly(Papa.parse(longSampleRawCsv)); }); + it('Pause and resume works (Regression Test for Bug #636)', function(done) { + this.timeout(30000); + var mod200Rows = [ + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Lorem ipsum dolor sit","42"] + ]; + var stepped = 0; + var dataRows = []; + Papa.parse(fs.createReadStream(__dirname + '/verylong-sample.csv'), { + step: function(results, parser) { + stepped++; + if (results) + { + parser.pause(); + parser.resume(); + if (results.data && stepped % 200 === 0) { + dataRows.push(results.data); + } + } + }, + complete: function() { + assert.strictEqual(2001, stepped); + assert.deepEqual(mod200Rows, dataRows); + done(); + } + }); + }); + it('asynchronously parsed CSV should be correctly parsed', function(done) { Papa.parse(longSampleRawCsv, { complete: function(parsedCsv) { diff --git a/tests/test-cases.js b/tests/test-cases.js index 07b8a53..d9dd459 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1780,6 +1780,49 @@ describe('Unparse Tests', function() { var CUSTOM_TESTS = [ + { + description: "Pause and resume works (Regression Test for Bug #636)", + disabled: !XHR_ENABLED, + timeout: 30000, + expected: [2001, [ + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Etiam a dolor vitae est vestibulum","84","DEF"], + ["Lorem ipsum dolor sit","42","ABC"], + ["Lorem ipsum dolor sit","42"] + ], 0], + run: function(callback) { + var stepped = 0; + var dataRows = []; + var errorCount = 0; + var output = []; + Papa.parse(BASE_PATH + "verylong-sample.csv", { + download: true, + step: function(results, parser) { + stepped++; + if (results) + { + parser.pause(); + parser.resume(); + if (results.data && stepped % 200 === 0) { + dataRows.push(results.data); + } + } + }, + complete: function() { + output.push(stepped); + output.push(dataRows); + output.push(errorCount); + callback(output); + } + }); + } + }, { description: "Complete is called with all results if neither step nor chunk is defined", expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']], @@ -2263,6 +2306,9 @@ var CUSTOM_TESTS = [ describe('Custom Tests', function() { function generateTest(test) { (test.disabled ? it.skip : it)(test.description, function(done) { + if(test.timeout) { + this.timeout(test.timeout); + } test.run(function(actual) { assert.deepEqual(JSON.stringify(actual), JSON.stringify(test.expected)); done(); From b74bd9e884b3bc2d94383f3908a33d59d1fc595e Mon Sep 17 00:00:00 2001 From: ilias bhallil Date: Thu, 21 Mar 2019 04:10:56 -0400 Subject: [PATCH 39/40] Add downloadRequestHeaders Doc to the website (#647) --- docs/docs.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/docs.html b/docs/docs.html index df87a72..261caa2 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -435,6 +435,7 @@ var csv = Papa.unparse({ complete: undefined, error: undefined, download: false, + downloadRequestHeaders: undefined, skipEmptyLines: false, chunk: undefined, fastMode: undefined, @@ -587,6 +588,19 @@ var csv = Papa.unparse({ If true, this indicates that the string you passed as the first argument to parse() is actually a URL from which to download a file and parse its contents. + + + downloadRequestHeaders + + + If defined, should be an object that describes the headers, example: + +
    +										downloadRequestHeaders: {
    +'Authorization': 'token 123345678901234567890',
    +}
    +									
    + skipEmptyLines From b28a552137545efc839a319510d7a0d41e121853 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 21 Mar 2019 09:14:01 +0100 Subject: [PATCH 40/40] Explain that docs are hosted on docs folder --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3270ee6..eca0188 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,8 @@ To learn how to use Papa Parse: - [Documentation](http://papaparse.com/docs) -The website is hosted on on [Github Pages](https://pages.github.com/). If -you want to contribute just clone the gh-branch of this repository and -open a pull request. +The website is hosted on on [Github Pages](https://pages.github.com/). It's content is also inclued on the docs folder of this repository. If +you want to contribute on it just clone the master of this repository and open a pull request. Papa Parse for Node