From 9b54b11fe2b6355569b27ec83bc39c996bacf949 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Tue, 17 Sep 2019 20:40:39 +1000 Subject: [PATCH 01/26] Fix Range Header processing logic for NetworkStreamer (#709) --- papaparse.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/papaparse.js b/papaparse.js index ae4b70d..b133d92 100755 --- a/papaparse.js +++ b/papaparse.js @@ -669,8 +669,6 @@ License: MIT if (IS_WORKER && xhr.status === 0) this._chunkError(); - else - this._start += this._config.chunkSize; }; this._chunkLoaded = function() @@ -684,7 +682,8 @@ License: MIT return; } - this._finished = !this._config.chunkSize || this._start > getFileSize(xhr); + this._start += xhr.responseText.length; + this._finished = !this._config.chunkSize || this._start >= getFileSize(xhr); this.parseChunk(xhr.responseText); }; From 5219809f1d83ffa611ebe7ed13e8224bcbcf3bd7 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 17 Sep 2019 14:14:42 +0200 Subject: [PATCH 02/26] Minor version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 264f72f..4769a19 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.0.2", + "version": "5.1.0", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index b133d92..b624e50 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.0.2 +v5.1.0 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 14c98ff..48c3ad9 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.0.2 +v5.1.0 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var r=(t=t||{}).dynamicTyping||!1;q(r)&&(t.dynamicTypingFunction=r,r={});if(t.dynamicTyping=r,t.transform=!!q(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var i=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(r=f.URL||f.webkitURL||null,i=s.toString(),b.BLOB_URL||(b.BLOB_URL=r.createObjectURL(new Blob(["(",i,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var r,i;return t.onmessage=_,t.id=h++,a[t.id]=t}();return i.userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=q(t.step),t.chunk=q(t.chunk),t.complete=q(t.complete),t.error=q(t.error),delete t.worker,void i.postMessage({input:e,config:t,workerId:i.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&q(e.read)&&q(e.on)?n=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var i=!1,_=!0,g=",",v="\r\n",n='"',s=n+n,r=!1,a=null;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(g=t.delimiter);("boolean"==typeof t.quotes||Array.isArray(t.quotes))&&(i=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(r=t.skipEmptyLines);"string"==typeof t.newline&&(v=t.newline);"string"==typeof t.quoteChar&&(n=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");a=t.columns}void 0!==t.escapeChar&&(s=t.escapeChar+n)}();var o=new RegExp(U(n),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,r);if("object"==typeof e[0])return u(a||h(e[0]),e,r)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:h(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],r);throw new Error("Unable to serialize unrecognized input");function h(e){if("object"!=typeof e)return[];var t=[];for(var r in e)t.push(r);return t}function u(e,t,r){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(q(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!q(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){q(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status?this._chunkError():this._start+=this._config.chunkSize}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._finished=!this._config.chunkSize||this._start>function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;u.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function m(e){u.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(g){var a,o,h,i=Math.pow(2,53),n=-i,s=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(q(g.step)){var p=g.step;g.step=function(e){if(c=e,_())m();else{if(m(),0===c.data.length)return;r+=e.data.length,g.preview&&r>g.preview?o.abort():p(c,t)}}}function v(e){return"greedy"===g.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function m(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),g.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[r]),g.transform&&(s=g.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return g.header&&(r>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+r,f+t):r=i.length/2?"\r\n":"\r"}(e,i)),h=!1,g.delimiter)q(g.delimiter)&&(g.delimiter=g.delimiter(e),c.meta.delimiter=g.delimiter);else{var n=function(e,t,r,i,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(g=M,M++;;){if(-1===(g=a.indexOf(O,g+1)))return t||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:M}),w();if(g===i-1)return w(a.substring(M,g).replace(_,O));if(O!==z||a[g+1]!==z){if(O===z||0===g||a[g-1]!==z){var y=E(-1===m?p:Math.min(p,m));if(a[g+1+y]===D){f.push(a.substring(M,g).replace(_,O)),a[M=g+1+y+e]!==O&&(g=a.indexOf(O,M)),p=a.indexOf(D,M),m=a.indexOf(I,M);break}var k=E(m);if(a.substr(g+1+k,n)===I){if(f.push(a.substring(M,g).replace(_,O)),C(g+1+k+n),p=a.indexOf(D,M),g=a.indexOf(O,M),o&&(S(),j))return R();if(L&&h.length>=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:M}),g++}}else g++}return w();function b(e){h.push(e),d=M}function E(e){var t=0;if(-1!==e){var r=a.substring(g+1,e);r&&""===r.trim()&&(t=r.length)}return t}function w(e){return t||(void 0===e&&(e=a.substr(M)),f.push(e),M=i,b(f),o&&S()),R()}function C(e){M=e,b(f),f=[],m=a.indexOf(I,M)}function R(e,t){return{data:t||!1?h[0]:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(r||0)}}}function S(){A(R(void 0,!0)),h=[],u=[]}function x(e,t,r){var i={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(q(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!q(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){q(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status&&this._chunkError()}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._start+=i.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;u.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function m(e){u.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(g){var a,o,h,i=Math.pow(2,53),n=-i,s=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(q(g.step)){var p=g.step;g.step=function(e){if(c=e,_())m();else{if(m(),0===c.data.length)return;r+=e.data.length,g.preview&&r>g.preview?o.abort():p(c,t)}}}function v(e){return"greedy"===g.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function m(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),g.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[r]),g.transform&&(s=g.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return g.header&&(r>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+r,f+t):r=i.length/2?"\r\n":"\r"}(e,i)),h=!1,g.delimiter)q(g.delimiter)&&(g.delimiter=g.delimiter(e),c.meta.delimiter=g.delimiter);else{var n=function(e,t,r,i,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(g=z,z++;;){if(-1===(g=a.indexOf(O,g+1)))return t||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:z}),w();if(g===i-1)return w(a.substring(z,g).replace(_,O));if(O!==M||a[g+1]!==M){if(O===M||0===g||a[g-1]!==M){var y=E(-1===m?p:Math.min(p,m));if(a[g+1+y]===D){f.push(a.substring(z,g).replace(_,O)),a[z=g+1+y+e]!==O&&(g=a.indexOf(O,z)),p=a.indexOf(D,z),m=a.indexOf(I,z);break}var k=E(m);if(a.substr(g+1+k,n)===I){if(f.push(a.substring(z,g).replace(_,O)),C(g+1+k+n),p=a.indexOf(D,z),g=a.indexOf(O,z),o&&(S(),j))return R();if(L&&h.length>=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:z}),g++}}else g++}return w();function b(e){h.push(e),d=z}function E(e){var t=0;if(-1!==e){var r=a.substring(g+1,e);r&&""===r.trim()&&(t=r.length)}return t}function w(e){return t||(void 0===e&&(e=a.substr(z)),f.push(e),z=i,b(f),o&&S()),R()}function C(e){z=e,b(f),f=[],m=a.indexOf(I,z)}function R(e,t){return{data:t||!1?h[0]:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(r||0)}}}function S(){A(R(void 0,!0)),h=[],u=[]}function x(e,t,r){var i={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t Date: Thu, 17 Oct 2019 13:34:18 -0600 Subject: [PATCH 03/26] Bump open from 0.0.5 to 7.0.0 (#721) Bumps [open](https://github.com/sindresorhus/open) from 0.0.5 to 7.0.0. - [Release notes](https://github.com/sindresorhus/open/releases) - [Commits](https://github.com/sindresorhus/open/commits/v7.0.0) Signed-off-by: dependabot[bot] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4769a19..b0d5bce 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "grunt-contrib-uglify": "^3.3.0", "mocha": "^5.2.0", "mocha-headless-chrome": "^2.0.1", - "open": "0.0.5", + "open": "7.0.0", "serve-static": "^1.7.1" }, "scripts": { From 6f7e43edd375133d813336cceef1dbda38584ebf Mon Sep 17 00:00:00 2001 From: Seito Tanaka Date: Fri, 18 Oct 2019 05:49:19 +0900 Subject: [PATCH 04/26] Fix step callback function when skipping empty lines (#714) Fixes #672 --- papaparse.js | 11 ++++++----- tests/test-cases.js | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/papaparse.js b/papaparse.js index b624e50..f1674f5 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1036,8 +1036,10 @@ License: MIT _stepCounter += results.data.length; if (_config.preview && _stepCounter > _config.preview) _parser.abort(); - else + else { + _results.data = _results.data[0]; userStep(_results, self); + } } }; } @@ -1706,11 +1708,10 @@ License: MIT } /** Returns an object with the results, errors, and meta. */ - function returnable(stopped, step) + function returnable(stopped) { - var isStep = step || false; return { - data: isStep ? data[0] : data, + data: data, errors: errors, meta: { delimiter: delim, @@ -1725,7 +1726,7 @@ License: MIT /** Executes the user's step function and resets data & errors. */ function doStep() { - step(returnable(undefined, true)); + step(returnable()); data = []; errors = []; } diff --git a/tests/test-cases.js b/tests/test-cases.js index 0c4ab04..58a85f0 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1973,6 +1973,22 @@ var CUSTOM_TESTS = [ }); } }, + { + description: "Data is correctly parsed with steps when skipping empty lines", + expected: [['A', 'b', 'c'], ['d', 'E', 'f']], + run: function(callback) { + var data = []; + Papa.parse('A,b,c\n\nd,E,f', { + skipEmptyLines: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, { description: "Step is called with the contents of the row", expected: ['A', 'b', 'c'], From e0b474dc384b4b63c0176a19f922c8f0c6287ecf Mon Sep 17 00:00:00 2001 From: jseter Date: Mon, 21 Oct 2019 09:48:53 -0400 Subject: [PATCH 05/26] Correct small typo (#723) --- papaparse.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/papaparse.js b/papaparse.js index f1674f5..22a6a7e 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1608,7 +1608,7 @@ License: MIT // we check, if we have quotes, because delimiter char may be part of field enclosed in quotes if (quoteSearch > nextDelim) { // we have quotes, so we try to find the next delimiter not enclosed in quotes and also next starting quote char - var nextDelimObj = getNextUnqotedDelimiter(nextDelim, quoteSearch, nextNewline); + var nextDelimObj = getNextUnquotedDelimiter(nextDelim, quoteSearch, nextNewline); // if we have next delimiter char which is not enclosed in quotes if (nextDelimObj && typeof nextDelimObj.nextDelim !== 'undefined') { @@ -1732,7 +1732,7 @@ License: MIT } /** Gets the delimiter character, which is not inside the quoted field */ - function getNextUnqotedDelimiter(nextDelim, quoteSearch, newLine) { + function getNextUnquotedDelimiter(nextDelim, quoteSearch, newLine) { var result = { nextDelim: undefined, quoteSearch: undefined @@ -1754,7 +1754,7 @@ License: MIT nextQuoteSearch = input.indexOf(quoteChar, nextQuoteSearch + 1); } // try to get the next delimiter position - result = getNextUnqotedDelimiter(nextNextDelim, nextQuoteSearch, newLine); + result = getNextUnquotedDelimiter(nextNextDelim, nextQuoteSearch, newLine); } else { result = { nextDelim: nextDelim, From e536351e7a90c93bf09def167ffb12f9b672c6ad Mon Sep 17 00:00:00 2001 From: jseter Date: Mon, 21 Oct 2019 13:54:20 -0400 Subject: [PATCH 06/26] Refactor substr calls to substring calls. (#725) --- papaparse.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/papaparse.js b/papaparse.js index 22a6a7e..9431770 100755 --- a/papaparse.js +++ b/papaparse.js @@ -699,7 +699,7 @@ License: MIT if (contentRange === null) { // no content range, then finish! return -1; } - return parseInt(contentRange.substr(contentRange.lastIndexOf('/') + 1)); + return parseInt(contentRange.substring(contentRange.lastIndexOf('/') + 1)); } } NetworkStreamer.prototype = Object.create(ChunkStreamer.prototype); @@ -788,8 +788,14 @@ License: MIT { if (this._finished) return; var size = this._config.chunkSize; - var chunk = size ? remaining.substr(0, size) : remaining; - remaining = size ? remaining.substr(size) : ''; + var chunk; + if(size) { + chunk = remaining.substring(0, size); + remaining = remaining.substring(size); + } else { + chunk = remaining; + remaining = ''; + } this._finished = !remaining; return this.parseChunk(chunk); }; @@ -1094,7 +1100,7 @@ License: MIT { _paused = true; _parser.abort(); - _input = _input.substr(_parser.getCharIndex()); + _input = _input.substring(_parser.getCharIndex()); }; this.resume = function() @@ -1332,7 +1338,7 @@ License: MIT function guessLineEndings(input, quoteChar) { - input = input.substr(0, 1024 * 1024); // max length 1 MB + input = input.substring(0, 1024 * 1024); // max length 1 MB // Replace all the text inside quotes var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); input = input.replace(re, ''); @@ -1450,7 +1456,7 @@ License: MIT cursor += newline.length; else if (ignoreLastRow) return returnable(); - if (comments && row.substr(0, commentsLen) === comments) + if (comments && row.substring(0, commentsLen) === comments) continue; if (stepIsFunction) { @@ -1553,7 +1559,7 @@ License: MIT var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline); // Closing quote followed by newline or 'unnecessary spaces + newLine' - if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, newlineLen) === newline) + if (input.substring(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen) === newline) { row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen); @@ -1592,7 +1598,7 @@ License: MIT } // Comment found at start of new line - if (comments && row.length === 0 && input.substr(cursor, commentsLen) === comments) + if (comments && row.length === 0 && input.substring(cursor, cursor + commentsLen) === comments) { if (nextNewline === -1) // Comment ends at EOF return returnable(); @@ -1684,7 +1690,7 @@ License: MIT if (ignoreLastRow) return returnable(); if (typeof value === 'undefined') - value = input.substr(cursor); + value = input.substring(cursor); row.push(value); cursor = inputLen; // important in case parsing is paused pushRow(row); From 7ad8dda68c6bbfd0e1115dd99b997b2997f836da Mon Sep 17 00:00:00 2001 From: jseter Date: Wed, 23 Oct 2019 13:36:38 -0400 Subject: [PATCH 07/26] Address deepEqual using compare by JSON strings. (#724) --- papaparse.js | 11 +++++++---- tests/test-cases.js | 8 ++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/papaparse.js b/papaparse.js index 9431770..d04eaae 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1364,12 +1364,15 @@ License: MIT function addError(type, code, msg, row) { - _results.errors.push({ + var error = { type: type, code: code, - message: msg, - row: row - }); + message: msg + }; + if(row !== undefined) { + error.row = row; + } + _results.errors.push(error); } } diff --git a/tests/test-cases.js b/tests/test-cases.js index 58a85f0..608b297 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -592,7 +592,7 @@ describe('Core Parser Tests', function() { function generateTest(test) { (test.disabled ? it.skip : it)(test.description, function() { var actual = new Papa.Parser(test.config).parse(test.input); - assert.deepEqual(JSON.stringify(actual.errors), JSON.stringify(test.expected.errors)); + assert.deepEqual(actual.errors, test.expected.errors); assert.deepEqual(actual.data, test.expected.data); }); } @@ -1475,7 +1475,7 @@ describe('Parse Tests', function() { if (test.expected.meta) { assert.deepEqual(actual.meta, test.expected.meta); } - assert.deepEqual(JSON.stringify(actual.errors), JSON.stringify(test.expected.errors)); + assert.deepEqual(actual.errors, test.expected.errors); assert.deepEqual(actual.data, test.expected.data); }); } @@ -1556,7 +1556,7 @@ describe('Parse Async Tests', function() { var config = test.config; config.complete = function(actual) { - assert.deepEqual(JSON.stringify(actual.errors), JSON.stringify(test.expected.errors)); + assert.deepEqual(actual.errors, test.expected.errors); assert.deepEqual(actual.data, test.expected.data); done(); }; @@ -2384,7 +2384,7 @@ describe('Custom Tests', function() { this.timeout(test.timeout); } test.run(function(actual) { - assert.deepEqual(JSON.stringify(actual), JSON.stringify(test.expected)); + assert.deepEqual(actual, test.expected); done(); }); }); From 47b356d6e06cdc2fc7d65c3710a2b5f7cd048b1a Mon Sep 17 00:00:00 2001 From: jseter Date: Thu, 24 Oct 2019 02:49:32 -0400 Subject: [PATCH 08/26] #727 update delimiter and newline index if they are earlier than the current position before tested. (#728) --- papaparse.js | 6 ++++++ tests/test-cases.js | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/papaparse.js b/papaparse.js index d04eaae..15d281b 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1539,6 +1539,12 @@ License: MIT continue; } + if(nextDelim !== -1 && nextDelim < (quoteSearch + 1)) { + nextDelim = input.indexOf(delim, (quoteSearch + 1)); + } + if(nextNewline !== -1 && nextNewline < (quoteSearch + 1)) { + nextNewline = input.indexOf(newline, (quoteSearch + 1)); + } // Check up to nextDelim or nextNewline, whichever is closest var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline); var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo); diff --git a/tests/test-cases.js b/tests/test-cases.js index 608b297..bc8e2f7 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1464,6 +1464,22 @@ var PARSE_TESTS = [ data: [['a', 'b'], ['c', 'd'], [' , ', ','], ['" "', '""']], errors: [] } + }, + { + description: "Quoted fields with spaces between closing quote and next delimiter and contains delimiter", + input: 'A,",B" ,C,D\nE,F,G,H', + expected: { + data: [['A', ',B', 'C', 'D'],['E', 'F', 'G', 'H']], + errors: [] + } + }, + { + description: "Quoted fields with spaces between closing quote and newline and contains newline", + input: 'a,b,"c\n" \nd,e,f', + expected: { + data: [['a', 'b', 'c\n'], ['d', 'e', 'f']], + errors: [] + } } ]; From a318396c9d7ac9baab4731c39e6ee748048f3eac Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 14 Nov 2019 09:23:03 +0100 Subject: [PATCH 09/26] Reword newline docs Fixes #737 --- docs/docs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index 647164e..207250e 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -318,7 +318,7 @@ newline - The newline sequence. Must be one of "\r", "\n", or "\r\n". + The character used to determine newline sequence. It defaults to "\r\n". From ae73d2a96639beec58a83326de6bd8e8ca0c02b3 Mon Sep 17 00:00:00 2001 From: Duc Tri Le Date: Wed, 20 Nov 2019 03:00:52 -0500 Subject: [PATCH 10/26] Use chunk size to determine the processed length Fixes #736 #743 --- papaparse.js | 8 ++- tests/test-cases.js | 102 ++++++++++++++++++++++++++++++++++++++ tests/verylong-sample.csv | 4 +- 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/papaparse.js b/papaparse.js index 15d281b..df5c66a 100755 --- a/papaparse.js +++ b/papaparse.js @@ -682,7 +682,8 @@ License: MIT return; } - this._start += xhr.responseText.length; + // Use chunckSize as it may be a diference on reponse lentgh due to characters with more than 1 byte + this._start += this._config.chunkSize ? this._config.chunkSize : xhr.responseText.length; this._finished = !this._config.chunkSize || this._start >= getFileSize(xhr); this.parseChunk(xhr.responseText); }; @@ -1100,7 +1101,10 @@ License: MIT { _paused = true; _parser.abort(); - _input = _input.substring(_parser.getCharIndex()); + + // If it is streaming via "chunking", the reader will start appending correctly already so no need to substring, + // otherwise we can get duplicate content within a row + _input = isFunction(_config.chunk) ? "" : _input.substring(_parser.getCharIndex()); }; this.resume = function() diff --git a/tests/test-cases.js b/tests/test-cases.js index bc8e2f7..4f6550a 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1897,6 +1897,108 @@ var CUSTOM_TESTS = [ }); } }, + { + description: "Pause and resume works for chunks with NetworkStreamer", + disabled: !XHR_ENABLED, + timeout: 30000, + expected: ["Etiam a dolor vitae est vestibulum", "84", "DEF"], + run: function(callback) { + var chunkNum = 0; + Papa.parse(BASE_PATH + "verylong-sample.csv", { + download: true, + chunkSize: 1000, + chunk: function(results, parser) { + chunkNum++; + parser.pause(); + + if (chunkNum === 2) { + callback(results.data[0]); + return; + } + + parser.resume(); + }, + complete: function() { + callback(new Error("Should have found matched row before parsing whole file")); + } + }); + } + }, + { + description: "Pause and resume works for chunks with FileStreamer", + disabled: !XHR_ENABLED, + timeout: 30000, + expected: ["Etiam a dolor vitae est vestibulum", "84", "DEF"], + run: function(callback) { + var chunkNum = 0; + var xhr = new XMLHttpRequest(); + xhr.onload = function() { + Papa.parse(new File([xhr.responseText], './verylong-sample.csv'), { + chunkSize: 1000, + chunk: function(results, parser) { + chunkNum++; + parser.pause(); + + if (chunkNum === 2) { + callback(results.data[0]); + return; + } + + parser.resume(); + }, + complete: function() { + callback(new Error("Should have found matched row before parsing whole file")); + } + }); + }; + + xhr.open("GET", BASE_PATH + "verylong-sample.csv"); + try { + xhr.send(); + } catch (err) { + callback(err); + return; + } + } + }, + { + description: "Pause and resume works for chunks with StringStreamer", + disabled: !XHR_ENABLED, + timeout: 30000, + // Test also with string as byte size may be diferent + expected: ["Etiam a dolor vitae est vestibulum", "84", "DEF"], + run: function(callback) { + var chunkNum = 0; + var xhr = new XMLHttpRequest(); + xhr.onload = function() { + Papa.parse(xhr.responseText, { + chunkSize: 1000, + chunk: function(results, parser) { + chunkNum++; + parser.pause(); + + if (chunkNum === 2) { + callback(results.data[0]); + return; + } + + parser.resume(); + }, + complete: function() { + callback(new Error("Should have found matched row before parsing whole file")); + } + }); + }; + + xhr.open("GET", BASE_PATH + "verylong-sample.csv"); + try { + xhr.send(); + } catch (err) { + callback(err); + return; + } + } + }, { description: "Complete is called with all results if neither step nor chunk is defined", expected: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']], diff --git a/tests/verylong-sample.csv b/tests/verylong-sample.csv index ad70393..14bc527 100644 --- a/tests/verylong-sample.csv +++ b/tests/verylong-sample.csv @@ -1,7 +1,7 @@ placeholder,meaning of life,TLD Lorem ipsum dolor sit,42,ABC Etiam a dolor vitae est vestibulum,84,DEF -Lorem ipsum dolor sit,42,ABC +"Lorem ipsum dolor sit",42,ABC Etiam a dolor vitae est vestibulum,84,DEF Etiam a dolor vitae est vestibulum,84,DEF Lorem ipsum dolor sit,42,ABC @@ -1998,4 +1998,4 @@ Lorem ipsum dolor sit,42,ABC Lorem ipsum dolor sit,42,ABC Etiam a dolor vitae est vestibulum,84,DEF Lorem ipsum dolor sit,42 -Lorem ipsum dolor sit,42,ABC \ No newline at end of file +Lorem ipsum dolor sit,42,ABC From 3497ded575f329095fad38c48e2196dd0e87ae4b Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 18 Dec 2019 12:52:43 +0100 Subject: [PATCH 11/26] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index b0d5bce..98dd9c4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.1.0", + "version": "5.1.1", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index df5c66a..1757c11 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.1.0 +v5.1.1 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 48c3ad9..e0dd98f 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.1.0 +v5.1.1 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var r=(t=t||{}).dynamicTyping||!1;q(r)&&(t.dynamicTypingFunction=r,r={});if(t.dynamicTyping=r,t.transform=!!q(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var i=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(r=f.URL||f.webkitURL||null,i=s.toString(),b.BLOB_URL||(b.BLOB_URL=r.createObjectURL(new Blob(["(",i,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var r,i;return t.onmessage=_,t.id=h++,a[t.id]=t}();return i.userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=q(t.step),t.chunk=q(t.chunk),t.complete=q(t.complete),t.error=q(t.error),delete t.worker,void i.postMessage({input:e,config:t,workerId:i.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&q(e.read)&&q(e.on)?n=new m(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,g=",",v="\r\n",s='"',a=s+s,r=!1,i=null;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(g=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(r=t.skipEmptyLines);"string"==typeof t.newline&&(v=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");i=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s)}();var o=new RegExp(U(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,r);if("object"==typeof e[0])return u(i||h(e[0]),e,r)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:h(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],r);throw new Error("Unable to serialize unrecognized input");function h(e){if("object"!=typeof e)return[];var t=[];for(var r in e)t.push(r);return t}function u(e,t,r){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(q(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!q(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){q(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)),i.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send()}catch(e){this._chunkError(e.message)}n&&0===i.status&&this._chunkError()}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._start+=i.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substr(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;u.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e=this._config.chunkSize,t=e?r.substr(0,e):r;return r=e?r.substr(e):"",this._finished=!r,this.parseChunk(t)}}}function m(e){u.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(g){var a,o,h,i=Math.pow(2,53),n=-i,s=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,r=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(q(g.step)){var p=g.step;g.step=function(e){if(c=e,_())m();else{if(m(),0===c.data.length)return;r+=e.data.length,g.preview&&r>g.preview?o.abort():p(c,t)}}}function v(e){return"greedy"===g.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function m(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),g.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[r]),g.transform&&(s=g.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return g.header&&(r>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+r,f+t):r=i.length/2?"\r\n":"\r"}(e,i)),h=!1,g.delimiter)q(g.delimiter)&&(g.delimiter=g.delimiter(e),c.meta.delimiter=g.delimiter);else{var n=function(e,t,r,i,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(g=z,z++;;){if(-1===(g=a.indexOf(O,g+1)))return t||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:z}),w();if(g===i-1)return w(a.substring(z,g).replace(_,O));if(O!==M||a[g+1]!==M){if(O===M||0===g||a[g-1]!==M){var y=E(-1===m?p:Math.min(p,m));if(a[g+1+y]===D){f.push(a.substring(z,g).replace(_,O)),a[z=g+1+y+e]!==O&&(g=a.indexOf(O,z)),p=a.indexOf(D,z),m=a.indexOf(I,z);break}var k=E(m);if(a.substr(g+1+k,n)===I){if(f.push(a.substring(z,g).replace(_,O)),C(g+1+k+n),p=a.indexOf(D,z),g=a.indexOf(O,z),o&&(S(),j))return R();if(L&&h.length>=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:z}),g++}}else g++}return w();function b(e){h.push(e),d=z}function E(e){var t=0;if(-1!==e){var r=a.substring(g+1,e);r&&""===r.trim()&&(t=r.length)}return t}function w(e){return t||(void 0===e&&(e=a.substr(z)),f.push(e),z=i,b(f),o&&S()),R()}function C(e){z=e,b(f),f=[],m=a.indexOf(I,z)}function R(e,t){return{data:t||!1?h[0]:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(r||0)}}}function S(){A(R(void 0,!0)),h=[],u=[]}function x(e,t,r){var i={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(q(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!q(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){q(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(_){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(q(_.step)){var p=_.step;_.step=function(e){if(c=e,m())g();else{if(g(),0===c.data.length)return;i+=e.data.length,_.preview&&i>_.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function v(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),_.transform&&(s=_.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return _.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,_.delimiter)q(_.delimiter)&&(_.delimiter=_.delimiter(e),c.meta.delimiter=_.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(_=M,M++;;){if(-1===(_=a.indexOf(O,_+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:M}),w();if(_===r-1)return w(a.substring(M,_).replace(m,O));if(O!==z||a[_+1]!==z){if(O===z||0===_||a[_-1]!==z){-1!==p&&p<_+1&&(p=a.indexOf(D,_+1)),-1!==g&&g<_+1&&(g=a.indexOf(I,_+1));var y=E(-1===g?p:Math.min(p,g));if(a[_+1+y]===D){f.push(a.substring(M,_).replace(m,O)),a[M=_+1+y+e]!==O&&(_=a.indexOf(O,M)),p=a.indexOf(D,M),g=a.indexOf(I,M);break}var k=E(g);if(a.substring(_+1+k,_+1+k+n)===I){if(f.push(a.substring(M,_).replace(m,O)),C(_+1+k+n),p=a.indexOf(D,M),_=a.indexOf(O,M),o&&(S(),j))return R();if(L&&h.length>=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:M}),_++}}else _++}return w();function b(e){h.push(e),d=M}function E(e){var t=0;if(-1!==e){var i=a.substring(_+1,e);i&&""===i.trim()&&(t=i.length)}return t}function w(e){return i||(void 0===e&&(e=a.substring(M)),f.push(e),M=r,b(f),o&&S()),R()}function C(e){M=e,b(f),f=[],g=a.indexOf(I,M)}function R(e){return{data:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(t||0)}}}function S(){A(R()),h=[],u=[]}function x(e,t,i){var r={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t Date: Tue, 3 Mar 2020 13:59:50 +0000 Subject: [PATCH 12/26] Using self instead of this to preserve binding. (#769) --- papaparse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 1757c11..58ba296 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1115,7 +1115,7 @@ License: MIT } else { // Bugfix: #636 In case the processing hasn't halted yet // wait for it to halt in order to resume - setTimeout(this.resume, 3); + setTimeout(self.resume, 3); } }; From e934deb1f61e2df8cdf6878513ada051b526d620 Mon Sep 17 00:00:00 2001 From: DanzelTaccayan Date: Sat, 21 Mar 2020 15:55:11 +0000 Subject: [PATCH 13/26] Support POST method when download is true --- docs/docs.html | 9 +++++++++ papaparse.js | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index 207250e..7a0557e 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -433,6 +433,7 @@ var csv = Papa.unparse({ error: undefined, download: false, downloadRequestHeaders: undefined, + downloadRequestBody: undefined, skipEmptyLines: false, chunk: undefined, fastMode: undefined, @@ -598,6 +599,14 @@ var csv = Papa.unparse({ } + + + downloadRequestBody + + + If defined and the download property is true, a POST request will be made instead of a GET request and the passed argument will be set as the body of the request. + + skipEmptyLines diff --git a/papaparse.js b/papaparse.js index 58ba296..7380a88 100755 --- a/papaparse.js +++ b/papaparse.js @@ -642,7 +642,7 @@ License: MIT xhr.onerror = bindFunction(this._chunkError, this); } - xhr.open('GET', this._input, !IS_WORKER); + xhr.open(this._config.downloadRequestBody ? 'POST' : 'GET', this._input, !IS_WORKER); // Headers can only be set when once the request state is OPENED if (this._config.downloadRequestHeaders) { @@ -661,7 +661,7 @@ License: MIT } try { - xhr.send(); + xhr.send(this._config.downloadRequestBody); } catch (err) { this._chunkError(err.message); From a4cf371ff291ccb6b269a2a5872317fe83c31267 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Sat, 21 Mar 2020 16:57:49 +0100 Subject: [PATCH 14/26] Improve downloadRequestBody documentation --- docs/docs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index 7a0557e..b7785de 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -604,7 +604,7 @@ var csv = Papa.unparse({ downloadRequestBody - If defined and the download property is true, a POST request will be made instead of a GET request and the passed argument will be set as the body of the request. + Use POST request on the URL of the download option. The value passed will be set as the body of the request. From 235a12758cd77266d2e98fd715f53536b34ad621 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 2 Apr 2020 15:12:02 +0200 Subject: [PATCH 15/26] Avoid ReDOS on float dynamic typing (#779) Fixes #777 --- papaparse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 7380a88..6c358e9 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1006,7 +1006,7 @@ License: MIT // One goal is to minimize the use of regular expressions... var MAX_FLOAT = Math.pow(2, 53); var MIN_FLOAT = -MAX_FLOAT; - var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; + var FLOAT = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)(e[-+]?\d+)?\s*$/; var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/; var self = this; var _stepCounter = 0; // Number of times step was called (number of rows parsed) From 4b192deef192e8025eabf0ac4a77f3c559b85baa Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 2 Apr 2020 15:14:30 +0200 Subject: [PATCH 16/26] Minor version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 98dd9c4..0339c21 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.1.1", + "version": "5.2.0", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index 6c358e9..70dedf0 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.1.1 +v5.2.0 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index e0dd98f..2876359 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.1.1 +v5.2.0 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;q(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!q(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var r=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),b.BLOB_URL||(b.BLOB_URL=i.createObjectURL(new Blob(["(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=m,t.id=h++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=q(t.step),t.chunk=q(t.chunk),t.complete=q(t.complete),t.error=q(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&q(e.read)&&q(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,m=!0,_=",",v="\r\n",s='"',a=s+s,i=!1,r=null;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(_=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(v=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(m=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s)}();var o=new RegExp(U(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||h(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:h(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function h(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function u(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(q(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!q(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){q(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open("GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send()}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(_){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(q(_.step)){var p=_.step;_.step=function(e){if(c=e,m())g();else{if(g(),0===c.data.length)return;i+=e.data.length,_.preview&&i>_.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function v(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),_.transform&&(s=_.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return _.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,_.delimiter)q(_.delimiter)&&(_.delimiter=_.delimiter(e),c.meta.delimiter=_.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(_=M,M++;;){if(-1===(_=a.indexOf(O,_+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:M}),w();if(_===r-1)return w(a.substring(M,_).replace(m,O));if(O!==z||a[_+1]!==z){if(O===z||0===_||a[_-1]!==z){-1!==p&&p<_+1&&(p=a.indexOf(D,_+1)),-1!==g&&g<_+1&&(g=a.indexOf(I,_+1));var y=E(-1===g?p:Math.min(p,g));if(a[_+1+y]===D){f.push(a.substring(M,_).replace(m,O)),a[M=_+1+y+e]!==O&&(_=a.indexOf(O,M)),p=a.indexOf(D,M),g=a.indexOf(I,M);break}var k=E(g);if(a.substring(_+1+k,_+1+k+n)===I){if(f.push(a.substring(M,_).replace(m,O)),C(_+1+k+n),p=a.indexOf(D,M),_=a.indexOf(O,M),o&&(S(),j))return R();if(L&&h.length>=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:M}),_++}}else _++}return w();function b(e){h.push(e),d=M}function E(e){var t=0;if(-1!==e){var i=a.substring(_+1,e);i&&""===i.trim()&&(t=i.length)}return t}function w(e){return i||(void 0===e&&(e=a.substring(M)),f.push(e),M=r,b(f),o&&S()),R()}function C(e){M=e,b(f),f=[],g=a.indexOf(I,M)}function R(e){return{data:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(t||0)}}}function S(){A(R()),h=[],u=[]}function x(e,t,i){var r={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(U(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!U(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)(e[-+]?\d+)?\s*$/,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(U(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function v(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)U(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(m=M,M++;;){if(-1===(m=a.indexOf(O,m+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:M}),E();if(m===r-1)return E(a.substring(M,m).replace(_,O));if(O!==z||a[m+1]!==z){if(O===z||0===m||a[m-1]!==z){-1!==p&&p=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:M}),m++}}else m++}return E();function b(e){h.push(e),d=M}function w(e){var t=0;if(-1!==e){var i=a.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=a.substring(M)),f.push(e),M=r,b(f),o&&S()),R()}function C(e){M=e,b(f),f=[],g=a.indexOf(I,M)}function R(e){return{data:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(t||0)}}}function S(){A(R()),h=[],u=[]}function x(e,t,i){var r={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t Date: Fri, 1 May 2020 20:37:02 +0300 Subject: [PATCH 17/26] Add test to check for empty field in the begining (#790) --- tests/test-cases.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test-cases.js b/tests/test-cases.js index 4f6550a..18ec380 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -732,6 +732,23 @@ var PARSE_TESTS = [ }] } }, + { + description: "Row with enough fields but blank field in the begining", + input: 'A,B,C\r\n,b1,c1\r\na2,b2,c2', + expected: { + data: [["A", "B", "C"], ['', 'b1', 'c1'], ['a2', 'b2', 'c2']], + errors: [] + } + }, + { + description: "Row with enough fields but blank field in the begining using headers", + input: 'A,B,C\r\n,b1,c1\r\n,b2,c2', + config: { header: true }, + expected: { + data: [{"A": "", "B": "b1", "C": "c1"}, {"A": "", "B": "b2", "C": "c2"}], + errors: [] + } + }, { description: "Row with enough fields but blank field at end", input: 'A,B,C\r\na,b,', From 6f997ef4fb6d34b38f8eb9aa97b120b954446a9a Mon Sep 17 00:00:00 2001 From: John Preston Date: Sun, 24 May 2020 10:31:52 +0100 Subject: [PATCH 18/26] Implement escapeFormulae option (#796) Closes #793 --- docs/docs.html | 8 ++++++++ papaparse.js | 10 ++++++++++ tests/test-cases.js | 31 ++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index b7785de..8d253a1 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -337,6 +337,14 @@ If data is an array of objects this option can be used to manually specify the keys (columns) you expect in the objects. If not set the keys of the first objects are used as column. + + + escapeFormulae + + + If true, field values that begin with =, +, -, or @, will be prepended with a ' to defend against injection attacks, because Excel and LibreOffice will automatically parse such cells as formulae. + +
diff --git a/papaparse.js b/papaparse.js index 70dedf0..ec49373 100755 --- a/papaparse.js +++ b/papaparse.js @@ -282,6 +282,9 @@ License: MIT /** the columns (keys) we expect when we unparse objects */ var _columns = null; + /** whether to prevent outputting cells that can be parsed as formulae by spreadsheet software (Excel and LibreOffice) */ + var _escapeFormulae = false; + unpackConfig(); var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g'); @@ -361,6 +364,9 @@ License: MIT if (_config.escapeChar !== undefined) { _escapedQuote = _config.escapeChar + _quoteChar; } + + if (typeof _config.escapeFormulae === 'boolean') + _escapeFormulae = _config.escapeFormulae; } @@ -447,6 +453,10 @@ License: MIT if (str.constructor === Date) return JSON.stringify(str).slice(1, 25); + if (_escapeFormulae === true && typeof str === "string" && (str.match(/^[=+\-@].*$/) !== null)) { + str = "'" + str; + } + var escapedQuoteStr = str.toString().replace(quoteCharRegex, _escapedQuote); var needsQuotes = (typeof _quotes === 'boolean' && _quotes) diff --git a/tests/test-cases.js b/tests/test-cases.js index 18ec380..bc0b333 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1842,7 +1842,36 @@ var UNPARSE_TESTS = [ input: [{a: 'foo', b: '"quoted"'}], config: {header: false}, expected: 'foo,"""quoted"""' - } + }, + { + description: "Escape formulae", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true }, + expected: 'Col1,Col2,Col3\r\n\'=danger,\'@danger,safe\r\nsafe=safe,\'+danger,"\'-danger, danger"\r\n\'+safe,\'@safe,"safe, safe"' + }, + { + description: "Don't escape formulae by default", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + expected: 'Col1,Col2,Col3\r\n=danger,@danger,safe\r\nsafe=safe,+danger,"-danger, danger"\r\n\'+safe,\'@safe,"safe, safe"' + }, + { + description: "Escape formulae with forced quotes", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true, quotes: true }, + expected: '"Col1","Col2","Col3"\r\n"\'=danger","\'@danger","safe"\r\n"safe=safe","\'+danger","\'-danger, danger"\r\n"\'+safe","\'@safe","safe, safe"' + }, + { + description: "Escape formulae with single-quote quoteChar and escapeChar", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true, quoteChar: "'", escapeChar: "'" }, + expected: 'Col1,Col2,Col3\r\n\'\'=danger,\'\'@danger,safe\r\nsafe=safe,\'\'+danger,\'\'\'-danger, danger\'\r\n\'\'+safe,\'\'@safe,\'safe, safe\'' + }, + { + description: "Escape formulae with single-quote quoteChar and escapeChar and forced quotes", + input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], + config: { escapeFormulae: true, quotes: true, quoteChar: "'", escapeChar: "'" }, + expected: '\'Col1\',\'Col2\',\'Col3\'\r\n\'\'\'=danger\',\'\'\'@danger\',\'safe\'\r\n\'safe=safe\',\'\'\'+danger\',\'\'\'-danger, danger\'\r\n\'\'\'+safe\',\'\'\'@safe\',\'safe, safe\'' + }, ]; describe('Unparse Tests', function() { From 018f5dfe419c68f4f084df3d4452190cd501a74a Mon Sep 17 00:00:00 2001 From: James Furey Date: Thu, 25 Jun 2020 06:20:05 +1000 Subject: [PATCH 19/26] Add index argument to transformHeader (#807) --- papaparse.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/papaparse.js b/papaparse.js index ec49373..4934472 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1189,10 +1189,10 @@ License: MIT if (!_results) return; - function addHeder(header) + function addHeader(header, i) { if (isFunction(_config.transformHeader)) - header = _config.transformHeader(header); + header = _config.transformHeader(header, i); _fields.push(header); } @@ -1200,13 +1200,13 @@ License: MIT if (Array.isArray(_results.data[0])) { for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) - _results.data[i].forEach(addHeder); + _results.data[i].forEach(addHeader); _results.data.splice(0, 1); } // if _results.data[0] is not an array, we are in a step where _results.data is the row. else - _results.data.forEach(addHeder); + _results.data.forEach(addHeader); } function shouldApplyDynamicTyping(field) { From ce858b3c414236e52de05e2d0285e08ef2272e0a Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 24 Jun 2020 22:26:11 +0200 Subject: [PATCH 20/26] Add docs and tests case for transformHeader with index --- docs/docs.html | 2 +- tests/test-cases.js | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index 8d253a1..c4fec01 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -508,7 +508,7 @@ var csv = Papa.unparse({ transformHeader - A function to apply on each header. Requires header to be true. The function receives the header as its first argument.
+ A function to apply on each header. Requires header to be true. The function receives the header as its first argument and the index as second.
Only available starting with version 5.0. diff --git a/tests/test-cases.js b/tests/test-cases.js index bc0b333..8d15b29 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -767,6 +767,15 @@ var PARSE_TESTS = [ errors: [] } }, + { + description: "transformHeader accepts and optional index attribute", + input: 'A,B,C\r\na,b,c', + config: { header: true, transformHeader: function(header, i) { return i % 2 ? header.toLowerCase() : header; } }, + expected: { + data: [{"A": "a", "b": "b", "C": "c"}], + errors: [] + } + }, { description: "Line ends with quoted field, first field of next line is empty using headers", input: 'a,b,"c"\r\nd,e,"f"\r\n,"h","i"\r\n,"k","l"', From 7b26173728ecfb500bc24ff6ece0de22aa5c82a4 Mon Sep 17 00:00:00 2001 From: Alexandre Saiz Verdaguer Date: Wed, 24 Jun 2020 22:29:17 +0200 Subject: [PATCH 21/26] add: MONEI, MoonMail, Wholesaler as lovers (#771) --- docs/resources/js/lovers.js | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index ea21149..7cb1aac 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -102,6 +102,23 @@ var peopleLovePapa = [ name: "Hua Explore", description: "The premier destination for information on Contemporary Chinese Art.", quote: "Papa makes processing data that galleries send us totally seamless." - } - + }, + { + link: "https://monei.net", + name: "MONEI", + description: "Digital payments made easy.", + quote: "With Papa life became much easier for us to manage huge csv payments files of our merchants." + }, + { + link: "https://moonmail.io", + name: "MoonMail", + description: "OmniChannel Communication Platform powered by AWS PinPoint", + quote: "Papa makes contact imports a plain sailing." + }, + { + link: "https://apps.shopify.com/wholesaler", + name: "Wholesaler for Shopify", + description: "Shopify App to offer Wholesaling within one unique Shopify store", + quote: "Super fast bulk Wholesale product price uploads. Love Papa!." + } ]; From aa333201af80f79abbbe2984de4a5dcf90f88fd6 Mon Sep 17 00:00:00 2001 From: Paul Schlattmann <3031146+pschlatt@users.noreply.github.com> Date: Wed, 8 Jul 2020 14:37:08 -0600 Subject: [PATCH 22/26] Improve gready comment in docs.html (#816) --- docs/docs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index c4fec01..a73f512 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -259,7 +259,7 @@ delimiter: ",", header: true, newline: "\r\n", - skipEmptyLines: false, //or 'greedy', + skipEmptyLines: false, //other option is 'greedy', meaning skip delimiters, quotes, and whitespace. columns: null //or array of strings } From 12bf28a62f805eebe9e221e4c9d8a7bc576b788a Mon Sep 17 00:00:00 2001 From: Akshay Raj Gollahalli Date: Thu, 23 Jul 2020 19:40:46 +1200 Subject: [PATCH 23/26] Add documentation for chunkSize (#818) closes #673 --- docs/docs.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/docs.html b/docs/docs.html index a73f512..fc2a26e 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -444,6 +444,7 @@ var csv = Papa.unparse({ downloadRequestBody: undefined, skipEmptyLines: false, chunk: undefined, + chunkSize: undefined, fastMode: undefined, beforeFirstChunk: undefined, withCredentials: undefined, @@ -631,6 +632,14 @@ var csv = Papa.unparse({ A callback function, identical to step, which activates streaming. However, this function is executed after every chunk of the file is loaded and parsed rather than every row. Works only with local and remote files. Do not use both chunk and step callbacks together. For the function signature, see the documentation for the step function. + + + chunkSize + + + Overrides Papa.LocalChunkSize and Papa.RemoteChunkSize. For more information see configurable. + + fastMode From 6616222db9d07260497b3697e66c0e5d783e1825 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 23 Jul 2020 09:42:06 +0200 Subject: [PATCH 24/26] Improve documentation wording --- docs/docs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index fc2a26e..e7911c9 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -637,7 +637,7 @@ var csv = Papa.unparse({ chunkSize - Overrides Papa.LocalChunkSize and Papa.RemoteChunkSize. For more information see configurable. + Overrides Papa.LocalChunkSize and Papa.RemoteChunkSize. See configurable section to know the usage of both parameters. From 8414f7645aa761857ab959438b0c2a70ad683162 Mon Sep 17 00:00:00 2001 From: Unnit Metaliya Date: Fri, 24 Jul 2020 03:21:45 -0400 Subject: [PATCH 25/26] Add Visa SOP Sample on lovers (#820) --- docs/resources/js/lovers.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 7cb1aac..530a164 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -120,5 +120,11 @@ var peopleLovePapa = [ name: "Wholesaler for Shopify", description: "Shopify App to offer Wholesaling within one unique Shopify store", quote: "Super fast bulk Wholesale product price uploads. Love Papa!." - } + }, + { + link: "https://www.unnitmetaliya.com/sop-sample/", + name: "Visa SOP Sample", + description: "Providing free guide to international students.", + quote: "Use Papa Parse for many of side projects. Super fast and works all the time. Love it!" + } ]; From 5747da6c9927642743d6d44d49689be2d2fa0b10 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 25 Aug 2020 18:13:47 +0200 Subject: [PATCH 26/26] Minor version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 0339c21..6404a35 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.2.0", + "version": "5.3.0", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index 4934472..e1825d1 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.2.0 +v5.3.0 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 2876359..a92afc1 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.2.0 +v5.3.0 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;U(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!U(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var r=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),b.BLOB_URL||(b.BLOB_URL=i.createObjectURL(new Blob(["(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=_,t.id=h++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=U(t.step),t.chunk=U(t.chunk),t.complete=U(t.complete),t.error=U(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&U(e.read)&&U(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",v="\r\n",s='"',a=s+s,i=!1,r=null;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(v=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s)}();var o=new RegExp(q(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||h(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:h(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function h(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function u(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(U(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!U(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)(e[-+]?\d+)?\s*$/,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(U(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function v(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)U(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(m=M,M++;;){if(-1===(m=a.indexOf(O,m+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:M}),E();if(m===r-1)return E(a.substring(M,m).replace(_,O));if(O!==z||a[m+1]!==z){if(O===z||0===m||a[m-1]!==z){-1!==p&&p=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:M}),m++}}else m++}return E();function b(e){h.push(e),d=M}function w(e){var t=0;if(-1!==e){var i=a.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=a.substring(M)),f.push(e),M=r,b(f),o&&S()),R()}function C(e){M=e,b(f),f=[],g=a.indexOf(I,M)}function R(e){return{data:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(t||0)}}}function S(){A(R()),h=[],u=[]}function x(e,t,i){var r={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(U(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!U(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(_){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)(e[-+]?\d+)?\s*$/,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(U(_.step)){var p=_.step;_.step=function(e){if(c=e,m())g();else{if(g(),0===c.data.length)return;i+=e.data.length,_.preview&&i>_.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function v(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),_.transform&&(s=_.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return _.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,_.delimiter)U(_.delimiter)&&(_.delimiter=_.delimiter(e),c.meta.delimiter=_.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(_=M,M++;;){if(-1===(_=a.indexOf(O,_+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:M}),E();if(_===r-1)return E(a.substring(M,_).replace(m,O));if(O!==z||a[_+1]!==z){if(O===z||0===_||a[_-1]!==z){-1!==p&&p<_+1&&(p=a.indexOf(D,_+1)),-1!==g&&g<_+1&&(g=a.indexOf(I,_+1));var y=w(-1===g?p:Math.min(p,g));if(a[_+1+y]===D){f.push(a.substring(M,_).replace(m,O)),a[M=_+1+y+e]!==O&&(_=a.indexOf(O,M)),p=a.indexOf(D,M),g=a.indexOf(I,M);break}var k=w(g);if(a.substring(_+1+k,_+1+k+n)===I){if(f.push(a.substring(M,_).replace(m,O)),C(_+1+k+n),p=a.indexOf(D,M),_=a.indexOf(O,M),o&&(S(),j))return R();if(L&&h.length>=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:M}),_++}}else _++}return E();function b(e){h.push(e),d=M}function w(e){var t=0;if(-1!==e){var i=a.substring(_+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=a.substring(M)),f.push(e),M=r,b(f),o&&S()),R()}function C(e){M=e,b(f),f=[],g=a.indexOf(I,M)}function R(e){return{data:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(t||0)}}}function S(){A(R()),h=[],u=[]}function x(e,t,i){var r={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t