Browse Source

support custom decoder option.

(used for cases when the encoding isn't supported by native buffer,  for example parsing 'SJIS'  in node)
pull/650/head
Ran Grizm 6 years ago
parent
commit
d3a3e59170
  1. 15
      docs/docs.html
  2. 27
      papaparse.js

15
docs/docs.html

@ -429,6 +429,7 @@ var csv = Papa.unparse({ @@ -429,6 +429,7 @@ var csv = Papa.unparse({
dynamicTyping: false,
preview: 0,
encoding: "",
decoder: undefined,
worker: false,
comments: false,
step: undefined,
@ -530,6 +531,20 @@ var csv = Papa.unparse({ @@ -530,6 +531,20 @@ var csv = Papa.unparse({
The encoding to use when opening local files. If specified, it must be a value supported by the FileReader API.
</td>
</tr>
<tr>
<td>
<code>decoder</code>
</td>
<td>
A function which allows custom decoding for the buffer string.
The function get Buffer and Encoding, and returns a string.
i.e assuming const iconv = require('iconv-lite'):
(buf , encoding) => {
if (iconv.encodingExists(encoding)) { return iconv.decode(buf, encoding) };
throw new TypeError('[ERR_UNKNOWN_ENCODING]', 'Unknown encoding: ' + encoding)
}
</td>
</tr>
<tr>
<td>
<code>worker</code>

27
papaparse.js

@ -852,7 +852,18 @@ License: MIT @@ -852,7 +852,18 @@ License: MIT
{
try
{
queue.push(typeof chunk === 'string' ? chunk : chunk.toString(this._config.encoding));
var encoding = this._config.encoding;
var decoder = this._config.decoder;
var encodedString = '';
if (typeof decoder === 'function') {
encodedString = decoder(chunk, encoding);
} else if (typeof chunk === 'string') {
encodedString = chunk;
} else {
encodedString = chunk.toString(encoding);
}
queue.push(encodedString);
if (parseOnData)
{
@ -941,7 +952,19 @@ License: MIT @@ -941,7 +952,19 @@ License: MIT
// when too many items have been added without their
// callback being invoked
parseCallbackQueue.push(bindFunction(function() {
this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding));
var encoding = config.encoding;
var decoder = config.decoder;
var encodedString = '';
if (typeof decoder === 'function') {
encodedString = decoder(chunk, encoding);
} else if (typeof chunk === 'string') {
encodedString = chunk;
} else {
encodedString = chunk.toString(encoding);
}
this.parseChunk(encodedString);
if (isFunction(callback)) {
return callback();
}

Loading…
Cancel
Save