Browse Source

support custom decoder option.

(used for cases when the encoding isn't supported by native buffer,  for example parsing 'SJIS'  in node)
pull/650/head
Ran Grizm 6 years ago
parent
commit
d3a3e59170
  1. 15
      docs/docs.html
  2. 27
      papaparse.js

15
docs/docs.html

@ -429,6 +429,7 @@ var csv = Papa.unparse({
dynamicTyping: false, dynamicTyping: false,
preview: 0, preview: 0,
encoding: "", encoding: "",
decoder: undefined,
worker: false, worker: false,
comments: false, comments: false,
step: undefined, step: undefined,
@ -530,6 +531,20 @@ var csv = Papa.unparse({
The encoding to use when opening local files. If specified, it must be a value supported by the FileReader API. The encoding to use when opening local files. If specified, it must be a value supported by the FileReader API.
</td> </td>
</tr> </tr>
<tr>
<td>
<code>decoder</code>
</td>
<td>
A function which allows custom decoding for the buffer string.
The function get Buffer and Encoding, and returns a string.
i.e assuming const iconv = require('iconv-lite'):
(buf , encoding) => {
if (iconv.encodingExists(encoding)) { return iconv.decode(buf, encoding) };
throw new TypeError('[ERR_UNKNOWN_ENCODING]', 'Unknown encoding: ' + encoding)
}
</td>
</tr>
<tr> <tr>
<td> <td>
<code>worker</code> <code>worker</code>

27
papaparse.js

@ -852,7 +852,18 @@ License: MIT
{ {
try try
{ {
queue.push(typeof chunk === 'string' ? chunk : chunk.toString(this._config.encoding)); var encoding = this._config.encoding;
var decoder = this._config.decoder;
var encodedString = '';
if (typeof decoder === 'function') {
encodedString = decoder(chunk, encoding);
} else if (typeof chunk === 'string') {
encodedString = chunk;
} else {
encodedString = chunk.toString(encoding);
}
queue.push(encodedString);
if (parseOnData) if (parseOnData)
{ {
@ -941,7 +952,19 @@ License: MIT
// when too many items have been added without their // when too many items have been added without their
// callback being invoked // callback being invoked
parseCallbackQueue.push(bindFunction(function() { parseCallbackQueue.push(bindFunction(function() {
this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding)); var encoding = config.encoding;
var decoder = config.decoder;
var encodedString = '';
if (typeof decoder === 'function') {
encodedString = decoder(chunk, encoding);
} else if (typeof chunk === 'string') {
encodedString = chunk;
} else {
encodedString = chunk.toString(encoding);
}
this.parseChunk(encodedString);
if (isFunction(callback)) { if (isFunction(callback)) {
return callback(); return callback();
} }

Loading…
Cancel
Save