Merge pull request #8620 from Rob--W/issue-8560-improve-png-compression

Improve compression of PNG images embedded in generated SVG files

examples/node/domstubs.js

@@ -41,24 +41,6 @@ function xmlEncode(s){
   return buf;
 }
 
-global.btoa = function btoa(chars) {
-  var digits =
-  'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
-  var buffer = '';
-    var i, n;
-    for (i = 0, n = chars.length; i < n; i += 3) {
-      var b1 = chars.charCodeAt(i) & 0xFF;
-      var b2 = chars.charCodeAt(i + 1) & 0xFF;
-      var b3 = chars.charCodeAt(i + 2) & 0xFF;
-      var d1 = b1 >> 2, d2 = ((b1 & 3) << 4) | (b2 >> 4);
-      var d3 = i + 1 < n ? ((b2 & 0xF) << 2) | (b3 >> 6) : 64;
-      var d4 = i + 2 < n ? (b3 & 0x3F) : 64;
-      buffer += (digits.charAt(d1) + digits.charAt(d2) +
-      digits.charAt(d3) + digits.charAt(d4));
-    }
-  return buffer;
-};
-
 function DOMElement(name) {
   this.nodeName = name;
   this.childNodes = [];
@@ -127,7 +109,7 @@ DOMElement.prototype = {
   },
 }
 
-global.document = {
+const document = {
   childNodes : [],
 
   get currentScript() {
@@ -171,4 +153,20 @@ Image.prototype = {
   }
 }
 
-global.Image = Image;
+exports.document = document;
+exports.Image = Image;
+
+var exported_symbols = Object.keys(exports);
+
+exports.setStubs = function(namespace) {
+  exported_symbols.forEach(function(key) {
+    console.assert(!(key in namespace), 'property should not be set: ' + key);
+    namespace[key] = exports[key];
+  });
+};
+exports.unsetStubs = function(namespace) {
+  exported_symbols.forEach(function(key) {
+    console.assert(key in namespace, 'property should be set: ' + key);
+    delete namespace[key];
+  });
+};

examples/node/pdf2svg.js

@@ -8,7 +8,7 @@
 var fs = require('fs');
 
 // HACK few hacks to let PDF.js be loaded not as a module in global space.
-require('./domstubs.js');
+require('./domstubs.js').setStubs(global);
 
 // Run `gulp dist-install` to generate 'pdfjs-dist' npm package files.
 var pdfjsLib = require('pdfjs-dist');

gulpfile.js

@@ -1044,6 +1044,7 @@ gulp.task('lib', ['buildnumber'], function () {
       'src/{pdf,pdf.worker}.js',
     ], { base: 'src/', }),
     gulp.src([
+      'examples/node/domstubs.js',
       'web/*.js',
       '!web/pdfjs.js',
       '!web/viewer.js',

src/display/svg.js

@@ -12,10 +12,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+/* globals __non_webpack_require__ */
 
 import {
   createObjectURL, FONT_IDENTITY_MATRIX, IDENTITY_MATRIX, ImageKind, isArray,
-  isNum, OPS, Util, warn
+  isNodeJS, isNum, OPS, Util, warn
 } from '../shared/util';
 
 var SVGGraphics = function() {
@@ -97,6 +98,89 @@ var convertImgDataToPng = (function convertImgDataToPngClosure() {
     return (b << 16) | a;
   }
 
+  /**
+   * @param {Uint8Array} literals The input data.
+   * @returns {Uint8Array} The DEFLATE-compressed data stream in zlib format.
+   *   This is the required format for compressed streams in the PNG format:
+   *   http://www.libpng.org/pub/png/spec/1.2/PNG-Compression.html
+   */
+  function deflateSync(literals) {
+    if (!isNodeJS()) {
+      // zlib is certainly not available outside of Node.js. We can either use
+      // the pako library for client-side DEFLATE compression, or use the canvas
+      // API of the browser to obtain a more optimal PNG file.
+      return deflateSyncUncompressed(literals);
+    }
+    try {
+      // NOTE: This implementation is far from perfect, but already way better
+      // than not applying any compression.
+      //
+      // A better algorithm will try to choose a good predictor/filter and
+      // then choose a suitable zlib compression strategy (e.g. 3,Z_RLE).
+      //
+      // Node v0.11.12 zlib.deflateSync is introduced (and returns a Buffer).
+      // Node v3.0.0   Buffer inherits from Uint8Array.
+      // Node v8.0.0   zlib.deflateSync accepts Uint8Array as input.
+      var input;
+        // eslint-disable-next-line no-undef
+      if (parseInt(process.versions.node) >= 8) {
+        input = literals;
+      } else {
+        // eslint-disable-next-line no-undef
+        input = new Buffer(literals);
+      }
+      var output = __non_webpack_require__('zlib')
+        .deflateSync(input, { level: 9, });
+      return output instanceof Uint8Array ? output : new Uint8Array(output);
+    } catch (e) {
+      warn('Not compressing PNG because zlib.deflateSync is unavailable: ' + e);
+    }
+
+    return deflateSyncUncompressed(literals);
+  }
+
+  // An implementation of DEFLATE with compression level 0 (Z_NO_COMPRESSION).
+  function deflateSyncUncompressed(literals) {
+    var len = literals.length;
+    var maxBlockLength = 0xFFFF;
+
+    var deflateBlocks = Math.ceil(len / maxBlockLength);
+    var idat = new Uint8Array(2 + len + deflateBlocks * 5 + 4);
+    var pi = 0;
+    idat[pi++] = 0x78; // compression method and flags
+    idat[pi++] = 0x9c; // flags
+
+    var pos = 0;
+    while (len > maxBlockLength) {
+      // writing non-final DEFLATE blocks type 0 and length of 65535
+      idat[pi++] = 0x00;
+      idat[pi++] = 0xff;
+      idat[pi++] = 0xff;
+      idat[pi++] = 0x00;
+      idat[pi++] = 0x00;
+      idat.set(literals.subarray(pos, pos + maxBlockLength), pi);
+      pi += maxBlockLength;
+      pos += maxBlockLength;
+      len -= maxBlockLength;
+    }
+
+    // writing non-final DEFLATE blocks type 0
+    idat[pi++] = 0x01;
+    idat[pi++] = len & 0xff;
+    idat[pi++] = len >> 8 & 0xff;
+    idat[pi++] = (~len & 0xffff) & 0xff;
+    idat[pi++] = (~len & 0xffff) >> 8 & 0xff;
+    idat.set(literals.subarray(pos), pi);
+    pi += literals.length - pos;
+
+    var adler = adler32(literals, 0, literals.length); // checksum
+    idat[pi++] = adler >> 24 & 0xff;
+    idat[pi++] = adler >> 16 & 0xff;
+    idat[pi++] = adler >> 8 & 0xff;
+    idat[pi++] = adler & 0xff;
+    return idat;
+  }
+
   function encode(imgData, kind, forceDataSchema) {
     var width = imgData.width;
     var height = imgData.height;
@@ -162,43 +246,7 @@ var convertImgDataToPng = (function convertImgDataToPngClosure() {
       0x00 // interlace method
     ]);
 
-    var len = literals.length;
+    var idat = deflateSync(literals);
-    var maxBlockLength = 0xFFFF;
-
-    var deflateBlocks = Math.ceil(len / maxBlockLength);
-    var idat = new Uint8Array(2 + len + deflateBlocks * 5 + 4);
-    var pi = 0;
-    idat[pi++] = 0x78; // compression method and flags
-    idat[pi++] = 0x9c; // flags
-
-    var pos = 0;
-    while (len > maxBlockLength) {
-      // writing non-final DEFLATE blocks type 0 and length of 65535
-      idat[pi++] = 0x00;
-      idat[pi++] = 0xff;
-      idat[pi++] = 0xff;
-      idat[pi++] = 0x00;
-      idat[pi++] = 0x00;
-      idat.set(literals.subarray(pos, pos + maxBlockLength), pi);
-      pi += maxBlockLength;
-      pos += maxBlockLength;
-      len -= maxBlockLength;
-    }
-
-    // writing non-final DEFLATE blocks type 0
-    idat[pi++] = 0x01;
-    idat[pi++] = len & 0xff;
-    idat[pi++] = len >> 8 & 0xff;
-    idat[pi++] = (~len & 0xffff) & 0xff;
-    idat[pi++] = (~len & 0xffff) >> 8 & 0xff;
-    idat.set(literals.subarray(pos), pi);
-    pi += literals.length - pos;
-
-    var adler = adler32(literals, 0, literals.length); // checksum
-    idat[pi++] = adler >> 24 & 0xff;
-    idat[pi++] = adler >> 16 & 0xff;
-    idat[pi++] = adler >> 8 & 0xff;
-    idat[pi++] = adler & 0xff;
 
     // PNG will consists: header, IHDR+data, IDAT+data, and IEND.
     var pngLength = PNG_HEADER.length + (CHUNK_WRAPPER_SIZE * 3) +

test/pdfs/.gitignore

@@ -288,3 +288,4 @@
 !font_ascent_descent.pdf
 !issue8097_reduced.pdf
 !transparent.pdf
+!xobject-image.pdf

test/pdfs/xobject-image.pdf

@@ -0,0 +1,61 @@
+%PDF-1.1
+1 0 obj
+<</Type/Catalog/Pages 2 0 R>>
+endobj
+2 0 obj
+<</Type/Pages/Count 1/Kids[3 0 R]/MediaBox [0 0 200 100]>>
+endobj
+3 0 obj
+<<
+ /Type/Page
+ /Parent 2 0 R
+ /Resources <<
+  /XObject << /SomeImage 4 0 R >>
+ >>
+ /Contents 5 0 R
+>>
+endobj
+4 0 obj
+<<
+ /Type/XObject
+ /Subtype/Image
+ /Width 200 % The width or height directly affects the image's file size.
+ /Height 100
+ /ColorSpace/DeviceRGB
+ /DecodeParms [] % Forces NativeImageDecoder.isSupported to return false.
+ /BitsPerComponent 8
+ /Length 580
+ /Filter [ /ASCIIHexDecode /DCTDecode ]
+>>
+% convert -size 1x1 xc:red jpeg:- | xxd -p -c40
+stream
+ffd8ffe000104a46494600010100000100010000ffdb004300030202020202030202020303030304
+060404040404080606050609080a0a090809090a0c0f0c0a0b0e0b09090d110d0e0f101011100a0c
+12131210130f101010ffdb00430103030304030408040408100b090b101010101010101010101010
+1010101010101010101010101010101010101010101010101010101010101010101010101010ffc0
+0011080001000103011100021101031101ffc40014000100000000000000000000000000000008ff
+c40014100100000000000000000000000000000000ffc40015010101000000000000000000000000
+00000709ffc40014110100000000000000000000000000000000ffda000c03010002110311003f00
+3a03154dffd9
+endstream
+endobj
+5 0 obj
+<</Length 14>>
+stream
+500 0 0 400 0 0 cm
+/SomeImage Do
+endstream
+endobj
+xref
+0 6
+0000000000 65535 f 
+0000000008 00000 n 
+0000000054 00000 n 
+0000000128 00000 n 
+0000000246 00000 n 
+0000001201 00000 n 
+trailer
+<</Root 1 0 R/Size 6>>
+startxref
+1281
+%%EOF

test/unit/clitests.json

@@ -7,6 +7,7 @@
     "cff_parser_spec.js",
     "cmap_spec.js",
     "crypto_spec.js",
+    "display_svg_spec.js",
     "document_spec.js",
     "dom_utils_spec.js",
     "evaluator_spec.js",

test/unit/display_svg_spec.js

@@ -0,0 +1,135 @@
+/* Copyright 2017 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* globals __non_webpack_require__ */
+
+import { isNodeJS, NativeImageDecoding } from '../../src/shared/util';
+import { setStubs, unsetStubs } from '../../examples/node/domstubs';
+import { buildGetDocumentParams } from './test_utils';
+import { getDocument } from '../../src/display/api';
+import { SVGGraphics } from '../../src/display/svg';
+
+// withZlib(true, callback); = run test with require('zlib') if possible.
+// withZlib(false, callback); = run test without require('zlib').deflateSync.
+// The return value of callback is returned as-is.
+function withZlib(isZlibRequired, callback) {
+  if (isZlibRequired) {
+    // We could try to polyfill zlib in the browser, e.g. using pako.
+    // For now, only support zlib functionality on Node.js
+    if (!isNodeJS()) {
+      throw new Error('zlib test can only be run in Node.js');
+    }
+
+    return callback();
+  }
+
+  if (!isNodeJS()) {
+    // Assume that require('zlib') is unavailable in non-Node.
+    return callback();
+  }
+
+  var zlib = __non_webpack_require__('zlib');
+  var deflateSync = zlib.deflateSync;
+  zlib.deflateSync = function() {
+    throw new Error('zlib.deflateSync is explicitly disabled for testing.');
+  };
+  try {
+    return callback();
+  } finally {
+    zlib.deflateSync = deflateSync;
+  }
+}
+
+describe('SVGGraphics', function () {
+  var loadingTask;
+  var page;
+  beforeAll(function(done) {
+    loadingTask = getDocument(buildGetDocumentParams('xobject-image.pdf', {
+      nativeImageDecoderSupport: NativeImageDecoding.DISPLAY,
+    }));
+    loadingTask.promise.then(function(doc) {
+      doc.getPage(1).then(function(firstPage) {
+        page = firstPage;
+        done();
+      });
+    });
+  });
+  afterAll(function(done) {
+    loadingTask.destroy().then(done);
+  });
+
+  describe('paintImageXObject', function() {
+    function getSVGImage() {
+      var svgGfx;
+      return page.getOperatorList().then(function(opList) {
+        var forceDataSchema = true;
+        svgGfx = new SVGGraphics(page.commonObjs, page.objs, forceDataSchema);
+        return svgGfx.loadDependencies(opList);
+      }).then(function() {
+        var svgImg;
+        // A mock to steal the svg:image element from paintInlineImageXObject.
+        var elementContainer = {
+          appendChild(element) {
+            svgImg = element;
+          },
+        };
+
+        // This points to the XObject image in xobject-image.pdf.
+        var xobjectObjId = { ref: 4, gen: 0, };
+        if (isNodeJS()) {
+          setStubs(global);
+        }
+        try {
+          svgGfx.paintImageXObject(xobjectObjId, elementContainer);
+        } finally {
+          if (isNodeJS()) {
+            unsetStubs(global);
+          }
+        }
+        return svgImg;
+      });
+    }
+
+    it('should produce a reasonably small svg:image', function() {
+      if (!isNodeJS()) {
+        pending('zlib.deflateSync is not supported in non-Node environments.');
+      }
+      withZlib(true, getSVGImage).then(function(svgImg) {
+        expect(svgImg.nodeName).toBe('svg:image');
+        expect(svgImg.getAttribute('width')).toBe('200px');
+        expect(svgImg.getAttribute('height')).toBe('100px');
+        var imgUrl = svgImg.getAttribute('xlink:href');
+        // forceDataSchema = true, so the generated URL should be a data:-URL.
+        expect(imgUrl).toMatch(/^data:image\/png;base64,/);
+        // Test whether the generated image has a reasonable file size.
+        // I obtained a data URL of size 366 with Node 8.1.3 and zlib 1.2.11.
+        // Without zlib (uncompressed), the size of the data URL was excessive
+        // (80247).
+        expect(imgUrl.length).toBeLessThan(367);
+      });
+    });
+
+    it('should produce a svg:image even if zlib is unavailable', function() {
+      withZlib(false, getSVGImage).then(function(svgImg) {
+        expect(svgImg.nodeName).toBe('svg:image');
+        expect(svgImg.getAttribute('width')).toBe('200px');
+        expect(svgImg.getAttribute('height')).toBe('100px');
+        var imgUrl = svgImg.getAttribute('xlink:href');
+        expect(imgUrl).toMatch(/^data:image\/png;base64,/);
+        // The size of our naively generated PNG file is excessive :(
+        expect(imgUrl.length).toBe(80247);
+      });
+    });
+  });
+});

test/unit/jasmine-boot.js

@@ -42,18 +42,31 @@
 
 function initializePDFJS(callback) {
   Promise.all([
-    'pdfjs/display/global', 'pdfjs-test/unit/annotation_spec',
+    'pdfjs/display/global',
-    'pdfjs-test/unit/api_spec', 'pdfjs-test/unit/bidi_spec',
+    'pdfjs-test/unit/annotation_spec',
-    'pdfjs-test/unit/cff_parser_spec', 'pdfjs-test/unit/cmap_spec',
+    'pdfjs-test/unit/api_spec',
-    'pdfjs-test/unit/crypto_spec', 'pdfjs-test/unit/custom_spec',
+    'pdfjs-test/unit/bidi_spec',
-    'pdfjs-test/unit/document_spec', 'pdfjs-test/unit/dom_utils_spec',
+    'pdfjs-test/unit/cff_parser_spec',
-    'pdfjs-test/unit/evaluator_spec', 'pdfjs-test/unit/fonts_spec',
+    'pdfjs-test/unit/cmap_spec',
-    'pdfjs-test/unit/function_spec', 'pdfjs-test/unit/metadata_spec',
+    'pdfjs-test/unit/crypto_spec',
-    'pdfjs-test/unit/murmurhash3_spec', 'pdfjs-test/unit/network_spec',
+    'pdfjs-test/unit/custom_spec',
-    'pdfjs-test/unit/parser_spec', 'pdfjs-test/unit/primitives_spec',
+    'pdfjs-test/unit/display_svg_spec',
-    'pdfjs-test/unit/stream_spec', 'pdfjs-test/unit/type1_parser_spec',
+    'pdfjs-test/unit/document_spec',
-    'pdfjs-test/unit/ui_utils_spec', 'pdfjs-test/unit/unicode_spec',
+    'pdfjs-test/unit/dom_utils_spec',
-    'pdfjs-test/unit/util_spec', 'pdfjs-test/unit/util_stream_spec'
+    'pdfjs-test/unit/evaluator_spec',
+    'pdfjs-test/unit/fonts_spec',
+    'pdfjs-test/unit/function_spec',
+    'pdfjs-test/unit/metadata_spec',
+    'pdfjs-test/unit/murmurhash3_spec',
+    'pdfjs-test/unit/network_spec',
+    'pdfjs-test/unit/parser_spec',
+    'pdfjs-test/unit/primitives_spec',
+    'pdfjs-test/unit/stream_spec',
+    'pdfjs-test/unit/type1_parser_spec',
+    'pdfjs-test/unit/ui_utils_spec',
+    'pdfjs-test/unit/unicode_spec',
+    'pdfjs-test/unit/util_spec',
+    'pdfjs-test/unit/util_stream_spec',
   ].map(function (moduleName) {
     return SystemJS.import(moduleName);
   })).then(function (modules) {