Better handling of encoding base64 uris

lib/convert.js

@@ -6,8 +6,8 @@ var path = require('path');
 var Promise = require('bluebird');
 var createGltf = require('./gltf');
 var loadObj = require('./obj');
+var writeUris = require('./writeUris');
 
-var fsExtraOutputFile = Promise.promisify(fsExtra.outputFile);
 var fsExtraOutputJson = Promise.promisify(fsExtra.outputJson);
 
 var defaultValue = Cesium.defaultValue;
@@ -86,7 +86,7 @@ function convert(objPath, gltfPath, options) {
             return createGltf(objData);
         })
         .then(function(gltf) {
-            return writeSeparateResources(gltf, gltfPath, separate, separateTextures);
+            return writeUris(gltf, gltfPath, separate, separateTextures);
         })
         .then(function(gltf) {
             if (bypassPipeline) {
@@ -97,73 +97,9 @@ function convert(objPath, gltfPath, options) {
         });
 }
 
-function deleteExtras(gltf) {
-    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
-    delete buffer.extras;
-
-    var images = gltf.images;
-    for (var id in images) {
-        if (images.hasOwnProperty(id)) {
-            var image = images[id];
-            delete image.extras;
-        }
-    }
-}
-
-function writeSeparateBuffer(gltf, gltfPath) {
-    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
-    var source = buffer.extras._obj2gltf.source;
-    var bufferName = path.basename(gltfPath, path.extname(gltfPath));
-    var bufferUri = bufferName + '.bin';
-    buffer.uri = bufferUri;
-    var bufferPath = path.join(path.dirname(gltfPath), bufferUri);
-    return convert._outputFile(bufferPath, source);
-}
-
-function writeSeparateTextures(gltf, gltfPath) {
-    var promises = [];
-    var images = gltf.images;
-    for (var id in images) {
-        if (images.hasOwnProperty(id)) {
-            var image = images[id];
-            var extras = image.extras._obj2gltf;
-            var imageUri = image.name + extras.extension;
-            image.uri = imageUri;
-            var imagePath = path.join(path.dirname(gltfPath), imageUri);
-            promises.push(convert._outputFile(imagePath, extras.source));
-        }
-    }
-    return Promise.all(promises);
-}
-
-function writeSeparateResources(gltf, gltfPath, separate, separateTextures) {
-    var promises = [];
-    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
-
-    if (separate || !defined(buffer.uri)) {
-        promises.push(writeSeparateBuffer(gltf, gltfPath));
-    }
-    if (separateTextures) {
-        promises.push(writeSeparateTextures(gltf, gltfPath));
-    }
-
-    deleteExtras(gltf);
-    return Promise.all(promises)
-        .then(function() {
-            return gltf;
-        });
-}
-
 /**
  * Exposed for testing
  *
  * @private
  */
 convert._outputJson = fsExtraOutputJson;
-
-/**
- * Exposed for testing
- *
- * @private
- */
-convert._outputFile = fsExtraOutputFile;

lib/gltf.js

@@ -137,10 +137,9 @@ function createGltf(objData) {
 
             gltf.images[imageId] = {
                 name : imageId,
-                uri : image.uri,
                 extras : {
                     _obj2gltf : {
-                        source : image.data,
+                        source : image.source,
                         extension : image.extension
                     }
                 }
@@ -317,15 +316,8 @@ function createGltf(objData) {
     buffers = buffers.concat(vertexBuffers, indexBuffers);
     var buffer = Buffer.concat(buffers);
 
-    // Buffers larger than ~192MB cannot be base64 encoded due to a NodeJS limitation. Source: https://github.com/nodejs/node/issues/4266
-    var bufferUri;
-    if (buffer.length <= 201326580) {
-        bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64');
-    }
-
     gltf.buffers[bufferId] = {
         byteLength : buffer.byteLength,
-        uri : bufferUri,
         extras : {
             _obj2gltf : {
                 source : buffer

lib/image.js

@@ -23,14 +23,11 @@ function loadImage(imagePath) {
     return fsReadFile(imagePath)
         .then(function(data) {
             var extension = path.extname(imagePath);
-            var uriType = getUriType(extension);
-            var uri = uriType + ';base64,' + data.toString('base64');
 
             var info = {
                 transparent : false,
-                data : data,
-                uri : uri,
                 format : getFormat(3),
+                source : data,
                 extension : extension
             };
 
@@ -81,20 +78,6 @@ function getChannels(colorType) {
     }
 }
 
-function getUriType(extension) {
-    switch (extension) {
-        case '.png':
-            return 'data:image/png';
-        case '.jpg':
-        case '.jpeg':
-            return 'data:image/jpeg';
-        case '.gif':
-            return 'data:image/gif';
-        default:
-            return 'data:image/' + extension.slice(1);
-    }
-}
-
 function getFormat(channels) {
     switch (channels) {
         case 1:

lib/writeUris.js

@@ -0,0 +1,126 @@
+'use strict';
+var fsExtra = require('fs-extra');
+var mime = require('mime');
+var path = require('path');
+var Promise = require('bluebird');
+
+var fsExtraOutputFile = Promise.promisify(fsExtra.outputFile);
+
+module.exports = writeUris;
+
+/**
+ * Write glTF resources as embedded data uris or external files.
+ *
+ * @param {Object} gltf The glTF asset.
+ * @param {String} gltfPath Path where the glTF will be saved.
+ * @param {Boolean} separateBuffers Writes out separate buffers.
+ * @param {Boolean} separateTextures Writes out separate textures.
+ * @returns {Promise} A promise that resolves to the glTF asset.
+ *
+ * @private
+ */
+function writeUris(gltf, gltfPath, separateBuffers, separateTextures) {
+    var promises = [];
+
+    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
+    var bufferByteLength = buffer.extras._obj2gltf.source.length;
+
+    var texturesByteLength = 0;
+    var images = gltf.images;
+    for (var id in images) {
+        if (images.hasOwnProperty(id)) {
+            texturesByteLength += images[id].extras._obj2gltf.source.length;
+        }
+    }
+
+    // Buffers larger than ~192MB cannot be base64 encoded due to a NodeJS limitation. Source: https://github.com/nodejs/node/issues/4266
+    var exceedsMaximum = (texturesByteLength + bufferByteLength > 201326580);
+
+    if (exceedsMaximum) {
+        console.log('Buffers and textures are too large to encode in the glTF, saving as separate resources.');
+    }
+
+    if (separateBuffers || exceedsMaximum) {
+        promises.push(writeSeparateBuffer(gltf, gltfPath));
+    } else {
+        writeEmbeddedBuffer(gltf);
+    }
+
+    if (separateTextures || exceedsMaximum) {
+        promises.push(writeSeparateTextures(gltf, gltfPath));
+    } else {
+        writeEmbeddedTextures(gltf);
+    }
+
+    deleteExtras(gltf);
+
+    return Promise.all(promises)
+        .then(function() {
+            return gltf;
+        });
+}
+
+function deleteExtras(gltf) {
+    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
+    delete buffer.extras;
+
+    var images = gltf.images;
+    for (var id in images) {
+        if (images.hasOwnProperty(id)) {
+            var image = images[id];
+            delete image.extras;
+        }
+    }
+}
+
+function writeSeparateBuffer(gltf, gltfPath) {
+    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
+    var source = buffer.extras._obj2gltf.source;
+    var bufferName = path.basename(gltfPath, path.extname(gltfPath));
+    var bufferUri = bufferName + '.bin';
+    buffer.uri = bufferUri;
+    var bufferPath = path.join(path.dirname(gltfPath), bufferUri);
+    return writeUris._outputFile(bufferPath, source);
+}
+
+function writeSeparateTextures(gltf, gltfPath) {
+    var promises = [];
+    var images = gltf.images;
+    for (var id in images) {
+        if (images.hasOwnProperty(id)) {
+            var image = images[id];
+            var extras = image.extras._obj2gltf;
+            var imageUri = image.name + extras.extension;
+            image.uri = imageUri;
+            var imagePath = path.join(path.dirname(gltfPath), imageUri);
+            promises.push(writeUris._outputFile(imagePath, extras.source));
+        }
+    }
+    return Promise.all(promises);
+}
+
+function writeEmbeddedBuffer(gltf) {
+    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
+    var source = buffer.extras._obj2gltf.source;
+    buffer.uri = 'data:application/octet-stream;base64,' + source.toString('base64');
+}
+
+function writeEmbeddedTextures(gltf) {
+    var promises = [];
+    var images = gltf.images;
+    for (var id in images) {
+        if (images.hasOwnProperty(id)) {
+            var image = images[id];
+            var extras = image.extras._obj2gltf;
+            image.uri = 'data:' + mime.lookup(extras.extension) + ';base64,' + extras.source.toString('base64');
+        }
+    }
+    return Promise.all(promises);
+}
+
+/**
+ * Exposed for testing
+ *
+ * @private
+ */
+writeUris._outputFile = fsExtraOutputFile;

package.json

@@ -31,6 +31,7 @@
     "event-stream": "^3.3.4",
     "fs-extra": "^2.0.0",
     "gltf-pipeline": "^0.1.0-alpha11",
+    "mime": "^1.3.4",
     "pngjs": "^3.0.1",
     "yargs": "^7.0.1"
   },

specs/lib/convertSpec.js

@@ -1,8 +1,8 @@
 'use strict';
-var fsExtra = require('fs-extra');
 var GltfPipeline = require('gltf-pipeline').Pipeline;
 var path = require('path');
 var convert = require('../../lib/convert');
+var writeUris = require('../../lib/writeUris');
 
 var objPath = 'specs/data/box-textured/box-textured.obj';
 var gltfPath = 'specs/data/box-textured/box-textured.gltf';
@@ -48,7 +48,7 @@ describe('convert', function() {
 
     it('sets options', function(done) {
         var spy1 = spyOn(GltfPipeline, 'processJSONToDisk');
-        var spy2 = spyOn(convert, '_outputFile');
+        var spy2 = spyOn(writeUris, '_outputFile');
         var textureCompressionOptions = {
             format : 'dxt1',
             quality : 10

specs/lib/gltfSpec.js

@@ -7,6 +7,7 @@ var clone = require('../../lib/clone.js');
 var createGltf = require('../../lib/gltf.js');
 var loadImage = require('../../lib/image.js');
 var loadObj = require('../../lib/obj.js');
+var writeUris = require('../../lib/writeUris.js');
 
 var WebGLConstants = Cesium.WebGLConstants;
 
@@ -19,19 +20,6 @@ var groupGltfUrl = 'specs/data/box-objects-groups-materials/box-objects-groups-m
 var diffuseTextureUrl = 'specs/data/box-textured/cesium.png';
 var transparentDiffuseTextureUrl = 'specs/data/box-complex-material/diffuse.png';
 
-function deleteExtras(gltf) {
-    var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]];
-    delete buffer.extras;
-
-    var images = gltf.images;
-    for (var id in images) {
-        if (images.hasOwnProperty(id)) {
-            var image = images[id];
-            delete image.extras;
-        }
-    }
-}
-
 describe('gltf', function() {
     var boxObjData;
     var groupObjData;
@@ -72,14 +60,14 @@ describe('gltf', function() {
     it('simple gltf', function() {
         var objData = clone(boxObjData, true);
         var gltf = createGltf(objData);
-        deleteExtras(gltf);
+        writeUris(gltf, boxGltfUrl, false, false);
         expect(gltf).toEqual(boxGltf);
     });
 
     it('multiple nodes, meshes, and primitives', function() {
         var objData = clone(groupObjData, true);
         var gltf = createGltf(objData);
-        deleteExtras(gltf);
+        writeUris(gltf, groupGltfUrl, false, false);
         expect(gltf).toEqual(groupGltf);
 
         expect(Object.keys(gltf.materials).length).toBe(3);
@@ -141,7 +129,8 @@ describe('gltf', function() {
 
         expect(image).toBeDefined();
         expect(image.name).toBe('cesium');
-        expect(image.uri.indexOf('data:image/png;base64,') >= 0).toBe(true);
+        expect(image.extras._obj2gltf.source).toBeDefined();
+        expect(image.extras._obj2gltf.extension).toBe('.png');
 
         expect(gltf.samplers.sampler).toEqual({
             magFilter : WebGLConstants.LINEAR,

specs/lib/imageSpec.js

@@ -19,9 +19,9 @@ describe('image', function() {
         expect(loadImage(pngImage)
             .then(function(info) {
                 expect(info.transparent).toBe(false);
-                expect(info.data).toBeDefined();
-                expect(info.uri.indexOf('data:image/png') === 0).toBe(true);
                 expect(info.format).toBe(WebGLConstants.RGB);
+                expect(info.source).toBeDefined();
+                expect(info.extension).toBe('.png');
             }), done).toResolve();
     });
 
@@ -29,9 +29,9 @@ describe('image', function() {
         expect(loadImage(jpgImage)
             .then(function(info) {
                 expect(info.transparent).toBe(false);
-                expect(info.data).toBeDefined();
-                expect(info.uri.indexOf('data:image/jpeg') === 0).toBe(true);
                 expect(info.format).toBe(WebGLConstants.RGB);
+                expect(info.source).toBeDefined();
+                expect(info.extension).toBe('.jpg');
             }), done).toResolve();
     });
 
@@ -39,9 +39,9 @@ describe('image', function() {
         expect(loadImage(jpegImage)
             .then(function(info) {
                 expect(info.transparent).toBe(false);
-                expect(info.data).toBeDefined();
-                expect(info.uri.indexOf('data:image/jpeg') === 0).toBe(true);
                 expect(info.format).toBe(WebGLConstants.RGB);
+                expect(info.source).toBeDefined();
+                expect(info.extension).toBe('.jpeg');
             }), done).toResolve();
     });
 
@@ -49,9 +49,9 @@ describe('image', function() {
         expect(loadImage(gifImage)
             .then(function(info) {
                 expect(info.transparent).toBe(false);
-                expect(info.data).toBeDefined();
-                expect(info.uri.indexOf('data:image/gif') === 0).toBe(true);
                 expect(info.format).toBe(WebGLConstants.RGB);
+                expect(info.source).toBeDefined();
+                expect(info.extension).toBe('.gif');
             }), done).toResolve();
     });
 
@@ -59,9 +59,9 @@ describe('image', function() {
         expect(loadImage(grayscaleImage)
             .then(function(info) {
                 expect(info.transparent).toBe(false);
-                expect(info.data).toBeDefined();
-                expect(info.uri.indexOf('data:image/png') === 0).toBe(true);
                 expect(info.format).toBe(WebGLConstants.ALPHA);
+                expect(info.source).toBeDefined();
+                expect(info.extension).toBe('.png');
             }), done).toResolve();
     });
 
@@ -69,9 +69,9 @@ describe('image', function() {
         expect(loadImage(transparentImage)
             .then(function(info) {
                 expect(info.transparent).toBe(true);
-                expect(info.data).toBeDefined();
-                expect(info.uri.indexOf('data:image/png') === 0).toBe(true);
                 expect(info.format).toBe(WebGLConstants.RGBA);
+                expect(info.source).toBeDefined();
+                expect(info.extension).toBe('.png');
             }), done).toResolve();
     });
 
@@ -79,9 +79,9 @@ describe('image', function() {
         expect(loadImage(opaqueAlphaImage)
             .then(function(info) {
                 expect(info.transparent).toBe(false);
-                expect(info.data).toBeDefined();
-                expect(info.uri.indexOf('data:image/png') === 0).toBe(true);
                 expect(info.format).toBe(WebGLConstants.RGBA);
+                expect(info.source).toBeDefined();
+                expect(info.extension).toBe('.png');
             }), done).toResolve();
     });