From 0bf726cea7a61722841c5eab31a12d4a90f92e66 Mon Sep 17 00:00:00 2001 From: Sean Lilley Date: Fri, 17 Mar 2017 15:44:01 -0400 Subject: [PATCH] Better handling of encoding base64 uris --- lib/convert.js | 68 +-------------------- lib/gltf.js | 10 +--- lib/image.js | 19 +----- lib/writeUris.js | 126 +++++++++++++++++++++++++++++++++++++++ package.json | 1 + specs/lib/convertSpec.js | 4 +- specs/lib/gltfSpec.js | 21 ++----- specs/lib/imageSpec.js | 28 ++++----- 8 files changed, 152 insertions(+), 125 deletions(-) create mode 100644 lib/writeUris.js diff --git a/lib/convert.js b/lib/convert.js index cc198fa..c34d8e4 100644 --- a/lib/convert.js +++ b/lib/convert.js @@ -6,8 +6,8 @@ var path = require('path'); var Promise = require('bluebird'); var createGltf = require('./gltf'); var loadObj = require('./obj'); +var writeUris = require('./writeUris'); -var fsExtraOutputFile = Promise.promisify(fsExtra.outputFile); var fsExtraOutputJson = Promise.promisify(fsExtra.outputJson); var defaultValue = Cesium.defaultValue; @@ -86,7 +86,7 @@ function convert(objPath, gltfPath, options) { return createGltf(objData); }) .then(function(gltf) { - return writeSeparateResources(gltf, gltfPath, separate, separateTextures); + return writeUris(gltf, gltfPath, separate, separateTextures); }) .then(function(gltf) { if (bypassPipeline) { @@ -97,73 +97,9 @@ function convert(objPath, gltfPath, options) { }); } -function deleteExtras(gltf) { - var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; - delete buffer.extras; - - var images = gltf.images; - for (var id in images) { - if (images.hasOwnProperty(id)) { - var image = images[id]; - delete image.extras; - } - } -} - -function writeSeparateBuffer(gltf, gltfPath) { - var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; - var source = buffer.extras._obj2gltf.source; - var bufferName = path.basename(gltfPath, path.extname(gltfPath)); - var bufferUri = bufferName + '.bin'; - buffer.uri = bufferUri; - var bufferPath = path.join(path.dirname(gltfPath), bufferUri); - return convert._outputFile(bufferPath, source); -} - -function writeSeparateTextures(gltf, gltfPath) { - var promises = []; - var images = gltf.images; - for (var id in images) { - if (images.hasOwnProperty(id)) { - var image = images[id]; - var extras = image.extras._obj2gltf; - var imageUri = image.name + extras.extension; - image.uri = imageUri; - var imagePath = path.join(path.dirname(gltfPath), imageUri); - promises.push(convert._outputFile(imagePath, extras.source)); - } - } - return Promise.all(promises); -} - -function writeSeparateResources(gltf, gltfPath, separate, separateTextures) { - var promises = []; - var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; - - if (separate || !defined(buffer.uri)) { - promises.push(writeSeparateBuffer(gltf, gltfPath)); - } - if (separateTextures) { - promises.push(writeSeparateTextures(gltf, gltfPath)); - } - - deleteExtras(gltf); - return Promise.all(promises) - .then(function() { - return gltf; - }); -} - /** * Exposed for testing * * @private */ convert._outputJson = fsExtraOutputJson; - -/** - * Exposed for testing - * - * @private - */ -convert._outputFile = fsExtraOutputFile; diff --git a/lib/gltf.js b/lib/gltf.js index c84df40..869276e 100644 --- a/lib/gltf.js +++ b/lib/gltf.js @@ -137,10 +137,9 @@ function createGltf(objData) { gltf.images[imageId] = { name : imageId, - uri : image.uri, extras : { _obj2gltf : { - source : image.data, + source : image.source, extension : image.extension } } @@ -317,15 +316,8 @@ function createGltf(objData) { buffers = buffers.concat(vertexBuffers, indexBuffers); var buffer = Buffer.concat(buffers); - // Buffers larger than ~192MB cannot be base64 encoded due to a NodeJS limitation. Source: https://github.com/nodejs/node/issues/4266 - var bufferUri; - if (buffer.length <= 201326580) { - bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64'); - } - gltf.buffers[bufferId] = { byteLength : buffer.byteLength, - uri : bufferUri, extras : { _obj2gltf : { source : buffer diff --git a/lib/image.js b/lib/image.js index 2387a6a..4974d9d 100644 --- a/lib/image.js +++ b/lib/image.js @@ -23,14 +23,11 @@ function loadImage(imagePath) { return fsReadFile(imagePath) .then(function(data) { var extension = path.extname(imagePath); - var uriType = getUriType(extension); - var uri = uriType + ';base64,' + data.toString('base64'); var info = { transparent : false, - data : data, - uri : uri, format : getFormat(3), + source : data, extension : extension }; @@ -81,20 +78,6 @@ function getChannels(colorType) { } } -function getUriType(extension) { - switch (extension) { - case '.png': - return 'data:image/png'; - case '.jpg': - case '.jpeg': - return 'data:image/jpeg'; - case '.gif': - return 'data:image/gif'; - default: - return 'data:image/' + extension.slice(1); - } -} - function getFormat(channels) { switch (channels) { case 1: diff --git a/lib/writeUris.js b/lib/writeUris.js new file mode 100644 index 0000000..286ed72 --- /dev/null +++ b/lib/writeUris.js @@ -0,0 +1,126 @@ +'use strict'; +var fsExtra = require('fs-extra'); +var mime = require('mime'); +var path = require('path'); +var Promise = require('bluebird'); + +var fsExtraOutputFile = Promise.promisify(fsExtra.outputFile); + +module.exports = writeUris; + +/** + * Write glTF resources as embedded data uris or external files. + * + * @param {Object} gltf The glTF asset. + * @param {String} gltfPath Path where the glTF will be saved. + * @param {Boolean} separateBuffers Writes out separate buffers. + * @param {Boolean} separateTextures Writes out separate textures. + * @returns {Promise} A promise that resolves to the glTF asset. + * + * @private + */ +function writeUris(gltf, gltfPath, separateBuffers, separateTextures) { + var promises = []; + + var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; + var bufferByteLength = buffer.extras._obj2gltf.source.length; + + var texturesByteLength = 0; + var images = gltf.images; + for (var id in images) { + if (images.hasOwnProperty(id)) { + texturesByteLength += images[id].extras._obj2gltf.source.length; + } + } + + // Buffers larger than ~192MB cannot be base64 encoded due to a NodeJS limitation. Source: https://github.com/nodejs/node/issues/4266 + var exceedsMaximum = (texturesByteLength + bufferByteLength > 201326580); + + if (exceedsMaximum) { + console.log('Buffers and textures are too large to encode in the glTF, saving as separate resources.'); + } + + if (separateBuffers || exceedsMaximum) { + promises.push(writeSeparateBuffer(gltf, gltfPath)); + } else { + writeEmbeddedBuffer(gltf); + } + + if (separateTextures || exceedsMaximum) { + promises.push(writeSeparateTextures(gltf, gltfPath)); + } else { + writeEmbeddedTextures(gltf); + } + + deleteExtras(gltf); + + return Promise.all(promises) + .then(function() { + return gltf; + }); +} + +function deleteExtras(gltf) { + var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; + delete buffer.extras; + + var images = gltf.images; + for (var id in images) { + if (images.hasOwnProperty(id)) { + var image = images[id]; + delete image.extras; + } + } +} + +function writeSeparateBuffer(gltf, gltfPath) { + var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; + var source = buffer.extras._obj2gltf.source; + var bufferName = path.basename(gltfPath, path.extname(gltfPath)); + var bufferUri = bufferName + '.bin'; + buffer.uri = bufferUri; + var bufferPath = path.join(path.dirname(gltfPath), bufferUri); + return writeUris._outputFile(bufferPath, source); +} + +function writeSeparateTextures(gltf, gltfPath) { + var promises = []; + var images = gltf.images; + for (var id in images) { + if (images.hasOwnProperty(id)) { + var image = images[id]; + var extras = image.extras._obj2gltf; + var imageUri = image.name + extras.extension; + image.uri = imageUri; + var imagePath = path.join(path.dirname(gltfPath), imageUri); + promises.push(writeUris._outputFile(imagePath, extras.source)); + } + } + return Promise.all(promises); +} + +function writeEmbeddedBuffer(gltf) { + var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; + var source = buffer.extras._obj2gltf.source; + buffer.uri = 'data:application/octet-stream;base64,' + source.toString('base64'); +} + +function writeEmbeddedTextures(gltf) { + var promises = []; + var images = gltf.images; + for (var id in images) { + if (images.hasOwnProperty(id)) { + var image = images[id]; + var extras = image.extras._obj2gltf; + image.uri = 'data:' + mime.lookup(extras.extension) + ';base64,' + extras.source.toString('base64'); + } + } + return Promise.all(promises); +} + +/** + * Exposed for testing + * + * @private + */ +writeUris._outputFile = fsExtraOutputFile; diff --git a/package.json b/package.json index 7a1516e..e1df602 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "event-stream": "^3.3.4", "fs-extra": "^2.0.0", "gltf-pipeline": "^0.1.0-alpha11", + "mime": "^1.3.4", "pngjs": "^3.0.1", "yargs": "^7.0.1" }, diff --git a/specs/lib/convertSpec.js b/specs/lib/convertSpec.js index f3da1f2..a409911 100644 --- a/specs/lib/convertSpec.js +++ b/specs/lib/convertSpec.js @@ -1,8 +1,8 @@ 'use strict'; -var fsExtra = require('fs-extra'); var GltfPipeline = require('gltf-pipeline').Pipeline; var path = require('path'); var convert = require('../../lib/convert'); +var writeUris = require('../../lib/writeUris'); var objPath = 'specs/data/box-textured/box-textured.obj'; var gltfPath = 'specs/data/box-textured/box-textured.gltf'; @@ -48,7 +48,7 @@ describe('convert', function() { it('sets options', function(done) { var spy1 = spyOn(GltfPipeline, 'processJSONToDisk'); - var spy2 = spyOn(convert, '_outputFile'); + var spy2 = spyOn(writeUris, '_outputFile'); var textureCompressionOptions = { format : 'dxt1', quality : 10 diff --git a/specs/lib/gltfSpec.js b/specs/lib/gltfSpec.js index 6e415b7..260a116 100644 --- a/specs/lib/gltfSpec.js +++ b/specs/lib/gltfSpec.js @@ -7,6 +7,7 @@ var clone = require('../../lib/clone.js'); var createGltf = require('../../lib/gltf.js'); var loadImage = require('../../lib/image.js'); var loadObj = require('../../lib/obj.js'); +var writeUris = require('../../lib/writeUris.js'); var WebGLConstants = Cesium.WebGLConstants; @@ -19,19 +20,6 @@ var groupGltfUrl = 'specs/data/box-objects-groups-materials/box-objects-groups-m var diffuseTextureUrl = 'specs/data/box-textured/cesium.png'; var transparentDiffuseTextureUrl = 'specs/data/box-complex-material/diffuse.png'; -function deleteExtras(gltf) { - var buffer = gltf.buffers[Object.keys(gltf.buffers)[0]]; - delete buffer.extras; - - var images = gltf.images; - for (var id in images) { - if (images.hasOwnProperty(id)) { - var image = images[id]; - delete image.extras; - } - } -} - describe('gltf', function() { var boxObjData; var groupObjData; @@ -72,14 +60,14 @@ describe('gltf', function() { it('simple gltf', function() { var objData = clone(boxObjData, true); var gltf = createGltf(objData); - deleteExtras(gltf); + writeUris(gltf, boxGltfUrl, false, false); expect(gltf).toEqual(boxGltf); }); it('multiple nodes, meshes, and primitives', function() { var objData = clone(groupObjData, true); var gltf = createGltf(objData); - deleteExtras(gltf); + writeUris(gltf, groupGltfUrl, false, false); expect(gltf).toEqual(groupGltf); expect(Object.keys(gltf.materials).length).toBe(3); @@ -141,7 +129,8 @@ describe('gltf', function() { expect(image).toBeDefined(); expect(image.name).toBe('cesium'); - expect(image.uri.indexOf('data:image/png;base64,') >= 0).toBe(true); + expect(image.extras._obj2gltf.source).toBeDefined(); + expect(image.extras._obj2gltf.extension).toBe('.png'); expect(gltf.samplers.sampler).toEqual({ magFilter : WebGLConstants.LINEAR, diff --git a/specs/lib/imageSpec.js b/specs/lib/imageSpec.js index 1d88553..9f6eb63 100644 --- a/specs/lib/imageSpec.js +++ b/specs/lib/imageSpec.js @@ -19,9 +19,9 @@ describe('image', function() { expect(loadImage(pngImage) .then(function(info) { expect(info.transparent).toBe(false); - expect(info.data).toBeDefined(); - expect(info.uri.indexOf('data:image/png') === 0).toBe(true); expect(info.format).toBe(WebGLConstants.RGB); + expect(info.source).toBeDefined(); + expect(info.extension).toBe('.png'); }), done).toResolve(); }); @@ -29,9 +29,9 @@ describe('image', function() { expect(loadImage(jpgImage) .then(function(info) { expect(info.transparent).toBe(false); - expect(info.data).toBeDefined(); - expect(info.uri.indexOf('data:image/jpeg') === 0).toBe(true); expect(info.format).toBe(WebGLConstants.RGB); + expect(info.source).toBeDefined(); + expect(info.extension).toBe('.jpg'); }), done).toResolve(); }); @@ -39,9 +39,9 @@ describe('image', function() { expect(loadImage(jpegImage) .then(function(info) { expect(info.transparent).toBe(false); - expect(info.data).toBeDefined(); - expect(info.uri.indexOf('data:image/jpeg') === 0).toBe(true); expect(info.format).toBe(WebGLConstants.RGB); + expect(info.source).toBeDefined(); + expect(info.extension).toBe('.jpeg'); }), done).toResolve(); }); @@ -49,9 +49,9 @@ describe('image', function() { expect(loadImage(gifImage) .then(function(info) { expect(info.transparent).toBe(false); - expect(info.data).toBeDefined(); - expect(info.uri.indexOf('data:image/gif') === 0).toBe(true); expect(info.format).toBe(WebGLConstants.RGB); + expect(info.source).toBeDefined(); + expect(info.extension).toBe('.gif'); }), done).toResolve(); }); @@ -59,9 +59,9 @@ describe('image', function() { expect(loadImage(grayscaleImage) .then(function(info) { expect(info.transparent).toBe(false); - expect(info.data).toBeDefined(); - expect(info.uri.indexOf('data:image/png') === 0).toBe(true); expect(info.format).toBe(WebGLConstants.ALPHA); + expect(info.source).toBeDefined(); + expect(info.extension).toBe('.png'); }), done).toResolve(); }); @@ -69,9 +69,9 @@ describe('image', function() { expect(loadImage(transparentImage) .then(function(info) { expect(info.transparent).toBe(true); - expect(info.data).toBeDefined(); - expect(info.uri.indexOf('data:image/png') === 0).toBe(true); expect(info.format).toBe(WebGLConstants.RGBA); + expect(info.source).toBeDefined(); + expect(info.extension).toBe('.png'); }), done).toResolve(); }); @@ -79,9 +79,9 @@ describe('image', function() { expect(loadImage(opaqueAlphaImage) .then(function(info) { expect(info.transparent).toBe(false); - expect(info.data).toBeDefined(); - expect(info.uri.indexOf('data:image/png') === 0).toBe(true); expect(info.format).toBe(WebGLConstants.RGBA); + expect(info.source).toBeDefined(); + expect(info.extension).toBe('.png'); }), done).toResolve(); });