From 10925599e1f098a8abb25533fd33917a05a1c753 Mon Sep 17 00:00:00 2001 From: Sean Lilley Date: Wed, 22 Jun 2016 10:07:08 -0400 Subject: [PATCH] Read obj with streams and handle large buffers --- .idea/jsLibraryMappings.xml | 1 + lib/convert.js | 5 +- lib/gltf.js | 26 +++- lib/mtl.js | 12 +- lib/obj.js | 228 +++++++++++++++++++++--------------- 5 files changed, 168 insertions(+), 104 deletions(-) diff --git a/.idea/jsLibraryMappings.xml b/.idea/jsLibraryMappings.xml index b45162c..2d8ad44 100644 --- a/.idea/jsLibraryMappings.xml +++ b/.idea/jsLibraryMappings.xml @@ -2,5 +2,6 @@ + \ No newline at end of file diff --git a/lib/convert.js b/lib/convert.js index 0d19efd..b84fd46 100644 --- a/lib/convert.js +++ b/lib/convert.js @@ -34,11 +34,12 @@ function convert(objFile, outputPath, options, done) { var gltfFile = path.join(outputPath, modelName + extension); parseObj(objFile, inputPath, function(data) { - createGltf(data, modelName, function(gltf) { + createGltf(data, inputPath, modelName, function(gltf) { var options = { binary : binary, embed : embed, - createDirectory : false + createDirectory : false, + basePath : inputPath }; gltfPipeline.processJSONToDisk(gltf, gltfFile, options, function(error) { if (error) { diff --git a/lib/gltf.js b/lib/gltf.js index 4200947..e0f2f3c 100644 --- a/lib/gltf.js +++ b/lib/gltf.js @@ -1,4 +1,5 @@ "use strict"; +var fs = require('fs-extra'); var path = require('path'); var Cesium = require('cesium'); var defined = Cesium.defined; @@ -7,7 +8,7 @@ var WebGLConstants = Cesium.WebGLConstants; module.exports = createGltf; -function createGltf(data, modelName, done) { +function createGltf(data, inputPath, modelName, done) { var vertexCount = data.vertexCount; var vertexArray = data.vertexArray; var positionMin = data.positionMin; @@ -171,7 +172,16 @@ function createGltf(data, modelName, done) { gltf.samplers[samplerId] = {}; // Use default values - var bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64'); + var bufferSeparate = false; + var bufferUri; + if (buffer.length > 201326580) { + // toString fails for buffers larger than ~192MB. Instead save the buffer to a .bin file. + // Source: https://github.com/nodejs/node/issues/4266 + bufferSeparate = true; + bufferUri = modelName + '.bin'; + } else { + bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64'); + } gltf.buffers[bufferId] = { byteLength : bufferByteLength, @@ -316,5 +326,15 @@ function createGltf(data, modelName, done) { } } - done(gltf); + if (bufferSeparate) { + var bufferPath = path.join(path.dirname(inputPath), modelName + '.bin'); + fs.writeFile(bufferPath, buffer, function(err) { + if (err) { + throw err; + } + done(gltf); + }) + } else { + done(gltf); + } } diff --git a/lib/mtl.js b/lib/mtl.js index cd7dc94..fecdf28 100644 --- a/lib/mtl.js +++ b/lib/mtl.js @@ -1,5 +1,6 @@ "use strict"; var fs = require('fs-extra'); +var defined = require('cesium').defined; module.exports = { getDefault : getDefault, @@ -13,7 +14,7 @@ function createMaterial() { diffuseColor : undefined, // Kd specularColor : undefined, // Ks specularShininess : undefined, // Ns - alpha : undefined, // d + alpha : undefined, // d / Tr ambientColorMap : undefined, // map_Ka emissionColorMap : undefined, // map_Ke diffuseColorMap : undefined, // map_Kd @@ -31,7 +32,7 @@ function getDefault() { } function parse(mtlPath, done) { - fs.readFile(mtlPath, 'utf-8', function (error, contents) { + fs.readFile(mtlPath, 'utf8', function (error, contents) { if (error) { console.log('Could not read material file at ' + mtlPath + '. Using default material instead.'); done({}); @@ -89,6 +90,9 @@ function parse(mtlPath, done) { } else if (/^d /i.test(line)) { value = line.substring(2).trim(); material.alpha = parseFloat(value); + } else if (/^Tr /i.test(line)) { + value = line.substring(3).trim(); + material.alpha = parseFloat(value); } else if (/^map_Ka /i.test(line)) { material.ambientColorMap = line.substring(7).trim(); } else if (/^map_Ke /i.test(line)) { @@ -105,6 +109,10 @@ function parse(mtlPath, done) { material.alphaMap = line.substring(6).trim(); } } + + if (defined(material.alpha)) { + material.diffuseColor[3] = material.alpha; + } done(materials); }); diff --git a/lib/obj.js b/lib/obj.js index 94d7717..21a7342 100644 --- a/lib/obj.js +++ b/lib/obj.js @@ -13,20 +13,12 @@ module.exports = parseObj; // OBJ regex patterns are from ThreeJS (https://github.com/mrdoob/three.js/blob/master/examples/js/loaders/OBJLoader.js) function parseObj(objFile, inputPath, done) { - fs.readFile(objFile, 'utf-8', function (error, contents) { - if (error) { - throw error; - } - - getMaterials(contents, inputPath, function (materials) { - getImages(inputPath, materials, function (images) { - processObj(contents, materials, images, done); - }); - }); + getObjInfo(objFile, inputPath, function(info, materials, images) { + processObj(objFile, info, materials, images, done); }); } -function processObj(contents, materials, images, done) { +function processObj(objFile, info, materials, images, done) { var i, length; // A vertex is specified by indexes into each of the attribute arrays, @@ -43,13 +35,8 @@ function processObj(contents, materials, images, done) { var positionMin = [Number.MAX_VALUE, Number.MAX_VALUE, Number.MAX_VALUE]; var positionMax = [Number.MIN_VALUE, Number.MIN_VALUE, Number.MIN_VALUE]; - var hasPositions = /^v\s/gm.test(contents); - var hasNormals = /^vn/gm.test(contents); - var hasUVs = /^vt/gm.test(contents); - - if (!hasPositions) { - throw new Error('Could not process OBJ file, no positions.'); - } + var hasNormals = info.hasNormals; + var hasUVs = info.hasUVs; var materialGroups = {}; // Map material to index array var currentIndexArray; @@ -167,75 +154,81 @@ function processObj(contents, materials, images, done) { // f vertex//normal vertex//normal vertex//normal ... var facePattern4 = /f( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))?/; - var lines = contents.split('\n'); - length = lines.length; - for (i = 0; i < length; ++i) { - var line = lines[i].trim(); - var result; - if ((line.length === 0) || (line.charAt(0) === '#')) { - continue; - } else if ((result = vertexPattern.exec(line)) !== null) { - positions.push( - parseFloat(result[1]), - parseFloat(result[2]), - parseFloat(result[3]) - ); - } else if ((result = normalPattern.exec(line) ) !== null) { - var nx = parseFloat(result[1]); - var ny = parseFloat(result[2]); - var nz = parseFloat(result[3]); - var normal = Cartesian3.normalize(new Cartesian3(nx, ny, nz), new Cartesian3()); - normals.push(normal.x, normal.y, normal.z); - } else if ((result = uvPattern.exec(line)) !== null) { - uvs.push( - parseFloat(result[1]), - parseFloat(result[2]) - ); + var stream = fs.createReadStream(objFile, 'utf8'); - } else if ((result = facePattern1.exec(line)) !== null) { - addFace( - result[1], result[1], undefined, undefined, - result[2], result[2], undefined, undefined, - result[3], result[3], undefined, undefined, - result[4], result[4], undefined, undefined - ); - } else if ((result = facePattern2.exec(line)) !== null) { - addFace( - result[1], result[2], result[3], undefined, - result[4], result[5], result[6], undefined, - result[7], result[8], result[9], undefined, - result[10], result[11], result[12], undefined - ); - } else if ((result = facePattern3.exec(line)) !== null) { - addFace( - result[1], result[2], result[3], result[4], - result[5], result[6], result[7], result[8], - result[9], result[10], result[11], result[12], - result[13], result[14], result[15], result[16] - ); - } else if ((result = facePattern4.exec(line)) !== null) { - addFace( - result[1], result[2], undefined, result[3], - result[4], result[5], undefined, result[6], - result[7], result[8], undefined, result[9], - result[10], result[11], undefined, result[12] - ); - } else if (/^usemtl /.test(line)) { - var materialName = line.substring(7).trim(); - useMaterial(materialName); + stream.on('data', function(chunk) { + var lines = chunk.split('\n'); + length = lines.length; + for (i = 0; i < length; ++i) { + var line = lines[i].trim(); + var result; + if ((line.length === 0) || (line.charAt(0) === '#')) { + continue; + } else if ((result = vertexPattern.exec(line)) !== null) { + positions.push( + parseFloat(result[1]), + parseFloat(result[2]), + parseFloat(result[3]) + ); + } else if ((result = normalPattern.exec(line) ) !== null) { + var nx = parseFloat(result[1]); + var ny = parseFloat(result[2]); + var nz = parseFloat(result[3]); + var normal = Cartesian3.normalize(new Cartesian3(nx, ny, nz), new Cartesian3()); + normals.push(normal.x, normal.y, normal.z); + } else if ((result = uvPattern.exec(line)) !== null) { + uvs.push( + parseFloat(result[1]), + parseFloat(result[2]) + ); + + } else if ((result = facePattern1.exec(line)) !== null) { + addFace( + result[1], result[1], undefined, undefined, + result[2], result[2], undefined, undefined, + result[3], result[3], undefined, undefined, + result[4], result[4], undefined, undefined + ); + } else if ((result = facePattern2.exec(line)) !== null) { + addFace( + result[1], result[2], result[3], undefined, + result[4], result[5], result[6], undefined, + result[7], result[8], result[9], undefined, + result[10], result[11], result[12], undefined + ); + } else if ((result = facePattern3.exec(line)) !== null) { + addFace( + result[1], result[2], result[3], result[4], + result[5], result[6], result[7], result[8], + result[9], result[10], result[11], result[12], + result[13], result[14], result[15], result[16] + ); + } else if ((result = facePattern4.exec(line)) !== null) { + addFace( + result[1], result[2], undefined, result[3], + result[4], result[5], undefined, result[6], + result[7], result[8], undefined, result[9], + result[10], result[11], undefined, result[12] + ); + } else if (/^usemtl /.test(line)) { + var materialName = line.substring(7).trim(); + useMaterial(materialName); + } } - } + }); - done({ - vertexCount : vertexCount, - vertexArray : vertexArray, - positionMin : positionMin, - positionMax : positionMax, - hasUVs : hasUVs, - hasNormals : hasNormals, - materialGroups : materialGroups, - materials : materials, - images : images + stream.on('end', function() { + done({ + vertexCount : vertexCount, + vertexArray : vertexArray, + positionMin : positionMin, + positionMax : positionMax, + hasUVs : hasUVs, + hasNormals : hasNormals, + materialGroups : materialGroups, + materials : materials, + images : images + }); }); } @@ -284,25 +277,66 @@ function getImages(inputPath, materials, done) { }); } -function getMaterials(contents, inputPath, done) { - var hasMaterialGroups = /^usemtl/gm.test(contents); +function getMaterials(mtlPath, hasMaterialGroups, done) { if (!hasMaterialGroups) { done({}); return; } - var mtllibMatches = contents.match(/^mtllib.*/gm); - if (mtllibMatches === null) { - done({}); - } else { - var mtlFile = mtllibMatches[0].substring(7).trim(); - var mtlPath = mtlFile; - if (!path.isAbsolute(mtlPath)) { - mtlPath = path.join(inputPath, mtlFile); - } - - Material.parse(mtlPath, function (materials) { + if (defined(mtlPath)) { + Material.parse(mtlPath, function(materials) { done(materials); }); + } else { + done({}); } } + +function getObjInfo(objFile, inputPath, done) { + var mtlPath = undefined; + var hasMaterialGroups = false; + var hasPositions = false; + var hasNormals = false; + var hasUVs = false; + + var stream = fs.createReadStream(objFile, 'utf8'); + stream.on('data', function(chunk) { + if (!defined(mtlPath)) { + var mtllibMatches = chunk.match(/^mtllib.*/gm); + if (mtllibMatches !== null) { + var mtlFile = mtllibMatches[0].substring(7).trim(); + mtlPath = mtlFile; + if (!path.isAbsolute(mtlPath)) { + mtlPath = path.join(inputPath, mtlFile); + } + } + } + if (!hasMaterialGroups) { + hasMaterialGroups = /^usemtl/gm.test(chunk); + } + if (!hasPositions) { + hasPositions = /^v\s/gm.test(chunk); + } + if (!hasNormals) { + hasNormals = /^vn/gm.test(chunk); + } + if (!hasUVs) { + hasUVs = /^vt/gm.test(chunk); + } + }); + + stream.on('end', function() { + if (!hasPositions) { + throw new Error('Could not process OBJ file, no positions.'); + } + var info = { + hasNormals : hasNormals, + hasUVs : hasUVs + }; + getMaterials(mtlPath, hasMaterialGroups, function(materials) { + getImages(inputPath, materials, function(images) { + done(info, materials, images); + }); + }); + }); +}