Read obj with streams and handle large buffers

.idea/jsLibraryMappings.xml

@@ -2,5 +2,6 @@
 <project version="4">
   <component name="JavaScriptLibraryMappings">
     <file url="file://$PROJECT_DIR$" libraries="{OBJ2GLTF node_modules}" />
+    <includedPredefinedLibrary name="Node.js Core" />
   </component>
 </project>

lib/convert.js

@@ -34,11 +34,12 @@ function convert(objFile, outputPath, options, done) {
     var gltfFile = path.join(outputPath, modelName + extension);
 
     parseObj(objFile, inputPath, function(data) {
-        createGltf(data, modelName, function(gltf) {
+        createGltf(data, inputPath, modelName, function(gltf) {
             var options = {
                 binary : binary,
                 embed : embed,
-                createDirectory : false
+                createDirectory : false,
+                basePath : inputPath
             };
             gltfPipeline.processJSONToDisk(gltf, gltfFile, options, function(error) {
                 if (error) {

lib/gltf.js

@@ -1,4 +1,5 @@
 "use strict";
+var fs = require('fs-extra');
 var path = require('path');
 var Cesium = require('cesium');
 var defined = Cesium.defined;
@@ -7,7 +8,7 @@ var WebGLConstants = Cesium.WebGLConstants;
 
 module.exports = createGltf;
 
-function createGltf(data, modelName, done) {
+function createGltf(data, inputPath, modelName, done) {
     var vertexCount = data.vertexCount;
     var vertexArray = data.vertexArray;
     var positionMin = data.positionMin;
@@ -171,7 +172,16 @@ function createGltf(data, modelName, done) {
 
     gltf.samplers[samplerId] = {}; // Use default values
 
-    var bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64');
+    var bufferSeparate = false;
+    var bufferUri;
+    if (buffer.length > 201326580) {
+        // toString fails for buffers larger than ~192MB. Instead save the buffer to a .bin file.
+        // Source: https://github.com/nodejs/node/issues/4266
+        bufferSeparate = true;
+        bufferUri = modelName + '.bin';
+    } else {
+        bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64');
+    }
 
     gltf.buffers[bufferId] = {
         byteLength : bufferByteLength,
@@ -316,5 +326,15 @@ function createGltf(data, modelName, done) {
         }
     }
 
+    if (bufferSeparate) {
+        var bufferPath = path.join(path.dirname(inputPath), modelName + '.bin');
+        fs.writeFile(bufferPath, buffer, function(err) {
+            if (err) {
+                throw err;
+            }
             done(gltf);
+        })
+    } else {
+        done(gltf);
+    }
 }

lib/mtl.js

@@ -1,5 +1,6 @@
 "use strict";
 var fs = require('fs-extra');
+var defined = require('cesium').defined;
 
 module.exports = {
     getDefault : getDefault,
@@ -13,7 +14,7 @@ function createMaterial() {
         diffuseColor : undefined,               // Kd
         specularColor : undefined,              // Ks
         specularShininess : undefined,          // Ns
-        alpha : undefined,                      // d
+        alpha : undefined,                      // d / Tr
         ambientColorMap : undefined,            // map_Ka
         emissionColorMap : undefined,           // map_Ke
         diffuseColorMap : undefined,            // map_Kd
@@ -31,7 +32,7 @@ function getDefault() {
 }
 
 function parse(mtlPath, done) {
-    fs.readFile(mtlPath, 'utf-8', function (error, contents) {
+    fs.readFile(mtlPath, 'utf8', function (error, contents) {
         if (error) {
             console.log('Could not read material file at ' + mtlPath + '. Using default material instead.');
             done({});
@@ -89,6 +90,9 @@ function parse(mtlPath, done) {
             } else if (/^d /i.test(line)) {
                 value = line.substring(2).trim();
                 material.alpha = parseFloat(value);
+            }  else if (/^Tr /i.test(line)) {
+                value = line.substring(3).trim();
+                material.alpha = parseFloat(value);
             } else if (/^map_Ka /i.test(line)) {
                 material.ambientColorMap = line.substring(7).trim();
             } else if (/^map_Ke /i.test(line)) {
@@ -106,6 +110,10 @@ function parse(mtlPath, done) {
             }
         }
         
+        if (defined(material.alpha)) {
+            material.diffuseColor[3] = material.alpha;
+        }
+
         done(materials);
     });
 }

lib/obj.js

@@ -13,20 +13,12 @@ module.exports = parseObj;
 // OBJ regex patterns are from ThreeJS (https://github.com/mrdoob/three.js/blob/master/examples/js/loaders/OBJLoader.js)
 
 function parseObj(objFile, inputPath, done) {
-    fs.readFile(objFile, 'utf-8', function (error, contents) {
+    getObjInfo(objFile, inputPath, function(info, materials, images) {
-        if (error) {
+        processObj(objFile, info, materials, images, done);
-            throw error;
-        }
-
-        getMaterials(contents, inputPath, function (materials) {
-            getImages(inputPath, materials, function (images) {
-                processObj(contents, materials, images, done);
-            });
-        });
     });
 }
 
-function processObj(contents, materials, images, done) {
+function processObj(objFile, info, materials, images, done) {
     var i, length;
 
     // A vertex is specified by indexes into each of the attribute arrays,
@@ -43,13 +35,8 @@ function processObj(contents, materials, images, done) {
     var positionMin = [Number.MAX_VALUE, Number.MAX_VALUE, Number.MAX_VALUE];
     var positionMax = [Number.MIN_VALUE, Number.MIN_VALUE, Number.MIN_VALUE];
 
-    var hasPositions = /^v\s/gm.test(contents);
+    var hasNormals = info.hasNormals;
-    var hasNormals = /^vn/gm.test(contents);
+    var hasUVs = info.hasUVs;
-    var hasUVs = /^vt/gm.test(contents);
-
-    if (!hasPositions) {
-        throw new Error('Could not process OBJ file, no positions.');
-    }
 
     var materialGroups = {}; // Map material to index array
     var currentIndexArray;
@@ -167,7 +154,10 @@ function processObj(contents, materials, images, done) {
     // f vertex//normal vertex//normal vertex//normal ...
     var facePattern4 = /f( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))?/;
 
-    var lines = contents.split('\n');
+    var stream = fs.createReadStream(objFile, 'utf8');
+
+    stream.on('data', function(chunk) {
+        var lines = chunk.split('\n');
         length = lines.length;
         for (i = 0; i < length; ++i) {
             var line = lines[i].trim();
@@ -225,7 +215,9 @@ function processObj(contents, materials, images, done) {
                 useMaterial(materialName);
             }
         }
+    });
 
+    stream.on('end', function() {
         done({
             vertexCount : vertexCount,
             vertexArray : vertexArray,
@@ -237,6 +229,7 @@ function processObj(contents, materials, images, done) {
             materials : materials,
             images : images
         });
+    });
 }
 
 function getImages(inputPath, materials, done) {
@@ -284,25 +277,66 @@ function getImages(inputPath, materials, done) {
     });
 }
 
-function getMaterials(contents, inputPath, done) {
+function getMaterials(mtlPath, hasMaterialGroups, done) {
-    var hasMaterialGroups = /^usemtl/gm.test(contents);
     if (!hasMaterialGroups) {
         done({});
         return;
     }
 
-    var mtllibMatches = contents.match(/^mtllib.*/gm);
+    if (defined(mtlPath)) {
-    if (mtllibMatches === null) {
+        Material.parse(mtlPath, function(materials) {
-        done({});
+            done(materials);
+        });
     } else {
+        done({});
+    }
+}
+
+function getObjInfo(objFile, inputPath, done) {
+    var mtlPath = undefined;
+    var hasMaterialGroups = false;
+    var hasPositions = false;
+    var hasNormals = false;
+    var hasUVs = false;
+
+    var stream = fs.createReadStream(objFile, 'utf8');
+    stream.on('data', function(chunk) {
+        if (!defined(mtlPath)) {
+            var mtllibMatches = chunk.match(/^mtllib.*/gm);
+            if (mtllibMatches !== null) {
                 var mtlFile = mtllibMatches[0].substring(7).trim();
-        var mtlPath = mtlFile;
+                mtlPath = mtlFile;
                 if (!path.isAbsolute(mtlPath)) {
                     mtlPath = path.join(inputPath, mtlFile);
                 }
-
-        Material.parse(mtlPath, function (materials) {
-            done(materials);
-        });
             }
+        }
+        if (!hasMaterialGroups) {
+            hasMaterialGroups = /^usemtl/gm.test(chunk);
+        }
+        if (!hasPositions) {
+            hasPositions = /^v\s/gm.test(chunk);
+        }
+        if (!hasNormals) {
+            hasNormals = /^vn/gm.test(chunk);
+        }
+        if (!hasUVs) {
+            hasUVs = /^vt/gm.test(chunk);
+        }
+    });
+
+    stream.on('end', function() {
+        if (!hasPositions) {
+            throw new Error('Could not process OBJ file, no positions.');
+        }
+        var info = {
+            hasNormals : hasNormals,
+            hasUVs : hasUVs
+        };
+        getMaterials(mtlPath, hasMaterialGroups, function(materials) {
+            getImages(inputPath, materials, function(images) {
+                done(info, materials, images);
+            });
+        });
+    });
 }