Read obj with streams and handle large buffers

.idea/jsLibraryMappings.xml

@@ -2,5 +2,6 @@
 <project version="4">
   <component name="JavaScriptLibraryMappings">
     <file url="file://$PROJECT_DIR$" libraries="{OBJ2GLTF node_modules}" />
+    <includedPredefinedLibrary name="Node.js Core" />
   </component>
 </project>

lib/convert.js

@@ -34,11 +34,12 @@ function convert(objFile, outputPath, options, done) {
     var gltfFile = path.join(outputPath, modelName + extension);
 
     parseObj(objFile, inputPath, function(data) {
-        createGltf(data, modelName, function(gltf) {
+        createGltf(data, inputPath, modelName, function(gltf) {
             var options = {
                 binary : binary,
                 embed : embed,
-                createDirectory : false
+                createDirectory : false,
+                basePath : inputPath
             };
             gltfPipeline.processJSONToDisk(gltf, gltfFile, options, function(error) {
                 if (error) {

lib/gltf.js

@@ -1,4 +1,5 @@
 "use strict";
+var fs = require('fs-extra');
 var path = require('path');
 var Cesium = require('cesium');
 var defined = Cesium.defined;
@@ -7,7 +8,7 @@ var WebGLConstants = Cesium.WebGLConstants;
 
 module.exports = createGltf;
 
-function createGltf(data, modelName, done) {
+function createGltf(data, inputPath, modelName, done) {
     var vertexCount = data.vertexCount;
     var vertexArray = data.vertexArray;
     var positionMin = data.positionMin;
@@ -171,7 +172,16 @@ function createGltf(data, modelName, done) {
 
     gltf.samplers[samplerId] = {}; // Use default values
 
-    var bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64');
+    var bufferSeparate = false;
+    var bufferUri;
+    if (buffer.length > 201326580) {
+        // toString fails for buffers larger than ~192MB. Instead save the buffer to a .bin file.
+        // Source: https://github.com/nodejs/node/issues/4266
+        bufferSeparate = true;
+        bufferUri = modelName + '.bin';
+    } else {
+        bufferUri = 'data:application/octet-stream;base64,' + buffer.toString('base64');
+    }
 
     gltf.buffers[bufferId] = {
         byteLength : bufferByteLength,
@@ -316,5 +326,15 @@ function createGltf(data, modelName, done) {
         }
     }
 
-    done(gltf);
+    if (bufferSeparate) {
+        var bufferPath = path.join(path.dirname(inputPath), modelName + '.bin');
+        fs.writeFile(bufferPath, buffer, function(err) {
+            if (err) {
+                throw err;
+            }
+            done(gltf);
+        })
+    } else {
+        done(gltf);
+    }
 }

lib/mtl.js

@@ -1,5 +1,6 @@
 "use strict";
 var fs = require('fs-extra');
+var defined = require('cesium').defined;
 
 module.exports = {
     getDefault : getDefault,
@@ -13,7 +14,7 @@ function createMaterial() {
         diffuseColor : undefined,               // Kd
         specularColor : undefined,              // Ks
         specularShininess : undefined,          // Ns
-        alpha : undefined,                      // d
+        alpha : undefined,                      // d / Tr
         ambientColorMap : undefined,            // map_Ka
         emissionColorMap : undefined,           // map_Ke
         diffuseColorMap : undefined,            // map_Kd
@@ -31,7 +32,7 @@ function getDefault() {
 }
 
 function parse(mtlPath, done) {
-    fs.readFile(mtlPath, 'utf-8', function (error, contents) {
+    fs.readFile(mtlPath, 'utf8', function (error, contents) {
         if (error) {
             console.log('Could not read material file at ' + mtlPath + '. Using default material instead.');
             done({});
@@ -89,6 +90,9 @@ function parse(mtlPath, done) {
             } else if (/^d /i.test(line)) {
                 value = line.substring(2).trim();
                 material.alpha = parseFloat(value);
+            }  else if (/^Tr /i.test(line)) {
+                value = line.substring(3).trim();
+                material.alpha = parseFloat(value);
             } else if (/^map_Ka /i.test(line)) {
                 material.ambientColorMap = line.substring(7).trim();
             } else if (/^map_Ke /i.test(line)) {
@@ -105,6 +109,10 @@ function parse(mtlPath, done) {
                 material.alphaMap = line.substring(6).trim();
             }
         }
+        
+        if (defined(material.alpha)) {
+            material.diffuseColor[3] = material.alpha;
+        }
 
         done(materials);
     });

lib/obj.js

@@ -13,20 +13,12 @@ module.exports = parseObj;
 // OBJ regex patterns are from ThreeJS (https://github.com/mrdoob/three.js/blob/master/examples/js/loaders/OBJLoader.js)
 
 function parseObj(objFile, inputPath, done) {
-    fs.readFile(objFile, 'utf-8', function (error, contents) {
+    getObjInfo(objFile, inputPath, function(info, materials, images) {
-        if (error) {
+        processObj(objFile, info, materials, images, done);
-            throw error;
-        }
-
-        getMaterials(contents, inputPath, function (materials) {
-            getImages(inputPath, materials, function (images) {
-                processObj(contents, materials, images, done);
-            });
-        });
     });
 }
 
-function processObj(contents, materials, images, done) {
+function processObj(objFile, info, materials, images, done) {
     var i, length;
 
     // A vertex is specified by indexes into each of the attribute arrays,
@@ -43,13 +35,8 @@ function processObj(contents, materials, images, done) {
     var positionMin = [Number.MAX_VALUE, Number.MAX_VALUE, Number.MAX_VALUE];
     var positionMax = [Number.MIN_VALUE, Number.MIN_VALUE, Number.MIN_VALUE];
 
-    var hasPositions = /^v\s/gm.test(contents);
+    var hasNormals = info.hasNormals;
-    var hasNormals = /^vn/gm.test(contents);
+    var hasUVs = info.hasUVs;
-    var hasUVs = /^vt/gm.test(contents);
-
-    if (!hasPositions) {
-        throw new Error('Could not process OBJ file, no positions.');
-    }
 
     var materialGroups = {}; // Map material to index array
     var currentIndexArray;
@@ -167,75 +154,81 @@ function processObj(contents, materials, images, done) {
     // f vertex//normal vertex//normal vertex//normal ...
     var facePattern4 = /f( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))( +(-?\d+)\/\/(-?\d+))?/;
 
-    var lines = contents.split('\n');
+    var stream = fs.createReadStream(objFile, 'utf8');
-    length = lines.length;
-    for (i = 0; i < length; ++i) {
-        var line = lines[i].trim();
-        var result;
-        if ((line.length === 0) || (line.charAt(0) === '#')) {
-            continue;
-        } else if ((result = vertexPattern.exec(line)) !== null) {
-            positions.push(
-                parseFloat(result[1]),
-                parseFloat(result[2]),
-                parseFloat(result[3])
-            );
-        } else if ((result = normalPattern.exec(line) ) !== null) {
-            var nx = parseFloat(result[1]);
-            var ny = parseFloat(result[2]);
-            var nz = parseFloat(result[3]);
-            var normal = Cartesian3.normalize(new Cartesian3(nx, ny, nz), new Cartesian3());
-            normals.push(normal.x, normal.y, normal.z);
-        } else if ((result = uvPattern.exec(line)) !== null) {
-            uvs.push(
-                parseFloat(result[1]),
-                parseFloat(result[2])
-            );
 
-        } else if ((result = facePattern1.exec(line)) !== null) {
+    stream.on('data', function(chunk) {
-            addFace(
+        var lines = chunk.split('\n');
-                result[1], result[1], undefined, undefined,
+        length = lines.length;
-                result[2], result[2], undefined, undefined,
+        for (i = 0; i < length; ++i) {
-                result[3], result[3], undefined, undefined,
+            var line = lines[i].trim();
-                result[4], result[4], undefined, undefined
+            var result;
-            );
+            if ((line.length === 0) || (line.charAt(0) === '#')) {
-        } else if ((result = facePattern2.exec(line)) !== null) {
+                continue;
-            addFace(
+            } else if ((result = vertexPattern.exec(line)) !== null) {
-                result[1], result[2], result[3], undefined,
+                positions.push(
-                result[4], result[5], result[6], undefined,
+                    parseFloat(result[1]),
-                result[7], result[8], result[9], undefined,
+                    parseFloat(result[2]),
-                result[10], result[11], result[12], undefined
+                    parseFloat(result[3])
-            );
+                );
-        } else if ((result = facePattern3.exec(line)) !== null) {
+            } else if ((result = normalPattern.exec(line) ) !== null) {
-            addFace(
+                var nx = parseFloat(result[1]);
-                result[1], result[2], result[3], result[4],
+                var ny = parseFloat(result[2]);
-                result[5], result[6], result[7], result[8],
+                var nz = parseFloat(result[3]);
-                result[9], result[10], result[11], result[12],
+                var normal = Cartesian3.normalize(new Cartesian3(nx, ny, nz), new Cartesian3());
-                result[13], result[14], result[15], result[16]
+                normals.push(normal.x, normal.y, normal.z);
-            );
+            } else if ((result = uvPattern.exec(line)) !== null) {
-        } else if ((result = facePattern4.exec(line)) !== null) {
+                uvs.push(
-            addFace(
+                    parseFloat(result[1]),
-                result[1], result[2], undefined, result[3],
+                    parseFloat(result[2])
-                result[4], result[5], undefined, result[6],
+                );
-                result[7], result[8], undefined, result[9],
+
-                result[10], result[11], undefined, result[12]
+            } else if ((result = facePattern1.exec(line)) !== null) {
-            );
+                addFace(
-        } else if (/^usemtl /.test(line)) {
+                    result[1], result[1], undefined, undefined,
-            var materialName = line.substring(7).trim();
+                    result[2], result[2], undefined, undefined,
-            useMaterial(materialName);
+                    result[3], result[3], undefined, undefined,
+                    result[4], result[4], undefined, undefined
+                );
+            } else if ((result = facePattern2.exec(line)) !== null) {
+                addFace(
+                    result[1], result[2], result[3], undefined,
+                    result[4], result[5], result[6], undefined,
+                    result[7], result[8], result[9], undefined,
+                    result[10], result[11], result[12], undefined
+                );
+            } else if ((result = facePattern3.exec(line)) !== null) {
+                addFace(
+                    result[1], result[2], result[3], result[4],
+                    result[5], result[6], result[7], result[8],
+                    result[9], result[10], result[11], result[12],
+                    result[13], result[14], result[15], result[16]
+                );
+            } else if ((result = facePattern4.exec(line)) !== null) {
+                addFace(
+                    result[1], result[2], undefined, result[3],
+                    result[4], result[5], undefined, result[6],
+                    result[7], result[8], undefined, result[9],
+                    result[10], result[11], undefined, result[12]
+                );
+            } else if (/^usemtl /.test(line)) {
+                var materialName = line.substring(7).trim();
+                useMaterial(materialName);
+            }
         }
-    }
+    });
 
-    done({
+    stream.on('end', function() {
-        vertexCount : vertexCount,
+        done({
-        vertexArray : vertexArray,
+            vertexCount : vertexCount,
-        positionMin : positionMin,
+            vertexArray : vertexArray,
-        positionMax : positionMax,
+            positionMin : positionMin,
-        hasUVs : hasUVs,
+            positionMax : positionMax,
-        hasNormals : hasNormals,
+            hasUVs : hasUVs,
-        materialGroups : materialGroups,
+            hasNormals : hasNormals,
-        materials : materials,
+            materialGroups : materialGroups,
-        images : images
+            materials : materials,
+            images : images
+        });
     });
 }
 
@@ -284,25 +277,66 @@ function getImages(inputPath, materials, done) {
     });
 }
 
-function getMaterials(contents, inputPath, done) {
+function getMaterials(mtlPath, hasMaterialGroups, done) {
-    var hasMaterialGroups = /^usemtl/gm.test(contents);
     if (!hasMaterialGroups) {
         done({});
         return;
     }
 
-    var mtllibMatches = contents.match(/^mtllib.*/gm);
+    if (defined(mtlPath)) {
-    if (mtllibMatches === null) {
+        Material.parse(mtlPath, function(materials) {
-        done({});
-    } else {
-        var mtlFile = mtllibMatches[0].substring(7).trim();
-        var mtlPath = mtlFile;
-        if (!path.isAbsolute(mtlPath)) {
-            mtlPath = path.join(inputPath, mtlFile);
-        }
-
-        Material.parse(mtlPath, function (materials) {
             done(materials);
         });
+    } else {
+        done({});
     }
 }
+
+function getObjInfo(objFile, inputPath, done) {
+    var mtlPath = undefined;
+    var hasMaterialGroups = false;
+    var hasPositions = false;
+    var hasNormals = false;
+    var hasUVs = false;
+
+    var stream = fs.createReadStream(objFile, 'utf8');
+    stream.on('data', function(chunk) {
+        if (!defined(mtlPath)) {
+            var mtllibMatches = chunk.match(/^mtllib.*/gm);
+            if (mtllibMatches !== null) {
+                var mtlFile = mtllibMatches[0].substring(7).trim();
+                mtlPath = mtlFile;
+                if (!path.isAbsolute(mtlPath)) {
+                    mtlPath = path.join(inputPath, mtlFile);
+                }
+            }
+        }
+        if (!hasMaterialGroups) {
+            hasMaterialGroups = /^usemtl/gm.test(chunk);
+        }
+        if (!hasPositions) {
+            hasPositions = /^v\s/gm.test(chunk);
+        }
+        if (!hasNormals) {
+            hasNormals = /^vn/gm.test(chunk);
+        }
+        if (!hasUVs) {
+            hasUVs = /^vt/gm.test(chunk);
+        }
+    });
+
+    stream.on('end', function() {
+        if (!hasPositions) {
+            throw new Error('Could not process OBJ file, no positions.');
+        }
+        var info = {
+            hasNormals : hasNormals,
+            hasUVs : hasUVs
+        };
+        getMaterials(mtlPath, hasMaterialGroups, function(materials) {
+            getImages(inputPath, materials, function(images) {
+                done(info, materials, images);
+            });
+        });
+    });
+}