Merge pull request #83 from CesiumGS/tile-content-extraction-fixes

Tile content extraction fixes

README.md

@@ -187,6 +187,22 @@ Extracts the glb models from a cmpt tile. If multiple models are found a number
 npx 3d-tiles-tools cmptToGlb -i ./specs/data/composite.cmpt -o ./output/extracted.glb
 ```
 
+#### splitCmpt
+
+Split a cmpt tile into its inner tiles. The output file name for each inner tile will be determined by appending a number to the given output file name, and an extension that depends on the type of the inner tile data.
+
+```
+npx 3d-tiles-tools cmptToGlb -i ./specs/data/compositeOfComposite.cmpt -o ./output/inner --recursive
+```
+
+For an input file `compositeOfComposite.cmpt` that contains a composite tile that contains one B3DM and one I3DM content, this will generate the files `inner_0.b3dm` and `inner_1.i3dm` in the output directory.
+
+Additional command line options:
+
+| Flag | Description | Required |
+| ---- | ----------- | -------- |
+|`--recursive`|Whether the split operation should be applied to inner tiles that are composite| No, default: `false` |
+
 
 #### convertB3dmToGlb
 

specs/tileFormats/TileFormatsSpec.ts

@@ -1,9 +1,25 @@
 import fs from "fs";
+import path from "path";
 import { Buffers } from "../../src/base/Buffers";
 
 import { TileFormats } from "../../src/tileFormats/TileFormats";
 import { SpecHelpers } from "../SpecHelpers";
 
+function createResolver(
+  input: string
+): (uri: string) => Promise<Buffer | undefined> {
+  const baseDir = path.dirname(input);
+  const resolver = async (uri: string): Promise<Buffer | undefined> => {
+    const externalGlbUri = path.resolve(baseDir, uri);
+    try {
+      return fs.readFileSync(externalGlbUri);
+    } catch (error) {
+      console.error(`Could not resolve ${uri} against ${baseDir}`);
+    }
+  };
+  return resolver;
+}
+
 describe("TileFormats", function () {
   it("reads B3DM (deprecated 1) from a buffer", function () {
     const p = "./specs/data/BatchedDeprecated1/batchedDeprecated1.b3dm";
@@ -115,30 +131,97 @@ describe("TileFormats", function () {
     expect(tileData.innerTileBuffers.length).toBe(2);
   });
 
-  it("extracts a single GLB buffers from B3DM", function () {
+  it("extracts a single GLB buffers from B3DM", async function () {
     const p = "./specs/data/contentTypes/content.b3dm";
     const tileDataBuffer = fs.readFileSync(p);
-
-    const glbBuffers = TileFormats.extractGlbBuffers(tileDataBuffer);
+    const externalGlbResolver = createResolver(p);
+    const glbBuffers = await TileFormats.extractGlbBuffers(
+      tileDataBuffer,
+      externalGlbResolver
+    );
     expect(glbBuffers.length).toBe(1);
   });
 
-  it("extracts multiple GLB buffers from CMPT", function () {
+  it("extracts multiple GLB buffers from CMPT", async function () {
     const p = "./specs/data/contentTypes/content.cmpt";
     const tileDataBuffer = fs.readFileSync(p);
-
-    const glbBuffers = TileFormats.extractGlbBuffers(tileDataBuffer);
+    const externalGlbResolver = createResolver(p);
+    const glbBuffers = await TileFormats.extractGlbBuffers(
+      tileDataBuffer,
+      externalGlbResolver
+    );
     expect(glbBuffers.length).toBe(2);
   });
 
-  it("extracts no GLB buffers from PNTS", function () {
-    const p = "./specs/data/contentTypes/content.pnts";
+  it("extracts a single GLB buffer from an I3DM that refers to an external GLB", async function () {
+    const p = "./specs/data/tileFormats/instancedGltfExternal.i3dm";
     const tileDataBuffer = fs.readFileSync(p);
+    const externalGlbResolver = createResolver(p);
+    const glbBuffers = await TileFormats.extractGlbBuffers(
+      tileDataBuffer,
+      externalGlbResolver
+    );
+    expect(glbBuffers.length).toBe(1);
+  });
 
-    const glbBuffers = TileFormats.extractGlbBuffers(tileDataBuffer);
+  it("extracts no GLB buffers from PNTS", async function () {
+    const p = "./specs/data/contentTypes/content.pnts";
+    const tileDataBuffer = fs.readFileSync(p);
+    const externalGlbResolver = createResolver(p);
+    const glbBuffers = await TileFormats.extractGlbBuffers(
+      tileDataBuffer,
+      externalGlbResolver
+    );
     expect(glbBuffers.length).toBe(0);
   });
 
+  it("properly omits padding bytes in extractGlbPayload for b3dmToGlb", async function () {
+    const p = "./specs/data/tileFormats/box.glb";
+
+    const inputGlbBuffer = fs.readFileSync(p);
+    const inputB3dmTileData =
+      TileFormats.createDefaultB3dmTileDataFromGlb(inputGlbBuffer);
+    const b3dmBuffer = TileFormats.createTileDataBuffer(inputB3dmTileData);
+
+    const outputB3dmTileData = TileFormats.readTileData(b3dmBuffer);
+    const outputGlbBuffer = TileFormats.extractGlbPayload(outputB3dmTileData);
+
+    // The input GLB is NOT 8-byte-aligned
+    expect(inputGlbBuffer.length % 8).not.toEqual(0);
+
+    // The payload from the generated B3DM tile data is
+    // aligned to 8 bytes
+    expect(outputB3dmTileData.payload.length % 8).toEqual(0);
+
+    // The GLB that is extracted from the B3DM tile data
+    // has the same length as the original input
+    expect(outputGlbBuffer.length).toEqual(inputGlbBuffer.length);
+  });
+
+  it("splits a composite with splitCmpt", async function () {
+    const p = "./specs/data/composite.cmpt";
+    const recursive = false;
+    const inputBuffer = fs.readFileSync(p);
+    const outputBuffers = await TileFormats.splitCmpt(inputBuffer, recursive);
+    expect(outputBuffers.length).toEqual(2);
+  });
+
+  it("splits a composite-of-composite into a single file with non-recursive splitCmpt", async function () {
+    const p = "./specs/data/compositeOfComposite.cmpt";
+    const recursive = false;
+    const inputBuffer = fs.readFileSync(p);
+    const outputBuffers = await TileFormats.splitCmpt(inputBuffer, recursive);
+    expect(outputBuffers.length).toEqual(1);
+  });
+
+  it("splits a composite-of-composite into a all 'leaf' tiles with recursive splitCmpt", async function () {
+    const p = "./specs/data/compositeOfComposite.cmpt";
+    const recursive = true;
+    const inputBuffer = fs.readFileSync(p);
+    const outputBuffers = await TileFormats.splitCmpt(inputBuffer, recursive);
+    expect(outputBuffers.length).toEqual(2);
+  });
+
   it("throws an error when trying to read tile data from a buffer that does not contain B3DM, I3DM, or PNTS", function () {
     const p = "./specs/data/contentTypes/content.cmpt";
     const tileDataBuffer = fs.readFileSync(p);

src/ToolsMain.ts

@@ -10,6 +10,7 @@ import { Tilesets } from "./tilesets/Tilesets";
 
 import { TileFormats } from "./tileFormats/TileFormats";
 import { TileDataLayouts } from "./tileFormats/TileDataLayouts";
+import { TileFormatError } from "./tileFormats/TileFormatError";
 
 import { ContentOps } from "./contentProcessing/ContentOps";
 import { GltfUtilities } from "./contentProcessing/GltfUtilities";
@@ -25,6 +26,8 @@ import { TilesetConverter } from "./tilesetProcessing/TilesetConverter";
 
 import { TilesetJsonCreator } from "./tilesetProcessing/TilesetJsonCreator";
 
+import { ContentDataTypeRegistry } from "./contentTypes/ContentDataTypeRegistry";
+
 import { Loggers } from "./logging/Loggers";
 const logger = Loggers.get("CLI");
 
@@ -52,7 +55,7 @@ export class ToolsMain {
     ToolsMain.ensureCanWrite(output, force);
     const inputBuffer = fs.readFileSync(input);
     const inputTileData = TileFormats.readTileData(inputBuffer);
-    const outputBuffer = inputTileData.payload;
+    const outputBuffer = TileFormats.extractGlbPayload(inputTileData);
     fs.writeFileSync(output, outputBuffer);
 
     logger.debug(`Executing b3dmToGlb DONE`);
@@ -98,13 +101,7 @@ export class ToolsMain {
     const inputBuffer = fs.readFileSync(input);
 
     // Prepare the resolver for external GLBs in I3DM
-    const baseDir = path.dirname(input);
-    const externalGlbResolver = async (
-      uri: string
-    ): Promise<Buffer | undefined> => {
-      const externalGlbUri = path.resolve(baseDir, uri);
-      return fs.readFileSync(externalGlbUri);
-    };
+    const externalGlbResolver = ToolsMain.createResolver(input);
     const outputBuffer = await TileFormatsMigration.convertI3dmToGlb(
       inputBuffer,
       externalGlbResolver
@@ -123,7 +120,17 @@ export class ToolsMain {
     ToolsMain.ensureCanWrite(output, force);
     const inputBuffer = fs.readFileSync(input);
     const inputTileData = TileFormats.readTileData(inputBuffer);
-    const outputBuffer = inputTileData.payload;
+    // Prepare the resolver for external GLBs in I3DM
+    const externalGlbResolver = ToolsMain.createResolver(input);
+    const outputBuffer = await TileFormats.obtainGlbPayload(
+      inputTileData,
+      externalGlbResolver
+    );
+    if (!outputBuffer) {
+      throw new TileFormatError(
+        `Could not resolve external GLB from I3DM file`
+      );
+    }
     fs.writeFileSync(output, outputBuffer);
 
     logger.debug(`Executing i3dmToGlb DONE`);
@@ -135,7 +142,11 @@ export class ToolsMain {
     logger.debug(`  force: ${force}`);
 
     const inputBuffer = fs.readFileSync(input);
-    const glbBuffers = TileFormats.extractGlbBuffers(inputBuffer);
+    const externalGlbResolver = ToolsMain.createResolver(input);
+    const glbBuffers = await TileFormats.extractGlbBuffers(
+      inputBuffer,
+      externalGlbResolver
+    );
     const glbsLength = glbBuffers.length;
     const glbPaths = Array<string>(glbsLength);
     if (glbsLength === 0) {
@@ -152,15 +163,54 @@ export class ToolsMain {
       const glbPath = glbPaths[i];
       ToolsMain.ensureCanWrite(glbPath, force);
       const glbBuffer = glbBuffers[i];
-      const upgradedOutputBuffer = await GltfUtilities.upgradeGlb(
-        glbBuffer,
-        undefined
-      );
-      fs.writeFileSync(glbPath, upgradedOutputBuffer);
+      fs.writeFileSync(glbPath, glbBuffer);
     }
 
     logger.debug(`Executing cmptToGlb DONE`);
   }
+
+  static async splitCmpt(
+    input: string,
+    output: string,
+    recursive: boolean,
+    force: boolean
+  ) {
+    logger.debug(`Executing splitCmpt`);
+    logger.debug(`  input: ${input}`);
+    logger.debug(`  output: ${output}`);
+    logger.debug(`  recursive: ${recursive}`);
+    logger.debug(`  force: ${force}`);
+
+    const inputBuffer = fs.readFileSync(input);
+    const outputBuffers = await TileFormats.splitCmpt(inputBuffer, recursive);
+    for (let i = 0; i < outputBuffers.length; i++) {
+      const outputBuffer = outputBuffers[i];
+      const prefix = Paths.replaceExtension(output, "");
+      const extension = await ToolsMain.determineFileExtension(outputBuffer);
+      const outputPath = `${prefix}_${i}.${extension}`;
+      ToolsMain.ensureCanWrite(outputPath, force);
+      fs.writeFileSync(outputPath, outputBuffer);
+    }
+
+    logger.debug(`Executing splitCmpt DONE`);
+  }
+
+  private static async determineFileExtension(data: Buffer): Promise<string> {
+    const type = await ContentDataTypeRegistry.findType("", data);
+    switch (type) {
+      case ContentDataTypes.CONTENT_TYPE_B3DM:
+        return "b3dm";
+      case ContentDataTypes.CONTENT_TYPE_I3DM:
+        return "i3dm";
+      case ContentDataTypes.CONTENT_TYPE_PNTS:
+        return "pnts";
+      case ContentDataTypes.CONTENT_TYPE_CMPT:
+        return "cmpt";
+    }
+    logger.warn("Could not determine type of inner tile");
+    return "UNKNOWN";
+  }
+
   static async glbToB3dm(input: string, output: string, force: boolean) {
     logger.debug(`Executing glbToB3dm`);
     logger.debug(`  input: ${input}`);
@@ -555,6 +605,34 @@ export class ToolsMain {
     logger.debug(`Executing createTilesetJson DONE`);
   }
 
+  /**
+   * Creates a function that can resolve URIs relative to
+   * the given input file.
+   *
+   * The function will resolve relative URIs against the
+   * base directory of the given input file name, and
+   * return the corresponding file data. If the data
+   * cannot be read, then the function will print an
+   * error message and return  `undefined`.
+   *
+   * @param input - The input file name
+   * @returns The resolver function
+   */
+  private static createResolver(
+    input: string
+  ): (uri: string) => Promise<Buffer | undefined> {
+    const baseDir = path.dirname(input);
+    const resolver = async (uri: string): Promise<Buffer | undefined> => {
+      const externalGlbUri = path.resolve(baseDir, uri);
+      try {
+        return fs.readFileSync(externalGlbUri);
+      } catch (error) {
+        logger.error(`Could not resolve ${uri} against ${baseDir}`);
+      }
+    };
+    return resolver;
+  }
+
   /**
    * Returns whether the specified file can be written.
    *

src/contentProcessing/ContentOps.ts

@@ -49,10 +49,15 @@ export class ContentOps {
    * to the results array, in unspecified order.
    *
    * @param inputBuffer - The input buffer
+   * @param externalGlbResolver - The function that will
+   * resolve external GLB references from I3DM files.
    * @returns The resulting buffers
    */
-  static cmptToGlbBuffers(inputBuffer: Buffer): Buffer[] {
-    return TileFormats.extractGlbBuffers(inputBuffer);
+  static async cmptToGlbBuffers(
+    inputBuffer: Buffer,
+    externalGlbResolver: (glbUri: string) => Promise<Buffer | undefined>
+  ): Promise<Buffer[]> {
+    return TileFormats.extractGlbBuffers(inputBuffer, externalGlbResolver);
   }
 
   /**

src/contentProcessing/GltfUtilities.ts

@@ -246,7 +246,7 @@ export class GltfUtilities {
    * @param glbBuffer - The buffer containing the binary glTF.
    * @returns A promise that resolves to the resulting binary glTF.
    */
-  static async replaceCesiumRtcExtension(glbBuffer: Buffer) {
+  static async replaceCesiumRtcExtension(glbBuffer: Buffer): Promise<Buffer> {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const customStage = (gltf: any, options: any) => {
       GltfUtilities.replaceCesiumRtcExtensionInternal(gltf);