glTF support

Example/Example.xcodeproj/project.pbxproj

@@ -64,11 +64,11 @@
 /* Begin PBXFileReference section */
 		067C5C4E2AAD6D7700F8FBB3 /* VRMKit */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = VRMKit; path = ..; sourceTree = "<group>"; };
 		06E116422F277AEA00D74CA4 /* MacExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = MacExample.app; sourceTree = BUILT_PRODUCTS_DIR; };
-		06E116512F277D1700D74CA4 /* AliciaSolid.vrm */ = {isa = PBXFileReference; lastKnownFileType = file; name = AliciaSolid.vrm; path = ../../Tests/VRMKitTests/Assets/AliciaSolid.vrm; sourceTree = "<group>"; };
+		06E116512F277D1700D74CA4 /* AliciaSolid.vrm */ = {isa = PBXFileReference; lastKnownFileType = file; name = AliciaSolid.vrm; path = ../../Tests/Assets/VRM/AliciaSolid.vrm; sourceTree = "<group>"; };
 		06F0BD6C2AAD81A30089488C /* WatchExample Watch App.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "WatchExample Watch App.app"; sourceTree = BUILT_PRODUCTS_DIR; };
 		245725D72146F47A003AA5D7 /* VRMExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VRMExample.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		96FABF60E748F3EF7D574461 /* VisionExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VisionExample.app; sourceTree = BUILT_PRODUCTS_DIR; };
-		AF000001AF000001AF000001 /* VRM1_Constraint_Twist_Sample.vrm */ = {isa = PBXFileReference; lastKnownFileType = file; name = VRM1_Constraint_Twist_Sample.vrm; path = ../../Tests/VRMKitTests/Assets/VRM1_Constraint_Twist_Sample.vrm; sourceTree = "<group>"; };
+		AF000001AF000001AF000001 /* VRM1_Constraint_Twist_Sample.vrm */ = {isa = PBXFileReference; lastKnownFileType = file; name = VRM1_Constraint_Twist_Sample.vrm; path = ../../Tests/Assets/VRM/VRM1_Constraint_Twist_Sample.vrm; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */

Package.swift

@@ -10,7 +10,10 @@ let package = Package(
         .library(name: "VRMRealityKit", targets: ["VRMRealityKit"])
     ],
     targets: [
-        .target(name: "VRMKit"),
+        .target(
+            name: "VRMKit",
+            exclude: ["Extensions/MoreCodable/LICENSE"]
+        ),
         .target(
             name: "VRMKitRuntime",
             dependencies: ["VRMKit"]
@@ -29,25 +32,26 @@ let package = Package(
         ),
 
         // Test-only helpers shared by the test targets.
-        .target(name: "VRMTestSupport", path: "Tests/VRMTestSupport"),
+        .target(
+            name: "VRMTestSupport",
+            path: "Tests/VRMTestSupport",
+            resources: [
+                .copy("../Assets/GLTF"),
+                .copy("../Assets/VRM")
+            ]
+        ),
 
         .testTarget(
             name: "VRMKitTests",
-            dependencies: ["VRMKit", "VRMTestSupport"],
-            resources: [.copy("Assets/AliciaSolid.vrm"), .copy("Assets/Seed-san.vrm")]
+            dependencies: ["VRMKit", "VRMTestSupport"]
         ),
         .testTarget(
             name: "VRMSceneKitTests",
-            dependencies: ["VRMSceneKit"],
-            resources: [
-                .copy("../VRMKitTests/Assets/AliciaSolid.vrm"),
-                .copy("../VRMKitTests/Assets/Seed-san.vrm")
-            ]
+            dependencies: ["VRMSceneKit", "VRMTestSupport"]
         ),
         .testTarget(
             name: "VRMRealityKitTests",
-            dependencies: ["VRMRealityKit", "VRMTestSupport"],
-            resources: [.copy("../VRMKitTests/Assets/AliciaSolid.vrm"), .copy("../VRMKitTests/Assets/Seed-san.vrm")]
+            dependencies: ["VRMRealityKit", "VRMTestSupport"]
         ),
     ]
 )

README.md

@@ -202,7 +202,8 @@ RealityKit constrains what the MToon renderer can express. Each case below logs
 
 </details>
 
-## Frame updates
+<details>
+<summary>Frame updates</summary>
 
 `VRMUpdateSystem` (a RealityKit `System` registered on load) calls `VRMEntity.update(deltaTime:)` on every render frame. To control the timing yourself, opt out and call it manually:
 
@@ -215,6 +216,39 @@ vrmEntity.update(deltaTime: deltaTime)
 
 To run your own animation code in a guaranteed order relative to the VRM update (e.g. posing joints that the same frame's skinning should reflect), put it in a custom `System` declared with `SystemDependency.before(VRMUpdateSystem.self)`.
 
+</details>
+
+<details>
+<summary>Render glTF / GLB</summary>
+
+VRMRealityKit can also render plain glTF assets (`.glb` and JSON `.gltf`, including external resources and data URIs).
+
+```swift
+let entity: GLTFEntity = try GLTFEntityLoader(withURL: url).loadEntity()
+content.add(entity)
+
+entity.animations          // [GLTFAnimation] — index, name, duration
+let controller = try entity.playAnimation(at: 0, loops: true)
+controller.speed = 2       // a negative speed plays backwards
+controller.seek(to: 0.5)
+controller.stop()
+```
+
+`loadEntity()` renders the asset's default scene, and throws when the glTF names none; pick one with `loadEntity(withSceneIndex:)`. A `clone(recursive:)` copy shares the loaded meshes and materials but not the animation bindings, so load the scene again for a second animatable instance.
+
+### RealityKit renderer limitations
+
+The renderer builds RealityKit meshes and materials, so a few parts of glTF have no place to go:
+
+- Only triangle primitives are drawn; `POINTS` and `LINES` primitives are skipped.
+- `COLOR_0` vertex colors are ignored: the `MeshResource.Part` buffers this renderer builds carry no vertex-color channel.
+- One UV set per material: the first UV-accessed texture decides both the `TEXCOORD_n` set and the single `KHR_texture_transform` every texture of that material is sampled with. An asset that merely lists `KHR_texture_transform` in `extensionsUsed` renders through that approximation and logs it; one that lists it in `extensionsRequired` and gives a material's textures different transforms is rejected instead of drawn wrong.
+- Tangents for a primitive without `TANGENT` are averaged from its UV gradients rather than generated with MikkTSpace, which the spec recommends, so a normal map baked against MikkTSpace can differ slightly along UV seams.
+- Blend shapes morph `POSITION` only, since RealityKit blend shapes have no `NORMAL` / `TANGENT` channel.
+- Skinning reads `JOINTS_0` / `WEIGHTS_0` only, so a vertex is driven by at most four joints; the further sets a glTF may carry (`JOINTS_1` and up) are ignored, which the spec allows and which can change the result of an animation.
+
+</details>
+
 # ToDo
 
 - [x] VRM 1.0 support
@@ -225,7 +259,7 @@ To run your own animation code in a guaranteed order relative to the VRM update
 - [ ] Improve rendering quality
 - [ ] Animation support (vrma)
 - [ ] VRM editing function
-- [ ] GLTF renderer support
+- [x] glTF renderer / animation support (RealityKit)
 
 # Contributing
 

Sources/VRMKit/BinaryGLTF.swift

@@ -5,8 +5,10 @@ import Foundation
 
 public struct BinaryGLTF {
     public let version: GLTF.Version
-    public let jsonData: GLTF /// chunk 0
-    public let binaryBuffer: Data? /// chunk1
+    /// Chunk 0.
+    public let jsonData: GLTF
+    /// Chunk 1.
+    public let binaryBuffer: Data?
 
     /// magic equals 0x46546C67. It is ASCII string glTF, and can be used to identify data as Binary glTF.
     static let magic: UInt32 = 0x46546C67
@@ -19,21 +21,16 @@ public struct BinaryGLTF {
 
 package extension BinaryGLTF {
     func bufferViewData(at index: Int, relativeTo rootDirectory: URL? = nil) throws -> (data: Data, stride: Int?) {
-        let bufferView = try jsonData.load(\.bufferViews, at: index)
-        let buffer = try bufferData(at: bufferView.buffer, relativeTo: rootDirectory)
-        let end = bufferView.byteOffset.addingReportingOverflow(bufferView.byteLength)
-        guard bufferView.byteOffset >= 0, bufferView.byteLength >= 0,
-              !end.overflow, end.partialValue <= buffer.count else {
-            throw VRMError._dataInconsistent(
-                "buffer view (offset: \(bufferView.byteOffset), length: \(bufferView.byteLength)) overruns its \(buffer.count) byte buffer"
-            )
-        }
-        return (buffer.subdata(in: bufferView.byteOffset..<end.partialValue), bufferView.byteStride)
+        try GLTFDocument(binary: self, rootDirectory: rootDirectory).bufferViewData(at: index)
     }
+}
 
-    func bufferData(at index: Int, relativeTo rootDirectory: URL? = nil) throws -> Data {
-        let gltfBuffer = try jsonData.load(\.buffers, at: index)
-        return try Data(buffer: gltfBuffer, relativeTo: rootDirectory, binaryBuffer: binaryBuffer)
+extension BinaryGLTF {
+    /// Whether the data starts with the GLB magic and so should be parsed as a
+    /// binary glTF container rather than as JSON.
+    public static func isGLB(_ data: Data) -> Bool {
+        guard data.count >= 4 else { return false }
+        return data.prefix(4).withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) }.littleEndian == magic
     }
 }
 
@@ -61,32 +58,59 @@ extension BinaryGLTF {
             )
         }
 
-        let chunk0Length = try reader.readUInt32()
-        let chunk0Type = try reader.readUInt32()
-        guard ChunkType(rawValue: chunk0Type) == .json else {
-            throw VRMError.notSupportedChunkType(chunk0Type)
-        }
-        let jsonData = try reader.readData(count: Int(chunk0Length))
-        let gltf = try JSONDecoder().decode(GLTF.self, from: jsonData)
-        // The GLB container version and the asset version are independent: a 2.x
-        // container can still declare an asset this parser does not implement.
-        guard gltf.asset.version.hasPrefix("2.") else {
-            throw VRMError._notSupported("glTF asset version \(gltf.asset.version) is not supported")
-        }
-        if let minVersion = gltf.asset.minVersion, minVersion != "2.0" {
-            throw VRMError._notSupported("glTF asset minVersion \(minVersion) is not supported")
+        // Chunk 0 holds the JSON, chunk 1 the optional BIN; every other chunk
+        // type is one this parser does not know and the spec says to skip.
+        var gltf: GLTF?
+        var binaryBuffer: Data?
+        var chunkIndex = 0
+        while reader.bytesRead + 8 <= Int(length) {
+            let chunkLength = try reader.readUInt32()
+            let chunkType = try reader.readUInt32()
+            // Every chunk starts and ends on a 4 byte boundary, so with the 12 byte
+            // header and 8 byte chunk headers the payloads are multiples of 4 too.
+            guard chunkLength.isMultiple(of: 4) else {
+                throw VRMError._dataInconsistent(
+                    "GLB chunk of \(chunkLength) bytes breaks the container's 4 byte alignment"
+                )
+            }
+            guard reader.bytesRead + Int(chunkLength) <= Int(length) else {
+                throw VRMError._dataInconsistent(
+                    "GLB chunk of \(chunkLength) bytes overruns the \(length) byte container"
+                )
+            }
+            let chunkData = try reader.readData(count: Int(chunkLength))
+            switch ChunkType(rawValue: chunkType) {
+            case .json:
+                guard chunkIndex == 0 else {
+                    throw VRMError._dataInconsistent("the JSON chunk must be the first GLB chunk")
+                }
+                gltf = try JSONDecoder().decode(GLTF.self, from: chunkData)
+            case .bin:
+                guard chunkIndex == 1 else {
+                    throw VRMError._dataInconsistent("the BIN chunk must be the second GLB chunk")
+                }
+                binaryBuffer = chunkData
+            case nil:
+                guard gltf != nil else {
+                    throw VRMError.notSupportedChunkType(chunkType)
+                }
+            }
+            chunkIndex += 1
         }
-        self.jsonData = gltf
 
-        if length > reader.bytesRead {
-            let chunk1Length = try reader.readUInt32()
-            let chunk1Type = try reader.readUInt32()
-            guard ChunkType(rawValue: chunk1Type) == .bin else {
-                throw VRMError.notSupportedChunkType(chunk1Type)
-            }
-            binaryBuffer = try reader.readData(count: Int(chunk1Length))
-        } else {
-            binaryBuffer = nil
+        // Anything left inside the declared length belongs to no chunk, so the
+        // container does not describe its own contents.
+        guard reader.bytesRead == Int(length) else {
+            throw VRMError._dataInconsistent(
+                "GLB has \(Int(length) - reader.bytesRead) bytes left over after its last chunk"
+            )
         }
+
+        let jsonData = try gltf ??? ._dataInconsistent("GLB carries no JSON chunk")
+        // The GLB container version and the asset version are independent: a 2.x
+        // container can still declare an asset this parser does not implement.
+        try jsonData.validateSupportedAssetVersion()
+        self.jsonData = jsonData
+        self.binaryBuffer = binaryBuffer
     }
 }

Sources/VRMKit/Extensions/Accessor+Data.swift

@@ -34,6 +34,30 @@ package extension GLTF.Accessor {
         return (componentsPerVector, bytesPerComponent, vectorSize)
     }
 
+    /// One component as a float. `normalized` accessors map integers onto [0, 1]
+    /// or [-1, 1], with the signed minimum clamped to -1 per spec.
+    func floatComponent(base: UnsafeRawPointer, offset: Int) -> Float {
+        switch componentType {
+        case .float:
+            return base.loadUnaligned(fromByteOffset: offset, as: Float.self)
+        case .unsignedByte:
+            let value = Float(base.load(fromByteOffset: offset, as: UInt8.self))
+            return normalized ? value / Float(UInt8.max) : value
+        case .byte:
+            let value = Float(base.load(fromByteOffset: offset, as: Int8.self))
+            return normalized ? Swift.max(-1, value / Float(Int8.max)) : value
+        case .unsignedShort:
+            let value = Float(base.loadUnaligned(fromByteOffset: offset, as: UInt16.self))
+            return normalized ? value / Float(UInt16.max) : value
+        case .short:
+            let value = Float(base.loadUnaligned(fromByteOffset: offset, as: Int16.self))
+            return normalized ? Swift.max(-1, value / Float(Int16.max)) : value
+        case .unsignedInt:
+            let value = Float(base.loadUnaligned(fromByteOffset: offset, as: UInt32.self))
+            return normalized ? value / Float(UInt32.max) : value
+        }
+    }
+
     /// The accessor's elements as tightly packed data: any buffer view stride is
     /// removed, an accessor without a buffer view yields the zeroes the spec
     /// defines for it, and a sparse substitution is applied on top.
@@ -72,7 +96,6 @@ private extension GLTF.Accessor {
 
     func packedBaseData(vectorSize: Int, provider: BufferViewProvider) throws -> Data {
         guard let bufferView else {
-            // The spec defines a bufferView-less accessor as all zeroes.
             return try Data(zeroedElementCount: count, elementSize: vectorSize)
         }
         let source = try provider(bufferView)

Sources/VRMKit/Extensions/Data+GLTF.swift

@@ -16,12 +16,48 @@ package extension Data {
     }
 
     init(gltfUrlString: String, relativeTo rootDirectory: URL?) throws {
-        if let base64Str = gltfUrlString.retrievedBase64EncodedString() {
-            self = try Data(base64Encoded: base64Str) ??? .dataInconsistent("failed to load base64 data")
+        if let data = try Data(dataURI: gltfUrlString) {
+            self = data
         } else {
-            let url = URL(fileURLWithPath: gltfUrlString, relativeTo: rootDirectory)
-            self = try Data(contentsOf: url)
+            self = try Data(contentsOf: URL(gltfUri: gltfUrlString, relativeTo: rootDirectory))
+        }
+    }
+
+    /// The bytes an RFC 2397 `data:` URI carries, or nil when the string is not
+    /// one. `;base64` is optional there: without it the data is percent-encoded.
+    init?(dataURI: String) throws {
+        guard dataURI.hasPrefix("data:") else { return nil }
+        let body = dataURI.dropFirst("data:".count)
+        guard let separator = body.firstIndex(of: ",") else {
+            throw VRMError._dataInconsistent("the data uri has no \",\" separating its media type from its data")
+        }
+        let payload = body[body.index(after: separator)...]
+        guard body[..<separator].hasSuffix(";base64") else {
+            self = try Data(percentEncoded: payload)
+                ??? .dataInconsistent("failed to decode the percent-encoded data uri")
+            return
         }
+        self = try Data(base64Encoded: String(payload)) ??? .dataInconsistent("failed to load base64 data")
+    }
+
+    /// Percent-decodes a URI component into the octets it stands for, which
+    /// unlike `removingPercentEncoding` survives data that is not text.
+    private init?(percentEncoded string: Substring) {
+        var bytes: [UInt8] = []
+        bytes.reserveCapacity(string.utf8.count)
+        var iterator = string.utf8.makeIterator()
+        while let byte = iterator.next() {
+            guard byte == UInt8(ascii: "%") else {
+                bytes.append(byte)
+                continue
+            }
+            guard let high = iterator.next()?.hexDigitValue,
+                  let low = iterator.next()?.hexDigitValue else {
+                return nil
+            }
+            bytes.append(high << 4 | low)
+        }
+        self.init(bytes)
     }
 
     /// All-zero data for a glTF accessor with no bufferView.
@@ -82,10 +118,15 @@ package extension Data {
     }
 }
 
-private extension String {
-    func retrievedBase64EncodedString() -> String? {
-        guard starts(with: "data:") else { return nil }
-        return components(separatedBy: ";base64,").last
+private extension UInt8 {
+    /// The value of one ASCII hexadecimal digit, or nil for anything else.
+    var hexDigitValue: UInt8? {
+        switch self {
+        case UInt8(ascii: "0")...UInt8(ascii: "9"): return self - UInt8(ascii: "0")
+        case UInt8(ascii: "a")...UInt8(ascii: "f"): return self - UInt8(ascii: "a") + 10
+        case UInt8(ascii: "A")...UInt8(ascii: "F"): return self - UInt8(ascii: "A") + 10
+        default: return nil
+        }
     }
 }
 
@@ -101,3 +142,36 @@ package extension Int {
         return end.partialValue
     }
 }
+
+package extension URL {
+    /// Resolves a glTF `uri` against the asset's directory.
+    ///
+    /// A `uri` is a URI reference, not a file path: its reserved characters are
+    /// percent-encoded, so `My%20Buffer.bin` names a file with a space in it.
+    /// Only local files are read — a glTF must not fetch resources off the
+    /// network on the caller's behalf.
+    init(gltfUri: String, relativeTo rootDirectory: URL?) throws {
+        // A uri that is not a valid URI reference is taken as a literal path,
+        // which is what exporters writing unencoded characters mean by it.
+        let uri = URL(string: gltfUri, relativeTo: rootDirectory)
+        guard let uri, uri.scheme != nil else {
+            // A relative uri names a file beside the glTF, so without that
+            // directory the only base left is the working directory of the
+            // process, which holds some unrelated file of the same name.
+            guard let rootDirectory else {
+                throw VRMError._dataInconsistent(
+                    """
+                    the glTF uri \"\(gltfUri)\" is relative to the directory of the asset, which this document was loaded without; \
+                    load it from a URL, or pass the directory its resources live in as rootDirectory
+                    """
+                )
+            }
+            self = URL(fileURLWithPath: uri?.relativePath ?? gltfUri, relativeTo: rootDirectory)
+            return
+        }
+        guard uri.isFileURL else {
+            throw VRMError._notSupported("the \(uri.scheme ?? "") scheme of the glTF uri \"\(gltfUri)\" is not loadable")
+        }
+        self = uri
+    }
+}