diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 522d41b1..47f02d02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,20 @@ concurrency: cancel-in-progress: true jobs: + check-mtoon-metallibs: + name: Check MToon metallibs are up to date + # Only compares the recorded build inputs, so the cheap Linux runner is enough. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # The bundled metallibs are stale when the shader sources or the compile + # settings change without re-running the build script. --check compares + # the recorded build inputs (metallib bytes vary across Metal toolchain + # versions) and needs no Metal toolchain, so it runs on the cheap runner + # ahead of the macOS jobs. + - name: Verify recorded metallib build inputs match sources + run: ./scripts/build-mtoon-metallibs.sh --check + test-package: name: Test package (${{ matrix.platform }}) runs-on: macos-26 diff --git a/.gitignore b/.gitignore index f90131b6..5c205132 100644 --- a/.gitignore +++ b/.gitignore @@ -55,5 +55,6 @@ xcuserdata/ *.xcscmblueprint UserInterfaceState.xcuserstate +docs .build diff --git a/Example/Example/RealityKitViewController.swift b/Example/Example/RealityKitViewController.swift index 5439d6b9..beb3cc09 100644 --- a/Example/Example/RealityKitViewController.swift +++ b/Example/Example/RealityKitViewController.swift @@ -1,6 +1,7 @@ import Combine import UIKit import RealityKit +import simd internal import VRMKit internal import VRMRealityKit @@ -9,8 +10,10 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg private var arView: ARView? private var updateSubscription: Cancellable? private var loadedEntity: VRMEntity? + private var loadedAnchor: AnchorEntity? private var cameraAnchor: AnchorEntity? private var cameraEntity: PerspectiveCamera? + private var lightEntity: DirectionalLight? private var expressionSegmentedControl: UISegmentedControl? private var orbitYaw: Float = 0 private var orbitPitch: Float = -0.1 @@ -18,6 +21,7 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg private var orbitTarget = SIMD3(0, 0.8, 0) private var currentModel: VRMExampleModel = .alicia private var currentExpression: ExampleExpression = .neutral + private var isMToonEnabled = true override func viewDidLoad() { super.viewDidLoad() @@ -61,12 +65,25 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg expressionSegmentedControl.translatesAutoresizingMaskIntoConstraints = false view.addSubview(expressionSegmentedControl) self.expressionSegmentedControl = expressionSegmentedControl - + + let mtoonLabel = UILabel() + mtoonLabel.text = "MToon" + let mtoonSwitch = UISwitch() + mtoonSwitch.isOn = isMToonEnabled + mtoonSwitch.addTarget(self, action: #selector(mtoonChanged(_:)), for: .valueChanged) + let mtoonControl = UIStackView(arrangedSubviews: [mtoonLabel, mtoonSwitch]) + mtoonControl.axis = .horizontal + mtoonControl.spacing = 8 + mtoonControl.translatesAutoresizingMaskIntoConstraints = false + view.addSubview(mtoonControl) + NSLayoutConstraint.activate([ segmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), segmentedControl.bottomAnchor.constraint(equalTo: view.safeAreaLayoutGuide.bottomAnchor, constant: -50), expressionSegmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), - expressionSegmentedControl.bottomAnchor.constraint(equalTo: segmentedControl.topAnchor, constant: -20) + expressionSegmentedControl.bottomAnchor.constraint(equalTo: segmentedControl.topAnchor, constant: -20), + mtoonControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), + mtoonControl.bottomAnchor.constraint(equalTo: expressionSegmentedControl.topAnchor, constant: -16) ]) } @@ -82,29 +99,38 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg loadedEntity?.setExampleExpression(currentExpression, value: 1.0) } + @objc private func mtoonChanged(_ sender: UISwitch) { + isMToonEnabled = sender.isOn + loadVRM(model: currentModel) + } + private func loadVRM(model: VRMExampleModel) { guard let arView = arView else { return } currentModel = model updateExpressionLabels() - if let loadedEntity = loadedEntity { - loadedEntity.entity.removeFromParent() - self.loadedEntity = nil + // Removing the anchor takes the whole model hierarchy out of the scene. + if let loadedAnchor { + arView.scene.removeAnchor(loadedAnchor) + self.loadedAnchor = nil } + loadedEntity = nil do { - let loader = try VRMEntityLoader(named: model.rawValue) + let loader = try VRMEntityLoader(named: model.rawValue, isMToonEnabled: isMToonEnabled) let vrmEntity = try loader.loadEntity() + vrmEntity.setMToonLightDirection(RealityKitExampleLighting.direction) let anchor = AnchorEntity(world: .zero) - vrmEntity.entity.transform.translation = SIMD3(0, -1.0, -1.5) - anchor.addChild(vrmEntity.entity) + vrmEntity.transform.translation = SIMD3(0, -1.0, -1.5) + anchor.addChild(vrmEntity) arView.scene.addAnchor(anchor) - normalizeScale(for: vrmEntity.entity) - updateOrbitTarget(for: vrmEntity.entity, adjustDistance: false) + setUpLight(in: arView) + normalizeScale(for: vrmEntity) + updateOrbitTarget(for: vrmEntity, adjustDistance: false) updateCameraTransform() - + let neck = vrmEntity.humanoid.node(for: .neck) let leftArm: Entity? let rightArm: Entity? @@ -116,7 +142,7 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg leftArm = vrmEntity.humanoid.node(for: .leftUpperArm) rightArm = vrmEntity.humanoid.node(for: .rightUpperArm) } - + let neckRotation = simd_quatf(angle: 20 * .pi / 180, axis: SIMD3(0, 0, 1)) let armRotation = simd_quatf(angle: 40 * .pi / 180, axis: SIMD3(0, 0, 1)) if let neck { @@ -129,17 +155,18 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg rightArm.transform.rotation = rightArm.transform.rotation * armRotation } vrmEntity.setExampleExpression(currentExpression, value: 1.0) - + loadedEntity = vrmEntity - + loadedAnchor = anchor + let rotationOffset = model.initialRotation var time: TimeInterval = 0 updateSubscription = arView.scene.subscribe(to: SceneEvents.Update.self) { [weak self] event in guard let loadedEntity = self?.loadedEntity else { return } - + time += event.deltaTime - + let cycle = time.truncatingRemainder(dividingBy: 1.0) let angle: Float if cycle < 0.5 { @@ -149,10 +176,8 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg let progress = Float(cycle - 0.5) / 0.5 angle = -0.5 + 0.5 * progress } - - loadedEntity.entity.transform.rotation = simd_quatf(angle: rotationOffset + angle, axis: SIMD3(0, 1, 0)) - - loadedEntity.update(at: event.deltaTime) + + loadedEntity.transform.rotation = simd_quatf(angle: rotationOffset + angle, axis: SIMD3(0, 1, 0)) } } catch { print(error) @@ -182,6 +207,21 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg updateCameraTransform() } + private func setUpLight(in arView: ARView) { + if lightEntity != nil { return } + let lightAnchor = AnchorEntity(world: .zero) + let light = DirectionalLight() + light.light.intensity = 1200 + // `direction` points at the light, so place the light there and aim it + // at the origin. + light.look(at: .zero, + from: RealityKitExampleLighting.direction, + relativeTo: nil) + lightAnchor.addChild(light) + arView.scene.addAnchor(lightAnchor) + lightEntity = light + } + private func setUpGestures() { guard let arView = arView else { return } @@ -249,7 +289,7 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg @objc private func handlePan(_ gesture: UIPanGestureRecognizer) { guard let arView = arView, let cameraEntity = cameraEntity else { return } let translation = gesture.translation(in: arView) - let panSpeed: Float = 0.002 * orbitDistance + let panSpeed = Float(0.002) * orbitDistance let transform = cameraEntity.transform.matrix let right = SIMD3(transform.columns.0.x, transform.columns.0.y, transform.columns.0.z) @@ -275,3 +315,8 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg return true } } + +private enum RealityKitExampleLighting { + /// Direction from the model toward the light, as `setMToonLightDirection(_:)` expects. + static let direction = simd_normalize(SIMD3(0, 0, -1)) +} diff --git a/Example/Example/ViewController.swift b/Example/Example/ViewController.swift index 519f4bc1..7705c44e 100644 --- a/Example/Example/ViewController.swift +++ b/Example/Example/ViewController.swift @@ -1,5 +1,6 @@ import UIKit import SceneKit +import simd internal import VRMSceneKit class ViewController: UIViewController { @@ -12,12 +13,12 @@ class ViewController: UIViewController { scnView.backgroundColor = UIColor.black } } - + private var vrmNode: VRMNode? private var expressionSegmentedControl: UISegmentedControl? private var currentModel: VRMExampleModel = .alicia private var currentExpression: ExampleExpression = .neutral - + override func viewDidLoad() { super.viewDidLoad() setupUI() @@ -31,7 +32,7 @@ class ViewController: UIViewController { segmentedControl.addTarget(self, action: #selector(segmentChanged(_:)), for: .valueChanged) segmentedControl.translatesAutoresizingMaskIntoConstraints = false view.addSubview(segmentedControl) - + let expressionItems = ExampleExpression.allCases.map { $0.displayName(for: currentModel) } let expressionSegmentedControl = UISegmentedControl(items: expressionItems) expressionSegmentedControl.selectedSegmentIndex = 0 @@ -39,11 +40,11 @@ class ViewController: UIViewController { expressionSegmentedControl.translatesAutoresizingMaskIntoConstraints = false view.addSubview(expressionSegmentedControl) self.expressionSegmentedControl = expressionSegmentedControl - + NSLayoutConstraint.activate([ segmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), segmentedControl.bottomAnchor.constraint(equalTo: view.safeAreaLayoutGuide.bottomAnchor, constant: -50), - + expressionSegmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), expressionSegmentedControl.bottomAnchor.constraint(equalTo: segmentedControl.topAnchor, constant: -20) ]) @@ -53,7 +54,7 @@ class ViewController: UIViewController { let model = VRMExampleModel.allCases[sender.selectedSegmentIndex] loadVRM(model: model) } - + @objc private func expressionSegmentChanged(_ sender: UISegmentedControl) { let expression = ExampleExpression.allCases[sender.selectedSegmentIndex] vrmNode?.setExampleExpression(currentExpression, value: 0.0) @@ -72,12 +73,11 @@ class ViewController: UIViewController { scnView.delegate = self let node = scene.vrmNode self.vrmNode = node - + let rotationOffset = CGFloat(model.initialRotation) node.eulerAngles = SCNVector3(0, rotationOffset, 0) - node.setExampleExpression(currentExpression, value: 1.0) - + node.humanoid.node(for: .neck)?.eulerAngles = SCNVector3(0, 0, 20 * CGFloat.pi / 180) let leftArm: SCNNode? let rightArm: SCNNode? @@ -91,7 +91,7 @@ class ViewController: UIViewController { } leftArm?.eulerAngles = SCNVector3(0, 0, 40 * CGFloat.pi / 180) rightArm?.eulerAngles = SCNVector3(0, 0, 40 * CGFloat.pi / 180) - + node.runAction(SCNAction.repeatForever(SCNAction.sequence([ SCNAction.rotateBy(x: 0, y: -0.5, z: 0, duration: 0.5), SCNAction.rotateBy(x: 0, y: 0.5, z: 0, duration: 0.5), @@ -120,9 +120,21 @@ class ViewController: UIViewController { cameraNode.position = SCNVector3(0, 0.8, -1.6) cameraNode.rotation = SCNVector4(0, 1, 0, Float.pi) + + let lightNode = SCNNode() + lightNode.light = SCNLight() + lightNode.light?.type = .directional + lightNode.light?.intensity = 1200 + lightNode.simdPosition = -SceneKitExampleLighting.direction + lightNode.look(at: SCNVector3Zero) + scene.rootNode.addChildNode(lightNode) } } +private enum SceneKitExampleLighting { + static let direction = simd_normalize(SIMD3(0.35, 0.55, 0.75)) +} + @available(*, deprecated, message: "Deprecated. Use VRMRealityKit instead.") extension ViewController: SCNSceneRendererDelegate { nonisolated func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { diff --git a/Example/MacExample/ContentView.swift b/Example/MacExample/ContentView.swift index 1e15c2ad..1b2c2189 100644 --- a/Example/MacExample/ContentView.swift +++ b/Example/MacExample/ContentView.swift @@ -15,12 +15,11 @@ internal import Combine internal import VRMKit struct ContentView: View { - @State private var realityKitViewModel = RealityKitContentViewModel() - @State private var sceneKitViewModel = SceneKitContentViewModel() @State private var selectedRenderer: MacExampleRenderer = .realityKit @State private var selectedModel: MacExampleModel = .alicia @State private var selectedExpression: MacExampleExpression = .neutral - + @State private var isMToonEnabled = true + var body: some View { VStack { HStack { @@ -44,18 +43,24 @@ struct ContentView: View { } } .pickerStyle(.segmented) + + Toggle("MToon", isOn: $isMToonEnabled) + .toggleStyle(.switch) + .disabled(selectedRenderer != .realityKit) } .padding([.top, .horizontal]) + // Only the selected renderer is mounted: keeping the other alive + // behind `opacity(0)` would hold on to its scene graph, GPU resources + // and 60 Hz timer for a view nobody can see. switch selectedRenderer { case .sceneKit: - SceneKitRendererView(viewModel: sceneKitViewModel, - selectedModel: selectedModel, + SceneKitRendererView(selectedModel: selectedModel, selectedExpression: selectedExpression) case .realityKit: - RealityKitRendererView(viewModel: realityKitViewModel, - selectedModel: selectedModel, - selectedExpression: selectedExpression) + RealityKitRendererView(selectedModel: selectedModel, + selectedExpression: selectedExpression, + isMToonEnabled: isMToonEnabled) } } .frame(minWidth: 800, minHeight: 600) @@ -63,17 +68,28 @@ struct ContentView: View { } private struct RealityKitRendererView: View { - let viewModel: RealityKitContentViewModel + @State private var viewModel = RealityKitContentViewModel() let selectedModel: MacExampleModel let selectedExpression: MacExampleExpression + let isMToonEnabled: Bool + + private var loadConfiguration: RealityKitLoadConfiguration { + RealityKitLoadConfiguration(model: selectedModel, isMToonEnabled: isMToonEnabled) + } var body: some View { RealityView { content in - content.add(viewModel.rootEntity) + content.add(viewModel.makeRenderRootEntity()) } + .background(Color.black) .frame(maxWidth: .infinity, maxHeight: .infinity) - .task(id: selectedModel) { - await viewModel.loadEntity(model: selectedModel, expression: selectedExpression) + .task(id: loadConfiguration) { + await viewModel.loadEntity(model: selectedModel, + expression: selectedExpression, + isMToonEnabled: isMToonEnabled) + } + .onAppear { + viewModel.resumeUpdates() } .onChange(of: selectedExpression) { _, expression in viewModel.setExpression(expression) @@ -89,8 +105,13 @@ private struct RealityKitRendererView: View { } } +private struct RealityKitLoadConfiguration: Hashable { + let model: MacExampleModel + let isMToonEnabled: Bool +} + private struct SceneKitRendererView: View { - let viewModel: SceneKitContentViewModel + @State private var viewModel = SceneKitContentViewModel() let selectedModel: MacExampleModel let selectedExpression: MacExampleExpression @@ -100,6 +121,9 @@ private struct SceneKitRendererView: View { .task(id: selectedModel) { await viewModel.loadScene(model: selectedModel, expression: selectedExpression) } + .onAppear { + viewModel.resumeUpdates() + } .onChange(of: selectedExpression) { _, expression in viewModel.setExpression(expression) } @@ -127,44 +151,70 @@ private struct ErrorMessageView: View { @MainActor @Observable final class RealityKitContentViewModel { - let rootEntity = Entity() + private var rootEntity = Entity() private(set) var errorMessage: String? private var vrmEntity: VRMEntity? + private var cameraEntity: PerspectiveCamera? + private var lightEntity: DirectionalLight? private var time: TimeInterval = 0 private var lastUpdateTime: Date? private var currentModel: MacExampleModel = .alicia private var currentExpression: MacExampleExpression = .neutral - + private var orbitDistance: Float = 2 + private var orbitTarget = SIMD3(0, 0.8, 0) + let updateTimer = Timer.publish(every: 1.0 / 60.0, on: .main, in: .common).autoconnect() - - func loadEntity(model: MacExampleModel, expression: MacExampleExpression) async { + + func makeRenderRootEntity() -> Entity { + let nextRootEntity = Entity() + if let cameraEntity { + nextRootEntity.addChild(cameraEntity) + } + if let lightEntity { + nextRootEntity.addChild(lightEntity) + } + if let vrmEntity { + nextRootEntity.addChild(vrmEntity) + } + rootEntity = nextRootEntity + return nextRootEntity + } + + func loadEntity( + model: MacExampleModel, + expression: MacExampleExpression, + isMToonEnabled: Bool + ) async { + await Task.yield() + guard !Task.isCancelled else { return } + do { errorMessage = nil - if let vrmEntity { - vrmEntity.entity.removeFromParent() - self.vrmEntity = nil - } - let loader = try VRMEntityLoader(named: model.rawValue) - let vrmEntity = try loader.loadEntity() - - vrmEntity.entity.transform.translation = SIMD3(0, -1, 0) - vrmEntity.entity.transform.rotation = simd_quatf(angle: model.initialRotation, axis: SIMD3(0, 1, 0)) - rootEntity.addChild(vrmEntity.entity) - - // Adjust pose - let neck = vrmEntity.humanoid.node(for: .neck) + let loader = try VRMEntityLoader(named: model.rawValue, isMToonEnabled: isMToonEnabled) + let nextVRMEntity = try loader.loadEntity() + + nextVRMEntity.transform.translation = SIMD3(0, -1, 0) + nextVRMEntity.transform.rotation = simd_quatf(angle: model.initialRotation, axis: SIMD3(0, 1, 0)) + nextVRMEntity.setMToonLightDirection(MacExampleLighting.towardLight) + setUpCamera() + setUpLight() + rootEntity.addChild(nextVRMEntity) + normalizeScale(for: nextVRMEntity) + updateCameraTransform() + + let neck = nextVRMEntity.humanoid.node(for: .neck) let leftArm: Entity? let rightArm: Entity? - switch vrmEntity.vrm { + switch nextVRMEntity.vrm { case .v1: - leftArm = vrmEntity.humanoid.node(for: .leftShoulder) - rightArm = vrmEntity.humanoid.node(for: .rightShoulder) + leftArm = nextVRMEntity.humanoid.node(for: .leftShoulder) + rightArm = nextVRMEntity.humanoid.node(for: .rightShoulder) case .v0: - leftArm = vrmEntity.humanoid.node(for: .leftUpperArm) - rightArm = vrmEntity.humanoid.node(for: .rightUpperArm) + leftArm = nextVRMEntity.humanoid.node(for: .leftUpperArm) + rightArm = nextVRMEntity.humanoid.node(for: .rightUpperArm) } - + let neckRotation = simd_quatf(angle: 20 * .pi / 180, axis: SIMD3(0, 0, 1)) let armRotation = simd_quatf(angle: 40 * .pi / 180, axis: SIMD3(0, 0, 1)) if let neck { @@ -176,12 +226,15 @@ final class RealityKitContentViewModel { if let rightArm { rightArm.transform.rotation = rightArm.transform.rotation * armRotation } - vrmEntity.setExampleExpression(expression, value: 1.0) - - self.vrmEntity = vrmEntity + apply(expression, replacing: nil, to: nextVRMEntity) + + let previousVRMEntity = self.vrmEntity + self.vrmEntity = nextVRMEntity + previousVRMEntity?.removeFromParent() self.currentModel = model self.currentExpression = expression - self.lastUpdateTime = Date() + self.time = 0 + resumeUpdates() } catch { errorMessage = error.localizedDescription print("VRM Load Error: \(error)") @@ -190,21 +243,25 @@ final class RealityKitContentViewModel { func setExpression(_ expression: MacExampleExpression) { guard expression != currentExpression else { return } - vrmEntity?.setExampleExpression(currentExpression, value: 0.0) + let previous = currentExpression currentExpression = expression - vrmEntity?.setExampleExpression(expression, value: 1.0) + guard let vrmEntity else { return } + apply(expression, replacing: previous, to: vrmEntity) + } + + func resumeUpdates() { + lastUpdateTime = Date() } - + func update() { guard let vrmEntity else { return } - + let now = Date() let deltaTime = lastUpdateTime.map { now.timeIntervalSince($0) } ?? (1.0 / 60.0) lastUpdateTime = now - + time += deltaTime - - // An animation that sways left and right + let cycle = time.truncatingRemainder(dividingBy: 1.0) let angle: Float if cycle < 0.5 { @@ -214,10 +271,69 @@ final class RealityKitContentViewModel { let progress = Float(cycle - 0.5) / 0.5 angle = -0.5 + 0.5 * progress } - - vrmEntity.entity.transform.rotation = simd_quatf(angle: currentModel.initialRotation + angle, - axis: SIMD3(0, 1, 0)) - vrmEntity.update(at: deltaTime) + + vrmEntity.transform.rotation = simd_quatf(angle: currentModel.initialRotation + angle, + axis: SIMD3(0, 1, 0)) + } + + private func setUpLight() { + if lightEntity == nil { + let light = DirectionalLight() + light.light.intensity = 1200 + rootEntity.addChild(light) + lightEntity = light + } + // `towardLight` points at the light, so place the light there and aim it + // at the origin. + lightEntity?.look(at: .zero, + from: MacExampleLighting.towardLight, + relativeTo: nil) + } + + private func setUpCamera() { + if cameraEntity == nil { + let camera = PerspectiveCamera() + rootEntity.addChild(camera) + cameraEntity = camera + } + updateCameraTransform() + } + + private func normalizeScale(for entity: Entity) { + let bounds = entity.visualBounds(relativeTo: nil) + let height = bounds.max.y - bounds.min.y + guard height > 0.001 else { return } + let targetHeight: Float = 2 + entity.transform.scale = SIMD3(repeating: targetHeight / height) + updateOrbitTarget(for: entity) + } + + private func updateOrbitTarget(for entity: Entity) { + let bounds = entity.visualBounds(relativeTo: nil) + orbitTarget = (bounds.min + bounds.max) * 0.5 + let extents = bounds.max - bounds.min + let maxExtent = max(extents.x, max(extents.y, extents.z)) + // Both renderers use a 60° vertical field of view, so pulling back by the + // model's largest extent reproduces the framing the SceneKit camera gets + // from sitting one body height away from the model. + orbitDistance = max(0.2, maxExtent) + } + + private func updateCameraTransform() { + guard let cameraEntity else { return } + let position = orbitTarget + SIMD3(0, 0, -orbitDistance) + cameraEntity.look(at: orbitTarget, from: position, relativeTo: nil) + } + + /// Both weights are sent together so the runtime re-applies its bindings once. + private func apply(_ expression: MacExampleExpression, + replacing previous: MacExampleExpression?, + to vrmEntity: VRMEntity) { + var weights: [MacExampleExpression: CGFloat] = [expression: 1.0] + if let previous, previous != expression { + weights[previous] = 0.0 + } + vrmEntity.setExampleExpressions(weights) } } @@ -252,26 +368,34 @@ final class SceneKitContentViewModel { let updateTimer = Timer.publish(every: 1.0 / 60.0, on: .main, in: .common).autoconnect() func loadScene(model: MacExampleModel, expression: MacExampleExpression) async { + if currentModel == model, let vrmNode { + apply(expression, replacing: currentExpression, to: vrmNode) + currentExpression = expression + resumeUpdates() + return + } + + await Task.yield() + guard !Task.isCancelled else { return } + do { errorMessage = nil - scene = nil - vrmNode = nil - time = 0 let loader = try VRMSceneLoader(named: model.rawValue) let scene = try loader.loadScene() setUpCamera(in: scene) let node = scene.vrmNode - node.eulerAngles = SCNVector3(0, CGFloat(model.sceneKitInitialRotation), 0) + node.eulerAngles = SCNVector3(0, CGFloat(model.initialRotation), 0) applyPose(to: node) - node.setExampleExpression(expression, value: 1.0) + apply(expression, replacing: nil, to: node) self.scene = scene self.vrmNode = node self.currentModel = model self.currentExpression = expression - self.lastUpdateTime = Date() + self.time = 0 + resumeUpdates() } catch { errorMessage = error.localizedDescription print("VRM Load Error: \(error)") @@ -280,9 +404,14 @@ final class SceneKitContentViewModel { func setExpression(_ expression: MacExampleExpression) { guard expression != currentExpression else { return } - vrmNode?.setExampleExpression(currentExpression, value: 0.0) + let previous = currentExpression currentExpression = expression - vrmNode?.setExampleExpression(expression, value: 1.0) + guard let vrmNode else { return } + apply(expression, replacing: previous, to: vrmNode) + } + + func resumeUpdates() { + lastUpdateTime = Date() } func update() { @@ -304,7 +433,7 @@ final class SceneKitContentViewModel { angle = -0.5 + 0.5 * progress } - vrmNode.eulerAngles = SCNVector3(0, CGFloat(currentModel.sceneKitInitialRotation + angle), 0) + vrmNode.eulerAngles = SCNVector3(0, CGFloat(currentModel.initialRotation + angle), 0) vrmNode.update(at: time) } @@ -331,9 +460,33 @@ final class SceneKitContentViewModel { cameraNode.position = SCNVector3(0, 0.8, -1.6) cameraNode.rotation = SCNVector4(0, 1, 0, Float.pi) scene.rootNode.addChildNode(cameraNode) + + let lightNode = SCNNode() + lightNode.light = SCNLight() + lightNode.light?.type = .directional + lightNode.light?.intensity = 1200 + lightNode.simdPosition = MacExampleLighting.towardLight + lightNode.look(at: SCNVector3Zero) + scene.rootNode.addChildNode(lightNode) + } + + /// Only the replaced expression needs clearing; this UI never has two active at once. + private func apply(_ expression: MacExampleExpression, + replacing previous: MacExampleExpression?, + to vrmNode: VRMNode) { + if let previous, previous != expression { + vrmNode.setExampleExpression(previous, value: 0.0) + } + vrmNode.setExampleExpression(expression, value: 1.0) } } +private enum MacExampleLighting { + /// Direction from the model toward the light, as `setMToonLightDirection(_:)` + /// expects. Both renderers place their directional light here so that this + /// example differs only in the renderer. + static let towardLight = simd_normalize(SIMD3(-0.35, -0.55, -0.75)) +} #Preview { ContentView() diff --git a/Example/MacExample/MacExampleModel.swift b/Example/MacExample/MacExampleModel.swift index 0c3ba828..38821f2c 100644 --- a/Example/MacExample/MacExampleModel.swift +++ b/Example/MacExample/MacExampleModel.swift @@ -32,13 +32,6 @@ enum MacExampleModel: String, CaseIterable, Identifiable { } var initialRotation: Float { - switch self { - case .alicia: return .pi - case .vrm1: return 0 - } - } - - var sceneKitInitialRotation: Float { switch self { case .alicia: return 0 case .vrm1: return .pi @@ -93,11 +86,21 @@ enum MacExampleExpression: String, CaseIterable, Identifiable { extension VRMEntity { func setExampleExpression(_ expression: MacExampleExpression, value: CGFloat) { + setExampleExpressions([expression: value]) + } + + /// Applies several example expressions at once. On VRM 1.0 this routes through + /// `setExpressions(_:)`, so the runtime re-applies its bindings only once. + func setExampleExpressions(_ weights: [MacExampleExpression: CGFloat]) { switch vrm { case .v0: - setBlendShape(value: value, for: .preset(expression.blendShapePreset)) + for (expression, value) in weights { + setBlendShape(value: value, for: .preset(expression.blendShapePreset)) + } case .v1: - setExpression(value: value, for: .preset(expression.expressionPreset)) + setExpressions(Dictionary(uniqueKeysWithValues: weights.map { + (ExpressionKey.preset($0.key.expressionPreset), $0.value) + })) } } } diff --git a/Example/VisionExample/ContentView.swift b/Example/VisionExample/ContentView.swift index a6dcd1d6..5a17b9a6 100644 --- a/Example/VisionExample/ContentView.swift +++ b/Example/VisionExample/ContentView.swift @@ -54,14 +54,9 @@ struct ImmersiveView: View { RealityView { content in content.add(viewModel.rootEntity) } - .task { + .task(id: appModel.selectedModelName) { await viewModel.loadEntity(model: appModel.selectedModelName) } - .onChange(of: appModel.selectedModelName) { _, newValue in - Task { - await viewModel.loadEntity(model: newValue) - } - } .onReceive(viewModel.updateTimer) { _ in viewModel.update() } @@ -86,7 +81,7 @@ final class ImmersiveViewModel { // Clean up previous if let current = vrmEntity { - current.entity.removeFromParent() + current.removeFromParent() vrmEntity = nil } @@ -94,12 +89,14 @@ final class ImmersiveViewModel { baseRotation = model.initialRotation do { + // visionOS has no CustomMaterial, so MToon always falls back to + // Unlit / PBR here. let loader = try VRMEntityLoader(named: modelName) let vrmEntity = try loader.loadEntity() - vrmEntity.entity.transform.translation = SIMD3(0, 0, -1.5) - vrmEntity.entity.transform.rotation = simd_quatf(angle: baseRotation, axis: SIMD3(0, 1, 0)) - rootEntity.addChild(vrmEntity.entity) + vrmEntity.transform.translation = SIMD3(0, 0, -1.5) + vrmEntity.transform.rotation = simd_quatf(angle: baseRotation, axis: SIMD3(0, 1, 0)) + rootEntity.addChild(vrmEntity) // Adjust pose let neck = vrmEntity.humanoid.node(for: .neck) @@ -155,7 +152,6 @@ final class ImmersiveViewModel { angle = -0.5 + 0.5 * progress } - vrmEntity.entity.transform.rotation = simd_quatf(angle: baseRotation + angle, axis: SIMD3(0, 1, 0)) - vrmEntity.update(at: deltaTime) + vrmEntity.transform.rotation = simd_quatf(angle: baseRotation + angle, axis: SIMD3(0, 1, 0)) } } diff --git a/Package.swift b/Package.swift index 9f850920..1be63f3b 100644 --- a/Package.swift +++ b/Package.swift @@ -21,17 +21,32 @@ let package = Package( ), .target( name: "VRMRealityKit", - dependencies: ["VRMKit", "VRMKitRuntime"] + dependencies: ["VRMKit", "VRMKitRuntime"], + // Shaders/MToon.metal is compiled offline into the per-platform + // metallibs under Resources by scripts/build-mtoon-metallibs.sh. + exclude: ["Shaders"], + resources: [.process("Resources")] ), + // Test-only helpers shared by the test targets. + .target(name: "VRMTestSupport", path: "Tests/VRMTestSupport"), + .testTarget( name: "VRMKitTests", - dependencies: ["VRMKit"], + dependencies: ["VRMKit", "VRMTestSupport"], resources: [.copy("Assets/AliciaSolid.vrm"), .copy("Assets/Seed-san.vrm")] ), .testTarget( name: "VRMSceneKitTests", dependencies: ["VRMSceneKit"], + resources: [ + .copy("../VRMKitTests/Assets/AliciaSolid.vrm"), + .copy("../VRMKitTests/Assets/Seed-san.vrm") + ] + ), + .testTarget( + name: "VRMRealityKitTests", + dependencies: ["VRMRealityKit", "VRMTestSupport"], resources: [.copy("../VRMKitTests/Assets/AliciaSolid.vrm"), .copy("../VRMKitTests/Assets/Seed-san.vrm")] ), ] diff --git a/README.md b/README.md index 50b17502..33a63354 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ For "VRM", please refer to [this page](https://dwango.github.io/en/vrm/). - visionOS 2.0+ - watchOS 8.0+ (Experimental) +VRMRealityKit requires iOS 18.0+ / macOS 15.0+ / visionOS 2.0+. + # Installation ## Swift Package Manager @@ -77,26 +79,26 @@ let vrmEntity = try loader.loadEntity() let arView = ARView(frame: .zero, cameraMode: .nonAR, automaticallyConfigureSession: false) let anchor = AnchorEntity(world: .zero) -anchor.addChild(vrmEntity.entity) +anchor.addChild(vrmEntity) arView.scene.addAnchor(anchor) ``` +`VRMEntity` is an `Entity`. Once it is in a scene, skinning, constraints and spring bones are updated every frame automatically. + ### Render VRM (SwiftUI) ```swift import RealityKit -import RealityKitContent +import SwiftUI import VRMKit import VRMRealityKit -import SwiftUI - struct ContentView: View { var body: some View { RealityView { content in - let loader = try VRMEntityLoader(named: "model.vrm") - let vrmEntity = try loader.loadEntity() - content.add(vrmEntity.entity) + guard let loader = try? VRMEntityLoader(named: "model.vrm"), + let vrmEntity = try? loader.loadEntity() else { return } + content.add(vrmEntity) } } } @@ -157,26 +159,8 @@ vrmEntity.setExpression(value: 1.0, for: .custom("customExpressionName")) Humanoid ```swift -switch vrmEntity.vrm { -case .v0: - vrmEntity.setBlendShape(value: 1.0, for: .preset(.fun)) -case .v1: - vrmEntity.setExpression(value: 1.0, for: .preset(.relaxed)) -} - let neckRotation = simd_quatf(angle: 20 * .pi / 180, axis: SIMD3(0, 0, 1)) -let armRotation = simd_quatf(angle: 40 * .pi / 180, axis: SIMD3(0, 0, 1)) -let (leftArm, rightArm): (Entity?, Entity?) -switch vrmEntity.vrm { -case .v1: - (leftArm, rightArm) = (vrmEntity.humanoid.node(for: .leftShoulder), vrmEntity.humanoid.node(for: .rightShoulder)) -case .v0: - (leftArm, rightArm) = (vrmEntity.humanoid.node(for: .leftUpperArm), vrmEntity.humanoid.node(for: .rightUpperArm)) -} - vrmEntity.humanoid.node(for: .neck)?.transform.rotation *= neckRotation -leftArm?.transform.rotation *= armRotation -rightArm?.transform.rotation *= armRotation ``` ### Read the thumbnail image @@ -187,13 +171,57 @@ let vrm = try loader.load(named: "model.vrm") let image = try loader.loadThumbnail(from: vrm) ``` +## MToon rendering + +VRMRealityKit renders MToon materials by default on iOS and macOS. visionOS falls back to Unlit / PBR materials because RealityKit's `CustomMaterial` is unavailable there. + +```swift +vrmEntity.setMToonLightDirection(SIMD3(0, 0, -1)) +vrmEntity.setMToonLightColor(SIMD3(1, 1, 1)) +vrmEntity.setMToonAmbientColor(SIMD3(0.1, 0.1, 0.1)) +``` + +
+Loader options and limitations + +```swift +let loader = try VRMEntityLoader( + named: "model.vrm", + isMToonEnabled: true, // false: disable MToon and use the legacy Unlit / PBR conversion + isOutlineEnabled: true // false: skip MToon outline entities +) +``` + +Outlines can be skipped while keeping the MToon surface shader. On visionOS both options fall back automatically because the required RealityKit APIs are unavailable. + +RealityKit constrains what the MToon renderer can express. Each case below logs a warning once per affected material. + +- `renderQueueOffsetNumber` is parsed but ignored, because RealityKit has no material-level draw-order hook. (`transparentWithZWrite` is supported through `CustomMaterial.writesDepth`, so a blended material can still write depth.) +- Textures requesting a UV set other than `TEXCOORD_0` use `TEXCOORD_0`, because custom meshes expose only that one. +- When UV-accessed texture slots specify different `KHR_texture_transform` values, the transform of the first UV-accessed slot — base color when the material has one — is applied to all of them, because `CustomMaterial` has a single material-level UV transform. Expression texture transform binds still update all UV-accessed textures together as required by VRMC_vrm. + +
+ +## Frame updates + +`VRMUpdateSystem` (a RealityKit `System` registered on load) calls `VRMEntity.update(deltaTime:)` on every render frame. To control the timing yourself, opt out and call it manually: + +```swift +vrmEntity.isAutomaticUpdateEnabled = false + +// Then, once per frame: +vrmEntity.update(deltaTime: deltaTime) +``` + +To run your own animation code in a guaranteed order relative to the VRM update (e.g. posing joints that the same frame's skinning should reflect), put it in a custom `System` declared with `SystemDependency.before(VRMUpdateSystem.self)`. + # ToDo - [x] VRM 1.0 support - [x] Decoding VRM 1.0 file - [x] Render an avatar by RealityKit (as VRM 0.x) - [x] Render an avatar by RealityKit (as VRM 1.x) -- [ ] VRM shaders support (MToon) +- [x] VRM shaders support (MToon, RealityKit) - [ ] Improve rendering quality - [ ] Animation support (vrma) - [ ] VRM editing function diff --git a/Sources/VRMKit/BinaryGLTF.swift b/Sources/VRMKit/BinaryGLTF.swift index 286073d1..a98a6530 100644 --- a/Sources/VRMKit/BinaryGLTF.swift +++ b/Sources/VRMKit/BinaryGLTF.swift @@ -9,7 +9,7 @@ public struct BinaryGLTF { public let binaryBuffer: Data? /// chunk1 /// magic equals 0x46546C67. It is ASCII string glTF, and can be used to identify data as Binary glTF. - static let magic = 0x46546C67 + static let magic: UInt32 = 0x46546C67 enum ChunkType: UInt32 { case json = 0x4E4F534A @@ -19,44 +19,72 @@ public struct BinaryGLTF { package extension BinaryGLTF { func bufferViewData(at index: Int, relativeTo rootDirectory: URL? = nil) throws -> (data: Data, stride: Int?) { - let bufferView = try jsonData.load(\.bufferViews)[index] + let bufferView = try jsonData.load(\.bufferViews, at: index) let buffer = try bufferData(at: bufferView.buffer, relativeTo: rootDirectory) - let data = buffer.subdata(in: bufferView.byteOffset..= 0, bufferView.byteLength >= 0, + !end.overflow, end.partialValue <= buffer.count else { + throw VRMError._dataInconsistent( + "buffer view (offset: \(bufferView.byteOffset), length: \(bufferView.byteLength)) overruns its \(buffer.count) byte buffer" + ) + } + return (buffer.subdata(in: bufferView.byteOffset.. Data { - let gltfBuffer = try jsonData.load(\.buffers)[index] + let gltfBuffer = try jsonData.load(\.buffers, at: index) return try Data(buffer: gltfBuffer, relativeTo: rootDirectory, binaryBuffer: binaryBuffer) } } extension BinaryGLTF { public init(data: Data) throws { - var offset = MemoryLayout.size // skip `magic` - let rawVersion: UInt32 = try read(data, offset: &offset, size: MemoryLayout.size) + var reader = BinaryReader(data) + let magic = try reader.readUInt32() + guard magic == Self.magic else { + throw VRMError._dataInconsistent("not a binary glTF file: magic is 0x\(String(magic, radix: 16))") + } + + let rawVersion = try reader.readUInt32() guard let version = GLTF.Version(rawValue: rawVersion), version == .two else { throw VRMError.notSupportedVersion(rawVersion) } self.version = version - let length: UInt32 = try read(data, offset: &offset, size: MemoryLayout.size) - let chunk0Length: UInt32 = try read(data, offset: &offset, size: MemoryLayout.size) - let chunk0Type: UInt32 = try read(data, offset: &offset, size: MemoryLayout.size) + // The header length covers the whole GLB, so fewer bytes than that means + // the file is truncated and the chunk walk below cannot be trusted. Bytes + // past it belong to no chunk and are ignored. + let length = try reader.readUInt32() + guard Int(length) <= data.count else { + throw VRMError._dataInconsistent( + "GLB header length \(length) overruns the \(data.count) byte file" + ) + } + + let chunk0Length = try reader.readUInt32() + let chunk0Type = try reader.readUInt32() guard ChunkType(rawValue: chunk0Type) == .json else { throw VRMError.notSupportedChunkType(chunk0Type) } - let jsonData = read(data, offset: &offset, size: Int(chunk0Length)) - let decoder = JSONDecoder() - self.jsonData = try decoder.decode(GLTF.self, from: jsonData) + let jsonData = try reader.readData(count: Int(chunk0Length)) + let gltf = try JSONDecoder().decode(GLTF.self, from: jsonData) + // The GLB container version and the asset version are independent: a 2.x + // container can still declare an asset this parser does not implement. + guard gltf.asset.version.hasPrefix("2.") else { + throw VRMError._notSupported("glTF asset version \(gltf.asset.version) is not supported") + } + if let minVersion = gltf.asset.minVersion, minVersion != "2.0" { + throw VRMError._notSupported("glTF asset minVersion \(minVersion) is not supported") + } + self.jsonData = gltf - if length > offset { - let chunk1Length: UInt32 = try read(data, offset: &offset, size: MemoryLayout.size) - let chunk1Type: UInt32 = try read(data, offset: &offset, size: MemoryLayout.size) + if length > reader.bytesRead { + let chunk1Length = try reader.readUInt32() + let chunk1Type = try reader.readUInt32() guard ChunkType(rawValue: chunk1Type) == .bin else { throw VRMError.notSupportedChunkType(chunk1Type) } - binaryBuffer = read(data, offset: &offset, size: Int(chunk1Length)) as Data + binaryBuffer = try reader.readData(count: Int(chunk1Length)) } else { binaryBuffer = nil } diff --git a/Sources/VRMKit/Extensions/Accessor+Data.swift b/Sources/VRMKit/Extensions/Accessor+Data.swift new file mode 100644 index 00000000..7143f8d6 --- /dev/null +++ b/Sources/VRMKit/Extensions/Accessor+Data.swift @@ -0,0 +1,156 @@ +import Foundation + +// https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/README.md#accessor + +/// Reads the bytes and stride of a glTF buffer view. Both loaders cache buffer +/// views on their own scene data, so accessor expansion goes through this. +package typealias BufferViewProvider = (Int) throws -> (bufferView: Data, stride: Int?) + +package func numberOfComponents(of type: GLTF.Accessor.`Type`) -> Int { + switch type { + case .SCALAR: return 1 + case .VEC2: return 2 + case .VEC3: return 3 + case .VEC4: return 4 + case .MAT2: return 4 + case .MAT3: return 9 + case .MAT4: return 16 + } +} + +package func bytes(of type: GLTF.Accessor.ComponentType) -> Int { + switch type { + case .byte, .unsignedByte: return 1 + case .short, .unsignedShort: return 2 + case .unsignedInt, .float: return 4 + } +} + +package extension GLTF.Accessor { + func components() -> (componentsPerVector: Int, bytesPerComponent: Int, vectorSize: Int) { + let componentsPerVector = numberOfComponents(of: type) + let bytesPerComponent = bytes(of: componentType) + let vectorSize = bytesPerComponent * componentsPerVector + return (componentsPerVector, bytesPerComponent, vectorSize) + } + + /// The accessor's elements as tightly packed data: any buffer view stride is + /// removed, an accessor without a buffer view yields the zeroes the spec + /// defines for it, and a sparse substitution is applied on top. + func packedData(bufferView provider: BufferViewProvider) throws -> Data { + let vectorSize = try unpaddedVectorSize() + var data = try packedBaseData(vectorSize: vectorSize, provider: provider) + if let sparse { + try apply(sparse, vectorSize: vectorSize, provider: provider, to: &data) + } + return data + } +} + +private extension GLTF.Accessor { + /// The element size, rejecting the matrix layouts glTF pads. MAT2 / MAT3 + /// columns are aligned to 4 bytes, so with small component types an element + /// is wider than its components — a layout no VRM asset uses, and one the + /// readers above would mis-slice. + func unpaddedVectorSize() throws -> Int { + let (componentsPerVector, bytesPerComponent, vectorSize) = components() + let columnCount: Int + switch type { + case .SCALAR, .VEC2, .VEC3, .VEC4: return vectorSize + case .MAT2: columnCount = 2 + case .MAT3: columnCount = 3 + case .MAT4: columnCount = 4 + } + let columnSize = bytesPerComponent * (componentsPerVector / columnCount) + guard columnSize.isMultiple(of: 4) else { + throw VRMError._notSupported( + "\(type) accessors with \(componentType) components pad their columns, which is not supported" + ) + } + return vectorSize + } + + func packedBaseData(vectorSize: Int, provider: BufferViewProvider) throws -> Data { + guard let bufferView else { + // The spec defines a bufferView-less accessor as all zeroes. + return try Data(zeroedElementCount: count, elementSize: vectorSize) + } + let source = try provider(bufferView) + return try source.bufferView.subdata(offset: byteOffset, + size: vectorSize, + stride: source.stride ?? vectorSize, + count: count) + } + + func apply(_ sparse: Sparse, + vectorSize: Int, + provider: BufferViewProvider, + to data: inout Data) throws { + guard sparse.count > 0 else { return } + let indices = try sparseIndices(sparse, provider: provider) + if let outOfRange = indices.first(where: { $0 < 0 || $0 >= count }) { + throw VRMError._dataInconsistent( + "sparse index \(outOfRange) is out of range for \(count) accessor elements" + ) + } + // Strictly increasing indices are what makes the substitution below + // unambiguous: repeated ones would leave the element they overlap on + // depending on the copy order. + guard zip(indices, indices.dropFirst()).allSatisfy({ $0 < $1 }) else { + throw VRMError._dataInconsistent("sparse indices must be strictly increasing") + } + let values = try sparseValues(sparse, vectorSize: vectorSize, provider: provider) + data.withUnsafeMutableBytes { rawDst in + guard let dst = rawDst.bindMemory(to: UInt8.self).baseAddress else { return } + values.withUnsafeBytes { rawSrc in + guard let src = rawSrc.bindMemory(to: UInt8.self).baseAddress else { return } + for (position, index) in indices.enumerated() { + memcpy(dst.advanced(by: index * vectorSize), + src.advanced(by: position * vectorSize), + vectorSize) + } + } + } + } + + func sparseIndices(_ sparse: Sparse, provider: BufferViewProvider) throws -> [Int] { + switch sparse.indices.componentType { + case .unsignedByte, .unsignedShort, .unsignedInt: break + case .byte, .short, .float: + throw VRMError._dataInconsistent( + "sparse indices cannot use \(sparse.indices.componentType) components" + ) + } + let source = try provider(sparse.indices.bufferView) + let bytesPerIndex = bytes(of: sparse.indices.componentType) + let indexData = try source.bufferView.subdata(offset: sparse.indices.byteOffset, + size: bytesPerIndex, + stride: source.stride ?? bytesPerIndex, + count: sparse.count) + var indices: [Int] = [] + indices.reserveCapacity(sparse.count) + indexData.withUnsafeBytes { raw in + guard let base = raw.baseAddress else { return } + for i in 0.. Data { + let source = try provider(sparse.values.bufferView) + return try source.bufferView.subdata(offset: sparse.values.byteOffset, + size: vectorSize, + stride: source.stride ?? vectorSize, + count: sparse.count) + } +} diff --git a/Sources/VRMKit/Extensions/BinaryReader.swift b/Sources/VRMKit/Extensions/BinaryReader.swift new file mode 100644 index 00000000..24b9e559 --- /dev/null +++ b/Sources/VRMKit/Extensions/BinaryReader.swift @@ -0,0 +1,36 @@ +import Foundation + +/// A bounds-checked, little-endian cursor over a `Data`. +/// +/// Every read is validated against the end of the data, so a truncated header or +/// an overrunning chunk length throws a ``VRMError`` instead of trapping. +struct BinaryReader { + private let data: Data + private var offset: Int + + init(_ data: Data) { + self.data = data + offset = data.startIndex + } + + /// The number of bytes consumed so far. + var bytesRead: Int { offset - data.startIndex } + + mutating func readUInt32() throws -> UInt32 { + // GLB is little-endian regardless of the host, so the value is + // assembled from the bytes instead of loaded. + let bytes = try readData(count: MemoryLayout.size) + return bytes.reduce(UInt32(0)) { $0 >> 8 | UInt32($1) << 24 } + } + + mutating func readData(count: Int) throws -> Data { + let end = offset.addingReportingOverflow(count) + guard count >= 0, !end.overflow, end.partialValue <= data.endIndex else { + throw VRMError._dataInconsistent( + "reading \(count) bytes at offset \(bytesRead) overruns the \(data.count) byte file" + ) + } + defer { offset = end.partialValue } + return data.subdata(in: offset.. Data { - let dataSize = size * count + /// All-zero data for a glTF accessor with no bufferView. + init(zeroedElementCount count: Int, elementSize: Int) throws { + guard count >= 0, elementSize > 0 else { + throw VRMError._dataInconsistent("invalid accessor size (count: \(count), size: \(elementSize))") + } + let byteCount = elementSize.multipliedReportingOverflow(by: count) + guard !byteCount.overflow else { + throw VRMError._dataInconsistent("accessor size overflows (count: \(count), size: \(elementSize))") + } + self.init(count: byteCount.partialValue) + } + + /// Copies `count` elements of `size` bytes each, `stride` bytes apart, + /// starting at `offset`, throwing when the described range overruns the + /// receiver. + func subdata(offset: Int, size: Int, stride: Int, count: Int) throws -> Data { + guard offset >= 0, size > 0, count >= 0, stride >= size else { + throw VRMError._dataInconsistent( + "invalid accessor layout (offset: \(offset), size: \(size), stride: \(stride), count: \(count))" + ) + } + guard count > 0 else { return Data() } + let packed = size.multipliedReportingOverflow(by: count) + guard let requiredBytes = Int.accessorExtent(offset: offset, stride: stride, count: count, size: size), + !packed.overflow else { + throw VRMError._dataInconsistent( + "accessor extent overflows (offset: \(offset), size: \(size), stride: \(stride), count: \(count))" + ) + } + let dataSize = packed.partialValue + guard requiredBytes <= self.count else { + throw VRMError._dataInconsistent( + "accessor needs \(requiredBytes) bytes but its buffer view holds \(self.count)" + ) + } + if stride == size { - if offset == 0 { return self } + if offset == 0, dataSize == self.count { return self } return subdata(in: offset.. Int? { + let span = stride.multipliedReportingOverflow(by: count - 1) + guard !span.overflow else { return nil } + let start = offset.addingReportingOverflow(span.partialValue) + guard !start.overflow else { return nil } + let end = start.partialValue.addingReportingOverflow(size) + guard !end.overflow else { return nil } + return end.partialValue + } +} diff --git a/Sources/VRMKit/Extensions/GlobalFunction.swift b/Sources/VRMKit/Extensions/GlobalFunction.swift index 9df4ffa2..858c82dd 100644 --- a/Sources/VRMKit/Extensions/GlobalFunction.swift +++ b/Sources/VRMKit/Extensions/GlobalFunction.swift @@ -1,17 +1,5 @@ import Foundation -func read(_ data: Data, offset: inout Int, size: Int) throws -> T { - defer { offset += size } - return try data.subdata(in: offset..<(offset+size)).withUnsafeBytes { - try $0.bindMemory(to: T.self).baseAddress?.pointee ??? VRMError._dataInconsistent("failed to read data") - } -} - -func read(_ data: Data, offset: inout Int, size: Int) -> Data { - defer { offset += size } - return data.subdata(in: offset..<(offset+size)) -} - infix operator ??? package func ???(lhs: T?, diff --git a/Sources/VRMKit/Extensions/MoreCodable/DictionaryDecoder.swift b/Sources/VRMKit/Extensions/MoreCodable/DictionaryDecoder.swift index 0f3fe2a3..5a548021 100644 --- a/Sources/VRMKit/Extensions/MoreCodable/DictionaryDecoder.swift +++ b/Sources/VRMKit/Extensions/MoreCodable/DictionaryDecoder.swift @@ -72,6 +72,15 @@ extension DictionaryDecoder { public func decode(_ type: T.Type, from container: Any) throws -> T { return try unbox(container, as: T.self) } + + /// Decodes an optional member of an untyped glTF object: a missing key + /// yields nil, while a present but malformed value still throws. + public func decodeIfPresent(_ type: T.Type, + from container: [String: Any], + forKey key: String) throws -> T? { + guard let value = container[key] else { return nil } + return try decode(type, from: value) + } } extension DictionaryDecoder { diff --git a/Sources/VRMKit/VRM/GLTF.swift b/Sources/VRMKit/VRM/GLTF.swift index 6e73d306..dd4f4bf7 100644 --- a/Sources/VRMKit/VRM/GLTF.swift +++ b/Sources/VRMKit/VRM/GLTF.swift @@ -52,10 +52,26 @@ extension GLTF { package extension GLTF { func load(_ keyPath: KeyPath) throws -> T { + try self[keyPath: keyPath] ??? .keyNotFound(Self.description(of: keyPath)) + } + + /// One element of a glTF array, throwing instead of trapping when the index + /// from the file is out of range. + func load(_ keyPath: KeyPath, at index: Int) throws -> T { + let values = try load(keyPath) + guard values.indices.contains(index) else { + throw VRMError._dataInconsistent( + "index \(index) is out of range for the \(values.count) elements of \(Self.description(of: keyPath))" + ) + } + return values[index] + } + + private static func description(of keyPath: KeyPath) -> String { if #available(macOS 13.3, iOS 16.4, watchOS 9.4, *) { - return try self[keyPath: keyPath] ??? .keyNotFound(keyPath.debugDescription) + return keyPath.debugDescription } else { - return try self[keyPath: keyPath] ??? .keyNotFound("\(keyPath)") + return "\(keyPath)" } } } diff --git a/Sources/VRMKit/VRM/VRM1.swift b/Sources/VRMKit/VRM/VRM1.swift index 7cd7d02f..d34dfba3 100644 --- a/Sources/VRMKit/VRM/VRM1.swift +++ b/Sources/VRMKit/VRM/VRM1.swift @@ -1,6 +1,11 @@ import Foundation public struct VRM1 { + /// The `VRMC_vrm` spec versions this type models. + public static func supports(specVersion: String) -> Bool { + specVersion == "1.0" || specVersion == "1.0-beta" + } + public let gltf: BinaryGLTF public let specVersion: String public let meta: Meta @@ -18,17 +23,20 @@ public struct VRM1 { let rawExtensions = try gltf.jsonData.extensions ??? .keyNotFound("extensions") let extensions = try rawExtensions.value as? [String: [String: Any]] ??? .dataInconsistent("extension type mismatch") let vrm = try extensions["VRMC_vrm"] ??? .keyNotFound("VRMC_vrm") - specVersion = vrm["specVersion"] as! String + specVersion = try vrm["specVersion"] as? String ??? .dataInconsistent("VRMC_vrm.specVersion is missing or not a string") + guard VRM1.supports(specVersion: specVersion) else { + throw VRMError._notSupported("VRMC_vrm specVersion \(specVersion)") + } let decoder = DictionaryDecoder() meta = try decoder.decode(Meta.self, from: try vrm["meta"] ??? .keyNotFound("meta")) humanoid = try decoder.decode(Humanoid.self, from: try vrm["humanoid"] ??? .keyNotFound("humanoid")) - firstPerson = vrm.keys.contains("firstPerson") ? try decoder.decode(FirstPerson.self, from: vrm["firstPerson"] ?? "".data(using: .utf8)!) : nil - lookAt = vrm.keys.contains("lookAt") ? try decoder.decode(LookAt.self, from: vrm["lookAt"] ?? "".data(using: .utf8)!) : nil - expressions = vrm.keys.contains("expressions") ? try decoder.decode(Expressions.self, from: vrm["expressions"] ?? "".data(using: .utf8)!) : nil - springBone = extensions.keys.contains("VRMC_springBone") ? try decoder.decode(SpringBone.self, from: extensions["VRMC_springBone"] ?? "".data(using: .utf8)!) : nil - self.extensions = vrm.keys.contains("extensions") ? try decoder.decode(CodableAny.self, from: vrm["extensions"] ?? "".data(using: .utf8)!) : nil - extras = vrm.keys.contains("extras") ? try decoder.decode(CodableAny.self, from: vrm["extras"] ?? "".data(using: .utf8)!) : nil + firstPerson = try decoder.decodeIfPresent(FirstPerson.self, from: vrm, forKey: "firstPerson") + lookAt = try decoder.decodeIfPresent(LookAt.self, from: vrm, forKey: "lookAt") + expressions = try decoder.decodeIfPresent(Expressions.self, from: vrm, forKey: "expressions") + springBone = try decoder.decodeIfPresent(SpringBone.self, from: extensions, forKey: "VRMC_springBone") + self.extensions = try decoder.decodeIfPresent(CodableAny.self, from: vrm, forKey: "extensions") + extras = try decoder.decodeIfPresent(CodableAny.self, from: vrm, forKey: "extras") } } @@ -155,6 +163,14 @@ public extension VRM1 { public let meshAnnotations: [MeshAnnotation] public let extensions: CodableAny? public let extras: CodableAny? + + // meshAnnotations is optional in practice, so decode it leniently. + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + meshAnnotations = try container.decodeIfPresent([MeshAnnotation].self, forKey: .meshAnnotations) ?? [] + extensions = try container.decodeIfPresent(CodableAny.self, forKey: .extensions) + extras = try container.decodeIfPresent(CodableAny.self, forKey: .extras) + } public struct MeshAnnotation: Codable { public let type: FirstPersonType diff --git a/Sources/VRMKitRuntime/BlendShapeBindings.swift b/Sources/VRMKitRuntime/BlendShapeBindings.swift index cd988f8e..499eeb13 100644 --- a/Sources/VRMKitRuntime/BlendShapeBindings.swift +++ b/Sources/VRMKitRuntime/BlendShapeBindings.swift @@ -1,4 +1,5 @@ import simd +import VRMKit /// Morph target binding shared by VRM 0.x BlendShape and VRM 1.0 Expression runtime clips. package struct BlendShapeBinding { @@ -33,6 +34,13 @@ package struct BlendShapeClip { self.values = values self.isBinary = isBinary } + + /// Clamps `value` to 0...1, rounding to the nearest of 0 and 1 for binary + /// groups as VRM 0.x defines them. + package func normalizedWeight(_ value: Double) -> Double { + let clamped = min(max(value, 0), 1) + return isBinary ? clamped.rounded() : clamped + } } /// Runtime clip for VRM 1.0 Expressions. @@ -41,6 +49,11 @@ package struct ExpressionClip { package let preset: ExpressionPreset? package let values: [BlendShapeBinding] package let isBinary: Bool + /// How this expression suppresses the blink / lookAt / mouth expressions + /// while it is active (VRMC_vrm `overrideBlink` / `overrideLookAt` / `overrideMouth`). + package let overrideBlink: ExpressionOverrideType + package let overrideLookAt: ExpressionOverrideType + package let overrideMouth: ExpressionOverrideType package var key: ExpressionKey { return preset.map(ExpressionKey.preset) ?? .custom(name) @@ -49,11 +62,110 @@ package struct ExpressionClip { package init(name: String, preset: ExpressionPreset?, values: [BlendShapeBinding], - isBinary: Bool) { + isBinary: Bool, + overrideBlink: ExpressionOverrideType = .none, + overrideLookAt: ExpressionOverrideType = .none, + overrideMouth: ExpressionOverrideType = .none) { self.name = name self.preset = preset self.values = values self.isBinary = isBinary + self.overrideBlink = overrideBlink + self.overrideLookAt = overrideLookAt + self.overrideMouth = overrideMouth + } + + /// Clamps `value` to 0...1. VRM 1.0 binary expressions are 1 only when the + /// weight is *greater than* 0.5, so exactly 0.5 stays 0. + package func normalizedWeight(_ value: Double) -> Double { + let clamped = min(max(value, 0), 1) + return isBinary ? (clamped > 0.5 ? 1 : 0) : clamped + } + + /// How this clip overrides `group`. + package func overrideType(for group: ExpressionOverrideGroup) -> ExpressionOverrideType { + switch group { + case .blink: return overrideBlink + case .lookAt: return overrideLookAt + case .mouth: return overrideMouth + } + } +} + +package typealias ExpressionOverrideType = VRM1.Expressions.Expression.ExpressionOverrideType + +/// Accumulates every active expression's override of one group, following +/// VRMC_vrm: `block` zeroes the group outright, while simultaneous `blend` +/// overrides *add up* before being saturated — they do not compose +/// multiplicatively. +package struct ExpressionOverrideState { + private var isBlocked = false + private var blendWeight: Double = 0 + + package init() {} + + package mutating func accumulate(_ type: ExpressionOverrideType, weight: Double) { + guard weight > 0 else { return } + switch type { + case .none: break + case .block: isBlocked = true + case .blend: blendWeight += weight + } + } + + /// The share of the overridden group's weight that survives. + package var factor: Double { + isBlocked ? 0 : 1 - min(blendWeight, 1) + } + + /// Whether the group receives any override effect at all. + package var isSuppressing: Bool { + factor < 1 + } +} + +/// The expression groups that VRMC_vrm expression overrides can suppress. +package enum ExpressionOverrideGroup: CaseIterable { + case blink + case lookAt + case mouth +} + +/// The ``ExpressionOverrideState`` of every group. Stored inline rather than in +/// a dictionary: face tracking recomputes this on every expression update. +package struct ExpressionOverrideStates { + private var blink = ExpressionOverrideState() + private var lookAt = ExpressionOverrideState() + private var mouth = ExpressionOverrideState() + + package init() {} + + package subscript(group: ExpressionOverrideGroup) -> ExpressionOverrideState { + switch group { + case .blink: return blink + case .lookAt: return lookAt + case .mouth: return mouth + } + } + + /// Accumulates `clip`'s overrides of every group but its own kind: a blink + /// expression's `overrideBlink` is invalid. + package mutating func accumulate(_ clip: ExpressionClip, + weight: Double, + excluding ownGroup: ExpressionOverrideGroup?) { + for group in ExpressionOverrideGroup.allCases where group != ownGroup { + let type = clip.overrideType(for: group) + switch group { + case .blink: blink.accumulate(type, weight: weight) + case .lookAt: lookAt.accumulate(type, weight: weight) + case .mouth: mouth.accumulate(type, weight: weight) + } + } + } + + /// Whether any group receives an override effect at all. + package var isSuppressingAnyGroup: Bool { + blink.isSuppressing || lookAt.isSuppressing || mouth.isSuppressing } } diff --git a/Sources/VRMKitRuntime/BlendShapeTypes.swift b/Sources/VRMKitRuntime/BlendShapeTypes.swift index 22e567f3..f661657e 100644 --- a/Sources/VRMKitRuntime/BlendShapeTypes.swift +++ b/Sources/VRMKitRuntime/BlendShapeTypes.swift @@ -82,6 +82,32 @@ public enum ExpressionKey: Hashable { } } +package extension ExpressionPreset { + /// The group whose weights VRMC_vrm expression overrides suppress, or nil + /// for presets no override applies to. + var overrideGroup: ExpressionOverrideGroup? { + switch self { + case .blink, .blinkLeft, .blinkRight: + return .blink + case .lookUp, .lookDown, .lookLeft, .lookRight: + return .lookAt + case .aa, .ih, .ou, .ee, .oh: + return .mouth + case .neutral, .happy, .angry, .sad, .relaxed, .surprised: + return nil + } + } +} + +package extension ExpressionKey { + var overrideGroup: ExpressionOverrideGroup? { + switch self { + case .preset(let preset): return preset.overrideGroup + case .custom: return nil + } + } +} + package extension BlendShapePreset { /// Compatibility bridge from VRM 0.x blend shape presets to VRM 1.0 expressions. var expressionPreset: ExpressionPreset? { diff --git a/Sources/VRMKitRuntime/Extensions/AnyNumeric+.swift b/Sources/VRMKitRuntime/Extensions/AnyNumeric+.swift new file mode 100644 index 00000000..92eed75d --- /dev/null +++ b/Sources/VRMKitRuntime/Extensions/AnyNumeric+.swift @@ -0,0 +1,79 @@ +import Foundation +import VRMKit + +/// A loosely-typed JSON number, as they appear in VRM extension dictionaries. +/// Swift's `Float` / `Double` / `Int` all bridge to `NSNumber`, so one cast +/// covers every numeric shape JSONSerialization produces; booleans bridge to +/// `NSNumber` too but are not numbers here. +private func jsonNumber(_ value: Any) -> NSNumber? { + guard let number = value as? NSNumber, CFGetTypeID(number) != CFBooleanGetTypeID() else { return nil } + return number +} + +/// Coerces a loosely-typed JSON number into a `Float`. Values a `Float` cannot +/// represent (`1e100` overflows to infinity) are rejected so callers fall back +/// to their defaults instead of feeding infinities or NaNs into the renderer. +package func numericFloatValue(_ value: Any) -> Float? { + guard let float = jsonNumber(value)?.floatValue, float.isFinite else { return nil } + return float +} + +/// Coerces a loosely-typed JSON number into an `Int` index. Only exact, +/// non-negative integers within `Int32` are accepted: glTF indices are small, +/// and `Int(_:)` traps on values a fixed-width integer cannot represent. The +/// number is read as a `Double` because `Float` rounds integers past 2^24 — +/// `16_777_217` and `Int32.max` would both pass a `Float` test as some other +/// value. +package func numericIndexValue(_ value: Any) -> Int? { + guard let number = jsonNumber(value), + let index = Int(exactly: number.doubleValue), + index >= 0, index <= Int(Int32.max) else { return nil } + return index +} + +package extension Array where Element == Any { + func float(at index: Int, default defaultValue: Float) -> Float { + guard indices.contains(index) else { return defaultValue } + return numericFloatValue(self[index]) ?? defaultValue + } +} + +package extension Dictionary where Key == String, Value == Any { + func float(_ key: String) -> Float? { + self[key].flatMap(numericFloatValue) + } + + func index(_ key: String) -> Int? { + self[key].flatMap(numericIndexValue) + } + + func simd2Value(forKey key: String, default defaultValue: SIMD2) -> SIMD2 { + guard let values = self[key] as? [Any] else { return defaultValue } + return SIMD2(values.float(at: 0, default: defaultValue.x), + values.float(at: 1, default: defaultValue.y)) + } + + func simd3(_ key: String) -> SIMD3? { + (self[key] as? [Any]).map { + SIMD3($0.float(at: 0, default: 0), + $0.float(at: 1, default: 0), + $0.float(at: 2, default: 0)) + } + } + + func simd4(_ key: String) -> SIMD4? { + (self[key] as? [Any]).map { + SIMD4($0.float(at: 0, default: 1), + $0.float(at: 1, default: 1), + $0.float(at: 2, default: 1), + $0.float(at: 3, default: 1)) + } + } +} + +package extension CodableAny { + /// The value as a JSON object, or an empty dictionary when it is not one. + var dictionaryValue: [String: Any] { + value as? [String: Any] ?? [:] + } +} diff --git a/Sources/VRMKitRuntime/Extensions/SIMD+.swift b/Sources/VRMKitRuntime/Extensions/SIMD+.swift index e19644cc..3bbb3754 100644 --- a/Sources/VRMKitRuntime/Extensions/SIMD+.swift +++ b/Sources/VRMKitRuntime/Extensions/SIMD+.swift @@ -1,4 +1,5 @@ import simd +import VRMKit package extension SIMD3 where Scalar == Float { init(_ values: [Double]?, `default` defaultValue: SIMD3) { @@ -11,6 +12,10 @@ package extension SIMD3 where Scalar == Float { self.init(Optional(values), default: defaultValue) } + init(_ color: Color3) { + self.init(color.r, color.g, color.b) + } + var normalized: SIMD3 { simd_normalize(self) } @@ -35,6 +40,21 @@ package extension SIMD4 where Scalar == Float { Float(values[safe: 2] ?? 0), Float(values[safe: 3] ?? Double(defaultAlpha))) } + + init(_ color: GLTF.Color4) { + self.init(color.r, color.g, color.b, color.a) + } + + init(_ values: [Double]?, `default` defaultValue: SIMD4) { + guard let values else { + self = defaultValue + return + } + self.init(Float(values[safe: 0] ?? Double(defaultValue.x)), + Float(values[safe: 1] ?? Double(defaultValue.y)), + Float(values[safe: 2] ?? Double(defaultValue.z)), + Float(values[safe: 3] ?? Double(defaultValue.w))) + } } package extension SIMD2 where Scalar == Float { @@ -52,16 +72,6 @@ package extension simd_quatf { package let quat_identity_float = simd_quatf(matrix_identity_float4x4) -package func cross(_ left: SIMD3, _ right: SIMD3) -> SIMD3 { - simd_cross(left, right) -} - -package func normal(_ v0: SIMD3, _ v1: SIMD3, _ v2: SIMD3) -> SIMD3 { - let e1 = v1 - v0 - let e2 = v2 - v0 - return simd_normalize(simd_cross(e1, e2)) -} - package extension simd_float4x4 { var translation: SIMD3 { SIMD3(columns.3.x, columns.3.y, columns.3.z) diff --git a/Sources/VRMKitRuntime/MToonMaterialDescriptor.swift b/Sources/VRMKitRuntime/MToonMaterialDescriptor.swift new file mode 100644 index 00000000..cb2937ca --- /dev/null +++ b/Sources/VRMKitRuntime/MToonMaterialDescriptor.swift @@ -0,0 +1,224 @@ +import Foundation +import simd +import VRMKit + +/// Canonical VRMC_materials_mtoon 1.0 material model. +/// +/// This type only knows MToon 1.0 semantics. VRM 0.x materials are converted +/// by ``VRM0MToonMigrator`` before they reach this descriptor, and renderer +/// specific constraints (RealityKit, ...) are applied by the renderer layer. +package struct MToonMaterialDescriptor { + package enum CullMode { + case none + case front + case back + } + + package enum OutlineWidthMode { + case none + case worldCoordinates + case screenCoordinates + } + + /// KHR_texture_transform-style UV transform (glTF top-left origin). + package struct UVTransform: Equatable { + package var scale: SIMD2 + package var offset: SIMD2 + package var rotation: Float + + package init(scale: SIMD2 = SIMD2(1, 1), + offset: SIMD2 = SIMD2(0, 0), + rotation: Float = 0) { + self.scale = scale + self.offset = offset + self.rotation = rotation + } + } + + package struct Texture { + package let index: Int + /// UV set this texture samples, honoring a `KHR_texture_transform` + /// `texCoord` override when present. + package let texCoord: Int + /// UV transform carried by the source format (`KHR_texture_transform` + /// for VRM 1.0, the Unity `_MainTex` scale/offset for VRM 0.x). + package let transform: UVTransform? + + package init(index: Int, texCoord: Int = 0, transform: UVTransform? = nil) { + self.index = index + self.texCoord = texCoord + self.transform = transform + } + } + + package let baseColorFactor: SIMD4 + package let emissiveFactor: SIMD3 + package let shadeColorFactor: SIMD4 + package let shadingShiftFactor: Float + package let shadingShiftTextureScale: Float + package let shadingToonyFactor: Float + package let giEqualizationFactor: Float + package let matcapFactor: SIMD3 + package let parametricRimColorFactor: SIMD4 + package let rimLightingMixFactor: Float + package let parametricRimFresnelPowerFactor: Float + package let parametricRimLiftFactor: Float + package let outlineWidthMode: OutlineWidthMode + package let outlineWidthFactor: Float + package let outlineColorFactor: SIMD4 + package let outlineLightingMixFactor: Float + package let uvAnimationScrollXSpeedFactor: Float + package let uvAnimationScrollYSpeedFactor: Float + package let uvAnimationRotationSpeedFactor: Float + package let transparentWithZWrite: Bool + package let renderQueueOffsetNumber: Int + package let alphaMode: GLTF.Material.AlphaMode + package let alphaCutoff: Float + package let cullMode: CullMode + package let normalScale: Float + package let baseColorTexture: Texture? + package let emissiveTexture: Texture? + package let shadeMultiplyTexture: Texture? + package let shadingShiftTexture: Texture? + package let normalTexture: Texture? + package let matcapTexture: Texture? + package let rimMultiplyTexture: Texture? + package let outlineWidthMultiplyTexture: Texture? + package let uvAnimationMaskTexture: Texture? + + // No initializers are declared in the struct body so that the implicit + // memberwise initializer stays available to VRM0MToonMigrator. +} + +package extension MToonMaterialDescriptor { + /// The `VRMC_materials_mtoon` spec versions this descriptor implements. + /// Anything else falls back to Unlit / PBR rather than being read with 1.0 + /// semantics. + static func supports(specVersion: String) -> Bool { + specVersion == "1.0" || specVersion == "1.0-beta" + } + + init?(material: GLTF.Material, materialProperty: VRM0.MaterialProperty?) { + if let mtoon = material.extensions?.materialsMToon { + guard Self.supports(specVersion: mtoon.specVersion) else { return nil } + self.init(vrm1: mtoon, material: material) + return + } + + guard let materialProperty, + materialProperty.vrmShader == .mToon || materialProperty.shader.lowercased().contains("mtoon") else { + return nil + } + self = VRM0MToonMigrator.migrate(property: materialProperty, material: material) + } +} + +package extension MToonMaterialDescriptor { + /// UV-accessed textures in the order renderers should consider them when + /// they can only honor a single material-level UV transform. + var uvAccessedTextures: [Texture] { + [baseColorTexture, shadeMultiplyTexture, shadingShiftTexture, normalTexture, + emissiveTexture, rimMultiplyTexture, outlineWidthMultiplyTexture, uvAnimationMaskTexture] + .compactMap { $0 } + } + + var hasOutline: Bool { + switch outlineWidthMode { + case .none: + return false + case .worldCoordinates, .screenCoordinates: + return outlineWidthFactor > 0 + } + } +} + +private extension MToonMaterialDescriptor { + init(vrm1 mtoon: GLTF.Material.MaterialExtensions.MaterialsMToon, material: GLTF.Material) { + let pbr = material.pbrMetallicRoughness + let baseColor = (pbr?.baseColorFactor).map(SIMD4.init) ?? SIMD4(1, 1, 1, 1) + let shadeColor = SIMD4(mtoon.shadeColorFactor, default: SIMD4(0, 0, 0, 1)) + let matcapFactor = SIMD3(mtoon.matcapFactor, default: SIMD3(1, 1, 1)) + let rimColor = SIMD4(mtoon.parametricRimColorFactor, default: SIMD4(0, 0, 0, 1)) + let outlineColor = SIMD4(mtoon.outlineColorFactor, default: SIMD4(0, 0, 0, 1)) + + self.baseColorFactor = baseColor + self.emissiveFactor = SIMD3(material.emissiveFactor) + self.shadeColorFactor = shadeColor + self.shadingShiftFactor = Float(mtoon.shadingShiftFactor ?? 0) + self.shadingShiftTextureScale = Float(mtoon.shadingShiftTexture?.scale ?? 1) + self.shadingToonyFactor = Float(mtoon.shadingToonyFactor ?? 0.9) + self.giEqualizationFactor = Float(mtoon.giEqualizationFactor ?? 0.9) + self.matcapFactor = matcapFactor + self.parametricRimColorFactor = rimColor + self.rimLightingMixFactor = Float(mtoon.rimLightingMixFactor ?? 1) + self.parametricRimFresnelPowerFactor = Float(mtoon.parametricRimFresnelPowerFactor ?? 5) + self.parametricRimLiftFactor = Float(mtoon.parametricRimLiftFactor ?? 0) + self.outlineWidthMode = .init(vrm1: mtoon.outlineWidthMode) + self.outlineWidthFactor = Float(mtoon.outlineWidthFactor ?? 0) + self.outlineColorFactor = outlineColor + self.outlineLightingMixFactor = Float(mtoon.outlineLightingMixFactor ?? 1) + self.uvAnimationScrollXSpeedFactor = Float(mtoon.uvAnimationScrollXSpeedFactor ?? 0) + self.uvAnimationScrollYSpeedFactor = Float(mtoon.uvAnimationScrollYSpeedFactor ?? 0) + self.uvAnimationRotationSpeedFactor = Float(mtoon.uvAnimationRotationSpeedFactor ?? 0) + self.transparentWithZWrite = mtoon.transparentWithZWrite ?? false + self.renderQueueOffsetNumber = mtoon.renderQueueOffsetNumber ?? 0 + self.alphaMode = material.alphaMode + self.alphaCutoff = material.alphaCutoff + self.cullMode = material.doubleSided ? .none : .back + self.normalScale = Float(material.normalTexture?.scale ?? 1) + self.baseColorTexture = pbr?.baseColorTexture.map(MToonMaterialDescriptor.Texture.init) + self.emissiveTexture = material.emissiveTexture.map(MToonMaterialDescriptor.Texture.init) + self.shadeMultiplyTexture = mtoon.shadeMultiplyTexture.map(MToonMaterialDescriptor.Texture.init) + self.shadingShiftTexture = mtoon.shadingShiftTexture.map(MToonMaterialDescriptor.Texture.init) + self.normalTexture = material.normalTexture.map(MToonMaterialDescriptor.Texture.init) + self.matcapTexture = mtoon.matcapTexture.map(MToonMaterialDescriptor.Texture.init) + self.rimMultiplyTexture = mtoon.rimMultiplyTexture.map(MToonMaterialDescriptor.Texture.init) + self.outlineWidthMultiplyTexture = mtoon.outlineWidthMultiplyTexture.map(MToonMaterialDescriptor.Texture.init) + self.uvAnimationMaskTexture = mtoon.uvAnimationMaskTexture.map(MToonMaterialDescriptor.Texture.init) + } +} + +private extension MToonMaterialDescriptor.OutlineWidthMode { + init(vrm1 mode: GLTF.Material.MaterialExtensions.MaterialsMToon.MaterialsMToonOutlineWidthMode?) { + switch mode { + case .some(.worldCoordinates): + self = .worldCoordinates + case .some(.screenCoordinates): + self = .screenCoordinates + case .some(.none), nil: + self = .none + } + } +} + +private extension MToonMaterialDescriptor.Texture { + /// Builds a texture reference from a glTF texture info, decoding + /// `KHR_texture_transform` (including its optional `texCoord` override). + init(index: Int, texCoord: Int, extensions: CodableAny?) { + guard let transform = extensions?.dictionaryValue["KHR_texture_transform"] as? [String: Any] else { + self.init(index: index, texCoord: texCoord) + return + } + self.init(index: index, + texCoord: transform.index("texCoord") ?? texCoord, + transform: .init(scale: transform.simd2Value(forKey: "scale", default: SIMD2(1, 1)), + offset: transform.simd2Value(forKey: "offset", default: SIMD2(0, 0)), + rotation: transform.float("rotation") ?? 0)) + } + + init(_ textureInfo: GLTF.TextureInfo) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord, extensions: textureInfo.extensions) + } + + init(_ textureInfo: GLTF.Material.NormalTextureInfo) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord, extensions: textureInfo.extensions) + } + + init(_ textureInfo: GLTF.Material.MaterialExtensions.MaterialsMToon.MaterialsMToonTextureInfo) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord ?? 0, extensions: textureInfo.extensions) + } + + init(_ textureInfo: GLTF.Material.MaterialExtensions.MaterialsMToon.MaterialsMToonShadingShiftTexture) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord ?? 0, extensions: textureInfo.extensions) + } +} diff --git a/Sources/VRMKitRuntime/VRM0MToonMigrator.swift b/Sources/VRMKitRuntime/VRM0MToonMigrator.swift new file mode 100644 index 00000000..6f4aaaea --- /dev/null +++ b/Sources/VRMKitRuntime/VRM0MToonMigrator.swift @@ -0,0 +1,221 @@ +import Foundation +import simd +import VRMKit + +/// Converts VRM 0.x MToon material properties into VRMC_materials_mtoon 1.0 +/// semantics, following UniVRM's `MToon10Migrator` as the migration oracle. +/// +/// This layer knows nothing about renderers; it only maps VRM 0.x Unity +/// shader properties onto the canonical ``MToonMaterialDescriptor``. +package enum VRM0MToonMigrator { + package static func migrate(property: VRM0.MaterialProperty, + material: GLTF.Material) -> MToonMaterialDescriptor { + let floats = property.floatProperties.dictionaryValue + let textures = property.textureProperties + let vectors = property.vectorProperties.dictionaryValue + let pbr = material.pbrMetallicRoughness + + // VRM 0.x stores Lit / Shade / Rim / Outline colors as sRGB, while + // MToon 1.0 factors are linear. UniVRM treats emission and the glTF + // baseColorFactor fallback as already linear. + let baseColor = vectors.simd4("_Color").map(srgbToLinear) + ?? (pbr?.baseColorFactor).map(SIMD4.init) + ?? SIMD4(1, 1, 1, 1) + let emissiveColor = vectors.simd3("_EmissionColor") ?? SIMD3(0, 0, 0) + let shadeColor = srgbToLinear(vectors.simd4("_ShadeColor") ?? SIMD4(0.97, 0.81, 0.86, 1)) + let rimColor = srgbToLinear(vectors.simd4("_RimColor") ?? SIMD4(0, 0, 0, 1)) + let outlineColor = srgbToLinear(vectors.simd4("_OutlineColor") ?? SIMD4(0, 0, 0, 1)) + + let alphaMode = GLTF.Material.AlphaMode(vrm0: property, fallback: material.alphaMode) + // MToon 0.x has no `_ZWRITE_ON` shader keyword: the render mode lives in + // `_BlendMode` (3 = TransparentWithZWrite), which is what MToon10Migrator + // reads. `_ZWrite` is derived state, so it only separates the two + // transparent modes -- opaque materials write depth as well. + let transparentWithZWrite: Bool + switch floats.float("_BlendMode") { + case .some(3): + transparentWithZWrite = true + case .some: + transparentWithZWrite = false + default: + transparentWithZWrite = alphaMode == .BLEND && floats.float("_ZWrite") == 1 + } + let cullMode: MToonMaterialDescriptor.CullMode + switch floats.float("_CullMode") { + case .some(0): + cullMode = .none + case .some(1): + cullMode = .front + case .some(2): + cullMode = .back + default: + cullMode = material.doubleSided ? .none : .back + } + let hasMToonNormalTexture = textures["_BumpMap"] != nil + let shadeShift0 = floats.float("_ShadeShift") ?? 0 + let shadeToony0 = floats.float("_ShadeToony") ?? 0.9 + let rangeMin = shadeShift0 + let rangeMax = simd_mix(Float(1), shadeShift0, shadeToony0) + + let outlineWidthMode = MToonMaterialDescriptor.OutlineWidthMode(vrm0: floats.float("_OutlineWidthMode") ?? 0) + let outlineWidthFactor: Float + switch outlineWidthMode { + case .none: + outlineWidthFactor = 0 + case .worldCoordinates: + // VRM 0.x expresses world-space outline width in centimeters. + outlineWidthFactor = (floats.float("_OutlineWidth") ?? 0) * 0.01 + case .screenCoordinates: + // UniVRM halves screen-space width during 0.x -> 1.0 migration. + outlineWidthFactor = (floats.float("_OutlineWidth") ?? 0) * 0.01 * 0.5 + } + let outlineLightingMixFactor: Float + switch floats.float("_OutlineColorMode") ?? 0 { + case 0: // FixedColor renders the outline unlit. + outlineLightingMixFactor = 0 + default: // MixedLighting keeps the source mix value. + outlineLightingMixFactor = floats.float("_OutlineLightingMix") ?? 1 + } + + // UniVRM keeps the _MainTex scale/offset as the material's texture + // transform during migration; MToon 0.x applied it to every texture. + let mainTransform = mainTextureTransform(vectors: vectors) + func texture(_ index: Int?) -> MToonMaterialDescriptor.Texture? { + index.map { .init(index: $0, texCoord: 0, transform: mainTransform) } + } + + return MToonMaterialDescriptor( + baseColorFactor: baseColor, + emissiveFactor: emissiveColor, + shadeColorFactor: shadeColor, + shadingShiftFactor: (-(rangeMax + rangeMin) / 2).clamped(to: -1 ... 1), + shadingShiftTextureScale: 1, + shadingToonyFactor: ((2 - (rangeMax - rangeMin)) / 2).clamped(to: 0 ... 1), + giEqualizationFactor: (1 - (floats.float("_IndirectLightIntensity") ?? 0.1)).clamped(to: 0 ... 1), + matcapFactor: SIMD3(1, 1, 1), + parametricRimColorFactor: rimColor, + // UniVRM migrates rim lighting mix destructively to 1.0 for + // visual compatibility; the 0.x source value is intentionally dropped. + rimLightingMixFactor: 1, + parametricRimFresnelPowerFactor: floats.float("_RimFresnelPower") ?? 1, + parametricRimLiftFactor: floats.float("_RimLift") ?? 0, + outlineWidthMode: outlineWidthMode, + outlineWidthFactor: outlineWidthFactor, + outlineColorFactor: outlineColor, + outlineLightingMixFactor: outlineLightingMixFactor, + uvAnimationScrollXSpeedFactor: floats.float("_UvAnimScrollX") ?? 0, + // UniVRM inverts the Y scroll direction during migration. + uvAnimationScrollYSpeedFactor: -(floats.float("_UvAnimScrollY") ?? 0), + uvAnimationRotationSpeedFactor: (floats.float("_UvAnimRotation") ?? 0) * 2 * Float.pi, + transparentWithZWrite: transparentWithZWrite, + // renderQueueOffsetNumber is a *relative* order among a model's + // transparent materials; a single material carries no such ordering, + // so Unity's absolute renderQueue cannot be migrated here. + renderQueueOffsetNumber: 0, + alphaMode: alphaMode, + alphaCutoff: floats.float("_Cutoff") ?? material.alphaCutoff, + cullMode: cullMode, + normalScale: hasMToonNormalTexture + ? (floats.float("_BumpScale") ?? 1) + : Float(material.normalTexture?.scale ?? 1), + baseColorTexture: texture(textures["_MainTex"]), + emissiveTexture: texture(textures["_EmissionMap"]), + shadeMultiplyTexture: texture(textures["_ShadeTexture"] ?? textures["_MainTex"]), + shadingShiftTexture: nil, + normalTexture: texture(textures["_BumpMap"]) + ?? material.normalTexture.map { .init(index: $0.index, texCoord: $0.texCoord) }, + matcapTexture: texture(textures["_SphereAdd"]), + rimMultiplyTexture: texture(textures["_RimTexture"]), + outlineWidthMultiplyTexture: texture(textures["_OutlineWidthTexture"]), + uvAnimationMaskTexture: texture(textures["_UvAnimMaskTexture"]) + ) + } + + /// Converts Unity `_MainTex` scale/offset (bottom-left UV origin) into + /// KHR_texture_transform semantics (top-left origin). Returns nil for the + /// identity transform. + static func mainTextureTransform(vectors: [String: Any]) -> MToonMaterialDescriptor.UVTransform? { + guard let values = vectors["_MainTex"] as? [Any], values.count >= 4 else { + return nil + } + let offsetX = values.float(at: 0, default: 0) + let offsetY = values.float(at: 1, default: 0) + let scaleX = values.float(at: 2, default: 1) + let scaleY = values.float(at: 3, default: 1) + guard offsetX != 0 || offsetY != 0 || scaleX != 1 || scaleY != 1 else { + return nil + } + return .init(scale: SIMD2(scaleX, scaleY), + offset: SIMD2(offsetX, 1 - offsetY - scaleY), + rotation: 0) + } + + static func srgbToLinear(_ value: Float) -> Float { + if value <= 0.04045 { + return value / 12.92 + } + return pow((value + 0.055) / 1.055, 2.4) + } + + static func srgbToLinear(_ value: SIMD4) -> SIMD4 { + SIMD4(srgbToLinear(value.x), + srgbToLinear(value.y), + srgbToLinear(value.z), + value.w) + } +} + +private extension MToonMaterialDescriptor.OutlineWidthMode { + init(vrm0 mode: Float) { + switch mode { + case 1: + self = .worldCoordinates + case 2: + self = .screenCoordinates + default: + self = .none + } + } +} + +package extension GLTF.Material.AlphaMode { + /// Resolves the glTF alpha mode from VRM 0.x Unity shader metadata. + /// `RenderType` takes priority over the keyword map, matching UniVRM. + init(vrm0 property: VRM0.MaterialProperty?, fallback: GLTF.Material.AlphaMode) { + guard let property else { + self = fallback + return + } + if let renderType = property.tagMap["RenderType"]?.lowercased() { + switch renderType { + case "opaque": + self = .OPAQUE + return + case "transparentcutout", "cutout": + self = .MASK + return + case "transparent": + self = .BLEND + return + default: + break + } + } + + if property.vrmShader == .unlitTransparent + || property.keywordMap["_ALPHAPREMULTIPLY_ON"] == true + || property.keywordMap["_ALPHABLEND_ON"] == true { + self = .BLEND + } else if property.keywordMap["_ALPHATEST_ON"] == true { + self = .MASK + } else { + self = fallback + } + } +} + +private extension Comparable { + func clamped(to range: ClosedRange) -> Self { + return min(max(self, range.lowerBound), range.upperBound) + } +} diff --git a/Sources/VRMRealityKit/CustomType/VRMEntity.swift b/Sources/VRMRealityKit/CustomType/VRMEntity.swift index 017b33d8..52623bab 100644 --- a/Sources/VRMRealityKit/CustomType/VRMEntity.swift +++ b/Sources/VRMRealityKit/CustomType/VRMEntity.swift @@ -1,42 +1,81 @@ #if canImport(RealityKit) import CoreGraphics import Foundation +import OSLog import RealityKit import simd import VRMKit import VRMKitRuntime @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) -struct BlendShapeNormalTangentComponent: Component { - let baseNormals: [SIMD3] - let baseTangents: [SIMD3] - let normalOffsets: [[SIMD3]] - let tangentOffsets: [[SIMD3]] +struct VRMMaterialIndexComponent: Component { + let materialIndex: Int } +/// Carries the loaded VRM on the entity, so the copies `clone(recursive:)` makes +/// through `init()` still answer ``VRMEntity/vrm``. @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) -struct VRMMaterialIndexComponent: Component { - let materialIndex: Int +struct VRMComponent: Component { + let vrm: VRM } +/// The root entity of a loaded VRM model, and the runtime that animates it. +/// +/// It is a plain `Entity`, so adding it to a scene is all its lifetime needs: +/// the parent entity owns it, and ``VRMUpdateSystem`` animates it for as long as +/// it stays in the scene. @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) -@MainActor -public final class VRMEntity { - public let vrm: VRM - public let entity: Entity - public let humanoid = Humanoid() +public final class VRMEntity: Entity { + private static let logger = Logger(subsystem: "dev.tattn.VRMKit", category: "MToon") + + /// The VRM this entity was loaded from. + /// + /// - Precondition: the entity came from a ``VRMEntityLoader``, not from + /// ``init()``. + public var vrm: VRM { + guard let vrm = components[VRMComponent.self]?.vrm else { + preconditionFailure("This VRMEntity carries no VRM. Load it with VRMEntityLoader.") + } + return vrm + } - private let enableNormalTangentBlendShape = false + public let humanoid = Humanoid() var blendShapeClips: [BlendShapeKey: BlendShapeClip] = [:] var expressionClips: [ExpressionKey: ExpressionClip] = [:] + private var expressionWeights: [ExpressionKey: Float] = [:] private var materialColorClips: [ExpressionKey: [MaterialColorBinding]] = [:] private var textureTransformClips: [ExpressionKey: [TextureTransformBinding]] = [:] private var firstPersonAnnotations: [FirstPersonAnnotation] = [] private var skinBindings: [SkinBinding] = [] - private var modelEntitiesByMaterialIndex: [Int: [ModelEntity]] = [:] + /// Everything the runtime tracks per glTF material. Keying this by material + /// index — rather than copying it onto every entity — is what keeps the + /// MToon parameter rows single-source. + private struct MaterialRuntimeState { + var modelEntities: [ModelEntity] = [] + var mtoonParameters: MToonMaterialParameters? + var needsMToonParameterFlush = false + } + + private var materialStates: [Int: MaterialRuntimeState] = [:] private var springBones: [VRMEntitySpringBone] = [] private var nodeConstraints: [NodeConstraintBinding] = [] + // Binding indexes and their baseline values are fully determined by the + // clips, so they are built once at load time instead of on every + // expression change. + private var morphBindingIndex: [MorphBindingKey: BlendShapeBinding] = [:] + private var colorBindingIndex: [MaterialColorBindingKey: MaterialColorBinding] = [:] + private var transformBindingIndex: [Int: TextureTransformBinding] = [:] + // Values last pushed to the render state, so re-applying every clip on each + // expression change only touches bindings whose value actually moved. + private var appliedMorphWeights: [MorphBindingKey: Float] = [:] + // Blend-shape target -> weight-set positions, resolved on first write. + private var blendShapeSlotCache: [MorphBindingKey: [BlendShapeSlot]] = [:] + private var appliedMaterialColors: [MaterialColorBindingKey: SIMD4] = [:] + private var appliedTextureTransforms: [Int: SIMD4] = [:] + private var mtoonLightDirection = MToonMaterialParameters.defaultLightDirection + private var mtoonLightColor = SIMD3(1, 1, 1) + private var mtoonAmbientColor = SIMD3(0, 0, 0) struct SkinBinding { let modelEntity: ModelEntity @@ -44,9 +83,44 @@ public final class VRMEntity { let jointEntities: [Entity] } + /// Registers the components and the system this entity relies on. RealityKit + /// instantiates registered systems for every scene, including scenes that + /// already exist, so registering on first use is enough; `static let` runs + /// the body exactly once. + @MainActor private static let registerRealityKitTypes: Void = { + VRMComponent.registerComponent() + VRMUpdateComponent.registerComponent() + VRMMaterialIndexComponent.registerComponent() + VRMUpdateSystem.registerSystem() + }() + init(vrm: VRM) { - self.vrm = vrm - self.entity = Entity() + super.init() + _ = Self.registerRealityKitTypes + components.set(VRMComponent(vrm: vrm)) + components.set(VRMUpdateComponent()) + } + + /// Required by `Entity`, which also builds the copies `clone(recursive:)` + /// returns. Such a copy inherits the ``VRMComponent`` but not the runtime + /// bindings, so it renders and ``update(deltaTime:)`` does nothing to it. + public required init() { + super.init() + _ = Self.registerRealityKitTypes + } + + /// Whether ``VRMUpdateSystem`` calls ``update(deltaTime:)`` automatically on + /// every render frame. Enabled by default. Disable it to take over the + /// per-frame timing and call ``update(deltaTime:)`` yourself. + public var isAutomaticUpdateEnabled: Bool { + get { components.has(VRMUpdateComponent.self) } + set { + if newValue { + components.set(VRMUpdateComponent()) + } else { + components.remove(VRMUpdateComponent.self) + } + } } func setUpHumanoid(nodes: [Entity?]) { @@ -58,12 +132,8 @@ public final class VRMEntity { } } + /// Called once per entity, right after its node hierarchy is built. func setUpBlendShapes(nodes: [Entity?], meshes: [Entity?], loader: VRMEntityLoader) throws { - blendShapeClips = [:] - expressionClips = [:] - materialColorClips = [:] - textureTransformClips = [:] - switch vrm { case .v0: blendShapeClips = vrm.blendShapeMaster.blendShapeGroups @@ -98,17 +168,30 @@ public final class VRMEntity { let runtimeClip = ExpressionClip(name: expressionClip.name, preset: expressionClip.preset, values: morphBindings, - isBinary: expressionClip.expression.isBinary ?? false) + isBinary: expressionClip.expression.isBinary ?? false, + overrideBlink: expressionClip.expression.overrideBlink ?? .none, + overrideLookAt: expressionClip.expression.overrideLookAt ?? .none, + overrideMouth: expressionClip.expression.overrideMouth ?? .none) expressionClips[runtimeClip.key] = runtimeClip let colorBindings: [MaterialColorBinding] = expressionClip.expression.materialColorBinds? .compactMap { bind in guard bind.targetValue.count >= 3 else { return nil } - guard let material = try? loader.material(withMaterialIndex: bind.material) else { return nil } + // A malformed bind (e.g. out-of-range material index) only + // invalidates that bind, never the whole model load. + guard let baseValue = try? loader.currentMaterialColor(withMaterialIndex: bind.material, + type: bind.type) else { + Self.logger.warning(""" + Skipping invalid MaterialColorBind. \ + expression=\(expressionClip.name, privacy: .public) \ + materialIndex=\(bind.material) + """) + return nil + } return MaterialColorBinding(materialIndex: bind.material, type: bind.type, targetValue: SIMD4(bind.targetValue, default: 1.0), - baseValue: material.currentColor(for: bind.type)) + baseValue: baseValue) } ?? [] if !colorBindings.isEmpty { materialColorClips[runtimeClip.key] = colorBindings @@ -116,11 +199,13 @@ public final class VRMEntity { let transformBindings: [TextureTransformBinding] = expressionClip.expression.textureTransformBinds? .compactMap { bind in - guard let material = try? loader.material(withMaterialIndex: bind.material) else { return nil } - let base = material.currentTextureTransform + guard let base = try? loader.currentTextureTransform(withMaterialIndex: bind.material) else { + return nil + } return TextureTransformBinding(materialIndex: bind.material, baseScale: base.scale, baseOffset: base.offset, + baseRotation: base.rotation, targetScale: SIMD2(bind.scale, default: 1.0), targetOffset: SIMD2(bind.offset, default: 0.0)) } ?? [] @@ -129,6 +214,29 @@ public final class VRMEntity { } } } + + buildExpressionBindingIndexes() + } + + /// Indexes every expression binding once, so `applyExpressions()` only + /// accumulates weights instead of rediscovering the bindings each time. + private func buildExpressionBindingIndexes() { + for clip in expressionClips.values { + for binding in clip.values { + let key = MorphBindingKey(mesh: binding.mesh, targetIndex: binding.index) + morphBindingIndex[key] = binding + } + } + for bindings in materialColorClips.values { + for binding in bindings { + colorBindingIndex[binding.key] = binding + } + } + for bindings in textureTransformClips.values { + for binding in bindings { + transformBindingIndex[binding.materialIndex] = binding + } + } } func setUpFirstPerson(nodes: [Entity?], meshes: [Entity?]) { @@ -257,70 +365,124 @@ public final class VRMEntity { initializeSkinPose(for: binding) } - func registerMaterialBinding(modelEntity: ModelEntity, materialIndex: Int) { - modelEntitiesByMaterialIndex[materialIndex, default: []].append(modelEntity) + /// Registers an entity as a renderer of `materialIndex`. The MToon parameter + /// rows are resolved once per material, so every entity sharing it shares them. + func registerMaterialBinding(modelEntity: ModelEntity, materialIndex: Int, loader: VRMEntityLoader) { + if materialStates[materialIndex] == nil { + materialStates[materialIndex] = MaterialRuntimeState( + mtoonParameters: try? loader.mtoonParameters(withMaterialIndex: materialIndex) + ) + } + materialStates[materialIndex]?.modelEntities.append(modelEntity) + } + + /// The MToon parameter rows a material renders with, or nil when it does + /// not render as MToon. + func mtoonParameters(forMaterialIndex index: Int) -> MToonMaterialParameters? { + materialStates[index]?.mtoonParameters } - public func update(at time: TimeInterval) { + /// Advances spring bones, node constraints, and skinning by one frame. + /// + /// ``VRMUpdateSystem`` calls this automatically once per render frame, so + /// there is normally no need to call it. To drive the timing manually, set + /// ``isAutomaticUpdateEnabled`` to `false` first — otherwise the model + /// advances twice per frame. + public func update(deltaTime: TimeInterval) { + let deltaTime = max(0, deltaTime) + // Skinning runs last so that this frame's constraint and spring-bone + // poses reach the skinned meshes in the same frame they are solved. nodeConstraints.forEach { $0.apply() } + springBones.forEach { $0.update(deltaTime: deltaTime) } updateSkinning() - springBones.forEach { $0.update(deltaTime: time) } + } + + /// Sets the explicit main light direction used by MToon CustomMaterial shaders. + /// The vector points from the surface toward the light, so a `DirectionalLight` + /// matching it is placed at `direction` and aimed at the model. + public func setMToonLightDirection(_ direction: SIMD3) { + let length = simd_length(direction) + let normalized = length > 0.001 ? direction / length : MToonMaterialParameters.defaultLightDirection + guard simd_distance(normalized, mtoonLightDirection) > 0.0001 else { return } + mtoonLightDirection = normalized + // The direction rides in custom.value rather than in a parameter row, so + // it reaches the materials without rebuilding their packed texture — and + // without clearing a rebuild an earlier row change is still waiting for. + for materialIndex in materialStates.keys { + guard var parameters = materialStates[materialIndex]?.mtoonParameters else { continue } + parameters.lightDirection = normalized + materialStates[materialIndex]?.mtoonParameters = parameters + applyMToonParameters(parameters, ofMaterial: materialIndex, parameterTexture: nil) + } + } + + /// Sets the explicit main light color used by MToon CustomMaterial shaders. The default is white. + public func setMToonLightColor(_ color: SIMD3) { + guard color != mtoonLightColor else { return } + mtoonLightColor = color + updateMToonLightingRows() + } + + /// Sets the explicit ambient color used by the MToon GI approximation. The default is black. + public func setMToonAmbientColor(_ color: SIMD3) { + guard color != mtoonAmbientColor else { return } + mtoonAmbientColor = color + updateMToonLightingRows() } private func updateSkinning() { + // Bindings that share a skeleton and model world transform — a mesh and + // its MToon outline twin, or primitives of the same skinned mesh — + // resolve to identical joint transforms, so solve them once per frame. + var solved: [String: (modelWorld: simd_float4x4, transforms: JointTransforms)] = [:] for binding in skinBindings { - updateSkinPose(for: binding) + let modelWorld = binding.modelEntity.transformMatrix(relativeTo: nil) + let transforms: JointTransforms + if let cached = solved[binding.skeleton.id], cached.modelWorld == modelWorld { + transforms = cached.transforms + } else { + transforms = jointTransforms(for: binding, modelWorld: modelWorld) + solved[binding.skeleton.id] = (modelWorld, transforms) + } + setSkinPose(transforms, for: binding) } } private func initializeSkinPose(for binding: SkinBinding) { - let transforms = jointTransforms(for: binding) - var pose = SkeletalPose(id: binding.skeleton.id, from: binding.skeleton) + let modelWorld = binding.modelEntity.transformMatrix(relativeTo: nil) + setSkinPose(jointTransforms(for: binding, modelWorld: modelWorld), for: binding) + } + + private func setSkinPose(_ transforms: JointTransforms, for binding: SkinBinding) { + let existing = binding.modelEntity.components[SkeletalPosesComponent.self] + var pose = existing?.poses[binding.skeleton.id] + ?? existing?.poses.default + ?? SkeletalPose(id: binding.skeleton.id, from: binding.skeleton) pose.jointTransforms = transforms - var component = binding.modelEntity.components[SkeletalPosesComponent.self] ?? SkeletalPosesComponent(poses: [pose]) + var component = existing ?? SkeletalPosesComponent(poses: [pose]) component.poses[pose.id] = pose component.poses.default = pose binding.modelEntity.components.set(component) } - private func updateSkinPose(for binding: SkinBinding) { - let transforms = jointTransforms(for: binding) - guard var component = binding.modelEntity.components[SkeletalPosesComponent.self] else { - initializeSkinPose(for: binding) - return - } - - if var pose = component.poses[binding.skeleton.id] ?? component.poses.default { - pose.jointTransforms = transforms - component.poses[pose.id] = pose - component.poses.default = pose - } else { - var pose = SkeletalPose(id: binding.skeleton.id, from: binding.skeleton) - pose.jointTransforms = transforms - component.poses[pose.id] = pose - component.poses.default = pose - } - - binding.modelEntity.components.set(component) - } - - private func jointTransforms(for binding: SkinBinding) -> JointTransforms { + private func jointTransforms(for binding: SkinBinding, + modelWorld: simd_float4x4) -> JointTransforms { let jointEntities = binding.jointEntities let joints = binding.skeleton.joints var transforms: [Transform] = [] transforms.reserveCapacity(jointEntities.count) - let modelWorld = binding.modelEntity.transformMatrix(relativeTo: nil) let modelWorldInverse = simd_inverse(modelWorld) + // Each joint's world matrix is also its children's parent matrix, so + // resolve them once instead of walking the ancestor chain twice. + let jointWorlds = jointEntities.map { $0.transformMatrix(relativeTo: nil) } for index in 0..() - for binding in clip.values { - let meshID = ObjectIdentifier(binding.mesh) - if seenMeshes.insert(meshID).inserted { - meshesToUpdate.append(binding.mesh) - } - } - for mesh in meshesToUpdate { - updateBlendShapeNormalsAndTangents(on: mesh) - } - } } public func blendShape(for key: BlendShapeKey) -> CGFloat { @@ -366,24 +515,42 @@ public final class VRMEntity { } public func setExpression(value: CGFloat, for key: ExpressionKey) { - guard let clip = expressionClip(for: key) else { return } - let normalized = max(0.0, min(1.0, clip.isBinary ? round(value) : value)) - for binding in clip.values { - let weight = Float(binding.weight / 100.0) * Float(normalized) - applyBlendShapeWeight(weight, targetIndex: binding.index, on: binding.mesh) - } - for binding in materialColorClip(for: key) { - binding.apply(value: Float(normalized), on: self) + guard storeExpressionWeight(value, for: key) else { return } + applyExpressions() + } + + /// Sets several expression weights and re-applies the result once. + /// + /// Prefer this over repeated ``setExpression(value:for:)`` calls when a single + /// frame changes more than one expression (face tracking, lip sync): applying + /// re-accumulates every active clip and can rebuild MToon parameter textures. + public func setExpressions(_ weights: [ExpressionKey: CGFloat]) { + var changed = false + for (key, value) in weights where storeExpressionWeight(value, for: key) { + changed = true } - for binding in textureTransformClip(for: key) { - binding.apply(value: Float(normalized), on: self) + guard changed else { return } + applyExpressions() + } + + /// Records the input weight for `key`, returning whether it actually moved. + private func storeExpressionWeight(_ value: CGFloat, for key: ExpressionKey) -> Bool { + guard let key = canonicalExpressionKey(for: key), + let clip = expressionClips[key] else { return false } + let normalized = clip.normalizedWeight(Double(value)) + let weight = normalized > 0 ? Float(normalized) : nil + guard weight != expressionWeights[key] else { return false } + if let weight { + expressionWeights[key] = weight + } else { + expressionWeights.removeValue(forKey: key) } + return true } public func expression(for key: ExpressionKey) -> CGFloat { - guard let clip = expressionClip(for: key), - let binding = clip.values.first else { return 0 } - return CGFloat(readBlendShapeWeight(targetIndex: binding.index, on: binding.mesh)) + guard let key = canonicalExpressionKey(for: key) else { return 0 } + return CGFloat(expressionWeights[key] ?? 0) } public func setFirstPersonRenderMode(_ mode: FirstPersonRenderMode) { @@ -396,204 +563,288 @@ public final class VRMEntity { fileprivate func applyMaterialColor(_ color: SIMD4, type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType, materialIndex: Int) { - guard let models = modelEntitiesByMaterialIndex[materialIndex] else { return } + // MToon owns its colors in the parameter rows; the packed texture is + // rebuilt once per material by flushDirtyMToonParameters(). + if mutateMToonParameters(ofMaterial: materialIndex, { $0.setColor(color, for: type) }) { + return + } let vrmColor = VRMColor(simd: color) - for modelEntity in models { - guard var component = modelEntity.components[ModelComponent.self] else { continue } - component.materials = component.materials.map { material in - material.settingColor(vrmColor, for: type) - } - modelEntity.components.set(component) + forEachModelEntity(ofMaterial: materialIndex) { component in + component.materials = component.materials.map { $0.settingColor(vrmColor, for: type) } } } fileprivate func applyTextureTransform(scale: SIMD2, offset: SIMD2, + rotation: Float, materialIndex: Int) { - guard let models = modelEntitiesByMaterialIndex[materialIndex] else { return } - let transform = MaterialParameterTypes.TextureCoordinateTransform(offset: offset, scale: scale) - for modelEntity in models { - guard var component = modelEntity.components[ModelComponent.self] else { continue } - component.materials = component.materials.map { material in - material.settingTextureTransform(transform) + // MToon applies the UV transform in its own shader from the parameter + // rows; writing RealityKit's material-level transform too would + // transform the primary UV twice. Fallback materials have no such + // shader, so they use the material-level transform. + if mutateMToonParameters(ofMaterial: materialIndex, { + $0.setTextureTransform(scale: scale, offset: offset, rotation: rotation) + }) { + return + } + forEachModelEntity(ofMaterial: materialIndex) { component in + component.materials = component.materials.map { + $0.settingTextureTransform(scale: scale, offset: offset, rotation: rotation) } - modelEntity.components.set(component) } } - private func expressionClip(for key: ExpressionKey) -> ExpressionClip? { - if let clip = expressionClips[key] { return clip } - if let legacyKey = key.legacyBlendShapeKey, - let expressionKey = legacyKey.expressionKey { - return expressionClips[expressionKey] - } - return nil + // MARK: - MToon runtime state + // + // MToon parameters describe a *material*, not an entity, so they are stored + // once per material index and pushed to every entity that renders with it. + // visionOS has no `CustomMaterial`, so no material has them and these all + // no-op there without platform conditionals of their own. + + /// Edits a material's MToon parameter rows, marking its packed texture for + /// rebuild. Returns false when the material does not render as MToon, which + /// is the caller's cue to fall back to the RealityKit material properties. + @discardableResult + private func mutateMToonParameters(ofMaterial materialIndex: Int, + _ mutate: (inout MToonMaterialParameters) -> Void) -> Bool { + guard var parameters = materialStates[materialIndex]?.mtoonParameters else { return false } + mutate(¶meters) + materialStates[materialIndex]?.mtoonParameters = parameters + materialStates[materialIndex]?.needsMToonParameterFlush = true + return true } - private func materialColorClip(for key: ExpressionKey) -> [MaterialColorBinding] { - if let clip = materialColorClips[key] { return clip } - if let legacyKey = key.legacyBlendShapeKey, - let expressionKey = legacyKey.expressionKey { - return materialColorClips[expressionKey] ?? [] + /// Rebuilds the packed parameter texture once per material whose rows + /// changed, instead of once per binding that touched it. + private func flushDirtyMToonParameters() { + for (materialIndex, state) in materialStates where state.needsMToonParameterFlush { + guard let parameters = state.mtoonParameters, + let parameterTexture = parameterTextureResource(for: parameters) else { + // The GPU still holds the previous values, so the material stays + // dirty and the next flush retries building its texture. + continue + } + applyMToonParameters(parameters, ofMaterial: materialIndex, parameterTexture: parameterTexture) + materialStates[materialIndex]?.needsMToonParameterFlush = false } - return [] } - private func textureTransformClip(for key: ExpressionKey) -> [TextureTransformBinding] { - if let clip = textureTransformClips[key] { return clip } - if let legacyKey = key.legacyBlendShapeKey, - let expressionKey = legacyKey.expressionKey { - return textureTransformClips[expressionKey] ?? [] + /// Pushes the entity-level light and ambient colors into every MToon + /// material's parameter rows. They are packed into the parameter texture, so + /// they take the same dirty-and-flush path as expression-driven row changes. + private func updateMToonLightingRows() { + for materialIndex in materialStates.keys { + mutateMToonParameters(ofMaterial: materialIndex) { parameters in + parameters.lightColor = SIMD4(mtoonLightColor, 1) + parameters.ambientColor = SIMD4(mtoonAmbientColor, 1) + } } - return [] + flushDirtyMToonParameters() } - private func modelEntities(in root: Entity) -> [ModelEntity] { - var result: [ModelEntity] = [] - var stack: [Entity] = [root] - while let entity = stack.popLast() { - if let modelEntity = entity as? ModelEntity { - result.append(modelEntity) + private func applyMToonParameters(_ parameters: MToonMaterialParameters, + ofMaterial materialIndex: Int, + parameterTexture: TextureResource?) { + forEachModelEntity(ofMaterial: materialIndex) { component in + component.materials = component.materials.map { + applyingMToonParameters(parameters, to: $0, parameterTexture: parameterTexture) } - stack.append(contentsOf: entity.children) } - return result } - private func applyBlendShapeWeight(_ weight: Float, targetIndex: Int, on mesh: Entity) { - let targetName = "blendShape_\(targetIndex)" - let models = modelEntities(in: mesh) - for modelEntity in models { - ensureBlendShapeComponent(on: modelEntity) - var weights = modelEntity.blendWeights - let names = modelEntity.blendWeightNames - guard !weights.isEmpty else { continue } - var didSet = false - if !names.isEmpty { - for setIndex in names.indices { - if let nameIndex = names[setIndex].firstIndex(of: targetName), - nameIndex < weights[setIndex].count { - weights[setIndex][nameIndex] = weight - didSet = true - } - } - } - if !didSet { - for setIndex in weights.indices { - guard targetIndex < weights[setIndex].count else { continue } - weights[setIndex][targetIndex] = weight - } - } - modelEntity.blendWeights = weights + /// Applies `edit` to the `ModelComponent` of every entity rendering with + /// `materialIndex`, writing the component back. + private func forEachModelEntity(ofMaterial materialIndex: Int, + _ edit: (inout ModelComponent) -> Void) { + guard let modelEntities = materialStates[materialIndex]?.modelEntities else { return } + for modelEntity in modelEntities { + guard var component = modelEntity.components[ModelComponent.self] else { continue } + edit(&component) + modelEntity.components.set(component) } } - private func updateBlendShapeNormalsAndTangents(on mesh: Entity) { - for modelEntity in modelEntities(in: mesh) { - applyNormalTangentMorphs(on: modelEntity) + /// MToon parameters live on `CustomMaterial`, which visionOS does not have, + /// so this is the single platform boundary of the runtime update path. + private func applyingMToonParameters(_ parameters: MToonMaterialParameters, + to material: any Material, + parameterTexture: TextureResource?) -> any Material { +#if os(visionOS) + return material +#else + guard var material = material as? CustomMaterial else { return material } + material.custom.value = parameters.customValue + if let parameterTexture { + material.custom.texture = CustomMaterial.Texture(parameterTexture) } + return material +#endif } - private func applyNormalTangentMorphs(on modelEntity: ModelEntity) { - guard let component = modelEntity.components[BlendShapeNormalTangentComponent.self] else { return } - let hasNormalOffsets = !component.normalOffsets.isEmpty - let hasTangentOffsets = !component.tangentOffsets.isEmpty - guard hasNormalOffsets || hasTangentOffsets else { return } + /// Packs the parameter rows into one GPU texture. Callers build it once per + /// material and share it across every entity that renders with it. + private func parameterTextureResource(for parameters: MToonMaterialParameters) -> TextureResource? { + do { + return try parameters.textureResource() + } catch { + Self.logger.error("Failed to update MToon parameter texture: \(error.localizedDescription, privacy: .public)") + return nil + } + } - let normals = hasNormalOffsets - ? applyOffsets(base: component.baseNormals, - offsets: component.normalOffsets, - weights: blendShapeWeights(for: modelEntity, - targetCount: component.normalOffsets.count)) - : nil - let tangents = hasTangentOffsets - ? applyOffsets(base: component.baseTangents, - offsets: component.tangentOffsets, - weights: blendShapeWeights(for: modelEntity, - targetCount: component.tangentOffsets.count)) - : nil - guard normals != nil || tangents != nil else { return } - guard let model = modelEntity.components[ModelComponent.self] else { return } - updateMeshBuffers(mesh: model.mesh, normals: normals, tangents: tangents) + private func canonicalExpressionKey(for key: ExpressionKey) -> ExpressionKey? { + if expressionClips[key] != nil { return key } + if let legacyKey = key.legacyBlendShapeKey, + let expressionKey = legacyKey.expressionKey, + expressionClips[expressionKey] != nil { + return expressionKey + } + return nil } - private func blendShapeWeights(for modelEntity: ModelEntity, targetCount: Int) -> [Float] { - guard let firstSet = modelEntity.blendWeights.first else { - return Array(repeating: 0, count: targetCount) + /// Applies VRMC_vrm expression overrides to the input weights. A binary + /// expression is suppressed outright rather than scaled, having no partial + /// state. + private func effectiveExpressionWeights() -> [ExpressionKey: Float] { + var states = ExpressionOverrideStates() + for (expressionKey, weight) in expressionWeights { + guard let clip = expressionClips[expressionKey] else { continue } + states.accumulate(clip, weight: Double(weight), excluding: expressionKey.overrideGroup) } - var result = Array(repeating: Float(0), count: targetCount) - let names = modelEntity.blendWeightNames.first ?? [] - if !names.isEmpty, names.count == firstSet.count { - for (index, name) in names.enumerated() { - guard let targetIndex = parseBlendShapeIndex(from: name), - targetIndex < targetCount, - index < firstSet.count else { continue } - result[targetIndex] = firstSet[index] + guard states.isSuppressingAnyGroup else { return expressionWeights } + + var result: [ExpressionKey: Float] = [:] + result.reserveCapacity(expressionWeights.count) + for (expressionKey, weight) in expressionWeights { + let state = expressionKey.overrideGroup.map { states[$0] } + guard let state, state.isSuppressing else { + result[expressionKey] = weight + continue } - } else { - let count = min(targetCount, firstSet.count) - for index in 0.. 0 { + result[expressionKey] = overridden } } return result } - private func parseBlendShapeIndex(from name: String) -> Int? { - let prefix = "blendShape_" - guard name.hasPrefix(prefix) else { return nil } - return Int(name.dropFirst(prefix.count)) + private func applyExpressions() { + let expressionWeights = effectiveExpressionWeights() + + var morphWeights: [MorphBindingKey: Float] = [:] + for (expressionKey, expressionWeight) in expressionWeights { + guard let clip = expressionClips[expressionKey] else { continue } + for binding in clip.values { + let key = MorphBindingKey(mesh: binding.mesh, targetIndex: binding.index) + morphWeights[key, default: 0] += Float(binding.weight / 100.0) * expressionWeight + } + } + for (key, binding) in morphBindingIndex { + let weight = morphWeights[key] ?? 0 + guard appliedMorphWeights[key] != weight else { continue } + appliedMorphWeights[key] = weight + applyBlendShapeWeight(weight, targetIndex: binding.index, on: binding.mesh) + } + + var colors: [MaterialColorBindingKey: SIMD4] = [:] + for (expressionKey, expressionWeight) in expressionWeights { + for binding in materialColorClips[expressionKey] ?? [] { + colors[binding.key, default: binding.baseValue] += + (binding.targetValue - binding.baseValue) * expressionWeight + } + } + for (key, binding) in colorBindingIndex { + let color = colors[key] ?? binding.baseValue + guard appliedMaterialColors[key] != color else { continue } + appliedMaterialColors[key] = color + applyMaterialColor(color, type: binding.type, materialIndex: binding.materialIndex) + } + + var scales: [Int: SIMD2] = [:] + var offsets: [Int: SIMD2] = [:] + for (expressionKey, expressionWeight) in expressionWeights { + for binding in textureTransformClips[expressionKey] ?? [] { + scales[binding.materialIndex, default: binding.baseScale] += + (binding.targetScale - binding.baseScale) * expressionWeight + offsets[binding.materialIndex, default: binding.baseOffset] += + (binding.targetOffset - binding.baseOffset) * expressionWeight + } + } + for (materialIndex, binding) in transformBindingIndex { + let scale = scales[materialIndex] ?? binding.baseScale + let offset = offsets[materialIndex] ?? binding.baseOffset + let applied = SIMD4(scale.x, scale.y, offset.x, offset.y) + guard appliedTextureTransforms[materialIndex] != applied else { continue } + appliedTextureTransforms[materialIndex] = applied + applyTextureTransform(scale: scale, + offset: offset, + rotation: binding.baseRotation, + materialIndex: materialIndex) + } + + flushDirtyMToonParameters() } - private func applyOffsets(base: [SIMD3], - offsets: [[SIMD3]], - weights: [Float]) -> [SIMD3]? { - guard !base.isEmpty, !offsets.isEmpty else { return nil } - guard offsets.count == weights.count else { return nil } - guard offsets.allSatisfy({ $0.count == base.count }) else { return nil } + /// Where one blend-shape target lives in a model entity's weight sets. + private struct BlendShapeSlot { + let modelEntity: ModelEntity + /// (weight set, index within it) pairs the target writes to. + let positions: [(set: Int, index: Int)] + } - var result = base - for targetIndex in 0..]?, - tangents: [SIMD3]?) { - guard normals != nil || tangents != nil else { return } - var contents = mesh.contents - var updatedModels = MeshModelCollection() - for model in contents.models { - var model = model - var updatedParts = MeshPartCollection() - for part in model.parts { - var part = part - let vertexCount = part.positions.count - if let normals, !normals.isEmpty, normals.count == vertexCount { - part.normals = MeshBuffer(normals) + /// Resolves the target's weight-set positions once per mesh, replacing a + /// blend-shape name lookup on every write. + private func blendShapeSlots(targetIndex: Int, on mesh: Entity) -> [BlendShapeSlot] { + let key = MorphBindingKey(mesh: mesh, targetIndex: targetIndex) + if let cached = blendShapeSlotCache[key] { + return cached + } + + let targetName = "blendShape_\(targetIndex)" + var slots: [BlendShapeSlot] = [] + for modelEntity in mesh.modelEntitiesInHierarchy { + ensureBlendShapeComponent(on: modelEntity) + let weights = modelEntity.blendWeights + guard !weights.isEmpty else { continue } + let names = modelEntity.blendWeightNames + var positions: [(set: Int, index: Int)] = [] + if !names.isEmpty { + for setIndex in names.indices { + if let nameIndex = names[setIndex].firstIndex(of: targetName), + nameIndex < weights[setIndex].count { + positions.append((setIndex, nameIndex)) + } } - if let tangents, !tangents.isEmpty, tangents.count == vertexCount { - part.tangents = MeshBuffer(tangents) + } + if positions.isEmpty { + // Meshes without blend-shape names address targets positionally. + for setIndex in weights.indices where targetIndex < weights[setIndex].count { + positions.append((setIndex, targetIndex)) } - updatedParts.insert(part) } - model.parts = updatedParts - updatedModels.insert(model) + guard !positions.isEmpty else { continue } + slots.append(BlendShapeSlot(modelEntity: modelEntity, positions: positions)) } - contents.models = updatedModels - try? mesh.replace(with: contents) + blendShapeSlotCache[key] = slots + return slots } private func readBlendShapeWeight(targetIndex: Int, on mesh: Entity) -> Float { let targetName = "blendShape_\(targetIndex)" - for modelEntity in modelEntities(in: mesh) { + for modelEntity in mesh.modelEntitiesInHierarchy { let weights = modelEntity.blendWeights if let firstSet = weights.first, targetIndex < firstSet.count { let names = modelEntity.blendWeightNames @@ -697,6 +948,26 @@ private struct NodeConstraintBinding { } } +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +/// Identifies one morph target on one mesh entity. Used both to accumulate +/// expression weights and to cache where that target lives in the blend-shape +/// weight sets. +private struct MorphBindingKey: Hashable { + let mesh: ObjectIdentifier + let targetIndex: Int + + init(mesh: Entity, targetIndex: Int) { + self.mesh = ObjectIdentifier(mesh) + self.targetIndex = targetIndex + } +} + +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +private struct MaterialColorBindingKey: Hashable { + let materialIndex: Int + let type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType +} + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) private struct MaterialColorBinding { let materialIndex: Int @@ -704,11 +975,8 @@ private struct MaterialColorBinding { let targetValue: SIMD4 let baseValue: SIMD4 - @MainActor - func apply(value: Float, on entity: VRMEntity) { - entity.applyMaterialColor(baseValue + (targetValue - baseValue) * value, - type: type, - materialIndex: materialIndex) + var key: MaterialColorBindingKey { + MaterialColorBindingKey(materialIndex: materialIndex, type: type) } } @@ -717,17 +985,9 @@ private struct TextureTransformBinding { let materialIndex: Int let baseScale: SIMD2 let baseOffset: SIMD2 + let baseRotation: Float let targetScale: SIMD2 let targetOffset: SIMD2 - - @MainActor - func apply(value: Float, on entity: VRMEntity) { - let scale = baseScale + (targetScale - baseScale) * value - let offset = baseOffset + (targetOffset - baseOffset) * value - entity.applyTextureTransform(scale: scale, - offset: offset, - materialIndex: materialIndex) - } } @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) @@ -737,6 +997,22 @@ private struct FirstPersonAnnotation { let hidesAutoInFirstPerson: Bool } +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +extension Entity { + /// Every `ModelEntity` in this entity's hierarchy, including itself. + var modelEntitiesInHierarchy: [ModelEntity] { + var result: [ModelEntity] = [] + var stack: [Entity] = [self] + while let entity = stack.popLast() { + if let modelEntity = entity as? ModelEntity { + result.append(modelEntity) + } + stack.append(contentsOf: entity.children) + } + return result + } +} + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) private extension Entity { func isSameOrDescendant(of ancestor: Entity?) -> Bool { @@ -752,55 +1028,82 @@ private extension Entity { } } +/// Materials whose UV transform VRMKit can read and write uniformly. +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +protocol TextureTransformableMaterial: Material { + var textureCoordinateTransform: MaterialParameterTypes.TextureCoordinateTransform { get set } +} + +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +extension UnlitMaterial: TextureTransformableMaterial {} + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) -private extension Material { +extension PhysicallyBasedMaterial: TextureTransformableMaterial {} + +#if !os(visionOS) +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +extension CustomMaterial: TextureTransformableMaterial {} +#endif + +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +extension Material { var currentTextureTransform: MaterialParameterTypes.TextureCoordinateTransform { - switch self { - case let material as UnlitMaterial: - return material.textureCoordinateTransform - case let material as PhysicallyBasedMaterial: - return material.textureCoordinateTransform - default: - return MaterialParameterTypes.TextureCoordinateTransform() - } + (self as? any TextureTransformableMaterial)?.textureCoordinateTransform + ?? MaterialParameterTypes.TextureCoordinateTransform() } func currentColor(for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> SIMD4 { switch self { case let material as UnlitMaterial: - return material.color.tint.simd + switch type { + case .color: + return material.color.tint.simd + case .emissionColor, .shadeColor, .matcapColor, .rimColor, .outlineColor: + return SIMD4(1, 1, 1, 1) + } case let material as PhysicallyBasedMaterial: switch type { case .color: return material.baseColor.tint.simd case .emissionColor: return material.emissiveColor.color.simd - case .shadeColor: - return material.baseColor.tint.simd - case .matcapColor, .rimColor: - return material.emissiveColor.color.simd - case .outlineColor: - return material.baseColor.tint.simd + // shadeColor / matcapColor / rimColor / outlineColor are MToon-only, + // so they have no meaning on the PBR fallback material. + case .shadeColor, .matcapColor, .rimColor, .outlineColor: + return SIMD4(1, 1, 1, 1) } default: return SIMD4(1, 1, 1, 1) } } + func settingTextureTransform(scale: SIMD2, offset: SIMD2, rotation: Float = 0) -> Material { + guard var material = self as? any TextureTransformableMaterial else { return self } + material.textureCoordinateTransform = MaterialParameterTypes.TextureCoordinateTransform(offset: offset, + scale: scale, + rotation: rotation) + return material + } + func settingColor(_ color: VRMColor, for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> Material { switch self { case var material as UnlitMaterial: - material.color.tint = color + switch type { + case .color: + material.color.tint = color + case .emissionColor, .shadeColor, .matcapColor, .rimColor, .outlineColor: + break + } return material case var material as PhysicallyBasedMaterial: switch type { - case .color, .shadeColor, .outlineColor: + case .color: material.baseColor.tint = color case .emissionColor: material.emissiveColor.color = color - case .matcapColor, .rimColor: - material.emissiveColor.color = color + case .shadeColor, .matcapColor, .rimColor, .outlineColor: + break } return material default: @@ -808,18 +1111,5 @@ private extension Material { } } - func settingTextureTransform(_ transform: MaterialParameterTypes.TextureCoordinateTransform) -> Material { - switch self { - case var material as UnlitMaterial: - material.textureCoordinateTransform = transform - return material - case var material as PhysicallyBasedMaterial: - material.textureCoordinateTransform = transform - return material - default: - return self - } - } } - #endif diff --git a/Sources/VRMRealityKit/CustomType/VRMUpdateSystem.swift b/Sources/VRMRealityKit/CustomType/VRMUpdateSystem.swift new file mode 100644 index 00000000..df97b1b1 --- /dev/null +++ b/Sources/VRMRealityKit/CustomType/VRMUpdateSystem.swift @@ -0,0 +1,35 @@ +#if canImport(RealityKit) +import Foundation +import RealityKit + +/// Marks a ``VRMEntity`` as driven by ``VRMUpdateSystem``, which advances its +/// skinning, constraints, and spring bones once per frame. +/// +/// ``VRMEntity`` attaches this component to itself automatically; the public +/// knob is ``VRMEntity/isAutomaticUpdateEnabled``, since the component does +/// nothing on any other entity. +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +struct VRMUpdateComponent: Component {} + +/// Calls ``VRMEntity/update(deltaTime:)`` for every ``VRMEntity`` in the scene +/// on each render frame. +/// +/// The system is registered automatically the first time a ``VRMEntity`` is +/// created, so a model animates as soon as it is added to a scene, without any +/// per-frame code on the caller's side. To run your own animation code in a +/// guaranteed order relative to the VRM update — for example, posing joints that +/// this frame's skinning should already reflect — declare it in a custom +/// `System` with `SystemDependency.before(VRMUpdateSystem.self)`. +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +public struct VRMUpdateSystem: System { + private static let query = EntityQuery(where: .has(VRMUpdateComponent.self)) + + public init(scene: Scene) {} + + public func update(context: SceneUpdateContext) { + for entity in context.entities(matching: Self.query, updatingSystemWhen: .rendering) { + (entity as? VRMEntity)?.update(deltaTime: context.deltaTime) + } + } +} +#endif diff --git a/Sources/VRMRealityKit/EntityData.swift b/Sources/VRMRealityKit/EntityData.swift index c47ba0d2..67aaa3ce 100644 --- a/Sources/VRMRealityKit/EntityData.swift +++ b/Sources/VRMRealityKit/EntityData.swift @@ -1,50 +1,45 @@ #if canImport(RealityKit) +import Foundation import RealityKit import VRMKit -#if !os(watchOS) -import QuartzCore -#endif @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) final class EntityData { + /// Loaded scenes, keyed by scene index. Each one owns its entity graph. var entities: [VRMEntity?] - var cameras: [Entity?] + // `nodes` / `meshes` hold Entity *instances*, which belong to a single + // scene, so `beginScene()` clears them between loads. Every other cache + // below holds values or GPU resources that are safe to share. var nodes: [Entity?] var skins: [MeshResource.Skeleton?] var skinJointRemaps: [[Int]?] -#if !os(watchOS) - var animationChannels: [[CAAnimation?]?] - var animationSamplers: [[CAAnimation?]?] -#endif var meshes: [Entity?] var accessors: [Any?] - var durations: [CFTimeInterval?] var bufferViews: [Data?] = [] - var buffers: [Data?] = [] var materials: [Material?] = [] var textures: [TextureResource?] = [] var images: [VRMImage?] = [] init(vrm: GLTF) { entities = Array(repeating: nil, count: vrm.scenes?.count ?? 0) - cameras = Array(repeating: nil, count: vrm.cameras?.count ?? 0) nodes = Array(repeating: nil, count: vrm.nodes?.count ?? 0) skins = Array(repeating: nil, count: vrm.skins?.count ?? 0) skinJointRemaps = Array(repeating: nil, count: vrm.skins?.count ?? 0) -#if !os(watchOS) - animationChannels = Array(repeating: nil, count: vrm.animations?.count ?? 0) - animationSamplers = Array(repeating: nil, count: vrm.animations?.count ?? 0) -#endif meshes = Array(repeating: nil, count: vrm.meshes?.count ?? 0) accessors = Array(repeating: nil, count: vrm.accessors?.count ?? 0) - durations = Array(repeating: nil, count: vrm.accessors?.count ?? 0) bufferViews = Array(repeating: nil, count: vrm.bufferViews?.count ?? 0) - buffers = Array(repeating: nil, count: vrm.buffers?.count ?? 0) materials = Array(repeating: nil, count: vrm.materials?.count ?? 0) textures = Array(repeating: nil, count: vrm.textures?.count ?? 0) images = Array(repeating: nil, count: vrm.images?.count ?? 0) } + /// Starts building a scene's entity graph, dropping the entity caches while + /// buffer views, accessors, materials, textures and skeletons stay warm. + func beginScene() { + nodes = Array(repeating: nil, count: nodes.count) + meshes = Array(repeating: nil, count: meshes.count) + } + enum EntityDataError: Error { case outOfRange(keyPath: String, index: Int, count: Int) } diff --git a/Sources/VRMRealityKit/GLTF2RealityKit/GLTF2RealityKit.swift b/Sources/VRMRealityKit/GLTF2RealityKit/GLTF2RealityKit.swift index 7ecea747..6e70c9e2 100644 --- a/Sources/VRMRealityKit/GLTF2RealityKit/GLTF2RealityKit.swift +++ b/Sources/VRMRealityKit/GLTF2RealityKit/GLTF2RealityKit.swift @@ -3,35 +3,6 @@ import CoreGraphics import RealityKit import VRMKit -func numberOfComponents(of type: GLTF.Accessor.`Type`) -> Int { - switch type { - case .SCALAR: return 1 - case .VEC2: return 2 - case .VEC3: return 3 - case .VEC4: return 4 - case .MAT2: return 4 - case .MAT3: return 9 - case .MAT4: return 16 - } -} - -func bytes(of type: GLTF.Accessor.ComponentType) -> Int { - switch type { - case .byte, .unsignedByte: return 1 - case .short, .unsignedShort: return 2 - case .unsignedInt, .float: return 4 - } -} - -extension GLTF.Accessor { - func components() -> (componentsPerVector: Int, bytesPerComponent: Int, vectorSize: Int) { - let componentsPerVector = numberOfComponents(of: type) - let bytesPerComponent = bytes(of: componentType) - let vectorSize = bytesPerComponent * componentsPerVector - return (componentsPerVector, bytesPerComponent, vectorSize) - } -} - extension GLTF.Vector3 { var simd: SIMD3 { SIMD3(x: x, y: y, z: z) diff --git a/Sources/VRMRealityKit/MToon/MToonMaterialParameters.swift b/Sources/VRMRealityKit/MToon/MToonMaterialParameters.swift new file mode 100644 index 00000000..6787ceca --- /dev/null +++ b/Sources/VRMRealityKit/MToon/MToonMaterialParameters.swift @@ -0,0 +1,349 @@ +#if canImport(RealityKit) +import Foundation +import Metal +import RealityKit +import simd +import VRMKit +import VRMKitRuntime + +/// Rows of the MToon parameter texture, in the order the shader indexes them. +/// This enum is the single source of truth for the layout: `MToon.metal` +/// mirrors it as `mtoonRow*` constants, and a test compares the two. +enum MToonParameterRow: Int, CaseIterable { + case baseColor + case shadeColor + case rimColor + case matcapColor + case outlineColor + case shadeParams + case rimParams + case outlineParams + case uvAnimation + case featureFlags + case extraFlags + case emissiveFactor + case lightColor + case ambientColor + case uvTransform + case uvTransformRotation + case normalParameters +} + +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +struct MToonMaterialParameters { + static let defaultLightDirection = simd_normalize(SIMD3(0.35, 0.55, 0.75)) + static let baseParameterRowCount = MToonParameterRow.allCases.count + static let samplerRowCount = MToonTextureSlot.allCases.count + static let textureRowCount = baseParameterRowCount + samplerRowCount + /// The glTF default sampler: REPEAT on both axes, linear magnification, + /// trilinear minification. + static let defaultSampler = SIMD4(0, 0, Float(MToonSamplerFilter.default.index), 0) + + var baseColor: SIMD4 + var shadeColor: SIMD4 + var rimColor: SIMD4 + var matcapColor: SIMD4 + var outlineColor: SIMD4 + var shadeParams: SIMD4 + var rimParams: SIMD4 + var outlineParams: SIMD4 + var uvAnimation: SIMD4 + var featureFlags: SIMD4 + var extraFlags: SIMD4 + var emissiveFactor: SIMD4 + var lightColor = SIMD4(1, 1, 1, 1) + var ambientColor = SIMD4(0, 0, 0, 1) + var uvTransform = SIMD4(1, 1, 0, 0) + var uvTransformRotation = SIMD4(1, 0, 0, 0) + var normalParameters: SIMD4 + var samplers = Array(repeating: MToonMaterialParameters.defaultSampler, + count: MToonMaterialParameters.samplerRowCount) + var lightDirection: SIMD3 = MToonMaterialParameters.defaultLightDirection + + init(_ mtoon: MToonMaterialDescriptor) { + baseColor = mtoon.baseColorFactor + shadeColor = mtoon.shadeColorFactor + rimColor = mtoon.parametricRimColorFactor + matcapColor = SIMD4(mtoon.matcapFactor, 1) + outlineColor = mtoon.outlineColorFactor + emissiveFactor = SIMD4(mtoon.emissiveFactor, 1) + // z keeps the rows a faithful copy of the material, but the shader has + // no use for giEqualizationFactor: equalizing GI needs a direction + // dependent ambient term, and VRMEntity exposes one uniform color. + shadeParams = SIMD4(mtoon.shadingShiftFactor, + mtoon.shadingToonyFactor, + mtoon.giEqualizationFactor, + mtoon.alphaCutoff) + rimParams = SIMD4(mtoon.parametricRimFresnelPowerFactor, + mtoon.parametricRimLiftFactor, + mtoon.rimLightingMixFactor, + 0) + // w is unused: the outline shaders only ever run on the outline material, + // which VRMEntityLoader creates for materials that have an outline. + outlineParams = SIMD4(mtoon.outlineWidthFactor, + mtoon.outlineWidthMode.mtoonRawValue, + mtoon.outlineLightingMixFactor, + 0) + uvAnimation = SIMD4(mtoon.uvAnimationScrollXSpeedFactor, + mtoon.uvAnimationScrollYSpeedFactor, + mtoon.uvAnimationRotationSpeedFactor, + mtoon.shadingShiftTextureScale) + featureFlags = SIMD4(mtoon.matcapTexture == nil ? 0 : 1, + mtoon.rimMultiplyTexture == nil ? 0 : 1, + mtoon.shadingShiftTexture == nil ? 0 : 1, + mtoon.uvAnimationMaskTexture == nil ? 0 : 1) + extraFlags = SIMD4(mtoon.normalTexture == nil ? 0 : 1, + mtoon.shadeMultiplyTexture == nil ? 0 : 1, + mtoon.emissiveTexture == nil ? 0 : 1, + mtoon.alphaMode.mtoonRawValue) + // y flags an outlineWidthMultiplyTexture so the outline geometry modifier + // can skip its mask sample when there is none. + normalParameters = SIMD4(mtoon.normalScale, + mtoon.outlineWidthMultiplyTexture == nil ? 0 : 1, + 0, + 0) + } + + // UV animation time is read from params.uniforms().time() on the GPU, + // so custom.value only carries the light direction. + var customValue: SIMD4 { + SIMD4(lightDirection, 0) + } + + mutating func setColor(_ color: SIMD4, + for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) { + switch type { + case .color: + baseColor = color + case .shadeColor: + shadeColor = color + case .matcapColor: + matcapColor = color + case .rimColor: + rimColor = color + case .outlineColor: + outlineColor = color + case .emissionColor: + emissiveFactor = SIMD4(color.x, color.y, color.z, emissiveFactor.w) + } + } + + func color(for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> SIMD4 { + switch type { + case .color: + return baseColor + case .shadeColor: + return shadeColor + case .matcapColor: + return matcapColor + case .rimColor: + return rimColor + case .outlineColor: + return outlineColor + case .emissionColor: + return emissiveFactor + } + } + + mutating func setTextureTransform(scale: SIMD2, + offset: SIMD2, + rotation: Float) { + uvTransform = SIMD4(scale.x, scale.y, offset.x, offset.y) + uvTransformRotation = SIMD4(cos(rotation), sin(rotation), 0, 0) + } + + /// The UV transform the shader currently applies. MToon materials own it in + /// the parameter rows rather than in `CustomMaterial.textureCoordinateTransform`. + var textureTransform: MaterialParameterTypes.TextureCoordinateTransform { + MaterialParameterTypes.TextureCoordinateTransform( + offset: SIMD2(uvTransform.z, uvTransform.w), + scale: SIMD2(uvTransform.x, uvTransform.y), + rotation: atan2(uvTransformRotation.y, uvTransformRotation.x) + ) + } + + mutating func setSampler(_ sampler: SIMD4, for slot: MToonTextureSlot) { + samplers[slot.rawValue] = sampler + } + + func value(for row: MToonParameterRow) -> SIMD4 { + switch row { + case .baseColor: return baseColor + case .shadeColor: return shadeColor + case .rimColor: return rimColor + case .matcapColor: return matcapColor + case .outlineColor: return outlineColor + case .shadeParams: return shadeParams + case .rimParams: return rimParams + case .outlineParams: return outlineParams + case .uvAnimation: return uvAnimation + case .featureFlags: return featureFlags + case .extraFlags: return extraFlags + case .emissiveFactor: return emissiveFactor + case .lightColor: return lightColor + case .ambientColor: return ambientColor + case .uvTransform: return uvTransform + case .uvTransformRotation: return uvTransformRotation + case .normalParameters: return normalParameters + } + } + + @MainActor + func textureResource() throws -> TextureResource { + // Sampler rows follow the base rows, indexed by MToonTextureSlot.rawValue. + let rows = MToonParameterRow.allCases.map(value(for:)) + samplers + precondition(rows.count == Self.textureRowCount) + let data = rows.withUnsafeBufferPointer { Data(buffer: $0) } + let mip = TextureResource.Contents.MipmapLevel.mip( + data: data, + bytesPerRow: MemoryLayout>.stride * rows.count + ) + return try TextureResource(dimensions: .dimensions(width: rows.count, height: 1), + format: .raw(pixelFormat: .rgba32Float), + contents: .init(mipmapLevels: [mip])) + } +} + +/// The filter half of a sampler parameter row. +/// +/// glTF's `magFilter` and `minFilter` are independent, and `minFilter` itself +/// encodes both the minification texel filter and the mip filter. `MToon.metal` +/// reads ``index`` and applies all three to the sample itself, because the 16 +/// constant samplers a Metal entry point allows are already spent on addressing +/// modes and shared with the shaders RealityKit generates. +struct MToonSamplerFilter { + enum TexelFilter: Int, CaseIterable { + case linear + case nearest + } + + /// Mirrors `MTLSamplerMipFilter`, including glTF's non-mipmapped filters. + enum MipFilter: Int, CaseIterable { + case none + case nearest + case linear + } + + var magnification: TexelFilter = .linear + var minification: TexelFilter = .linear + var mip: MipFilter = .linear + + /// The encoding `mtoonFilteredSample` in `MToon.metal` decodes. + var index: Int { + (magnification.rawValue * TexelFilter.allCases.count + minification.rawValue) + * MipFilter.allCases.count + mip.rawValue + } + + /// The glTF default sampler: linear magnification and trilinear minification. + static let `default` = MToonSamplerFilter() + + static let count = TexelFilter.allCases.count * TexelFilter.allCases.count * MipFilter.allCases.count +} + +extension MToonSamplerFilter.MipFilter { + init(_ mipFilter: MTLSamplerMipFilter) { + switch mipFilter { + case .nearest: self = .nearest + case .linear: self = .linear + case .notMipmapped: self = .none + @unknown default: self = .linear + } + } +} + +/// MToon texture slots. The raw value is also the sampler parameter row the +/// shader reads for this slot (see `mtoonSamplerParameter` in MToon.metal). +enum MToonTextureSlot: Int, CaseIterable { + case base + case shade + case shadingShift + case normal + case matcap + case emissive + case rim + case outlineWidth + case uvAnimationMask + + /// Neutral texture bound when a material omits this slot, so the shader can + /// sample unconditionally. + enum Fallback { + case white + case neutralNormal + } + + var semantic: TextureResource.Semantic { + switch self { + case .shadingShift, .outlineWidth, .uvAnimationMask: + return .raw + case .normal: + return .normal + case .base, .shade, .matcap, .emissive, .rim: + return .color + } + } + + var fallback: Fallback { + switch self { + case .normal: + return .neutralNormal + case .base, .shade, .shadingShift, .matcap, .emissive, .rim, .outlineWidth, .uvAnimationMask: + return .white + } + } +} + +extension MToonMaterialDescriptor { + /// The descriptor texture bound to `slot`. This is the single place the + /// slot → MToon texture pairing is written down. + func texture(for slot: MToonTextureSlot) -> Texture? { + switch slot { + case .base: return baseColorTexture + case .shade: return shadeMultiplyTexture + case .shadingShift: return shadingShiftTexture + case .normal: return normalTexture + case .matcap: return matcapTexture + case .emissive: return emissiveTexture + case .rim: return rimMultiplyTexture + case .outlineWidth: return outlineWidthMultiplyTexture + case .uvAnimationMask: return uvAnimationMaskTexture + } + } +} + +#if !os(visionOS) +extension MToonMaterialDescriptor.CullMode { + var faceCulling: CustomMaterial.FaceCulling { + switch self { + case .none: return .none + case .front: return .front + case .back: return .back + } + } +} +#endif + +private extension MToonMaterialDescriptor.OutlineWidthMode { + /// Encoding read back by MToon.metal as `outlineParams.y` (> 1.5h means screen space). + var mtoonRawValue: Float { + switch self { + case .none: return 0 + case .worldCoordinates: return 1 + case .screenCoordinates: return 2 + } + } +} + +private extension GLTF.Material.AlphaMode { + var mtoonRawValue: Float { + switch self { + case .OPAQUE: + return 0 + case .MASK: + return 1 + case .BLEND: + return 2 + } + } +} + +#endif diff --git a/Sources/VRMRealityKit/MToon/MToonShaderLibraryLoader.swift b/Sources/VRMRealityKit/MToon/MToonShaderLibraryLoader.swift new file mode 100644 index 00000000..a15ba2c7 --- /dev/null +++ b/Sources/VRMRealityKit/MToon/MToonShaderLibraryLoader.swift @@ -0,0 +1,88 @@ +#if canImport(RealityKit) +import Foundation +import Metal + +enum MToonShaderLibraryLoaderError: Error { + case noMetalDevice + case unsupportedPlatform + case resourceMissing(String) + case loadFailed(String, Error) + case requiredFunctionsMissing(Set) +} + +/// Loads the precompiled platform-specific MToon Metal library bundled as a +/// package resource. +/// +/// The metallibs are built offline by scripts/build-mtoon-metallibs.sh so that +/// nothing depends on the consumer's build system compiling the package's +/// .metal source, which `swift build` does not support. +@MainActor +enum MToonShaderLibraryLoader { + static let requiredFunctions: Set = [ + "mtoonSurface", + "mtoonOutlineSurface", + "mtoonOutlineGeometry" + ] + + static var resourceName: String? { +#if os(macOS) && !targetEnvironment(macCatalyst) + return "MToon-macos" +#elseif os(iOS) && targetEnvironment(simulator) + return "MToon-iossim" +#elseif os(iOS) && !targetEnvironment(macCatalyst) + return "MToon-ios" +#else + // No precompiled MToon library is bundled for this platform + // (e.g. Mac Catalyst, visionOS); MToon rendering falls back to UnlitMaterial. + return nil +#endif + } + + /// The bundle the precompiled metallibs ship in. + static var resourceBundle: Bundle { .module } + + // Both success and failure are cached so repeated material creation does + // not recreate Metal devices or re-attempt a load that cannot succeed. + private static var cachedResult: Result? + + static func loadDefault() throws -> MTLLibrary { + if let cachedResult { + return try cachedResult.get() + } + let result = Result { () throws -> MTLLibrary in + // Check the statically-known failure before creating a device. + guard resourceName != nil else { + throw MToonShaderLibraryLoaderError.unsupportedPlatform + } + guard let device = MTLCreateSystemDefaultDevice() else { + throw MToonShaderLibraryLoaderError.noMetalDevice + } + return try load(device: device) + } + cachedResult = result + return try result.get() + } + + static func load(device: MTLDevice) throws -> MTLLibrary { + guard let resourceName else { + throw MToonShaderLibraryLoaderError.unsupportedPlatform + } + guard let libraryURL = resourceBundle.url(forResource: resourceName, withExtension: "metallib") else { + throw MToonShaderLibraryLoaderError.resourceMissing(resourceName) + } + + let library: MTLLibrary + do { + library = try device.makeLibrary(URL: libraryURL) + } catch { + throw MToonShaderLibraryLoaderError.loadFailed(resourceName, error) + } + + let missing = requiredFunctions.subtracting(Set(library.functionNames)) + guard missing.isEmpty else { + throw MToonShaderLibraryLoaderError.requiredFunctionsMissing(missing) + } + return library + } +} +#endif diff --git a/Sources/VRMRealityKit/Resources/MToon-ios.metallib b/Sources/VRMRealityKit/Resources/MToon-ios.metallib new file mode 100644 index 00000000..dfd36d55 Binary files /dev/null and b/Sources/VRMRealityKit/Resources/MToon-ios.metallib differ diff --git a/Sources/VRMRealityKit/Resources/MToon-iossim.metallib b/Sources/VRMRealityKit/Resources/MToon-iossim.metallib new file mode 100644 index 00000000..543b8064 Binary files /dev/null and b/Sources/VRMRealityKit/Resources/MToon-iossim.metallib differ diff --git a/Sources/VRMRealityKit/Resources/MToon-macos.metallib b/Sources/VRMRealityKit/Resources/MToon-macos.metallib new file mode 100644 index 00000000..90c8d0bc Binary files /dev/null and b/Sources/VRMRealityKit/Resources/MToon-macos.metallib differ diff --git a/Sources/VRMRealityKit/Shaders/MToon.metal b/Sources/VRMRealityKit/Shaders/MToon.metal new file mode 100644 index 00000000..e204c7e6 --- /dev/null +++ b/Sources/VRMRealityKit/Shaders/MToon.metal @@ -0,0 +1,614 @@ +#include + +#include "MToonCore.h" + +using namespace metal; + +// This file is the RealityKit adapter for the MToon specification math in +// MToonCore.h. Functions named realityKitApproximate* are approximations +// imposed by RealityKit's CustomMaterial constraints, not MToon semantics. + +// Metal allows at most 16 constant samplers per shader entry point, and the +// budget is shared with the shaders RealityKit generates. Exceeding it only +// shows up at runtime, as a pipeline that never builds, so the samplers below +// are spent on addressing alone: a coordinate wrapped in the shader and handed +// to a clamping sampler cannot filter across a REPEAT seam. glTF's +// magnification, minification and mip filters are applied to the sample +// instead; see mtoonFilteredSample. +constexpr sampler mtoonParameterSampler(coord::normalized, + address::clamp_to_edge, + filter::nearest, + mip_filter::none); + +#define MTOON_SAMPLER(name, sMode, tMode) \ + constexpr sampler name(coord::normalized, \ + s_address::sMode, t_address::tMode, \ + mag_filter::linear, min_filter::linear, mip_filter::linear); + +MTOON_SAMPLER(mtoonRepeatRepeat, repeat, repeat) +MTOON_SAMPLER(mtoonRepeatClamp, repeat, clamp_to_edge) +MTOON_SAMPLER(mtoonRepeatMirror, repeat, mirrored_repeat) +MTOON_SAMPLER(mtoonClampRepeat, clamp_to_edge, repeat) +MTOON_SAMPLER(mtoonClampClamp, clamp_to_edge, clamp_to_edge) +MTOON_SAMPLER(mtoonClampMirror, clamp_to_edge, mirrored_repeat) +MTOON_SAMPLER(mtoonMirrorRepeat, mirrored_repeat, repeat) +MTOON_SAMPLER(mtoonMirrorClamp, mirrored_repeat, clamp_to_edge) +MTOON_SAMPLER(mtoonMirrorMirror, mirrored_repeat, mirrored_repeat) + +constant float mtoonParameterTextureWidth = 26.0; + +// Parameter rows, mirroring MToonParameterRow on the Swift side. +constant float mtoonRowBaseColor = 0.0; +constant float mtoonRowShadeColor = 1.0; +constant float mtoonRowRimColor = 2.0; +constant float mtoonRowMatcapColor = 3.0; +constant float mtoonRowOutlineColor = 4.0; +constant float mtoonRowShadeParams = 5.0; +constant float mtoonRowRimParams = 6.0; +constant float mtoonRowOutlineParams = 7.0; +constant float mtoonRowUvAnimation = 8.0; +constant float mtoonRowFeatureFlags = 9.0; +constant float mtoonRowExtraFlags = 10.0; +constant float mtoonRowEmissiveFactor = 11.0; +constant float mtoonRowLightColor = 12.0; +constant float mtoonRowAmbientColor = 13.0; +constant float mtoonRowUvTransform = 14.0; +constant float mtoonRowUvTransformRotation = 15.0; +constant float mtoonRowNormalParameters = 16.0; +constant float mtoonSamplerParameterStart = 17.0; + +// Sampler parameter slots, mirroring MToonTextureSlot on the Swift side. +constant float mtoonSamplerSlotBase = 0.0; +constant float mtoonSamplerSlotShade = 1.0; +constant float mtoonSamplerSlotShadingShift = 2.0; +constant float mtoonSamplerSlotNormal = 3.0; +constant float mtoonSamplerSlotMatcap = 4.0; +constant float mtoonSamplerSlotEmissive = 5.0; +constant float mtoonSamplerSlotRim = 6.0; +constant float mtoonSamplerSlotOutlineWidth = 7.0; +constant float mtoonSamplerSlotUvAnimationMask = 8.0; + +// The parameter texture is a 1-row lookup table, so sample it at an explicit +// LOD: an implicit-LOD sample would need derivatives from uniform control flow, +// which prevents the compiler from sinking these fetches into the branches that +// actually consume them. +half4 mtoonParameter(realitykit::texture::textures textures, float row) +{ + return textures.custom().sample(mtoonParameterSampler, + float2((row + 0.5) / mtoonParameterTextureWidth, 0.5), + level(0)); +} + +half4 mtoonSamplerParameter(realitykit::texture::textures textures, float slot) +{ + return mtoonParameter(textures, mtoonSamplerParameterStart + slot); +} + +// The LOD a sample resolves to. The query needs the screen-space derivatives only +// a fragment function has, so specializing -- rather than branching -- keeps the +// derivative instruction out of the code the vertex stage links against. +template +struct MToonSampledLOD { + static float of(texture2d texture, sampler textureSampler, float2 uv); +}; + +template <> +struct MToonSampledLOD { + static float of(texture2d texture, sampler textureSampler, float2 uv) + { + return texture.calculate_clamped_lod(textureSampler, uv); + } +}; + +template <> +struct MToonSampledLOD { + static float of(texture2d, sampler, float2) + { + return 0.0; + } +}; + +// glTF's filter modes, applied to the sample rather than baked into sampler +// state. `filterIndex` is MToonSamplerFilter.index on the Swift side: +// (magnification * 2 + minification) * 3 + mip, where the texel filters are +// 0 = linear, 1 = nearest and the mip filter is 0 = none, 1 = nearest, +// 2 = linear. +template +half4 mtoonFilteredSample(texture2d texture, + sampler textureSampler, + float2 uv, + int filterIndex) +{ + const int mipFilter = filterIndex % 3; + const bool nearestMinification = (filterIndex / 3) % 2 != 0; + const bool nearestMagnification = filterIndex / 6 != 0; + + // Sampling at an explicit LOD leaves the mip filter to this function. The + // outline geometry modifier has no implicit LOD at all, so it samples level 0. + const float sampledLod = MToonSampledLOD::of(texture, textureSampler, uv); + // glTF's NEAREST and LINEAR minFilters do not mipmap, so they are level 0; + // the MIPMAP_NEAREST filters take the nearest level; the MIPMAP_LINEAR + // filters blend the two levels the fractional LOD falls between. + const float lod = mipFilter == 0 ? 0.0 : (mipFilter == 1 ? round(sampledLod) : sampledLod); + // Magnification and minification are independent glTF filters; the LOD the + // sampler resolved is what decides which of the two applies here. + const bool nearest = sampledLod > 0.0 ? nearestMinification : nearestMagnification; + + float2 sampleUV = uv; + if (nearest) { + // A linear sampler returns one texel exactly when the coordinate sits at + // that texel's centre, so NEAREST costs a coordinate snap instead of a + // sampler of its own. It is approximate only where two levels are blended. + const float2 levelSize = float2(texture.get_width(uint(lod)), texture.get_height(uint(lod))); + sampleUV = (floor(uv * levelSize) + 0.5) / levelSize; + } + return texture.sample(textureSampler, sampleUV, level(lod)); +} + +// The sampler parameter row is (wrapS, wrapT, filterIndex, 0). The wrap modes +// are encoded as 0 = repeat, 1 = clamp to edge, 2 = mirrored repeat by +// VRMEntityLoader.mtoonWrapMode(_:), and filterIndex by MToonSamplerFilter. +#define MTOON_SAMPLE_CASE(name, index) \ + case (index): return mtoonFilteredSample(texture, name, uv, filterIndex); + +template +half4 mtoonWrappedSample(texture2d texture, float2 uv, half4 samplerParameters) +{ + const int wrapS = int(float(samplerParameters.x) + 0.5); + const int wrapT = int(float(samplerParameters.y) + 0.5); + const int filterIndex = int(float(samplerParameters.z) + 0.5); + switch (wrapS * 3 + wrapT) { + MTOON_SAMPLE_CASE(mtoonRepeatRepeat, 0) + MTOON_SAMPLE_CASE(mtoonRepeatClamp, 1) + MTOON_SAMPLE_CASE(mtoonRepeatMirror, 2) + MTOON_SAMPLE_CASE(mtoonClampRepeat, 3) + MTOON_SAMPLE_CASE(mtoonClampClamp, 4) + MTOON_SAMPLE_CASE(mtoonClampMirror, 5) + MTOON_SAMPLE_CASE(mtoonMirrorRepeat, 6) + MTOON_SAMPLE_CASE(mtoonMirrorClamp, 7) + MTOON_SAMPLE_CASE(mtoonMirrorMirror, 8) + default: return mtoonFilteredSample(texture, mtoonRepeatRepeat, uv, filterIndex); + } +} + +// Fragment-stage sampling: the LOD comes from the screen-space derivatives. +half4 mtoonSample(texture2d texture, float2 uv, half4 samplerParameters) +{ + return mtoonWrappedSample(texture, uv, samplerParameters); +} + +// Vertex-stage sampling for the geometry modifier, which has no derivatives. +half4 mtoonVertexSample(texture2d texture, float2 uv, half4 samplerParameters) +{ + return mtoonWrappedSample(texture, uv, samplerParameters); +} + +// RealityKit tone maps every CustomMaterial draw; this inverts it. +constant float mtoonRealityKitInverseToneMap[65] = { + 0.0000, 0.0040, 0.0075, 0.0106, 0.0135, 0.0169, 0.0209, 0.0251, + 0.0298, 0.0344, 0.0395, 0.0451, 0.0512, 0.0574, 0.0641, 0.0714, + 0.0791, 0.0873, 0.0958, 0.1047, 0.1142, 0.1247, 0.1365, 0.1488, + 0.1615, 0.1747, 0.1885, 0.2025, 0.2170, 0.2324, 0.2499, 0.2682, + 0.2872, 0.3068, 0.3272, 0.3482, 0.3699, 0.3924, 0.4158, 0.4400, + 0.4661, 0.4939, 0.5225, 0.5523, 0.5805, 0.6128, 0.6528, 0.6925, + 0.7323, 0.7721, 0.8151, 0.8649, 0.9147, 0.9694, 1.0304, 1.0875, + 1.1516, 1.2234, 1.3086, 1.3939, 1.4796, 1.5861, 1.7011, 1.8490, + 2.0000 +}; + +float realityKitInverseToneMapChannel(float target) +{ + const float encoded = target <= 0.0031308f + ? target * 12.92f + : 1.055f * metal::pow(target, 1.0f / 2.4f) - 0.055f; + const float scaled = saturate(encoded) * 64.0f; + const int index = min(int(scaled), 63); + return mix(mtoonRealityKitInverseToneMap[index], + mtoonRealityKitInverseToneMap[index + 1], + scaled - float(index)); +} + +float3 realityKitInverseToneMap(float3 color) +{ + return float3(realityKitInverseToneMapChannel(color.x), + realityKitInverseToneMapChannel(color.y), + realityKitInverseToneMapChannel(color.z)); +} + +// MToon's rim term is modulated by the *lighting*, never by the surface's own +// base/shade colors, so mtoonDirectLighting()'s result cannot be reused here. +// RealityKit does not hand a CustomMaterial the scene's evaluated irradiance, +// so the runtime's explicit light stands in: toon-shaded direct light plus the +// ambient term. +float3 realityKitApproximateRimLighting(float3 lightColor, float3 giColor, float shading) +{ + return lightColor * shading + giColor; +} + +// RealityKit does not expose the fully evaluated lit term to the outline +// pass; use the runtime light color as the lit approximation. +float3 realityKitApproximateOutlineLighting(float3 lightColor, float outlineLightingMix) +{ + return mix(float3(1.0), lightColor, saturate(outlineLightingMix)); +} + +// Converts RealityKit's mesh UV (v pointing up, as VRMEntityLoader writes it) +// into the glTF / MToon UV space that KHR_texture_transform, MToon UV animation +// and Metal texture sampling all share (v pointing down). +// +// This runs once, *before* any UV math: MToon's animation and +// KHR_texture_transform are both defined in glTF UV space, so flipping +// afterwards would invert Y offsets and the rotation direction, and shift +// anything with a Y scale. +float2 mtoonTextureUV(float2 uv) +{ + return float2(uv.x, 1.0 - uv.y); +} + +float2 mtoonTransformedUV(float2 uv, half4 uvTransform, half4 uvTransformRotation) +{ + float2 transformed = uv * float2(uvTransform.xy); + float c = float(uvTransformRotation.x); + float s = float(uvTransformRotation.y); + transformed = float2(transformed.x * c - transformed.y * s, + transformed.x * s + transformed.y * c); + return transformed + float2(uvTransform.zw); +} + +float3 mtoonLightDirection(float4 customValue) +{ + // VRMEntity always sends a normalized direction, so this only guards against + // an uninitialized custom value; renormalizing would cost every fragment. + if (all(customValue.xyz == 0.0)) { + return float3(0.0, 0.0, 1.0); + } + return customValue.xyz; +} + +float3 mtoonShadingNormal(realitykit::surface_parameters params, + float2 uv, + half4 extraFlags, + half normalScale, + half4 normalSampler) +{ + float3 geometryNormal = normalize(params.geometry().normal()); + if (extraFlags.x < 0.5h) { + return geometryNormal; + } + half3 tangentNormal = realitykit::unpack_normal(mtoonSample(params.textures().normal(), uv, normalSampler).rgb, + normalScale); + float3 rawTangent = params.geometry().tangent(); + float3 rawBitangent = params.geometry().bitangent(); + if (dot(rawTangent, rawTangent) < 0.000001 || dot(rawBitangent, rawBitangent) < 0.000001) { + return geometryNormal; + } + float3 tangent = normalize(rawTangent); + float3 bitangent = normalize(rawBitangent); + return normalize(tangent * float(tangentNormal.x) + + bitangent * float(tangentNormal.y) + + geometryNormal * float(tangentNormal.z)); +} + +float mtoonAlpha(float alphaMode, float baseAlpha, float cutoff) +{ + if (alphaMode < 0.5) { + return 1.0; + } + if (alphaMode < 1.5) { + if (baseAlpha < cutoff) { + discard_fragment(); + } + return 1.0; + } + return baseAlpha; +} + +// Both surface entry points resolve opacity and write their result the same way. +float mtoonOpacity(float opacityThreshold, + half4 baseSample, + half4 baseColorFactor, + half4 extraFlags, + half4 shadeParams) +{ + const float cutoff = opacityThreshold > 0.0 ? opacityThreshold : float(shadeParams.w); + return mtoonAlpha(float(extraFlags.w), float(baseSample.a * baseColorFactor.a), cutoff); +} + +template +float2 mtoonAnimatedUVImpl(realitykit::texture::textures textures, + float time, + float2 uv, + half4 uvAnimation, + half4 featureFlags, + half4 uvAnimationMaskSampler, + half4 uvTransform, + half4 uvTransformRotation) +{ + // Most materials animate nothing, so skip the mask sample and the rotation. + if (all(uvAnimation.xyz == 0.0h)) { + return uv; + } + + // `uv` is already in glTF UV space, so the mask only needs the transform. + float mask = 1.0; + if (featureFlags.w > 0.5h) { + float2 maskUV = mtoonTransformedUV(uv, uvTransform, uvTransformRotation); + mask = float(mtoonWrappedSample(textures.ambient_occlusion(), maskUV, uvAnimationMaskSampler).b); + } + + // Scrolling without rotation is the common case, so the rotation is its own + // branch rather than a sin/cos of a zero angle. + float2 animated = uv; + if (uvAnimation.z != 0.0h) { + float angle = float(uvAnimation.z) * time * mask; + float2 center = float2(0.5, 0.5); + float2 centered = uv - center; + float s = sin(angle); + float c = cos(angle); + animated = float2(centered.x * c - centered.y * s, + centered.x * s + centered.y * c) + center; + } + return animated + float2(float(uvAnimation.x), float(uvAnimation.y)) * time * mask; +} + +float2 mtoonAnimatedUV(realitykit::texture::textures textures, + float time, + float2 uv, + half4 uvAnimation, + half4 featureFlags, + half4 uvAnimationMaskSampler, + half4 uvTransform, + half4 uvTransformRotation) +{ + return mtoonAnimatedUVImpl(textures, time, uv, uvAnimation, featureFlags, + uvAnimationMaskSampler, uvTransform, uvTransformRotation); +} + +// The geometry modifier's counterpart: same animation, sampled at level 0. +float2 mtoonVertexAnimatedUV(realitykit::texture::textures textures, + float time, + float2 uv, + half4 uvAnimation, + half4 featureFlags, + half4 uvAnimationMaskSampler, + half4 uvTransform, + half4 uvTransformRotation) +{ + return mtoonAnimatedUVImpl(textures, time, uv, uvAnimation, featureFlags, + uvAnimationMaskSampler, uvTransform, uvTransformRotation); +} + +[[visible]] +void mtoonSurface(realitykit::surface_parameters params) +{ + auto textures = params.textures(); + auto surface = params.surface(); + auto material = params.material_constants(); + + half4 baseColorFactor = mtoonParameter(textures, mtoonRowBaseColor); + half4 shadeColorFactor = mtoonParameter(textures, mtoonRowShadeColor); + half4 rimColorFactor = mtoonParameter(textures, mtoonRowRimColor); + half4 matcapFactor = mtoonParameter(textures, mtoonRowMatcapColor); + half4 shadeParams = mtoonParameter(textures, mtoonRowShadeParams); + half4 rimParams = mtoonParameter(textures, mtoonRowRimParams); + half4 uvAnimation = mtoonParameter(textures, mtoonRowUvAnimation); + half4 featureFlags = mtoonParameter(textures, mtoonRowFeatureFlags); + half4 extraFlags = mtoonParameter(textures, mtoonRowExtraFlags); + half4 emissiveFactor = mtoonParameter(textures, mtoonRowEmissiveFactor); + half4 lightColorParameter = mtoonParameter(textures, mtoonRowLightColor); + half4 giColorParameter = mtoonParameter(textures, mtoonRowAmbientColor); + half4 uvTransform = mtoonParameter(textures, mtoonRowUvTransform); + half4 uvTransformRotation = mtoonParameter(textures, mtoonRowUvTransformRotation); + half4 normalParameters = mtoonParameter(textures, mtoonRowNormalParameters); + half4 baseSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotBase); + half4 shadeSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotShade); + half4 shadingShiftSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotShadingShift); + half4 normalSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotNormal); + half4 matcapSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotMatcap); + half4 emissiveSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotEmissive); + half4 rimSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotRim); + + half4 uvAnimationMaskSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotUvAnimationMask); + // UV animation time comes from RealityKit's per-frame uniforms; no CPU-side + // material update is required to advance the animation. + float2 uv = mtoonAnimatedUV(textures, + params.uniforms().time(), + mtoonTextureUV(params.geometry().uv0()), + uvAnimation, + featureFlags, + uvAnimationMaskSampler, + uvTransform, + uvTransformRotation); + uv = mtoonTransformedUV(uv, uvTransform, uvTransformRotation); + + half4 baseSample = mtoonSample(textures.base_color(), uv, baseSampler); + half4 shadeSample = extraFlags.y > 0.5h + ? mtoonSample(textures.roughness(), uv, shadeSampler) + : half4(1.0h); + + float shift = float(shadeParams.x); + if (featureFlags.z > 0.5h) { + half shadingShift = mtoonSample(textures.specular(), uv, shadingShiftSampler).r; + shift += float(shadingShift) * float(uvAnimation.w); + } + + float3 normal = mtoonShadingNormal(params, uv, extraFlags, normalParameters.x, normalSampler); + float3 lightDirection = mtoonLightDirection(params.uniforms().custom_parameter()); + float shadingToony = clamp(float(shadeParams.y), 0.0, 1.0); + float shading = mtoonShading(normal, lightDirection, shift, shadingToony); + + float3 litColor = float3(baseSample.rgb * baseColorFactor.rgb); + float3 shadeColor = float3(shadeSample.rgb * shadeColorFactor.rgb); + float3 lightColor = float3(lightColorParameter.rgb); + // MToon equalizes GI between the raw normal-direction sample and a + // direction-independent one. VRMEntity exposes a single uniform ambient + // color, so both samples are that color and the equalization is the identity. + float3 giColor = float3(giColorParameter.rgb); + + float3 direct = mtoonDirectLighting(litColor, shadeColor, shading, lightColor); + float3 indirect = mtoonIndirectLighting(litColor, giColor); + float3 color = direct + indirect; + + // Without a matcap and with a black parametric rim color the whole rim term + // is zero, so skip it (the majority of MToon materials). + if (featureFlags.x > 0.5h || any(rimColorFactor.rgb > 0.0h)) { + float3 rim = float3(0.0); + // `normal` and view_direction() are both world-space, which is what + // lets the matcap, the parametric rim and the shading term share one + // normal without any change of basis. + float3 viewDirection = normalize(params.geometry().view_direction()); + if (featureFlags.x > 0.5h) { + float2 matcapUV = mtoonTextureUV(mtoonMatcapUV(normal, viewDirection)); + rim += float3(mtoonSample(textures.metallic(), matcapUV, matcapSampler).rgb * matcapFactor.rgb); + } + + if (any(rimColorFactor.rgb > 0.0h)) { + float parametricRim = mtoonParametricRim(normal, viewDirection, float(rimParams.x), float(rimParams.y)); + rim += parametricRim * float3(rimColorFactor.rgb); + } + + if (featureFlags.y > 0.5h) { + rim *= float3(mtoonSample(textures.clearcoat_roughness(), uv, rimSampler).rgb); + } + float3 rimLighting = realityKitApproximateRimLighting(lightColor, giColor, shading); + rim *= mix(float3(1.0), rimLighting, clamp(float(rimParams.z), 0.0, 1.0)); + color += rim; + } + + float3 emissiveTexture = extraFlags.z > 0.5h + ? float3(mtoonSample(textures.emissive_color(), uv, emissiveSampler).rgb) + : float3(1.0); + color += float3(emissiveFactor.rgb) * emissiveTexture; + + float opacity = mtoonOpacity(material.opacity_threshold(), baseSample, baseColorFactor, extraFlags, shadeParams); + + surface.set_base_color(half3(0.0h)); + surface.set_emissive_color(half3(realityKitInverseToneMap(color))); + surface.set_opacity(half(opacity)); + surface.set_roughness(1.0h); + surface.set_metallic(0.0h); +} + +[[visible]] +void mtoonOutlineSurface(realitykit::surface_parameters params) +{ + auto textures = params.textures(); + auto surface = params.surface(); + auto material = params.material_constants(); + half4 outlineColor = mtoonParameter(textures, mtoonRowOutlineColor); + half4 shadeParams = mtoonParameter(textures, mtoonRowShadeParams); + half4 outlineParams = mtoonParameter(textures, mtoonRowOutlineParams); + half4 uvAnimation = mtoonParameter(textures, mtoonRowUvAnimation); + half4 featureFlags = mtoonParameter(textures, mtoonRowFeatureFlags); + half4 extraFlags = mtoonParameter(textures, mtoonRowExtraFlags); + half4 lightColorParameter = mtoonParameter(textures, mtoonRowLightColor); + half4 uvTransform = mtoonParameter(textures, mtoonRowUvTransform); + half4 uvTransformRotation = mtoonParameter(textures, mtoonRowUvTransformRotation); + half4 baseSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotBase); + half4 uvAnimationMaskSampler = mtoonSamplerParameter(textures, mtoonSamplerSlotUvAnimationMask); + + // Opaque outlines have opacity 1 regardless of the base texture, so the UV + // chain and the base-color sample only run for MASK / BLEND materials. + float opacity = 1.0; + if (extraFlags.w > 0.5h) { + // UV animation time comes from RealityKit's per-frame uniforms. + float2 uv = mtoonAnimatedUV(textures, + params.uniforms().time(), + mtoonTextureUV(params.geometry().uv0()), + uvAnimation, + featureFlags, + uvAnimationMaskSampler, + uvTransform, + uvTransformRotation); + uv = mtoonTransformedUV(uv, uvTransform, uvTransformRotation); + half4 baseSample = mtoonSample(textures.base_color(), uv, baseSampler); + half4 baseColorFactor = mtoonParameter(textures, mtoonRowBaseColor); + opacity = mtoonOpacity(material.opacity_threshold(), baseSample, baseColorFactor, extraFlags, shadeParams); + } + float3 outlineLit = realityKitApproximateOutlineLighting(float3(lightColorParameter.rgb), float(outlineParams.z)); + float3 finalColor = float3(outlineColor.rgb) * outlineLit; + + surface.set_base_color(half3(0.0h)); + surface.set_emissive_color(half3(realityKitInverseToneMap(finalColor))); + surface.set_opacity(half(opacity)); + surface.set_roughness(1.0h); + surface.set_metallic(0.0h); +} + +// RealityKit's geometry_parameters exposes projection matrices but no viewport +// height, so screen-coordinate outline width is treated as a fraction of +// normalized screen height rather than a pixel count. +float realityKitApproximateScreenOutlineWidth(realitykit::geometry_parameters params, float width, float3 modelNormal) +{ + float4x4 modelToView = params.uniforms().model_to_view(); + float4x4 viewToProjection = params.uniforms().view_to_projection(); + float4 viewPosition = modelToView * float4(params.geometry().model_position(), 1.0); + float3 viewNormal = normalize((modelToView * float4(modelNormal, 0.0)).xyz); + float4 clipPosition = viewToProjection * viewPosition; + float4 offsetClipPosition = viewToProjection * (viewPosition + float4(viewNormal, 0.0)); + float clipW = clipPosition.w; + if (abs(clipW) < mtoonEpsilon) { + clipW = clipW < 0.0 ? -mtoonEpsilon : mtoonEpsilon; + } + float offsetClipW = offsetClipPosition.w; + if (abs(offsetClipW) < mtoonEpsilon) { + offsetClipW = offsetClipW < 0.0 ? -mtoonEpsilon : mtoonEpsilon; + } + float2 ndc = clipPosition.xy / clipW; + float2 offsetNdc = offsetClipPosition.xy / offsetClipW; + float ndcPerModelUnit = length(offsetNdc - ndc); + if (ndcPerModelUnit < mtoonEpsilon) { + return 0.0; + } + return (width * 2.0) / ndcPerModelUnit; +} + +[[visible]] +void mtoonOutlineGeometry(realitykit::geometry_parameters params) +{ + half4 outlineParams = mtoonParameter(params.textures(), mtoonRowOutlineParams); + + // Without an outlineWidthMultiplyTexture the mask is a 1x1 white fallback, + // so skip the UV work and the fetch entirely. + float widthMask = 1.0; + half4 normalParameters = mtoonParameter(params.textures(), mtoonRowNormalParameters); + if (normalParameters.y > 0.5h) { + half4 uvTransform = mtoonParameter(params.textures(), mtoonRowUvTransform); + half4 uvTransformRotation = mtoonParameter(params.textures(), mtoonRowUvTransformRotation); + half4 uvAnimation = mtoonParameter(params.textures(), mtoonRowUvAnimation); + half4 featureFlags = mtoonParameter(params.textures(), mtoonRowFeatureFlags); + half4 uvAnimationMaskSampler = mtoonSamplerParameter(params.textures(), mtoonSamplerSlotUvAnimationMask); + // Computed locally for the width mask only: mtoonOutlineSurface applies + // the UV animation and transform itself, so writing the transformed UV + // back to uv0 would apply it twice. + float2 widthUV = mtoonVertexAnimatedUV(params.textures(), + params.uniforms().time(), + mtoonTextureUV(params.geometry().uv0()), + uvAnimation, + featureFlags, + uvAnimationMaskSampler, + uvTransform, + uvTransformRotation); + widthUV = mtoonTransformedUV(widthUV, uvTransform, uvTransformRotation); + + half4 outlineWidthSampler = mtoonSamplerParameter(params.textures(), mtoonSamplerSlotOutlineWidth); + widthMask = float(mtoonVertexSample(params.textures().clearcoat(), widthUV, outlineWidthSampler).g); + } + float width = max(0.0, float(outlineParams.x)) * widthMask; + float3 modelNormal = normalize(params.geometry().normal()); + if (outlineParams.y > 1.5h) { + // Screen coordinates: the width is resolved into a model-space offset. + params.geometry().set_model_position_offset( + modelNormal * realityKitApproximateScreenOutlineWidth(params, width, modelNormal)); + return; + } + // World coordinates: MToon defines the width as a distance in meters, so it + // must not inherit the entity's scale. Offsetting in world space keeps the + // outline the same thickness under any (including non-uniform) scale. + float3 worldNormal = params.uniforms().normal_to_world() * modelNormal; + float worldNormalLength = length(worldNormal); + if (worldNormalLength < mtoonEpsilon) { + return; + } + params.geometry().set_world_position_offset(worldNormal * (width / worldNormalLength)); +} diff --git a/Sources/VRMRealityKit/Shaders/MToonCore.h b/Sources/VRMRealityKit/Shaders/MToonCore.h new file mode 100644 index 00000000..4e9d8c80 --- /dev/null +++ b/Sources/VRMRealityKit/Shaders/MToonCore.h @@ -0,0 +1,67 @@ +#ifndef MTOON_CORE_H +#define MTOON_CORE_H + +// Pure VRMC_materials_mtoon 1.0 math. This header must stay free of +// RealityKit types so that the MToon specification layer can be read and +// verified independently of RealityKit-specific approximations, which live +// in MToon.metal as realityKitApproximate* functions. + +#include + +constant float mtoonEpsilon = 0.00001; + +inline float mtoonLinearstep(float minValue, float maxValue, float value) +{ + return metal::saturate((value - minValue) / metal::max(maxValue - minValue, mtoonEpsilon)); +} + +// https://github.com/vrm-c/vrm-specification/tree/master/specification/VRMC_materials_mtoon-1.0#shading-shift +inline float mtoonShading(float3 normal, float3 lightDirection, float shadingShift, float shadingToony) +{ + return mtoonLinearstep(-1.0 + shadingToony, + 1.0 - shadingToony, + metal::dot(normal, lightDirection) + shadingShift); +} + +// Direct lighting: base and shade colors are mixed by the shading value and +// modulated by the light color. +inline float3 mtoonDirectLighting(float3 litColor, float3 shadeColor, float shading, float3 lightColor) +{ + return metal::mix(shadeColor, litColor, shading) * lightColor; +} + +// Global illumination: the lit color is modulated by the (equalized) GI color. +inline float3 mtoonIndirectLighting(float3 litColor, float3 giColor) +{ + return litColor * giColor; +} + +// Matcap UV, in the specification's UV convention (v pointing up). +// +// The basis is built from the view direction rather than from a view matrix, so +// `normal` and `viewDirection` only have to agree with each other — both are +// world-space here, the same space the rim and shading terms use. +// https://github.com/vrm-c/vrm-specification/tree/master/specification/VRMC_materials_mtoon-1.0#matcap +inline metal::float2 mtoonMatcapUV(float3 normal, float3 viewDirection) +{ + float3 worldViewX = float3(viewDirection.z, 0.0, -viewDirection.x); + const float horizontalLength = metal::length(worldViewX); + if (horizontalLength < mtoonEpsilon) { + // Looking straight along world up or down leaves no horizontal axis to + // build the basis from; the matcap centre is the stable choice. + return metal::float2(0.5, 0.5); + } + worldViewX /= horizontalLength; + const float3 worldViewY = metal::cross(viewDirection, worldViewX); + return metal::float2(metal::dot(worldViewX, normal), + metal::dot(worldViewY, normal)) * 0.495 + 0.5; +} + +// Parametric rim term before the rim-multiply texture and lighting mix. +inline float mtoonParametricRim(float3 normal, float3 viewDirection, float rimFresnelPower, float rimLift) +{ + const float rimBase = metal::saturate(1.0 - metal::dot(normal, viewDirection) + rimLift); + return metal::pow(rimBase, metal::max(rimFresnelPower, mtoonEpsilon)); +} + +#endif diff --git a/Sources/VRMRealityKit/Shaders/MToonMetallibInputs.txt b/Sources/VRMRealityKit/Shaders/MToonMetallibInputs.txt new file mode 100644 index 00000000..be274ef2 --- /dev/null +++ b/Sources/VRMRealityKit/Shaders/MToonMetallibInputs.txt @@ -0,0 +1,8 @@ +msl-std=metal2.4 +flags=-Wall -Wextra -Werror +target=macosx|macos-|-mmacosx-version-min=12.0|MToon-macos.metallib +target=iphoneos|ios-|-mios-version-min=15.0|MToon-ios.metallib +target=iphonesimulator|ios-|-miphonesimulator-version-min=15.0|MToon-iossim.metallib +fe9fe1508ea72c766b169f7fa1318840b7b48d1de0f6065b2c16e1ad0665089c Sources/VRMRealityKit/Shaders/MToonCore.h +bc474b74ef0b2f812ec1e9c4120916531b92065a963bc1cf46e629c0cd7b28d5 Sources/VRMRealityKit/Shaders/MToon.metal +83dc27b262e43b5c83b6ee8b4c8b5702acc533eb3a7deda108ac548786d7bce9 scripts/build-mtoon-metallibs.sh diff --git a/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift b/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift index 9cf5236d..745c1d72 100644 --- a/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift +++ b/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift @@ -4,19 +4,46 @@ import VRMKit @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) extension VRMEntityLoader { - public convenience init(withURL url: URL, rootDirectory: URL? = nil) throws { + /// Loads a VRM from a file URL. + /// + /// - Parameters: + /// - url: VRM file location. + /// - rootDirectory: Optional base directory for external glTF resources. + /// - isMToonEnabled: When `false`, MToon is fully disabled and Unlit / PBR fallbacks are used. + /// - isOutlineEnabled: Controls creation of MToon outline entities. + public convenience init(withURL url: URL, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true) throws { let vrm = try VRMLoader().load(withURL: url) - self.init(vrm: vrm, rootDirectory: rootDirectory) + self.init(vrm: vrm, + rootDirectory: rootDirectory, + isMToonEnabled: isMToonEnabled, + isOutlineEnabled: isOutlineEnabled) } - public convenience init(named: String, rootDirectory: URL? = nil) throws { + /// Loads a bundled VRM resource. + public convenience init(named: String, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true) throws { let vrm = try VRMLoader().load(named: named) - self.init(vrm: vrm, rootDirectory: rootDirectory) + self.init(vrm: vrm, + rootDirectory: rootDirectory, + isMToonEnabled: isMToonEnabled, + isOutlineEnabled: isOutlineEnabled) } - public convenience init(withData data: Data, rootDirectory: URL? = nil) throws { + /// Loads a VRM from in-memory data. + public convenience init(withData data: Data, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true) throws { let vrm = try VRMLoader().load(withData: data) - self.init(vrm: vrm, rootDirectory: rootDirectory) + self.init(vrm: vrm, + rootDirectory: rootDirectory, + isMToonEnabled: isMToonEnabled, + isOutlineEnabled: isOutlineEnabled) } } #endif diff --git a/Sources/VRMRealityKit/VRMEntityLoader.swift b/Sources/VRMRealityKit/VRMEntityLoader.swift index cd362e38..10be070e 100644 --- a/Sources/VRMRealityKit/VRMEntityLoader.swift +++ b/Sources/VRMRealityKit/VRMEntityLoader.swift @@ -1,7 +1,9 @@ #if canImport(RealityKit) import CoreGraphics +import Foundation import RealityKit import Metal +import OSLog import VRMKit import VRMKitRuntime @@ -15,35 +17,68 @@ open class VRMEntityLoader { private var rootDirectory: URL? = nil private let entityName: String? private weak var currentEntity: VRMEntity? + private static let logger = Logger(subsystem: "dev.tattn.VRMKit", category: "MToon") private var textureCacheBySemantic: [TextureResource.Semantic: [Int: TextureResource]] = [:] private var metallicRoughnessCache: [Int: (metal: TextureResource, rough: TextureResource)] = [:] private var samplerCache: [Int: MaterialParameters.Texture.Sampler] = [:] - private var enableNormalTangentBlendShape = false // NOTE: Setting this to true currently has no effect + private var fallbackTextureCache: [MToonTextureSlot.Fallback: TextureResource] = [:] + private var mtoonDescriptorCache: [Int: MToonMaterialDescriptor?] = [:] +#if !os(visionOS) + /// Everything the RealityKit adapter derives from one MToon material. A + /// non-nil state is what "this material renders as MToon" means, and it is + /// shared by the surface material, the outline material and the parameters. + private struct MToonState { + let descriptor: MToonMaterialDescriptor + let parameters: MToonMaterialParameters + let parameterTexture: CustomMaterial.Texture + let library: MTLLibrary + } - public init(vrm: VRM, rootDirectory: URL? = nil) { + private var mtoonStateCache: [Int: MToonState?] = [:] + private var mtoonOutlineMaterialCache: [Int: Material?] = [:] +#endif + private var loggedMToonUVLimitations: Set = [] + private var loggedMToonLibraryError = false + /// When `false`, MToon materials are not created and the loader falls back to Unlit / PBR materials. + /// visionOS always uses the fallback because `CustomMaterial` is unavailable there. + public let isMToonEnabled: Bool + /// Controls creation of MToon's inverted-hull outline entities. + /// visionOS does not create MToon outlines because `CustomMaterial` is unavailable there. + public let isOutlineEnabled: Bool + public init(vrm: VRM, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true) { self.vrm = vrm self.gltf = vrm.gltf.jsonData self.rootDirectory = rootDirectory self.entityName = vrm.meta.title self.entityData = EntityData(vrm: gltf) + self.isMToonEnabled = isMToonEnabled + self.isOutlineEnabled = isOutlineEnabled } public func loadEntity() throws -> VRMEntity { return try loadEntity(withSceneIndex: gltf.scene) } + /// Loads one scene of the glTF as its own entity graph. + /// + /// Each scene builds its own entities, so a node shared by two scenes becomes + /// a separate `Entity` in each. Buffers, materials and textures are reused. public func loadEntity(withSceneIndex index: Int) throws -> VRMEntity { if let cache = try entityData.load(\.entities, index: index) { return cache } - let gltfScene = try gltf.load(\.scenes)[index] + let gltfScene = try gltf.load(\.scenes, at: index) + entityData.beginScene() let vrmEntity = VRMEntity(vrm: vrm) if let entityName { - vrmEntity.entity.name = entityName + vrmEntity.name = entityName } currentEntity = vrmEntity defer { currentEntity = nil } for node in gltfScene.nodes ?? [] { - vrmEntity.entity.addChild(try self.node(withNodeIndex: node)) + vrmEntity.addChild(try self.node(withNodeIndex: node)) } vrmEntity.setUpHumanoid(nodes: entityData.nodes) try vrmEntity.setUpBlendShapes(nodes: entityData.nodes, meshes: entityData.meshes, loader: self) @@ -64,7 +99,7 @@ open class VRMEntityLoader { func node(withNodeIndex index: Int) throws -> Entity { if let cache = try entityData.load(\.nodes, index: index) { return cache } - let gltfNode = try gltf.load(\.nodes)[index] + let gltfNode = try gltf.load(\.nodes, at: index) let entity = Entity() entity.name = gltfNode.name ?? "node_\(index)" @@ -95,7 +130,7 @@ open class VRMEntityLoader { } private func applyCamera(withCameraIndex index: Int, to entity: Entity) throws { - let gltfCamera = try gltf.load(\.cameras)[index] + let gltfCamera = try gltf.load(\.cameras, at: index) switch gltfCamera.type { case .perspective: let perspective = try gltfCamera.perspective ??? .keyNotFound("perspective") @@ -133,7 +168,7 @@ open class VRMEntityLoader { return clone } - let gltfMesh = try gltf.load(\.meshes)[index] + let gltfMesh = try gltf.load(\.meshes, at: index) let meshEntity = Entity() meshEntity.name = gltfMesh.name ?? "mesh_\(index)" @@ -158,8 +193,8 @@ open class VRMEntityLoader { let sharedTargets = targetsByPositionAccessor[positionAccessor] { resolvedPrimitive.targets = sharedTargets } - if let modelEntity = try modelEntity(withPrimitive: resolvedPrimitive, skinIndex: skinIndex) { - meshEntity.addChild(modelEntity) + if let primitiveEntity = try modelEntity(withPrimitive: resolvedPrimitive, skinIndex: skinIndex) { + meshEntity.addChild(primitiveEntity) } } @@ -176,7 +211,7 @@ open class VRMEntityLoader { return meshEntity } - private func modelEntity(withPrimitive primitive: GLTF.Mesh.Primitive, skinIndex: Int?) throws -> ModelEntity? { + private func modelEntity(withPrimitive primitive: GLTF.Mesh.Primitive, skinIndex: Int?) throws -> Entity? { guard supportsTriangles(primitive.mode) else { return nil } let attributes = primitive.attributes.rawValue @@ -186,50 +221,40 @@ open class VRMEntityLoader { let positions = try vector3s(positionIndex) - var normals: [SIMD3]? - if let normalIndex = attributes[.NORMAL] { - normals = try vector3s(normalIndex) - } - var tangents: [SIMD3]? - if enableNormalTangentBlendShape, let tangentIndex = attributes[.TANGENT] { - let rawTangents = try vector4s(tangentIndex) - tangents = rawTangents.map { SIMD3($0.x, $0.y, $0.z) } - } - let texcoords: [SIMD2]? = { - if let uvIndex = attributes[.TEXCOORD_0] { - return try? vector2s(uvIndex) + // glTF requires every vertex attribute of a primitive to hold as many + // elements as POSITION, and nothing downstream re-checks it. + func vertexAttribute(_ key: GLTF.Mesh.Primitive.AttributeKey, + _ read: (Int) throws -> [Element]) throws -> [Element]? { + guard let accessorIndex = attributes[key] else { return nil } + let values = try read(accessorIndex) + guard values.count == positions.count else { + throw VRMError._dataInconsistent( + "\(key) has \(values.count) elements but POSITION has \(positions.count)" + ) } - return nil - }() + return values + } + + let normals = try vertexAttribute(.NORMAL, vector3s) + let rawTangents = try vertexAttribute(.TANGENT, vector4s) + let texcoords = try vertexAttribute(.TEXCOORD_0, vector2s) let jointRemap: [Int]? = { guard let skinIndex else { return nil } return try? jointIndexRemap(forSkinIndex: skinIndex) }() - let skinJointInfluences: ([SIMD4], [SIMD4])? = { + let skinJointInfluences: ([SIMD4], [SIMD4])? = try { guard skinIndex != nil, - let jointsIndex = attributes[.JOINTS_0], - let weightsIndex = attributes[.WEIGHTS_0] else { - return nil - } - guard let joints = try? vector4UInts(jointsIndex), - let weights = try? vector4s(weightsIndex) else { + let joints = try vertexAttribute(.JOINTS_0, jointIndices), + let weights = try vertexAttribute(.WEIGHTS_0, jointWeights) else { return nil } return (joints, weights) }() + // Only POSITION morphs are applied: RealityKit blend shapes drive vertex + // positions, and NORMAL / TANGENT targets have no equivalent channel. var targetOffsets: [[SIMD3]] = [] - var normalOffsets: [[SIMD3]] = [] - var tangentOffsets: [[SIMD3]] = [] if let targets = primitive.targets, !targets.isEmpty { - let hasNormalTargets = enableNormalTangentBlendShape && targets.contains { $0[.NORMAL] != nil } - let hasTangentTargets = enableNormalTangentBlendShape && targets.contains { $0[.TANGENT] != nil } targetOffsets.reserveCapacity(targets.count) - if hasNormalTargets { - normalOffsets.reserveCapacity(targets.count) - } - if hasTangentTargets { - tangentOffsets.reserveCapacity(targets.count) - } for target in targets { if let positionAccessor = target[.POSITION] { let offsets = try vector3s(positionAccessor) @@ -240,28 +265,6 @@ open class VRMEntityLoader { } else { targetOffsets.append(Array(repeating: .zero, count: positions.count)) } - if hasNormalTargets { - if let normalAccessor = target[.NORMAL] { - let offsets = try vector3s(normalAccessor) - guard offsets.count == positions.count else { - throw VRMError._dataInconsistent("blend shape normal target count \(offsets.count) does not match vertex count \(positions.count)") - } - normalOffsets.append(offsets) - } else { - normalOffsets.append(Array(repeating: .zero, count: positions.count)) - } - } - if hasTangentTargets { - if let tangentAccessor = target[.TANGENT] { - let offsets = try vector3s(tangentAccessor) - guard offsets.count == positions.count else { - throw VRMError._dataInconsistent("blend shape tangent target count \(offsets.count) does not match vertex count \(positions.count)") - } - tangentOffsets.append(offsets) - } else { - tangentOffsets.append(Array(repeating: .zero, count: positions.count)) - } - } } } @@ -271,86 +274,96 @@ open class VRMEntityLoader { } else { indexData = (0..= positions.count { + throw VRMError._dataInconsistent( + "triangle index \(maxIndex) is out of range for \(positions.count) vertices" + ) + } + + // NORMAL is optional in glTF; everything else is used as-is. + let finalNormals = normals ?? smoothNormals(positions: positions, indices: indexData) + let finalTexcoords = texcoords ?? [] + let tangentFrame = tangentFrame(rawTangents: rawTangents, + positions: positions, + normals: finalNormals, + texcoords: finalTexcoords, + indices: indexData, + materialIndex: primitive.material) + let finalJoints = skinJointInfluences?.0 ?? [] + let finalWeights = skinJointInfluences?.1 ?? [] + + // One unbuildable material must not fail the whole model: the primitive + // renders with the default material instead. + let material = primitive.material.flatMap { materialIndex -> Material? in + do { + return try self.material(withMaterialIndex: materialIndex) + } catch { + Self.logger.error("Failed to build the material \(materialIndex, privacy: .public); falling back to the default material: \(String(describing: error), privacy: .public)") + return nil } - return defaultMaterial() - }() + } ?? defaultMaterial() let hasSkinning = skinIndex != nil && !finalJoints.isEmpty - let hasBlendShapes = !finalTargetOffsets.isEmpty + let hasBlendShapes = !targetOffsets.isEmpty let mesh: MeshResource var boundSkeleton: MeshResource.Skeleton? if let skinIndex, hasSkinning { let influences = try makeJointInfluences(joints: finalJoints, weights: finalWeights, - vertexCount: finalPositions.count, + vertexCount: positions.count, jointIndexRemap: jointRemap) let skinSkeleton = try skeleton(withSkinIndex: skinIndex) - mesh = try meshResource(positions: finalPositions, + mesh = try meshResource(positions: positions, normals: finalNormals, - tangents: finalTangents, + tangentFrame: tangentFrame, texcoords: finalTexcoords, - indices: finalIndexData, - blendShapeOffsets: finalTargetOffsets, + indices: indexData, + blendShapeOffsets: targetOffsets, skeleton: skinSkeleton, jointInfluences: influences) boundSkeleton = skinSkeleton } else { - mesh = try meshResource(positions: finalPositions, + mesh = try meshResource(positions: positions, normals: finalNormals, - tangents: finalTangents, + tangentFrame: tangentFrame, texcoords: finalTexcoords, - indices: finalIndexData, - blendShapeOffsets: finalTargetOffsets, + indices: indexData, + blendShapeOffsets: targetOffsets, skeleton: nil, jointInfluences: nil) } - let modelEntity = ModelEntity(mesh: mesh, materials: [material]) - if let materialIndex = primitive.material { - modelEntity.components.set(VRMMaterialIndexComponent(materialIndex: materialIndex)) - } - if hasBlendShapes { - let mapping = BlendShapeWeightsMapping(meshResource: mesh) - modelEntity.components.set(BlendShapeWeightsComponent(weightsMapping: mapping)) - } - if enableNormalTangentBlendShape, - !finalNormalOffsets.isEmpty || !finalTangentOffsets.isEmpty { - let component = BlendShapeNormalTangentComponent(baseNormals: finalNormals, - baseTangents: finalTangents, - normalOffsets: finalNormalOffsets, - tangentOffsets: finalTangentOffsets) - modelEntity.components.set(component) + // The blend-shape mapping is derived from the mesh, so the model entity + // and its outline twin share one instance. + let blendShapeMapping = hasBlendShapes ? BlendShapeWeightsMapping(meshResource: mesh) : nil + + func makeEntity(materials: [Material]) throws -> ModelEntity { + let entity = ModelEntity(mesh: mesh, materials: materials) + if let materialIndex = primitive.material { + entity.components.set(VRMMaterialIndexComponent(materialIndex: materialIndex)) + } + if let blendShapeMapping { + entity.components.set(BlendShapeWeightsComponent(weightsMapping: blendShapeMapping)) + } + if let skinIndex, let boundSkeleton { + try registerSkinBinding(modelEntity: entity, skinIndex: skinIndex, skeleton: boundSkeleton) + } + return entity } - if let skinIndex, let boundSkeleton { - try registerSkinBinding(modelEntity: modelEntity, skinIndex: skinIndex, skeleton: boundSkeleton) + + let modelEntity = try makeEntity(materials: [material]) + if let materialIndex = primitive.material, + let outlineMaterial = try mtoonOutlineMaterial(withMaterialIndex: materialIndex) { + let outlineEntity = try makeEntity(materials: [outlineMaterial]) + outlineEntity.name = "\(modelEntity.name)_outline" + let container = Entity() + container.name = "\(modelEntity.name)_container" + container.addChild(outlineEntity) + container.addChild(modelEntity) + return container } return modelEntity } @@ -364,14 +377,25 @@ open class VRMEntityLoader { } } + /// glTF requires a TRIANGLES primitive to hold a non-zero multiple of three + /// indices and a strip / fan to hold at least three, so a primitive that does + /// not fails the load rather than being quietly trimmed into a valid one. private func triangulatedIndices(for mode: GLTF.Mesh.Primitive.Mode, - indices: [UInt32]) -> [UInt32] { + indices: [UInt32]) throws -> [UInt32] { switch mode { case .TRIANGLES: - let count = indices.count / 3 * 3 - return Array(indices.prefix(count)) + guard !indices.isEmpty, indices.count.isMultiple(of: 3) else { + throw VRMError._dataInconsistent( + "a TRIANGLES primitive needs a non-zero multiple of 3 indices, but has \(indices.count)" + ) + } + return indices case .TRIANGLE_STRIP: - guard indices.count >= 3 else { return [] } + guard indices.count >= 3 else { + throw VRMError._dataInconsistent( + "a TRIANGLE_STRIP primitive needs at least 3 indices, but has \(indices.count)" + ) + } var result: [UInt32] = [] result.reserveCapacity((indices.count - 2) * 3) for i in 0..<(indices.count - 2) { @@ -386,7 +410,11 @@ open class VRMEntityLoader { } return result case .TRIANGLE_FAN: - guard indices.count >= 3 else { return [] } + guard indices.count >= 3 else { + throw VRMError._dataInconsistent( + "a TRIANGLE_FAN primitive needs at least 3 indices, but has \(indices.count)" + ) + } let base = indices[0] var result: [UInt32] = [] result.reserveCapacity((indices.count - 2) * 3) @@ -395,64 +423,47 @@ open class VRMEntityLoader { } return result case .POINTS, .LINES, .LINE_LOOP, .LINE_STRIP: - return [] + // Filtered out by supportsTriangles() before the indices are read. + throw VRMError._notSupported("\(mode) primitives have no triangles") } } func material(withMaterialIndex index: Int) throws -> Material { if let cache = try entityData.load(\.materials, index: index) { return cache } - let materials = try gltf.load(\.materials) - guard materials.indices.contains(index) else { - throw VRMError._dataInconsistent("Material index \(index) out of bounds") + let (gltfMaterial, materialProperty) = try materialSource(withMaterialIndex: index) +#if !os(visionOS) + do { + // Building the state reads the extension's textures and samplers, so + // it fails on the same malformed files the material build does. + if let state = try mtoonState(withMaterialIndex: index) { + let material = try customMToonMaterial(state) + entityData.materials[index] = material + return material + } + } catch { + // The caller falls back to a non-MToon material, so the state has + // to go with it: otherwise expression binds would keep writing + // parameter rows that nothing on screen reads. + discardMToonState(withMaterialIndex: index) + Self.logger.error("Failed to build the MToon material \(index, privacy: .public); falling back to Unlit / PBR: \(String(describing: error), privacy: .public)") } - let gltfMaterial = materials[index] +#endif - let materialProperty: VRM0.MaterialProperty? = { - guard case .v0(let vrm0) = vrm, - let name = gltfMaterial.name else { return nil } - return vrm0.materialPropertyNameMap[name] - }() let shaderName = materialProperty?.shader.lowercased() + let isMToon = try mtoonDescriptor(withMaterialIndex: index) != nil // MToon / Unlit variants are not PBR, so use UnlitMaterial for consistent rendering // This matches SceneKit's behavior which uses lightingModel = .constant - let isMToon = shaderName?.contains("mtoon") == true || gltfMaterial.extensions?.materialsMToon != nil let isUnlit = shaderName?.contains("unlit") == true || gltfMaterial.extensions?.materialsUnlit != nil let useUnlit = isMToon || isUnlit - let hasAlphaPremultiply = materialProperty?.keywordMap["_ALPHAPREMULTIPLY_ON"] == true - let hasAlphaBlend = materialProperty?.keywordMap["_ALPHABLEND_ON"] == true - let hasAlphaTest = materialProperty?.keywordMap["_ALPHATEST_ON"] == true - let forceBlend = materialProperty?.vrmShader == .unlitTransparent || hasAlphaPremultiply || hasAlphaBlend - let resolvedAlphaMode: GLTF.Material.AlphaMode = { - if let renderType = materialProperty?.tagMap["RenderType"]?.lowercased() { - switch renderType { - case "opaque": - return .OPAQUE - case "transparentcutout", "cutout": - return .MASK - case "transparent": - return .BLEND - default: - break - } - } - if forceBlend { return .BLEND } - if hasAlphaTest { return .MASK } - return gltfMaterial.alphaMode - }() - - let tint: VRMColor = { - guard let pbr = gltfMaterial.pbrMetallicRoughness else { - return .white - } - let factor = pbr.baseColorFactor - return VRMColor(red: CGFloat(factor.r), - green: CGFloat(factor.g), - blue: CGFloat(factor.b), - alpha: CGFloat(factor.a)) - }() + let resolvedAlphaMode = GLTF.Material.AlphaMode(vrm0: materialProperty, + fallback: gltfMaterial.alphaMode) + let tint = gltfMaterial.pbrMetallicRoughness + .map { VRMColor(simd: SIMD4($0.baseColorFactor)) } ?? .white if useUnlit { - var material = UnlitMaterial() + // RealityKit tone maps everything it draws, and that curve visibly + // darkens flat art, so the unlit path opts out of it. + var material = UnlitMaterial(applyPostProcessToneMap: false) if let pbr = gltfMaterial.pbrMetallicRoughness, let baseTexture = pbr.baseColorTexture { let textureParam = try materialTexture(withTextureIndex: baseTexture.index, semantic: .color) @@ -522,6 +533,243 @@ open class VRMEntityLoader { return material } +#if !os(visionOS) + private func customMToonMaterial(_ state: MToonState) throws -> Material { + // RealityKit has no material-level draw-order hook, so MToon's + // renderQueueOffsetNumber is not honored here. + let mtoon = state.descriptor + let surface = CustomMaterial.SurfaceShader(named: "mtoonSurface", in: state.library) + var material = try CustomMaterial(surfaceShader: surface, lightingModel: .unlit) + // MToon needs more textures than CustomMaterial has semantic channels, + // so the extra slots ride on unrelated channels; MToon.metal reads them + // back through the same mapping. + material.baseColor = .init(tint: .white, texture: try mtoonTexture(mtoon, slot: .base)) + material.roughness.texture = try mtoonTexture(mtoon, slot: .shade) + material.specular.texture = try mtoonTexture(mtoon, slot: .shadingShift) + material.metallic.texture = try mtoonTexture(mtoon, slot: .matcap) + material.normal.texture = try mtoonTexture(mtoon, slot: .normal) + material.emissiveColor = .init(color: .white, texture: try mtoonTexture(mtoon, slot: .emissive)) + material.clearcoatRoughness.texture = try mtoonTexture(mtoon, slot: .rim) + material.clearcoat.texture = try mtoonTexture(mtoon, slot: .outlineWidth) + material.ambientOcclusion.texture = try mtoonTexture(mtoon, slot: .uvAnimationMask) + + applyAlphaMode(mtoon.alphaMode, alphaCutoff: mtoon.alphaCutoff, to: &material) + applyDepthWrite(mtoon, to: &material) + material.faceCulling = mtoon.cullMode.faceCulling + applyMToonParameters(state, to: &material) + return material + } + + private func customMToonOutlineMaterial(_ state: MToonState) throws -> Material { + let mtoon = state.descriptor + let surface = CustomMaterial.SurfaceShader(named: "mtoonOutlineSurface", in: state.library) + let geometry = CustomMaterial.GeometryModifier(named: "mtoonOutlineGeometry", in: state.library) + var material = try CustomMaterial(surfaceShader: surface, + geometryModifier: geometry, + lightingModel: .unlit) + // The inverted hull is rendered with front faces culled. + material.faceCulling = .front + material.baseColor = .init(tint: .white, texture: try mtoonTexture(mtoon, slot: .base)) + material.clearcoat.texture = try mtoonTexture(mtoon, slot: .outlineWidth) + material.ambientOcclusion.texture = try mtoonTexture(mtoon, slot: .uvAnimationMask) + applyAlphaMode(mtoon.alphaMode, alphaCutoff: mtoon.alphaCutoff, to: &material) + applyDepthWrite(mtoon, to: &material) + applyMToonParameters(state, to: &material) + return material + } + + /// MToon.metal applies the UV transform from the parameter rows, so + /// `textureCoordinateTransform` is deliberately left at identity here: + /// setting it too would transform the primary UV a second time. + private func applyMToonParameters(_ state: MToonState, to material: inout CustomMaterial) { + material.custom.value = state.parameters.customValue + material.custom.texture = state.parameterTexture + } + + /// Resolves the descriptor's texture for `slot`, falling back to the slot's + /// neutral texture when the material does not provide one. + private func mtoonTexture(_ descriptor: MToonMaterialDescriptor, + slot: MToonTextureSlot) throws -> CustomMaterial.Texture { + guard let texture = descriptor.texture(for: slot) else { + return CustomMaterial.Texture(try fallbackTextureResource(slot.fallback)) + } + return CustomMaterial.Texture(try self.texture(withTextureIndex: texture.index, semantic: slot.semantic)) + } +#endif + + func currentMaterialColor(withMaterialIndex index: Int, + type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) throws -> SIMD4 { + if let color = try mtoonParameters(withMaterialIndex: index)?.color(for: type) { + return color + } + return try material(withMaterialIndex: index).currentColor(for: type) + } + + /// The UV transform a `textureTransformBind` starts from. MToon keeps it in + /// its parameter rows, everything else in the RealityKit material. + func currentTextureTransform(withMaterialIndex index: Int) throws -> MaterialParameterTypes.TextureCoordinateTransform { + if let transform = try mtoonParameters(withMaterialIndex: index)?.textureTransform { + return transform + } + return try material(withMaterialIndex: index).currentTextureTransform + } + + func mtoonParameters(withMaterialIndex index: Int) throws -> MToonMaterialParameters? { +#if os(visionOS) + return nil +#else + // A nil state means the material does not render as MToon (disabled, + // no metallib, or not an MToon material). + return try mtoonState(withMaterialIndex: index)?.parameters +#endif + } + +#if !os(visionOS) + private func mtoonState(withMaterialIndex index: Int) throws -> MToonState? { + if let cached = mtoonStateCache[index] { + return cached + } + let state = try makeMToonState(withMaterialIndex: index) + mtoonStateCache[index] = state + return state + } + + /// Records that the material does not render as MToon after all, so every + /// MToon-derived path — surface, outline, runtime parameters — agrees. + private func discardMToonState(withMaterialIndex index: Int) { + mtoonStateCache.updateValue(nil, forKey: index) + mtoonOutlineMaterialCache.updateValue(nil, forKey: index) + } + + private func makeMToonState(withMaterialIndex index: Int) throws -> MToonState? { + guard isMToonEnabled, + let descriptor = try mtoonDescriptor(withMaterialIndex: index), + let library = mtoonShaderLibrary() else { + return nil + } + let textureTransform = mtoonTextureTransform(withMaterialIndex: index, descriptor: descriptor) + let parameters = try mtoonParameters(for: descriptor, textureTransform: textureTransform) + logMToonUnsupportedFeatures(for: descriptor, index: index) + return MToonState(descriptor: descriptor, + parameters: parameters, + parameterTexture: CustomMaterial.Texture(try parameters.textureResource()), + library: library) + } + + /// MToon features that this renderer cannot express are reported once per + /// material instead of being dropped silently. + private func logMToonUnsupportedFeatures(for descriptor: MToonMaterialDescriptor, index: Int) { + if descriptor.renderQueueOffsetNumber != 0 { + Self.logger.warning("MToon material \(index, privacy: .public) requests renderQueueOffsetNumber \(descriptor.renderQueueOffsetNumber); RealityKit has no material-level draw-order hook, so it is ignored.") + } + } +#endif + + private func mtoonParameters(for descriptor: MToonMaterialDescriptor, + textureTransform: MaterialParameterTypes.TextureCoordinateTransform) throws -> MToonMaterialParameters { + var parameters = MToonMaterialParameters(descriptor) + parameters.setTextureTransform(scale: textureTransform.scale, + offset: textureTransform.offset, + rotation: textureTransform.rotation) + for slot in MToonTextureSlot.allCases { + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.texture(for: slot)), for: slot) + } + return parameters + } + + /// Custom meshes expose only TEXCOORD_0 and `CustomMaterial` has a single + /// material-level UV transform, so the descriptor's first UV-accessed + /// transform is applied to every MToon texture. + private func mtoonTextureTransform(withMaterialIndex index: Int, + descriptor: MToonMaterialDescriptor) -> MaterialParameterTypes.TextureCoordinateTransform { + let textures = descriptor.uvAccessedTextures + // The first UV-accessed slot wins even when it has no transform of its + // own; letting a later slot's transform stand in would shift it. + let selectedTransform = textures.first?.transform + .map { MaterialParameterTypes.TextureCoordinateTransform(offset: $0.offset, + scale: $0.scale, + rotation: $0.rotation) } + ?? MaterialParameterTypes.TextureCoordinateTransform() + let usesUnsupportedTexCoord = textures.contains { $0.texCoord != 0 } + let hasDifferentTransforms = textures.contains { + textureTransform($0.transform ?? .init(), differsFrom: selectedTransform) + } + if (usesUnsupportedTexCoord || hasDifferentTransforms), + loggedMToonUVLimitations.insert(index).inserted { + if usesUnsupportedTexCoord { + Self.logger.warning("MToon material \(index, privacy: .public) requests a nonzero texCoord; RealityKit uses TEXCOORD_0 on supported deployment targets.") + } + if hasDifferentTransforms { + Self.logger.warning("MToon material \(index, privacy: .public) has per-texture KHR_texture_transform values; RealityKit uses the first UV-accessed transform for all MToon textures.") + } + } + return selectedTransform + } + + private func textureTransform(_ lhs: MToonMaterialDescriptor.UVTransform, + differsFrom rhs: MaterialParameterTypes.TextureCoordinateTransform) -> Bool { + lhs.scale != rhs.scale || lhs.offset != rhs.offset || abs(lhs.rotation - rhs.rotation) > 0.000_001 + } + + private func mtoonOutlineMaterial(withMaterialIndex index: Int) throws -> Material? { +#if os(visionOS) + return nil +#else + guard isOutlineEnabled else { + return nil + } + if let cached = mtoonOutlineMaterialCache[index] { + return cached + } + let material: Material? + if let state = try mtoonState(withMaterialIndex: index), state.descriptor.hasOutline { + material = try customMToonOutlineMaterial(state) + } else { + material = nil + } + mtoonOutlineMaterialCache[index] = material + return material +#endif + } + + /// Memoized because both the material-type decision and the MToon state + /// need it, and building it runs the whole VRM 0.x migration. + private func mtoonDescriptor(withMaterialIndex index: Int) throws -> MToonMaterialDescriptor? { + if let cached = mtoonDescriptorCache[index] { + return cached + } + let descriptor = try makeMToonDescriptor(withMaterialIndex: index) + mtoonDescriptorCache[index] = descriptor + return descriptor + } + + private func makeMToonDescriptor(withMaterialIndex index: Int) throws -> MToonMaterialDescriptor? { + let (gltfMaterial, materialProperty) = try materialSource(withMaterialIndex: index) + return MToonMaterialDescriptor(material: gltfMaterial, materialProperty: materialProperty) + } + + /// The glTF material and, for VRM 0.x, the Unity material property that + /// describes it. The VRM 0.x name lookup lives here only. + private func materialSource(withMaterialIndex index: Int) throws -> (GLTF.Material, VRM0.MaterialProperty?) { + let gltfMaterial = try gltf.load(\.materials, at: index) + guard case .v0(let vrm0) = vrm, let name = gltfMaterial.name else { + return (gltfMaterial, nil) + } + return (gltfMaterial, vrm0.materialPropertyNameMap[name]) + } + + private func mtoonShaderLibrary() -> MTLLibrary? { + do { + return try MToonShaderLibraryLoader.loadDefault() + } catch { + if !loggedMToonLibraryError { + loggedMToonLibraryError = true + Self.logger.error("Failed to load bundled MToon shader library: \(String(describing: error), privacy: .public)") + } + return nil + } + } + func texture(withTextureIndex index: Int, semantic: TextureResource.Semantic = .color) throws -> TextureResource { if semantic == .color, let cache = try entityData.load(\.textures, index: index) { return cache @@ -529,16 +777,14 @@ open class VRMEntityLoader { if semantic != .color, let cache = textureCacheBySemantic[semantic]?[index] { return cache } - let gltfTexture = try gltf.load(\.textures)[index] + let gltfTexture = try gltf.load(\.textures, at: index) let image = try image(withImageIndex: gltfTexture.source) let cgImage = try image.cgImage ??? .dataInconsistent("failed to load cgImage") let texture = try TextureResource(image: cgImage, options: .init(semantic: semantic)) if semantic == .color { entityData.textures[index] = texture } else { - var cache = textureCacheBySemantic[semantic] ?? [:] - cache[index] = texture - textureCacheBySemantic[semantic] = cache + textureCacheBySemantic[semantic, default: [:]][index] = texture } return texture } @@ -550,41 +796,108 @@ open class VRMEntityLoader { return MaterialParameters.Texture(texture, sampler: sampler) } + /// The neutral 1x1 texture bound when a material omits an MToon slot. + private func fallbackTextureResource(_ fallback: MToonTextureSlot.Fallback) throws -> TextureResource { + if let cached = fallbackTextureCache[fallback] { + return cached + } + let texture: TextureResource + switch fallback { + case .white: + texture = try solidColorTextureResource(rgba: [255, 255, 255, 255], semantic: .color) + case .neutralNormal: + texture = try solidColorTextureResource(rgba: [128, 128, 255, 255], semantic: .normal) + } + fallbackTextureCache[fallback] = texture + return texture + } + + private func solidColorTextureResource(rgba: [UInt8], + semantic: TextureResource.Semantic) throws -> TextureResource { + guard let provider = CGDataProvider(data: Data(rgba) as CFData), + let image = CGImage(width: 1, + height: 1, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: 4, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent) else { + throw VRMError._dataInconsistent("failed to create 1x1 \(semantic) texture") + } + return try TextureResource(image: image, options: .init(semantic: semantic)) + } + private func sampler(withTextureIndex index: Int) throws -> MaterialParameters.Texture.Sampler { if let cache = samplerCache[index] { return cache } - let gltfTexture = try gltf.load(\.textures)[index] let descriptor = MTLSamplerDescriptor() - if let samplerIndex = gltfTexture.sampler { - let sampler = try gltf.load(\.samplers)[samplerIndex] - applySampler(sampler, to: descriptor) - } else { - applyDefaultSampler(to: descriptor) - } + applySampler(try gltfSampler(withTextureIndex: index), to: descriptor) let sampler = MaterialParameters.Texture.Sampler(descriptor) samplerCache[index] = sampler return sampler } - private func applySampler(_ sampler: GLTF.Sampler, to descriptor: MTLSamplerDescriptor) { - let magFilter = sampler.magFilter ?? .LINEAR - let minFilter = sampler.minFilter ?? .LINEAR_MIPMAP_LINEAR - descriptor.magFilter = metalFilter(magFilter) - let (min, mip) = metalFilters(minFilter) - descriptor.minFilter = min - descriptor.mipFilter = mip - descriptor.sAddressMode = metalWrap(sampler.wrapS) - descriptor.tAddressMode = metalWrap(sampler.wrapT) + /// The glTF sampler a texture references, or nil when it relies on the + /// glTF default sampler. + private func gltfSampler(withTextureIndex index: Int) throws -> GLTF.Sampler? { + let textures = try gltf.load(\.textures) + guard textures.indices.contains(index) else { + throw VRMError._dataInconsistent("Texture index \(index) out of bounds") + } + guard let samplerIndex = textures[index].sampler else { + return nil + } + let samplers = try gltf.load(\.samplers) + guard samplers.indices.contains(samplerIndex) else { + throw VRMError._dataInconsistent("Sampler index \(samplerIndex) out of bounds") + } + return samplers[samplerIndex] } - private func applyDefaultSampler(to descriptor: MTLSamplerDescriptor) { - descriptor.magFilter = metalFilter(.LINEAR) - let (min, mip) = metalFilters(.LINEAR_MIPMAP_LINEAR) + /// A nil sampler means the texture relies on the glTF defaults. + private func applySampler(_ sampler: GLTF.Sampler?, to descriptor: MTLSamplerDescriptor) { + descriptor.magFilter = metalFilter(sampler?.magFilter ?? .LINEAR) + let (min, mip) = metalFilters(sampler?.minFilter ?? .LINEAR_MIPMAP_LINEAR) descriptor.minFilter = min descriptor.mipFilter = mip - descriptor.sAddressMode = metalWrap(.REPEAT) - descriptor.tAddressMode = metalWrap(.REPEAT) + descriptor.sAddressMode = metalWrap(sampler?.wrapS ?? .REPEAT) + descriptor.tAddressMode = metalWrap(sampler?.wrapT ?? .REPEAT) + } + + private func mtoonSamplerParameters(for texture: MToonMaterialDescriptor.Texture?) throws -> SIMD4 { + guard let texture, + let sampler = try gltfSampler(withTextureIndex: texture.index) else { + return MToonMaterialParameters.defaultSampler + } + return mtoonSamplerParameters(sampler) + } + + /// (wrapS, wrapT, filterIndex, 0), matching the sampler rows `MToon.metal` + /// selects its samplers with. + private func mtoonSamplerParameters(_ sampler: GLTF.Sampler) -> SIMD4 { + let (minFilter, mipFilter) = metalFilters(sampler.minFilter ?? .LINEAR_MIPMAP_LINEAR) + let filter = MToonSamplerFilter( + magnification: metalFilter(sampler.magFilter ?? .LINEAR) == .nearest ? .nearest : .linear, + minification: minFilter == .nearest ? .nearest : .linear, + mip: MToonSamplerFilter.MipFilter(mipFilter) + ) + return SIMD4(mtoonWrapMode(sampler.wrapS), + mtoonWrapMode(sampler.wrapT), + Float(filter.index), + 0) + } + + private func mtoonWrapMode(_ wrap: GLTF.Sampler.Wrap) -> Float { + switch wrap { + case .REPEAT: return 0 + case .CLAMP_TO_EDGE: return 1 + case .MIRRORED_REPEAT: return 2 + } } private func metalFilter(_ filter: GLTF.Sampler.MagFilter) -> MTLSamplerMinMagFilter { @@ -621,7 +934,7 @@ open class VRMEntityLoader { func image(withImageIndex index: Int) throws -> VRMImage { if let cache = try entityData.load(\.images, index: index) { return cache } - let gltfImage = try gltf.load(\.images)[index] + let gltfImage = try gltf.load(\.images, at: index) let image = try VRMImage.from(gltfImage, relativeTo: rootDirectory) { index in try self.bufferView(withBufferViewIndex: index).bufferView } @@ -631,7 +944,7 @@ open class VRMEntityLoader { func bufferView(withBufferViewIndex index: Int) throws -> (bufferView: Data, stride: Int?) { if let cache = try entityData.load(\.bufferViews, index: index) { - let gltfBufferView = try gltf.load(\.bufferViews)[index] + let gltfBufferView = try gltf.load(\.bufferViews, at: index) return (cache, gltfBufferView.byteStride) } let result = try vrm.gltf.bufferViewData(at: index, relativeTo: rootDirectory) @@ -644,7 +957,7 @@ open class VRMEntityLoader { if let cache = metallicRoughnessCache[index] { resources = cache } else { - let gltfTexture = try gltf.load(\.textures)[index] + let gltfTexture = try gltf.load(\.textures, at: index) let image = try image(withImageIndex: gltfTexture.source) let textures = try createMetallicRoughnessTextures(from: image) metallicRoughnessCache[index] = textures @@ -731,38 +1044,57 @@ open class VRMEntityLoader { return image } + /// The single glTF alpha-mode → RealityKit blending decision, shared by + /// every material type this loader builds. + private struct AlphaModeSettings { + let isTransparent: Bool + let opacityThreshold: Float? + + init(_ mode: GLTF.Material.AlphaMode, alphaCutoff: Float) { + switch mode { + case .OPAQUE: + (isTransparent, opacityThreshold) = (false, nil) + case .MASK: + (isTransparent, opacityThreshold) = (false, alphaCutoff) + case .BLEND: + (isTransparent, opacityThreshold) = (true, nil) + } + } + } + private func applyAlphaMode(_ mode: GLTF.Material.AlphaMode, alphaCutoff: Float, to material: inout UnlitMaterial) { - switch mode { - case .OPAQUE: - material.blending = .opaque - material.opacityThreshold = nil - case .MASK: - material.blending = .opaque - material.opacityThreshold = alphaCutoff - case .BLEND: - material.blending = .transparent(opacity: .init(scale: 1.0)) - material.opacityThreshold = nil - } + let settings = AlphaModeSettings(mode, alphaCutoff: alphaCutoff) + material.blending = settings.isTransparent ? .transparent(opacity: .init(scale: 1.0)) : .opaque + material.opacityThreshold = settings.opacityThreshold } private func applyAlphaMode(_ mode: GLTF.Material.AlphaMode, alphaCutoff: Float, to material: inout PhysicallyBasedMaterial) { - switch mode { - case .OPAQUE: - material.blending = .opaque - material.opacityThreshold = nil - case .MASK: - material.blending = .opaque - material.opacityThreshold = alphaCutoff - case .BLEND: - material.blending = .transparent(opacity: .init(scale: 1.0)) - material.opacityThreshold = nil - } + let settings = AlphaModeSettings(mode, alphaCutoff: alphaCutoff) + material.blending = settings.isTransparent ? .transparent(opacity: .init(scale: 1.0)) : .opaque + material.opacityThreshold = settings.opacityThreshold } +#if !os(visionOS) + private func applyAlphaMode(_ mode: GLTF.Material.AlphaMode, + alphaCutoff: Float, + to material: inout CustomMaterial) { + let settings = AlphaModeSettings(mode, alphaCutoff: alphaCutoff) + material.blending = settings.isTransparent ? .transparent(opacity: .init(scale: 1.0)) : .opaque + material.opacityThreshold = settings.opacityThreshold + } + + /// MToon's `transparentWithZWrite` asks a blended material to still write + /// depth. Only BLEND materials can turn depth writing off, so every other + /// alpha mode keeps it on. + private func applyDepthWrite(_ mtoon: MToonMaterialDescriptor, to material: inout CustomMaterial) { + material.writesDepth = mtoon.alphaMode != .BLEND || mtoon.transparentWithZWrite + } +#endif + private struct AccessorSlice { let data: Data let componentsPerVector: Int @@ -776,31 +1108,14 @@ open class VRMEntityLoader { if let cache = try entityData.load(\.accessors, index: index) as? AccessorSlice { return cache } - let accessor = try gltf.load(\.accessors)[index] - let (componentsPerVector, bytesPerComponent, vectorSize) = accessor.components() - - let baseData: Data = try { - if let bufferViewIndex = accessor.bufferView { - let bufferView = try self.bufferView(withBufferViewIndex: bufferViewIndex) - let dataStride = bufferView.stride ?? vectorSize - return bufferView.bufferView.subdata(offset: accessor.byteOffset, - size: vectorSize, - stride: dataStride, - count: accessor.count) - } - return Data(count: vectorSize * accessor.count) - }() - - var data = baseData - if let sparse = accessor.sparse { - try applySparse(sparse: sparse, - accessorCount: accessor.count, - vectorSize: vectorSize, - data: &data) + let accessors = try gltf.load(\.accessors) + guard accessors.indices.contains(index) else { + throw VRMError._dataInconsistent("accessor index \(index) is out of range for \(accessors.count) accessors") } - + let accessor = accessors[index] + let (componentsPerVector, bytesPerComponent, _) = accessor.components() let slice = AccessorSlice( - data: data, + data: try accessor.packedData(bufferView: { try self.bufferView(withBufferViewIndex: $0) }), componentsPerVector: componentsPerVector, bytesPerComponent: bytesPerComponent, count: accessor.count, @@ -811,68 +1126,6 @@ open class VRMEntityLoader { return slice } - private func applySparse(sparse: GLTF.Accessor.Sparse, - accessorCount: Int, - vectorSize: Int, - data: inout Data) throws { - guard sparse.count > 0 else { return } - let indices = try sparseIndices(sparse: sparse) - let values = try sparseValues(sparse: sparse, vectorSize: vectorSize) - let count = min(indices.count, sparse.count) - data.withUnsafeMutableBytes { rawDst in - guard let dst = rawDst.bindMemory(to: UInt8.self).baseAddress else { return } - values.withUnsafeBytes { rawSrc in - guard let src = rawSrc.bindMemory(to: UInt8.self).baseAddress else { return } - for i in 0..= 0, index < accessorCount else { continue } - let dstPos = index * vectorSize - let srcPos = i * vectorSize - memcpy(dst.advanced(by: dstPos), src.advanced(by: srcPos), vectorSize) - } - } - } - } - - private func sparseIndices(sparse: GLTF.Accessor.Sparse) throws -> [Int] { - let bufferView = try self.bufferView(withBufferViewIndex: sparse.indices.bufferView) - let bytesPerIndex = bytes(of: sparse.indices.componentType) - let stride = bufferView.stride ?? bytesPerIndex - let indexData = bufferView.bufferView.subdata(offset: sparse.indices.byteOffset, - size: bytesPerIndex, - stride: stride, - count: sparse.count) - var indices: [Int] = [] - indices.reserveCapacity(sparse.count) - indexData.withUnsafeBytes { raw in - guard let base = raw.baseAddress else { return } - for i in 0.. Data { - let bufferView = try self.bufferView(withBufferViewIndex: sparse.values.bufferView) - let stride = bufferView.stride ?? vectorSize - return bufferView.bufferView.subdata(offset: sparse.values.byteOffset, - size: vectorSize, - stride: stride, - count: sparse.count) - } - private func vector2s(_ accessorIndex: Int) throws -> [SIMD2] { let slice = try accessorSlice(accessorIndex) guard slice.componentsPerVector == 2 else { @@ -971,16 +1224,9 @@ open class VRMEntityLoader { slice.data.withUnsafeBytes { raw in guard let base = raw.baseAddress else { return } for i in 0.. UInt32 { + componentType: GLTF.Accessor.ComponentType) -> UInt32? { switch componentType { case .unsignedByte: return UInt32(base.load(fromByteOffset: offset, as: UInt8.self)) @@ -1033,12 +1282,8 @@ open class VRMEntityLoader { return UInt32(base.load(fromByteOffset: offset, as: UInt16.self)) case .unsignedInt: return base.load(fromByteOffset: offset, as: UInt32.self) - case .byte: - return UInt32(Int32(base.load(fromByteOffset: offset, as: Int8.self))) - case .short: - return UInt32(Int32(base.load(fromByteOffset: offset, as: Int16.self))) - case .float: - return UInt32(base.load(fromByteOffset: offset, as: Float.self)) + case .byte, .short, .float: + return nil } } @@ -1053,18 +1298,20 @@ open class VRMEntityLoader { guard let base = raw.baseAddress else { return } for i in 0..(x, y, z, w)) } } @@ -1074,6 +1321,43 @@ open class VRMEntityLoader { return result } + /// JOINTS_n as glTF defines it: unsigned byte or short indices into the skin. + /// A wider or signed component type means the file indexes joints in a way + /// its own skin does not describe. + private func jointIndices(_ accessorIndex: Int) throws -> [SIMD4] { + let slice = try accessorSlice(accessorIndex) + switch slice.componentType { + case .unsignedByte, .unsignedShort: + return try vector4UInts(accessorIndex) + case .byte, .short, .unsignedInt, .float: + throw VRMError._dataInconsistent( + "JOINTS_0 must use unsigned byte or short components, not \(slice.componentType)" + ) + } + } + + /// WEIGHTS_n as glTF defines it: float, or normalized unsigned byte / short. + /// An unnormalized integer accessor would arrive as raw counts rather than + /// the 0...1 weights the influences are built from. + private func jointWeights(_ accessorIndex: Int) throws -> [SIMD4] { + let slice = try accessorSlice(accessorIndex) + switch slice.componentType { + case .float: + return try vector4s(accessorIndex) + case .unsignedByte, .unsignedShort: + guard slice.normalized else { + throw VRMError._dataInconsistent( + "WEIGHTS_0 with \(slice.componentType) components must be normalized" + ) + } + return try vector4s(accessorIndex) + case .byte, .short, .unsignedInt: + throw VRMError._dataInconsistent( + "WEIGHTS_0 must use float or normalized unsigned byte / short components, not \(slice.componentType)" + ) + } + } + private func makeJointInfluences(joints: [SIMD4], weights: [SIMD4], vertexCount: Int, @@ -1088,6 +1372,17 @@ open class VRMEntityLoader { var influences: [MeshJointInfluence] = [] influences.reserveCapacity(joints.count * 4) let remap = jointIndexRemap + // JOINTS_0 comes straight from the file, so an out-of-range index has to + // fail the load rather than reach the remap table or the skeleton. + func remapped(_ jointIndex: UInt32) throws -> Int { + guard let remap else { return Int(jointIndex) } + guard remap.indices.contains(Int(jointIndex)) else { + throw VRMError._dataInconsistent( + "joint index \(jointIndex) is out of range for \(remap.count) skin joints" + ) + } + return remap[Int(jointIndex)] + } for i in 0.. MeshResource.Skeleton { if let cache = try entityData.load(\.skins, index: index) { return cache } - let skin = try gltf.load(\.skins)[index] + let skin = try gltf.load(\.skins, at: index) let nodes = try gltf.load(\.nodes) let (parentIndices, order, remap) = computeSkinJointOrdering(skin: skin, nodes: nodes) entityData.skinJointRemaps[index] = remap - let inverseBindMatrices: [simd_float4x4] = { - guard let accessorIndex = skin.inverseBindMatrices else { - return Array(repeating: matrix_identity_float4x4, count: skin.joints.count) + // glTF defines an absent inverseBindMatrices as identity per joint, but a + // present one has to cover every joint: silently substituting identities + // for a broken accessor would bind the mesh to the wrong rest pose. + let inverseBindMatrices: [simd_float4x4] + if let accessorIndex = skin.inverseBindMatrices { + inverseBindMatrices = try matrix4s(accessorIndex) + guard inverseBindMatrices.count >= skin.joints.count else { + throw VRMError._dataInconsistent( + "inverseBindMatrices has \(inverseBindMatrices.count) elements for \(skin.joints.count) skin joints" + ) } - return (try? matrix4s(accessorIndex)) ?? Array(repeating: matrix_identity_float4x4, count: skin.joints.count) - }() + } else { + inverseBindMatrices = Array(repeating: matrix_identity_float4x4, count: skin.joints.count) + } var joints: [MeshResource.Skeleton.Joint] = [] joints.reserveCapacity(order.count) @@ -1178,7 +1477,7 @@ open class VRMEntityLoader { let parentNew = parentOld.map { remap[$0] } let restTransform = transform(from: node) - let ibm = oldIndex < inverseBindMatrices.count ? inverseBindMatrices[oldIndex] : matrix_identity_float4x4 + let ibm = inverseBindMatrices[oldIndex] joints.append(.init(name: name, parentIndex: parentNew, inverseBindPoseMatrix: ibm, @@ -1192,7 +1491,7 @@ open class VRMEntityLoader { private func jointIndexRemap(forSkinIndex index: Int) throws -> [Int] { if let cache = try entityData.load(\.skinJointRemaps, index: index) { return cache } - let skin = try gltf.load(\.skins)[index] + let skin = try gltf.load(\.skins, at: index) let nodes = try gltf.load(\.nodes) let (_, _, remap) = computeSkinJointOrdering(skin: skin, nodes: nodes) entityData.skinJointRemaps[index] = remap @@ -1257,36 +1556,9 @@ open class VRMEntityLoader { return (parentIndices, order, remap) } - private func applySkinning(to mesh: MeshResource, - skinIndex: Int, - jointInfluences: MeshResource.JointInfluences) throws -> MeshResource.Skeleton { - let skeleton = try skeleton(withSkinIndex: skinIndex) - var contents = mesh.contents - var skeletons = contents.skeletons - _ = skeletons.insert(skeleton) - contents.skeletons = skeletons - - var updatedModels = MeshModelCollection() - for model in contents.models { - var model = model - var updatedParts = MeshPartCollection() - for part in model.parts { - var part = part - part.skeletonID = skeleton.id - part.jointInfluences = jointInfluences - updatedParts.insert(part) - } - model.parts = updatedParts - updatedModels.insert(model) - } - contents.models = updatedModels - try mesh.replace(with: contents) - return skeleton - } - private func meshResource(positions: [SIMD3], normals: [SIMD3], - tangents: [SIMD3], + tangentFrame: TangentFrame, texcoords: [SIMD2], indices: [UInt32], blendShapeOffsets: [[SIMD3]], @@ -1297,8 +1569,9 @@ open class VRMEntityLoader { if !normals.isEmpty { part.normals = MeshBuffer(normals) } - if !tangents.isEmpty { - part.tangents = MeshBuffer(tangents) + if !tangentFrame.tangents.isEmpty { + part.tangents = MeshBuffer(tangentFrame.tangents) + part.bitangents = MeshBuffer(tangentFrame.bitangents) } if !texcoords.isEmpty { part.textureCoordinates = MeshBuffer(texcoords) @@ -1341,7 +1614,7 @@ open class VRMEntityLoader { skinIndex: Int, skeleton: MeshResource.Skeleton) throws { guard let vrmEntity = currentEntity else { return } - let skin = try gltf.load(\.skins)[skinIndex] + let skin = try gltf.load(\.skins, at: skinIndex) var jointEntities = try skin.joints.map { try node(withNodeIndex: $0) } if let remap = try? jointIndexRemap(forSkinIndex: skinIndex), remap.count == jointEntities.count { var ordered: [Entity] = Array(repeating: jointEntities[0], count: jointEntities.count) @@ -1357,13 +1630,13 @@ open class VRMEntityLoader { private func registerMaterialBindings(in root: Entity) { guard let vrmEntity = currentEntity else { return } - var stack: [Entity] = [root] - while let entity = stack.popLast() { - if let modelEntity = entity as? ModelEntity, - let materialIndex = modelEntity.components[VRMMaterialIndexComponent.self]?.materialIndex { - vrmEntity.registerMaterialBinding(modelEntity: modelEntity, materialIndex: materialIndex) + for modelEntity in root.modelEntitiesInHierarchy { + guard let materialIndex = modelEntity.components[VRMMaterialIndexComponent.self]?.materialIndex else { + continue } - stack.append(contentsOf: entity.children) + vrmEntity.registerMaterialBinding(modelEntity: modelEntity, + materialIndex: materialIndex, + loader: self) } } @@ -1376,8 +1649,68 @@ open class VRMEntityLoader { translation: node.translation.simd) } - private func estimateNormals(positions: [SIMD3], indices: [UInt32]) -> [SIMD3] { - var normals = [SIMD3](repeating: .zero, count: positions.count) + /// A complete tangent basis. RealityKit stores tangents and bitangents as two + /// independent mesh buffers and derives neither from the other, so both are + /// filled together or left empty together. + private struct TangentFrame { + static let empty = TangentFrame(tangents: [], bitangents: []) + + let tangents: [SIMD3] + let bitangents: [SIMD3] + } + + /// The tangent basis a normal map needs: taken from glTF `TANGENT` when the + /// primitive has one, and otherwise derived from the UVs. Meshes whose + /// material never samples a normal map get no basis, since nothing reads it. + private func tangentFrame(rawTangents: [SIMD4]?, + positions: [SIMD3], + normals: [SIMD3], + texcoords: [SIMD2], + indices: [UInt32], + materialIndex: Int?) -> TangentFrame { + if let rawTangents { + // glTF stores handedness in w; the bitangent it selects is what makes + // the basis match the normal map the asset was authored against. + var tangents = [SIMD3](repeating: .zero, count: positions.count) + var bitangents = tangents + for i in 0..(raw.x, raw.y, raw.z) + tangents[i] = tangent + bitangents[i] = simd_cross(normals[i], tangent) * (raw.w < 0 ? -1 : 1) + } + return TangentFrame(tangents: tangents, bitangents: bitangents) + } + guard texcoords.count == positions.count, + let materialIndex, + materialUsesNormalTexture(withMaterialIndex: materialIndex) else { + return .empty + } + return generatedTangentFrame(positions: positions, + normals: normals, + texcoords: texcoords, + indices: indices) + } + + /// Whether the material samples a normal map. The MToon descriptor is asked + /// first because VRM 0.x carries its normal map in Unity's `_BumpMap`, which + /// the migration surfaces there and not on the glTF material. + private func materialUsesNormalTexture(withMaterialIndex index: Int) -> Bool { + if let descriptor = try? mtoonDescriptor(withMaterialIndex: index), descriptor.normalTexture != nil { + return true + } + guard let (gltfMaterial, _) = try? materialSource(withMaterialIndex: index) else { return false } + return gltfMaterial.normalTexture != nil + } + + /// Per-triangle UV gradients accumulated per vertex, then orthonormalized + /// against the normal — the standard basis a normal map is authored against. + private func generatedTangentFrame(positions: [SIMD3], + normals: [SIMD3], + texcoords: [SIMD2], + indices: [UInt32]) -> TangentFrame { + var tangentSums = [SIMD3](repeating: .zero, count: positions.count) + var bitangentSums = tangentSums let triangleCount = indices.count / 3 for i in 0.. 1e-12 else { continue } + let scale = 1 / determinant + let tangent = (edge1 * deltaUV2.y - edge2 * deltaUV1.y) * scale + let bitangent = (edge2 * deltaUV1.x - edge1 * deltaUV2.x) * scale + tangentSums[i0] += tangent + tangentSums[i1] += tangent + tangentSums[i2] += tangent + bitangentSums[i0] += bitangent + bitangentSums[i1] += bitangent + bitangentSums[i2] += bitangent + } + + var tangents = [SIMD3](repeating: .zero, count: positions.count) + var bitangents = tangents + for i in 0..) -> SIMD2 { + SIMD2(delta.x, -delta.y) + } + + /// Gram-Schmidt against the normal, falling back to any perpendicular axis + /// for vertices no usable triangle reached. + private func orthonormalizedTangent(_ tangent: SIMD3, normal: SIMD3) -> SIMD3 { + let projected = tangent - normal * simd_dot(normal, tangent) + if simd_length_squared(projected) > 1e-12 { + return simd_normalize(projected) + } + let axis = abs(normal.x) < 0.9 ? SIMD3(1, 0, 0) : SIMD3(0, 1, 0) + return simd_normalize(simd_cross(normal, axis)) + } + + /// Vertex normals for a primitive that ships none, accumulated from the + /// unnormalized face normals: their length is twice the triangle area, which + /// is the weighting a smooth normal wants. + /// + /// This is a deliberate departure from glTF, which asks for flat normals — + /// and for TANGENT to be ignored — when NORMAL is absent. Flat normals need + /// a vertex per triangle, and splitting the vertices would break the index + /// buffer that blend-shape targets, joint influences and the outline twin all + /// address. Every VRM ships NORMAL, so this only ever runs for a plain glTF, + /// where a smooth-shaded model is the better failure mode than none at all. + private func smoothNormals(positions: [SIMD3], indices: [UInt32]) -> [SIMD3] { + var normals = [SIMD3](repeating: .zero, count: positions.count) + let triangleCount = indices.count / 3 + for i in 0.. 1e-24 else { continue } + normals[i0] += faceNormal + normals[i1] += faceNormal + normals[i2] += faceNormal } - for i in 0.. 1e-24 { + normals[i] = simd_normalize(normals[i]) } return normals } @@ -1406,46 +1811,6 @@ open class VRMEntityLoader { return material } - private struct PreparedMeshBuffers { - let positions: [SIMD3] - let normals: [SIMD3] - let tangents: [SIMD3] - let texcoords: [SIMD2] - let joints: [SIMD4] - let weights: [SIMD4] - let targetOffsets: [[SIMD3]] - let normalOffsets: [[SIMD3]] - let tangentOffsets: [[SIMD3]] - let indexData: [UInt32] - } - - private func prepareVertexData(positions: [SIMD3], - normals: [SIMD3]?, - tangents: [SIMD3]?, - texcoords: [SIMD2]?, - joints: [SIMD4]?, - weights: [SIMD4]?, - targetOffsets: [[SIMD3]], - normalOffsets: [[SIMD3]], - tangentOffsets: [[SIMD3]], - indexData: [UInt32]) -> PreparedMeshBuffers { - let finalNormals: [SIMD3] - if let normals { - finalNormals = normals - } else { - finalNormals = estimateNormals(positions: positions, indices: indexData) - } - return PreparedMeshBuffers(positions: positions, - normals: finalNormals, - tangents: tangents ?? [], - texcoords: texcoords ?? [], - joints: joints ?? [], - weights: weights ?? [], - targetOffsets: targetOffsets, - normalOffsets: normalOffsets, - tangentOffsets: tangentOffsets, - indexData: indexData) - } } #endif diff --git a/Sources/VRMSceneKit/CustomType/VRMNode.swift b/Sources/VRMSceneKit/CustomType/VRMNode.swift index ef71e321..1d871a65 100644 --- a/Sources/VRMSceneKit/CustomType/VRMNode.swift +++ b/Sources/VRMSceneKit/CustomType/VRMNode.swift @@ -235,7 +235,7 @@ open class VRMNode: SCNNode { return } guard let clip = blendShapeClips[key] else { return } - let value: CGFloat = clip.isBinary ? round(value) : value + let value = CGFloat(clip.normalizedWeight(Double(value))) for binding in clip.values { let weight = CGFloat(binding.weight / 100.0) for morpher in binding.mesh.allMorphers { @@ -246,7 +246,7 @@ open class VRMNode: SCNNode { public func setExpression(value: CGFloat, for key: ExpressionKey) { guard let clip = expressionClip(for: key) else { return } - let value = max(0.0, min(1.0, clip.isBinary ? round(value) : value)) + let value = CGFloat(clip.normalizedWeight(Double(value))) for binding in clip.values { let weight = CGFloat(binding.weight / 100.0) for morpher in binding.mesh.allMorphers { diff --git a/Sources/VRMSceneKit/GLTF2SCN/GLTF2SCN.swift b/Sources/VRMSceneKit/GLTF2SCN/GLTF2SCN.swift index 72d6cd91..5bddb679 100644 --- a/Sources/VRMSceneKit/GLTF2SCN/GLTF2SCN.swift +++ b/Sources/VRMSceneKit/GLTF2SCN/GLTF2SCN.swift @@ -16,35 +16,6 @@ func semantic(of key: GLTF.Mesh.Primitive.AttributeKey) -> SCNGeometrySource.Sem } } -func numberOfComponents(of type: GLTF.Accessor.`Type`) -> Int { - switch type { - case .SCALAR: return 1 - case .VEC2: return 2 - case .VEC3: return 3 - case .VEC4: return 4 - case .MAT2: return 4 - case .MAT3: return 9 - case .MAT4: return 16 - } -} - -func bytes(of type: GLTF.Accessor.ComponentType) -> Int { - switch type { - case .byte, .unsignedByte: return 1 - case .short, .unsignedShort: return 2 - case .unsignedInt, .float: return 4 - } -} - -extension GLTF.Accessor { - func components() -> (componentsPerVector: Int, bytesPerComponent: Int, vectorSize: Int) { - let componentsPerVector = numberOfComponents(of: type) - let bytesPerComponent = bytes(of: componentType) - let vectorSize = bytesPerComponent * componentsPerVector - return (componentsPerVector, bytesPerComponent, vectorSize) - } -} - extension SCNGeometryPrimitiveType { func primitiveCount(ofCount count: Int) -> Int { switch self { diff --git a/Sources/VRMSceneKit/GLTF2SCN/SCNGeometryElement+GLTF.swift b/Sources/VRMSceneKit/GLTF2SCN/SCNGeometryElement+GLTF.swift index 8fb29ab7..a5450a0f 100644 --- a/Sources/VRMSceneKit/GLTF2SCN/SCNGeometryElement+GLTF.swift +++ b/Sources/VRMSceneKit/GLTF2SCN/SCNGeometryElement+GLTF.swift @@ -11,16 +11,7 @@ extension SCNGeometryElement { if usesFloatComponents { throw VRMError._dataInconsistent("index accessor cannot use float components") } if accessor.type != .SCALAR { throw VRMError._dataInconsistent("accessor type is not SCALAR") } - let (bufferView, dataStride): (Data, Int) = try { - if let bufferViewIndex = accessor.bufferView { - let bufferView = try loader.bufferView(withBufferViewIndex: bufferViewIndex) - return (bufferView.bufferView, bufferView.stride ?? bytesPerComponent) - } else { - return (Data(count: bytesPerComponent * accessor.count), bytesPerComponent) - } - }() - - self.init(data: bufferView.subdata(offset: accessor.byteOffset, size: bytesPerComponent, stride: dataStride, count: accessor.count), + self.init(data: try accessor.packedData(bufferView: { try loader.bufferView(withBufferViewIndex: $0) }), primitiveType: primitiveType, primitiveCount: primitiveType.primitiveCount(ofCount: accessor.count), bytesPerIndex: bytesPerComponent) diff --git a/Sources/VRMSceneKit/GLTF2SCN/SCNGeometrySource+GLTF.swift b/Sources/VRMSceneKit/GLTF2SCN/SCNGeometrySource+GLTF.swift index 415dfc8c..54936144 100644 --- a/Sources/VRMSceneKit/GLTF2SCN/SCNGeometrySource+GLTF.swift +++ b/Sources/VRMSceneKit/GLTF2SCN/SCNGeometrySource+GLTF.swift @@ -5,117 +5,14 @@ import SceneKit extension SCNGeometrySource { convenience init(accessor: GLTF.Accessor, semantic: SCNGeometrySource.Semantic, loader: VRMSceneLoader) throws { let (componentsPerVector, bytesPerComponent, vectorSize) = accessor.components() - if let sparse = accessor.sparse { - var data = try baseDataForSparse(accessor: accessor, vectorSize: vectorSize, loader: loader) - try applySparse(sparse: sparse, - accessorCount: accessor.count, - vectorSize: vectorSize, - loader: loader, - data: &data) - self.init(data: data, - semantic: semantic, - vectorCount: accessor.count, - usesFloatComponents: accessor.componentType == .float, - componentsPerVector: componentsPerVector, - bytesPerComponent: bytesPerComponent, - dataOffset: 0, - dataStride: vectorSize) - } else { - let (bufferView, dataStride): (Data, Int) = try { - if let bufferViewIndex = accessor.bufferView { - let bufferView = try loader.bufferView(withBufferViewIndex: bufferViewIndex) - return (bufferView.bufferView, bufferView.stride ?? vectorSize) - } else { - return (Data(count: vectorSize * accessor.count), vectorSize) - } - }() - - self.init(data: bufferView, - semantic: semantic, - vectorCount: accessor.count, - usesFloatComponents: accessor.componentType == .float, - componentsPerVector: componentsPerVector, - bytesPerComponent: bytesPerComponent, - dataOffset: accessor.byteOffset, - dataStride: dataStride) - } - } -} - -private func baseDataForSparse(accessor: GLTF.Accessor, - vectorSize: Int, - loader: VRMSceneLoader) throws -> Data { - if let bufferViewIndex = accessor.bufferView { - let bufferView = try loader.bufferView(withBufferViewIndex: bufferViewIndex) - let dataStride = bufferView.stride ?? vectorSize - return bufferView.bufferView.subdata(offset: accessor.byteOffset, - size: vectorSize, - stride: dataStride, - count: accessor.count) - } - return Data(count: vectorSize * accessor.count) -} - -private func applySparse(sparse: GLTF.Accessor.Sparse, - accessorCount: Int, - vectorSize: Int, - loader: VRMSceneLoader, - data: inout Data) throws { - guard sparse.count > 0 else { return } - let indices = try sparseIndices(sparse: sparse, loader: loader) - let values = try sparseValues(sparse: sparse, vectorSize: vectorSize, loader: loader) - let count = min(indices.count, sparse.count) - data.withUnsafeMutableBytes { rawDst in - guard let dst = rawDst.bindMemory(to: UInt8.self).baseAddress else { return } - values.withUnsafeBytes { rawSrc in - guard let src = rawSrc.bindMemory(to: UInt8.self).baseAddress else { return } - for i in 0..= 0, index < accessorCount else { continue } - let dstPos = index * vectorSize - let srcPos = i * vectorSize - memcpy(dst.advanced(by: dstPos), src.advanced(by: srcPos), vectorSize) - } - } - } -} - -private func sparseIndices(sparse: GLTF.Accessor.Sparse, loader: VRMSceneLoader) throws -> [Int] { - let bufferView = try loader.bufferView(withBufferViewIndex: sparse.indices.bufferView) - let bytesPerIndex = bytes(of: sparse.indices.componentType) - let stride = bufferView.stride ?? bytesPerIndex - let indexData = bufferView.bufferView.subdata(offset: sparse.indices.byteOffset, - size: bytesPerIndex, - stride: stride, - count: sparse.count) - var indices: [Int] = [] - indices.reserveCapacity(sparse.count) - indexData.withUnsafeBytes { raw in - guard let base = raw.baseAddress else { return } - for i in 0.. Data { - let bufferView = try loader.bufferView(withBufferViewIndex: sparse.values.bufferView) - let stride = bufferView.stride ?? vectorSize - return bufferView.bufferView.subdata(offset: sparse.values.byteOffset, - size: vectorSize, - stride: stride, - count: sparse.count) } diff --git a/Sources/VRMSceneKit/VRMSceneLoader.swift b/Sources/VRMSceneKit/VRMSceneLoader.swift index 88409424..a8a5bb09 100644 --- a/Sources/VRMSceneKit/VRMSceneLoader.swift +++ b/Sources/VRMSceneKit/VRMSceneLoader.swift @@ -25,7 +25,7 @@ open class VRMSceneLoader { public func loadScene(withSceneIndex index: Int) throws -> VRMScene { if let cache = try sceneData.load(\.scenes, index: index) { return cache } - let gltfScene = try gltf.load(\.scenes)[index] + let gltfScene = try gltf.load(\.scenes, at: index) let vrmNode = VRMNode(vrm: vrm) for node in gltfScene.nodes ?? [] { @@ -50,7 +50,7 @@ open class VRMSceneLoader { func node(withNodeIndex index: Int) throws -> SCNNode { if let cache = try sceneData.load(\.nodes, index: index) { return cache } - let gltfNode = try gltf.load(\.nodes)[index] + let gltfNode = try gltf.load(\.nodes, at: index) let gltfSkins = try? gltf.load(\.skins) let scnNode = try SCNNode(node: gltfNode, skins: gltfSkins, loader: self) sceneData.nodes[index] = scnNode @@ -59,7 +59,7 @@ open class VRMSceneLoader { func camera(withCameraIndex index: Int) throws -> SCNCamera { if let cache = try sceneData.load(\.cameras, index: index) { return cache } - let gltfCamera = try gltf.load(\.cameras)[index] + let gltfCamera = try gltf.load(\.cameras, at: index) let camera = try SCNCamera(camera: gltfCamera) sceneData.cameras[index] = camera return camera @@ -67,7 +67,7 @@ open class VRMSceneLoader { func mesh(withMeshIndex index: Int) throws -> SCNNode { if let cache = try sceneData.load(\.meshes, index: index) { return cache } - let gltfMesh = try gltf.load(\.meshes)[index] + let gltfMesh = try gltf.load(\.meshes, at: index) let mesh = try SCNNode(mesh: gltfMesh, loader: self) sceneData.meshes[index] = mesh return mesh @@ -77,7 +77,7 @@ open class VRMSceneLoader { return try attributes.compactMap { attribute, index in guard attribute != .COLOR_0 else { return nil } // FIXME if let cache = try sceneData.load(\.accessors, index: index) as? SCNGeometrySource { return cache } - let gltfAccessor = try gltf.load(\.accessors)[index] + let gltfAccessor = try gltf.load(\.accessors, at: index) let geometrySource = try SCNGeometrySource(accessor: gltfAccessor, semantic: semantic(of: attribute), loader: self) sceneData.accessors[index] = geometrySource return geometrySource @@ -86,7 +86,7 @@ open class VRMSceneLoader { func indexAccessor(withAccessorIndex index: Int, mode: GLTF.Mesh.Primitive.Mode) throws -> SCNGeometryElement { if let cache = try sceneData.load(\.accessors, index: index) as? SCNGeometryElement { return cache } - let gltfAccessor = try gltf.load(\.accessors)[index] + let gltfAccessor = try gltf.load(\.accessors, at: index) let geometryElement = try SCNGeometryElement(accessor: gltfAccessor, mode: mode, loader: self) sceneData.accessors[index] = geometryElement return geometryElement @@ -94,7 +94,7 @@ open class VRMSceneLoader { func inverseBindMatrix(withAccessorIndex index: Int) throws -> [InverseBindMatrix] { if let cache = try sceneData.load(\.accessors, index: index) as? [InverseBindMatrix] { return cache } - let gltfAccessor = try gltf.load(\.accessors)[index] + let gltfAccessor = try gltf.load(\.accessors, at: index) let ibm = try [InverseBindMatrix](accessor: gltfAccessor, loader: self) sceneData.accessors[index] = ibm return ibm @@ -112,7 +112,7 @@ open class VRMSceneLoader { func bufferView(withBufferViewIndex index: Int) throws -> (bufferView: Data, stride: Int?) { if let cache = try sceneData.load(\.bufferViews, index: index) { - let gltfBufferView = try gltf.load(\.bufferViews)[index] + let gltfBufferView = try gltf.load(\.bufferViews, at: index) return (cache, gltfBufferView.byteStride) } let result = try vrm.gltf.bufferViewData(at: index, relativeTo: rootDirectory) @@ -161,10 +161,10 @@ open class VRMSceneLoader { func texture(withTextureIndex index: Int) throws -> SCNMaterialProperty { if let cache = try sceneData.load(\.textures, index: index) { return cache } - let gltfTexture = try gltf.load(\.textures)[index] + let gltfTexture = try gltf.load(\.textures, at: index) let texture = SCNMaterialProperty(contents: try image(withImageIndex: gltfTexture.source)) if let sampler = gltfTexture.sampler { - texture.setSampler(try gltf.load(\.samplers)[sampler]) + texture.setSampler(try gltf.load(\.samplers, at: sampler)) } else { texture.wrapS = .repeat texture.wrapT = .repeat @@ -175,7 +175,7 @@ open class VRMSceneLoader { func image(withImageIndex index: Int) throws -> VRMImage { if let cache = try sceneData.load(\.images, index: index) { return cache } - let gltfImage = try gltf.load(\.images)[index] + let gltfImage = try gltf.load(\.images, at: index) let image = try VRMImage.from(gltfImage, relativeTo: rootDirectory) { index in try self.bufferView(withBufferViewIndex: index).bufferView } diff --git a/Tests/VRMKitTests/BinaryGLTFTests.swift b/Tests/VRMKitTests/BinaryGLTFTests.swift index c22f3486..f37564da 100644 --- a/Tests/VRMKitTests/BinaryGLTFTests.swift +++ b/Tests/VRMKitTests/BinaryGLTFTests.swift @@ -1,5 +1,6 @@ import XCTest import VRMKit +import VRMTestSupport class BinaryGLTFTests: XCTestCase { @@ -13,5 +14,180 @@ class BinaryGLTFTests: XCTestCase { XCTAssertEqual(json.asset.generator, "UniGLTF") XCTAssertEqual(json.asset.version, "2.0") } - + + func testStridedSubdataCopiesBytes() throws { + let data = Data([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + let strided = try data.subdata(offset: 1, size: 2, stride: 4, count: 2) + XCTAssertEqual(Array(strided), [1, 2, 5, 6]) + } + + func testTightlyPackedSubdataReturnsOnlyTheRequestedRange() throws { + let data = Data([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + let packed = try data.subdata(offset: 2, size: 2, stride: 2, count: 3) + XCTAssertEqual(Array(packed), [2, 3, 4, 5, 6, 7]) + } + + /// An accessor that overruns its buffer view must fail the load instead of + /// reading out of bounds. + func testSubdataRejectsRangesBeyondTheBuffer() { + let data = Data([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + XCTAssertThrowsError(try data.subdata(offset: 0, size: 4, stride: 4, count: 3)) + XCTAssertThrowsError(try data.subdata(offset: 9, size: 2, stride: 2, count: 1)) + XCTAssertThrowsError(try data.subdata(offset: 8, size: 2, stride: 4, count: 2)) + XCTAssertThrowsError(try data.subdata(offset: -1, size: 2, stride: 2, count: 1)) + // The last byte landing exactly on the end is still valid. + XCTAssertNoThrow(try data.subdata(offset: 8, size: 2, stride: 2, count: 1)) + } + + /// The extent computation itself must not trap: a hostile glTF can name + /// counts and strides whose product overflows Int. + func testSubdataRejectsOverflowingExtentsWithoutTrapping() { + let data = Data([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + XCTAssertThrowsError(try data.subdata(offset: 0, size: 8, stride: 8, count: .max)) + XCTAssertThrowsError(try data.subdata(offset: .max, size: 1, stride: 1, count: 2)) + XCTAssertThrowsError(try data.subdata(offset: 0, size: .max, stride: .max, count: 3)) + XCTAssertThrowsError(try data.subdata(offset: .max - 1, size: 4, stride: 4, count: 1)) + } + + /// A buffer view that overruns its buffer has to throw before any accessor + /// can be sliced out of it. + func testBufferViewDataRejectsRangesBeyondTheBuffer() throws { + let overrunning = try binaryGLTF(withFirstBufferView: ["byteOffset": 0, "byteLength": Int(UInt32.max)]) + XCTAssertThrowsError(try overrunning.bufferViewData(at: 0)) + + let overflowing = try binaryGLTF(withFirstBufferView: ["byteOffset": Int.max, "byteLength": 16]) + XCTAssertThrowsError(try overflowing.bufferViewData(at: 0)) + + let negative = try binaryGLTF(withFirstBufferView: ["byteOffset": -1, "byteLength": 16]) + XCTAssertThrowsError(try negative.bufferViewData(at: 0)) + } + + func testBufferViewDataRejectsAnIndexBeyondTheFile() throws { + let binaryGltf = try BinaryGLTF(data: Resources.aliciaSolid.data) + XCTAssertThrowsError(try binaryGltf.bufferViewData(at: binaryGltf.jsonData.bufferViews?.count ?? 0)) + } + + /// The fixture with `byteOffset` / `byteLength` of its first buffer view + /// replaced, leaving the rest of the file intact. + private func binaryGLTF(withFirstBufferView fields: [String: Any]) throws -> BinaryGLTF { + let data = try Resources.aliciaSolid.rewritingJSON { json in + guard var bufferViews = json["bufferViews"] as? [[String: Any]], !bufferViews.isEmpty else { + throw GLBRewriter.Error.invalidJSON + } + bufferViews[0].merge(fields) { _, new in new } + json["bufferViews"] = bufferViews + } + return try BinaryGLTF(data: data) + } + + /// A file whose magic is not `glTF` has to fail the load. + func testRejectsAFileThatIsNotBinaryGLTF() { + var notGLB = Resources.aliciaSolid.data + notGLB.writeUInt32LE(0x12345678, at: 0) + XCTAssertThrowsError(try BinaryGLTF(data: notGLB)) + } + + func testRejectsATruncatedHeader() { + let data = Resources.aliciaSolid.data + // Cut short at each header field in turn: magic, version, total length, + // chunk 0 length, chunk 0 type, and mid-way through the JSON chunk. + for count in [0, 4, 8, 12, 16, 20, 24] { + XCTAssertThrowsError(try BinaryGLTF(data: data.prefix(count)), + "a \(count) byte file must not load") + } + } + + func testRejectsChunkLengthsBeyondTheFile() { + var jsonOverrun = Resources.aliciaSolid.data + jsonOverrun.writeUInt32LE(.max, at: 12) + XCTAssertThrowsError(try BinaryGLTF(data: jsonOverrun)) + + var binaryOverrun = Resources.aliciaSolid.data + let binaryChunkOffset = 20 + Int(binaryOverrun.uint32LE(at: 12)) + binaryOverrun.writeUInt32LE(.max, at: binaryChunkOffset) + XCTAssertThrowsError(try BinaryGLTF(data: binaryOverrun)) + } + + /// A buffer view naming a buffer the file does not have has to throw rather + /// than trap on the subscript. + func testBufferViewDataRejectsAnIndexBeyondTheBuffers() throws { + let binaryGltf = try binaryGLTF(withFirstBufferView: ["buffer": 99]) + XCTAssertThrowsError(try binaryGltf.bufferViewData(at: 0)) + } + + /// The header length names the size of the whole GLB, so a file shorter than + /// it is truncated and its chunk offsets cannot be trusted. + func testRejectsAHeaderLengthBeyondTheFile() { + var overrunLength = Resources.aliciaSolid.data + overrunLength.writeUInt32LE(UInt32(overrunLength.count + 4), at: 8) + XCTAssertThrowsError(try BinaryGLTF(data: overrunLength)) + } + + /// Bytes past the header length are not part of any chunk, so a file that + /// carries them (exporter padding, a slice of a larger container) still loads. + func testLoadsAFileWithBytesPastTheHeaderLength() throws { + var trailingBytes = Resources.aliciaSolid.data + trailingBytes.append(contentsOf: [0, 0, 0, 0]) + let binaryGltf = try BinaryGLTF(data: trailingBytes) + XCTAssertEqual(binaryGltf.jsonData.asset.version, "2.0") + XCTAssertNotNil(binaryGltf.binaryBuffer) + } + + /// The GLB container version and the asset version are independent, so a 2.0 + /// container can still declare an asset this parser does not implement. + func testRejectsAnAssetVersionItDoesNotImplement() throws { + let futureVersion = try Resources.aliciaSolid.rewritingJSON { json in + var asset = json["asset"] as? [String: Any] ?? [:] + asset["version"] = "3.0" + json["asset"] = asset + } + XCTAssertThrowsError(try BinaryGLTF(data: futureVersion)) + + let futureMinVersion = try Resources.aliciaSolid.rewritingJSON { json in + var asset = json["asset"] as? [String: Any] ?? [:] + asset["minVersion"] = "2.1" + json["asset"] = asset + } + XCTAssertThrowsError(try BinaryGLTF(data: futureMinVersion)) + } + + /// glTF aligns MAT2 / MAT3 columns to 4 bytes, so with narrow components an + /// element is wider than its components and the packed reader would mis-slice + /// it. Those layouts are rejected; the MAT4 float layout VRM uses is not. + func testRejectsMatrixAccessorsWhoseColumnsArePadded() throws { + let padded = try accessor(#"{"bufferView": 0, "count": 1, "componentType": 5121, "type": "MAT3"}"#) + XCTAssertThrowsError(try padded.packedData(bufferView: { _ in (Data(repeating: 0, count: 48), nil) })) + + let unpadded = try accessor(#"{"bufferView": 0, "count": 1, "componentType": 5126, "type": "MAT4"}"#) + let data = try unpadded.packedData(bufferView: { _ in (Data(repeating: 0, count: 64), nil) }) + XCTAssertEqual(data.count, 64) + } + + /// Sparse indices substitute elements by position, so the spec requires them + /// strictly increasing: a repeated index leaves the element it overlaps on + /// depending on the copy order. + func testSparseAccessorRequiresStrictlyIncreasingIndices() throws { + let sparse = try accessor(""" + {"count": 4, "componentType": 5126, "type": "SCALAR", + "sparse": {"count": 2, + "indices": {"bufferView": 0, "componentType": 5121}, + "values": {"bufferView": 1}}} + """) + func provider(indices: [UInt8]) -> BufferViewProvider { + { $0 == 0 ? (Data(indices), nil) : (Data(repeating: 1, count: 8), nil) } + } + XCTAssertThrowsError(try sparse.packedData(bufferView: provider(indices: [2, 2]))) + XCTAssertThrowsError(try sparse.packedData(bufferView: provider(indices: [2, 1]))) + XCTAssertNoThrow(try sparse.packedData(bufferView: provider(indices: [1, 2]))) + } + + private func accessor(_ json: String) throws -> GLTF.Accessor { + try JSONDecoder().decode(GLTF.Accessor.self, from: Data(json.utf8)) + } + + func testZeroedAccessorDataRejectsOverflowingSizes() { + XCTAssertThrowsError(try Data(zeroedElementCount: .max, elementSize: 12)) + XCTAssertThrowsError(try Data(zeroedElementCount: -1, elementSize: 12)) + XCTAssertEqual(try Data(zeroedElementCount: 3, elementSize: 4).count, 12) + } } diff --git a/Tests/VRMKitTests/Resources.swift b/Tests/VRMKitTests/Resources.swift index 9fbd7612..29d41090 100644 --- a/Tests/VRMKitTests/Resources.swift +++ b/Tests/VRMKitTests/Resources.swift @@ -1,4 +1,5 @@ import Foundation +import VRMTestSupport enum Resources { case aliciaSolid @@ -15,3 +16,29 @@ enum Resources { } } } + +extension Resources { + /// The fixture with its glTF JSON rewritten, so tests can feed the loaders + /// malformed or unusual files without shipping extra assets. + func rewritingJSON(_ modify: (inout [String: Any]) throws -> Void) throws -> Data { + try GLBRewriter.rewritingJSON(of: data, modify) + } + + /// The fixture with `extensions.VRMC_vrm.specVersion` replaced. Passing nil + /// removes the key entirely. + func withVRMCSpecVersion(_ specVersion: Any?) throws -> Data { + try rewritingJSON { json in + guard var extensions = json["extensions"] as? [String: Any], + var vrm = extensions["VRMC_vrm"] as? [String: Any] else { + throw GLBRewriter.Error.invalidJSON + } + if let specVersion { + vrm["specVersion"] = specVersion + } else { + vrm.removeValue(forKey: "specVersion") + } + extensions["VRMC_vrm"] = vrm + json["extensions"] = extensions + } + } +} diff --git a/Tests/VRMKitTests/VRM1Tests.swift b/Tests/VRMKitTests/VRM1Tests.swift index 0947fb3f..ae57eb45 100644 --- a/Tests/VRMKitTests/VRM1Tests.swift +++ b/Tests/VRMKitTests/VRM1Tests.swift @@ -12,6 +12,24 @@ class VRM1Tests: XCTestCase { func testSpecVersion() { XCTAssertEqual(vrm.specVersion, "1.0") } + + /// A missing or mistyped specVersion has to surface as a thrown error rather + /// than trapping in the initializer. + func testMalformedSpecVersionThrowsInsteadOfCrashing() throws { + XCTAssertThrowsError(try VRM1(data: try Resources.seedSan.withVRMCSpecVersion(nil))) + XCTAssertThrowsError(try VRM1(data: try Resources.seedSan.withVRMCSpecVersion(1.0))) + XCTAssertThrowsError(try VRM1(data: try Resources.seedSan.withVRMCSpecVersion(["1.0"]))) + } + + func testUnsupportedSpecVersionIsRejected() throws { + XCTAssertTrue(VRM1.supports(specVersion: "1.0")) + XCTAssertTrue(VRM1.supports(specVersion: "1.0-beta")) + XCTAssertFalse(VRM1.supports(specVersion: "2.0")) + XCTAssertFalse(VRM1.supports(specVersion: "1.0-draft")) + + XCTAssertNoThrow(try VRM1(data: try Resources.seedSan.withVRMCSpecVersion("1.0-beta"))) + XCTAssertThrowsError(try VRM1(data: try Resources.seedSan.withVRMCSpecVersion("2.0"))) + } func testMeta() { diff --git a/Tests/VRMRealityKitTests/ApproximateEquality.swift b/Tests/VRMRealityKitTests/ApproximateEquality.swift new file mode 100644 index 00000000..cab1fe91 --- /dev/null +++ b/Tests/VRMRealityKitTests/ApproximateEquality.swift @@ -0,0 +1,20 @@ +import simd + +/// Shared approximate-equality helpers for float comparisons in tests. +extension Float { + func isApproximatelyEqual(to other: Float, tolerance: Float = 0.0001) -> Bool { + abs(self - other) < tolerance + } +} + +extension SIMD where Scalar == Float { + func isApproximatelyEqual(to other: Self, tolerance: Float = 0.0001) -> Bool { + indices.allSatisfy { abs(self[$0] - other[$0]) < tolerance } + } +} + +extension simd_float4x4 { + func isApproximatelyEqual(to other: simd_float4x4, tolerance: Float = 0.0001) -> Bool { + (0..<4).allSatisfy { self[$0].isApproximatelyEqual(to: other[$0], tolerance: tolerance) } + } +} diff --git a/Tests/VRMRealityKitTests/MToonMaterialDescriptorTests.swift b/Tests/VRMRealityKitTests/MToonMaterialDescriptorTests.swift new file mode 100644 index 00000000..28d36fd0 --- /dev/null +++ b/Tests/VRMRealityKitTests/MToonMaterialDescriptorTests.swift @@ -0,0 +1,490 @@ +import Foundation +import Testing +@testable import VRMKit +@testable import VRMKitRuntime + +@Suite +struct MToonMaterialDescriptorTests { + /// JSON numbers reach the VRM extensions as `NSNumber`, and glTF indices + /// have to survive that intact: `Float` rounds `16_777_217` to its neighbour + /// and `Int32.max` past its own range. + @Test + func testIndexCoercionAcceptsOnlyExactNonNegativeInt32Values() { + #expect(numericIndexValue(0) == 0) + #expect(numericIndexValue(7) == 7) + #expect(numericIndexValue(16_777_217) == 16_777_217) + #expect(numericIndexValue(Int(Int32.max)) == Int(Int32.max)) + #expect(numericIndexValue(Int(Int32.max) + 1) == nil) + #expect(numericIndexValue(-1) == nil) + #expect(numericIndexValue(1.5) == nil) + #expect(numericIndexValue(Double.nan) == nil) + #expect(numericIndexValue(true) == nil) + #expect(numericIndexValue("3") == nil) + } + + @Test + func testVRM0DefaultValuesMigrateToMToon10Domain() throws { + let descriptor = try #require(MToonMaterialDescriptor(material: material(), + materialProperty: vrm0MaterialProperty())) + + #expect(descriptor.shadingToonyFactor.isApproximatelyEqual(to: 0.95)) + #expect(descriptor.shadingShiftFactor.isApproximatelyEqual(to: -0.05)) + #expect(descriptor.giEqualizationFactor.isApproximatelyEqual(to: 0.9)) + // UniVRM migrates rim lighting mix destructively to 1.0. + #expect(descriptor.rimLightingMixFactor == 1) + } + + @Test + func testVRM0RimLightingMixIsMigratedDestructivelyLikeUniVRM() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_RimLightingMix": 0.3}"#) + )) + + #expect(descriptor.rimLightingMixFactor == 1) + } + + @Test + func testVRM0LitShadeRimOutlineColorsAreConvertedToLinear() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(vectors: #""" + { + "_Color": [0.5, 0.5, 0.5, 0.5], + "_ShadeColor": [0.5, 0.5, 0.5, 1.0], + "_RimColor": [0.5, 0.5, 0.5, 1.0], + "_OutlineColor": [0.5, 0.5, 0.5, 1.0], + "_EmissionColor": [0.5, 0.5, 0.5, 1.0] + } + """#) + )) + let linearHalf = VRM0MToonMigrator.srgbToLinear(0.5) + + #expect(linearHalf.isApproximatelyEqual(to: 0.21404114)) + #expect(descriptor.baseColorFactor.isApproximatelyEqual(to: SIMD4(linearHalf, linearHalf, linearHalf, 0.5))) + #expect(descriptor.shadeColorFactor.isApproximatelyEqual(to: SIMD4(linearHalf, linearHalf, linearHalf, 1))) + #expect(descriptor.parametricRimColorFactor.isApproximatelyEqual(to: SIMD4(linearHalf, linearHalf, linearHalf, 1))) + #expect(descriptor.outlineColorFactor.isApproximatelyEqual(to: SIMD4(linearHalf, linearHalf, linearHalf, 1))) + // Emission is already linear in VRM 0.x and must be passed through. + #expect(descriptor.emissiveFactor.isApproximatelyEqual(to: SIMD3(0.5, 0.5, 0.5))) + } + + @Test + func testVRM0UVAnimationScrollYIsInverted() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_UvAnimScrollX": 0.1, "_UvAnimScrollY": 0.25}"#) + )) + + #expect(descriptor.uvAnimationScrollXSpeedFactor.isApproximatelyEqual(to: 0.1)) + #expect(descriptor.uvAnimationScrollYSpeedFactor.isApproximatelyEqual(to: -0.25)) + } + + @Test + func testVRM0ScreenOutlineWidthIsHalved() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_OutlineWidthMode": 2, "_OutlineWidth": 10}"#) + )) + + #expect(descriptor.outlineWidthMode == .screenCoordinates) + #expect(descriptor.outlineWidthFactor.isApproximatelyEqual(to: 10 * 0.01 * 0.5)) + } + + @Test + func testVRM0FixedOutlineColorModeRendersUnlit() throws { + let fixed = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_OutlineColorMode": 0, "_OutlineLightingMix": 0.8}"#) + )) + let mixed = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_OutlineColorMode": 1, "_OutlineLightingMix": 0.8}"#) + )) + + #expect(fixed.outlineLightingMixFactor == 0) + #expect(mixed.outlineLightingMixFactor.isApproximatelyEqual(to: 0.8)) + } + + @Test + func testVRM0OutOfRangeAndBooleanNumbersFallBackToDefaults() throws { + // `1e100` overflows `Float` and JSON booleans bridge to `NSNumber`: + // neither is a usable value, and converting them to `Int` would trap. + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #""" + { + "_OutlineWidthMode": 1e100, + "_OutlineColorMode": 1e100, + "_OutlineLightingMix": 0.8, + "_ShadeToony": true + } + """#) + )) + + #expect(descriptor.outlineWidthMode == .none) + #expect(descriptor.outlineWidthFactor == 0) + #expect(descriptor.outlineLightingMixFactor == 0) + #expect(descriptor.shadingToonyFactor.isApproximatelyEqual(to: 0.95)) + } + + @Test + func testVRM1OutOfRangeTextureTransformNumbersFallBackToDefaults() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(#""" + { + "emissiveTexture": { + "index": 5, + "texCoord": 1, + "extensions": { + "KHR_texture_transform": { "texCoord": 1e100, "rotation": 1e100 } + } + }, + "extensions": { + "VRMC_materials_mtoon": { + "specVersion": "1.0" + } + } + } + """#), + materialProperty: nil + )) + + // The unusable override is ignored, so the texture info's own texCoord wins. + #expect(descriptor.emissiveTexture?.texCoord == 1) + #expect(descriptor.emissiveTexture?.transform?.rotation == 0) + } + + @Test + func testVRM0MainTextureTransformIsMigrated() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_MainTex": 2, "_ShadeTexture": 4}"#, + vectors: #"{"_MainTex": [0.1, 0.2, 0.5, 0.4]}"#) + )) + + let transform = try #require(descriptor.baseColorTexture?.transform) + #expect(transform.scale == SIMD2(0.5, 0.4)) + #expect(transform.offset.isApproximatelyEqual(to: SIMD2(0.1, 1 - 0.2 - 0.4))) + #expect(transform.rotation == 0) + // MToon 0.x applies the material's _MainTex ST to every texture. + #expect(descriptor.shadeMultiplyTexture?.transform == transform) + } + + @Test + func testVRM0IdentityMainTextureTransformStaysNil() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_MainTex": 2}"#, + vectors: #"{"_MainTex": [0, 0, 1, 1]}"#) + )) + + #expect(descriptor.baseColorTexture?.transform == nil) + } + + @Test + func testVRM0MigratesOutlineWidthAndUvRotationUnits() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #""" + { + "_OutlineWidthMode": 1, + "_OutlineWidth": 3.5, + "_UvAnimRotation": 0.25 + } + """#) + )) + + #expect(descriptor.outlineWidthMode == .worldCoordinates) + #expect(descriptor.outlineWidthFactor.isApproximatelyEqual(to: 0.035)) + #expect(descriptor.uvAnimationRotationSpeedFactor.isApproximatelyEqual(to: 0.5 * Float.pi)) + } + + @Test + func testVRM0RenderQueueIsNotMigratedToAnOffset() throws { + // renderQueueOffsetNumber is a relative order among a model's transparent + // materials, so a per-material conversion of Unity's absolute renderQueue + // would fabricate an ordering; VRM 0.x stays neutral. + for renderQueue in [0, 2508, 2980, 2994, 3000, 3200] { + let transparent = try descriptor(renderQueue: renderQueue, + keywordMap: #"{"_ALPHABLEND_ON": true}"#, + tagMap: #"{"RenderType": "Transparent"}"#) + #expect(transparent.renderQueueOffsetNumber == 0) + + let zWrite = try descriptor(renderQueue: renderQueue, + floats: #"{"_BlendMode": 3, "_ZWrite": 1}"#, + keywordMap: #"{"_ALPHABLEND_ON": true}"#, + tagMap: #"{"RenderType": "Transparent"}"#) + #expect(zWrite.renderQueueOffsetNumber == 0) + // The TransparentWithZWrite render mode still reaches the descriptor. + #expect(zWrite.transparentWithZWrite) + } + } + + /// MToon 0.x carries its render mode in `_BlendMode`, not in a shader + /// keyword, so only mode 3 migrates to transparentWithZWrite. + @Test + func testVRM0TransparentWithZWriteComesFromTheBlendMode() throws { + let transparentTags = #"{"RenderType": "Transparent"}"# + let blend = #"{"_ALPHABLEND_ON": true}"# + + for blendMode in [0, 1, 2] { + let other = try descriptor(renderQueue: 3000, + floats: #"{"_BlendMode": \#(blendMode), "_ZWrite": 1}"#, + keywordMap: blend, + tagMap: transparentTags) + #expect(!other.transparentWithZWrite, "_BlendMode \(blendMode) is not TransparentWithZWrite") + } + + // Without `_BlendMode`, `_ZWrite` is the only signal left, and it only + // separates the transparent modes. + let zWriteOnly = try descriptor(renderQueue: 3000, + floats: #"{"_ZWrite": 1}"#, + keywordMap: blend, + tagMap: transparentTags) + #expect(zWriteOnly.transparentWithZWrite) + + let opaqueZWrite = try descriptor(renderQueue: 2000, + floats: #"{"_ZWrite": 1}"#, + keywordMap: "{}", + tagMap: #"{"RenderType": "Opaque"}"#) + #expect(!opaqueZWrite.transparentWithZWrite) + } + + @Test + func testVRM0BaseColorFallsBackToLinearGltfFactor() throws { + // _Color is a Unity sRGB color and needs converting; the glTF + // baseColorFactor fallback is already a linear multiplier and must not + // be converted a second time. + let gltfMaterial = try material(#""" + {"pbrMetallicRoughness": {"baseColorFactor": [0.5, 0.5, 0.5, 1.0]}} + """#) + let fallback = try #require(MToonMaterialDescriptor(material: gltfMaterial, + materialProperty: vrm0MaterialProperty())) + #expect(fallback.baseColorFactor.isApproximatelyEqual(to: SIMD4(0.5, 0.5, 0.5, 1))) + + let unityColor = try #require(MToonMaterialDescriptor( + material: gltfMaterial, + materialProperty: vrm0MaterialProperty(vectors: #"{"_Color": [0.5, 0.5, 0.5, 1.0]}"#) + )) + #expect(unityColor.baseColorFactor.x < 0.5) + #expect(unityColor.baseColorFactor.x.isApproximatelyEqual(to: VRM0MToonMigrator.srgbToLinear(0.5))) + #expect(unityColor.baseColorFactor.w.isApproximatelyEqual(to: 1)) + } + + @Test + func testUnsupportedMToonSpecVersionFallsBackToNonMToon() throws { + // A future revision must not be reinterpreted with 1.0 semantics; the + // renderer falls back to Unlit / PBR instead. + for specVersion in ["1.0", "1.0-beta"] { + let supported = try material(#""" + {"extensions": {"VRMC_materials_mtoon": {"specVersion": "\#(specVersion)"}}} + """#) + #expect(MToonMaterialDescriptor(material: supported, materialProperty: nil) != nil, + "specVersion \(specVersion) should be supported") + } + for specVersion in ["2.0", "0.9", ""] { + let unsupported = try material(#""" + {"extensions": {"VRMC_materials_mtoon": {"specVersion": "\#(specVersion)"}}} + """#) + #expect(MToonMaterialDescriptor(material: unsupported, materialProperty: nil) == nil, + "specVersion \(specVersion) should not be treated as MToon 1.0") + } + } + + @Test + func testVRM0EmissiveFieldsUseEmissionProperties() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_EmissionMap": 3}"#, + vectors: #"{"_EmissionColor": [0.2, 0.3, 0.4, 1.0]}"#) + )) + + #expect(descriptor.emissiveFactor.isApproximatelyEqual(to: SIMD3(0.2, 0.3, 0.4))) + #expect(descriptor.emissiveTexture?.index == 3) + #expect(descriptor.emissiveTexture?.texCoord == 0) + } + + @Test + func testVRM0ShadeTextureTakesPriorityOverMainTexture() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_MainTex": 2, "_ShadeTexture": 4}"#) + )) + + #expect(descriptor.baseColorTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture?.index == 4) + #expect(descriptor.shadeMultiplyTexture?.texCoord == 0) + } + + @Test + func testVRM0ShadeTextureFallsBackToMainTexture() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_MainTex": 2}"#) + )) + + #expect(descriptor.baseColorTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture?.texCoord == 0) + } + + @Test + func testVRM1MissingShadeTextureRemainsNil() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(#""" + { + "pbrMetallicRoughness": { + "baseColorTexture": { "index": 2, "texCoord": 1 } + }, + "extensions": { + "VRMC_materials_mtoon": { + "specVersion": "1.0" + } + } + } + """#), + materialProperty: nil + )) + + #expect(descriptor.baseColorTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture == nil) + } + + @Test + func testVRM1MToon10Defaults() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: vrm1Material(), + materialProperty: nil + )) + + #expect(descriptor.shadeColorFactor == SIMD4(0, 0, 0, 1)) + #expect(descriptor.parametricRimFresnelPowerFactor == 5) + #expect(descriptor.cullMode == .back) + #expect(descriptor.normalScale == 1) + } + + @Test + func testVRM0CullModePreservesFrontBackAndDisabledValues() throws { + let expected: [(Float, MToonMaterialDescriptor.CullMode)] = [ + (0, .none), + (1, .front), + (2, .back) + ] + + for (value, cullMode) in expected { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_CullMode": \#(value)}"#) + )) + #expect(descriptor.cullMode == cullMode) + } + } + + @Test + func testVRM0InvalidCullModeUsesMaterialFallback() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_CullMode": 1.5}"#) + )) + + #expect(descriptor.cullMode == .back) + } + + @Test + func testVRM1DoubleSidedDisablesCulling() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(#"{"doubleSided": true, "extensions": {"VRMC_materials_mtoon": {"specVersion": "1.0"}}}"#), + materialProperty: nil + )) + + #expect(descriptor.cullMode == .none) + } + + @Test + func testNormalScaleUsesVRM0AndVRM1MaterialValues() throws { + let vrm0 = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_BumpScale": 0.4}"#, + textures: #"{"_BumpMap": 1}"#) + )) + let vrm1 = try #require(MToonMaterialDescriptor( + material: material(#"{"normalTexture": {"index": 1, "scale": 0.65}, "extensions": {"VRMC_materials_mtoon": {"specVersion": "1.0"}}}"#), + materialProperty: nil + )) + + #expect(vrm0.normalScale.isApproximatelyEqual(to: 0.4)) + #expect(vrm1.normalScale.isApproximatelyEqual(to: 0.65)) + } + + @Test + func testVRM1EmissiveFieldsAndMatcapDefaultUseGltfMaterial() throws { + let gltfMaterial = try material(#""" + { + "emissiveFactor": [0.6, 0.7, 0.8], + "emissiveTexture": { "index": 5, "texCoord": 1 }, + "extensions": { + "VRMC_materials_mtoon": { + "specVersion": "1.0" + } + } + } + """#) + let descriptor = try #require(MToonMaterialDescriptor(material: gltfMaterial, materialProperty: nil)) + + #expect(descriptor.emissiveFactor.isApproximatelyEqual(to: SIMD3(0.6, 0.7, 0.8))) + #expect(descriptor.emissiveTexture?.index == 5) + #expect(descriptor.emissiveTexture?.texCoord == 1) + #expect(descriptor.matcapFactor.isApproximatelyEqual(to: SIMD3(1, 1, 1))) + } + + private func descriptor(renderQueue: Int, + floats: String = "{}", + keywordMap: String, + tagMap: String) throws -> MToonMaterialDescriptor { + return try #require(MToonMaterialDescriptor( + material: material(#"{"alphaMode": "OPAQUE"}"#), + materialProperty: vrm0MaterialProperty(renderQueue: renderQueue, + floats: floats, + keywordMap: keywordMap, + tagMap: tagMap) + )) + } + + private func material(_ json: String = "{}") throws -> GLTF.Material { + return try JSONDecoder().decode(GLTF.Material.self, from: Data(json.utf8)) + } + + private func vrm1Material() throws -> GLTF.Material { + return try material(#""" + { + "extensions": { + "VRMC_materials_mtoon": { + "specVersion": "1.0" + } + } + } + """#) + } + + private func vrm0MaterialProperty(renderQueue: Int = 0, + floats: String = "{}", + keywordMap: String = "{}", + tagMap: String = "{}", + textures: String = "{}", + vectors: String = "{}") throws -> VRM0.MaterialProperty { + let json = #""" + { + "name": "MToon", + "shader": "VRM/MToon", + "renderQueue": \#(renderQueue), + "floatProperties": \#(floats), + "keywordMap": \#(keywordMap), + "tagMap": \#(tagMap), + "textureProperties": \#(textures), + "vectorProperties": \#(vectors) + } + """# + return try JSONDecoder().decode(VRM0.MaterialProperty.self, from: Data(json.utf8)) + } +} diff --git a/Tests/VRMRealityKitTests/TestSupport.swift b/Tests/VRMRealityKitTests/TestSupport.swift new file mode 100644 index 00000000..0753f9d3 --- /dev/null +++ b/Tests/VRMRealityKitTests/TestSupport.swift @@ -0,0 +1,173 @@ +#if canImport(RealityKit) +import Foundation +import RealityKit +import Testing +import VRMKit +import VRMTestSupport +@testable import VRMRealityKit + +/// Shared fixtures and helpers for the VRMRealityKit test target. +enum TestSupport { + /// The bundled Seed-san VRM 1.0 fixture, read once per test process. + static let seedSanData: Data = { + guard let url = Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), + let data = try? Data(contentsOf: url) else { + fatalError("Failed to load Seed-san.vrm resource from test bundle.") + } + return data + }() + + /// The bundled AliciaSolid VRM 0.x fixture, read once per test process. + /// The VRM 0.x loading paths need a 0.x model; Seed-san is 1.0. + static let aliciaSolidData: Data = { + guard let url = Bundle.module.url(forResource: "AliciaSolid", withExtension: "vrm"), + let data = try? Data(contentsOf: url) else { + fatalError("Failed to load AliciaSolid.vrm resource from test bundle.") + } + return data + }() + + /// Rewrites the fixture's glTF JSON in memory. Loaders accept the returned + /// data directly, so no temporary files are written. `name` identifies the + /// variant in failure messages. + static func modifiedSeedSanData(name: String, + modify: (inout [String: Any]) throws -> Void) throws -> Data { + do { + return try GLBRewriter.rewritingJSON(of: seedSanData, modify) + } catch let error as GLBRewriter.Error { + throw VRMError.dataInconsistent("Invalid Seed-san fixture data for '\(name)': \(error)") + } + } + + /// Rewrites the VRM 0.x fixture's glTF JSON in memory. + static func modifiedAliciaSolidData(name: String, + modify: (inout [String: Any]) throws -> Void) throws -> Data { + do { + return try GLBRewriter.rewritingJSON(of: aliciaSolidData, modify) + } catch let error as GLBRewriter.Error { + throw VRMError.dataInconsistent("Invalid AliciaSolid fixture data for '\(name)': \(error)") + } + } + + /// Rewrites a single glTF material of the fixture. Wraps the repeated + /// unwrap/mutate/write-back dance the material tests all need. + static func modifiedSeedSanMaterial(name: String, + index: Int = 0, + modify: (inout [String: Any]) throws -> Void) throws -> Data { + try modifiedSeedSanData(name: name) { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(index) else { + throw VRMError.dataInconsistent("Missing Seed-san material \(index) for fixture '\(name)'") + } + try modify(&materials[index]) + json["materials"] = materials + } + } + + /// Rewrites a material's `VRMC_materials_mtoon` extension. + static func modifiedSeedSanMToonExtension(name: String, + index: Int = 0, + modify: (inout [String: Any]) throws -> Void) throws -> Data { + try modifiedSeedSanMaterial(name: name, index: index) { material in + guard var extensions = material["extensions"] as? [String: Any], + var mtoon = extensions["VRMC_materials_mtoon"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san MToon extension for fixture '\(name)'") + } + try modify(&mtoon) + extensions["VRMC_materials_mtoon"] = mtoon + material["extensions"] = extensions + } + } + + /// Rewrites the fixture's `VRMC_vrm` preset expressions. + static func modifiedSeedSanExpressions(name: String, + modify: (inout [String: Any]) throws -> Void) throws -> Data { + try modifiedSeedSanData(name: name) { json in + try modifyExpressionPresets(in: &json, modify) + } + } + + /// The unwrap/mutate/write-back dance for `extensions.VRMC_vrm.expressions.preset`. + static func modifyExpressionPresets(in json: inout [String: Any], + _ modify: (inout [String: Any]) throws -> Void) throws { + guard var extensions = json["extensions"] as? [String: Any], + var vrm = extensions["VRMC_vrm"] as? [String: Any], + var expressions = vrm["expressions"] as? [String: Any], + var preset = expressions["preset"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san expression fixture data") + } + try modify(&preset) + expressions["preset"] = preset + vrm["expressions"] = expressions + extensions["VRMC_vrm"] = vrm + json["extensions"] = extensions + } + + /// The MToon shader sources concatenated, read once per test process. + /// They are repository sources, not bundle resources: the shaders are + /// compiled offline into the bundled metallibs. + static let mtoonShaderSource: String = { + let shaders = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("Sources/VRMRealityKit/Shaders") + return ["MToonCore.h", "MToon.metal"] + .compactMap { try? String(contentsOf: shaders.appendingPathComponent($0), encoding: .utf8) } + .joined(separator: "\n") + }() + + static let expectedCustomMaterialMessage: Comment = + "Expected default MToon rendering to load a CustomMaterial. Run scripts/build-mtoon-metallibs.sh and verify the package resources." + + @MainActor + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + static func modelEntities(in root: Entity) -> [ModelEntity] { + root.modelEntitiesInHierarchy + } + +#if !os(visionOS) + /// Whether any model entity in the hierarchy renders with a CustomMaterial. + /// visionOS has no `CustomMaterial`, so this only exists where MToon does. + @MainActor + @available(iOS 18.0, macOS 15.0, *) + static func hasCustomMaterial(in root: Entity) -> Bool { + modelEntities(in: root) + .flatMap { $0.components[ModelComponent.self]?.materials ?? [] } + .contains { $0 is CustomMaterial } + } +#endif + + /// Whether any of the entity's materials carries MToon runtime state. + @MainActor + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + static func hasMToonParameters(in vrmEntity: VRMEntity) -> Bool { + materialIndexes(in: vrmEntity).contains { vrmEntity.mtoonParameters(forMaterialIndex: $0) != nil } + } + + /// Every glTF material index rendered by the hierarchy, in ascending order. + @MainActor + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + static func materialIndexes(in root: Entity) -> [Int] { + Set(modelEntities(in: root).compactMap { $0.components[VRMMaterialIndexComponent.self]?.materialIndex }) + .sorted() + } + +#if !os(visionOS) + static func isTransparent(_ blending: CustomMaterial.Blending) -> Bool { + if case .transparent = blending { + return true + } + return false + } + + static func isOpaque(_ blending: CustomMaterial.Blending) -> Bool { + if case .opaque = blending { + return true + } + return false + } +#endif + +} +#endif diff --git a/Tests/VRMRealityKitTests/VRM1RealityKitTests.swift b/Tests/VRMRealityKitTests/VRM1RealityKitTests.swift new file mode 100644 index 00000000..94b2968f --- /dev/null +++ b/Tests/VRMRealityKitTests/VRM1RealityKitTests.swift @@ -0,0 +1,1429 @@ +#if canImport(RealityKit) +import Foundation +import Metal +import RealityKit +import Testing +import VRMKit +@testable import VRMRealityKit + +@Suite +@MainActor +struct VRM1RealityKitTests { + +#if !os(visionOS) + @Test + func testVRM1MToonCustomMaterialUsesParameterTexture() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let material = try vrmLoader.material(withMaterialIndex: 0) + let customMaterial = try #require(material as? CustomMaterial, + TestSupport.expectedCustomMaterialMessage) + + #expect(customMaterial.custom.texture != nil) + #expect(customMaterial.normal.texture != nil) + #expect(customMaterial.roughness.texture != nil) + #expect(customMaterial.emissiveColor.texture != nil) + #expect(customMaterial.clearcoat.texture != nil) + #expect(customMaterial.clearcoatRoughness.texture != nil) + + let direction = MToonMaterialParameters.defaultLightDirection + #expect(customMaterial.custom.value.isApproximatelyEqual(to: SIMD4(direction, 0))) + } + + @Test + func testVRM1MToonRenderingCanBeDisabled() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let defaultLoader = try VRMEntityLoader(withData: seedSan) + let defaultMaterial = try defaultLoader.material(withMaterialIndex: 0) + _ = try #require(defaultMaterial as? CustomMaterial, + TestSupport.expectedCustomMaterialMessage) + + let disabledLoader = try VRMEntityLoader(withData: seedSan, isMToonEnabled: false) + let disabledMaterial = try disabledLoader.material(withMaterialIndex: 0) + #expect(!(disabledMaterial is CustomMaterial)) + #expect(disabledMaterial is UnlitMaterial) + + let disabledEntity = try disabledLoader.loadEntity() + #expect(!TestSupport.hasCustomMaterial(in: disabledEntity)) + #expect(!TestSupport.hasMToonParameters(in: disabledEntity)) + } + + @Test + func testMToonParameterTextureRowsMatchMetalConstant() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let vrmEntity = try vrmLoader.loadEntity() + let parameters = try firstMToonParameters(in: vrmEntity) + let texture = try parameters.textureResource() + let shader = TestSupport.mtoonShaderSource + + #expect(MToonMaterialParameters.baseParameterRowCount == 17) + #expect(MToonMaterialParameters.samplerRowCount == MToonTextureSlot.allCases.count) + #expect(MToonMaterialParameters.textureRowCount == 26) + #expect(parameters.samplers.count == MToonMaterialParameters.samplerRowCount) + #expect(texture.width == MToonMaterialParameters.textureRowCount) + #expect(texture.height == 1) + + // The shader's row constants must match MToonParameterRow exactly: + // extract them instead of restating the literals, so any reordering or + // insertion on either side fails here. + let shaderConstants = shaderFloatConstants(in: shader) + #expect(shaderConstants["mtoonParameterTextureWidth"] == Float(MToonMaterialParameters.textureRowCount)) + #expect(shaderConstants["mtoonSamplerParameterStart"] == Float(MToonMaterialParameters.baseParameterRowCount)) + for row in MToonParameterRow.allCases { + let name = shaderConstantName(prefix: "mtoonRow", case: row) + #expect(shaderConstants[name] == Float(row.rawValue), + "Shader constant \(name) does not match MToonParameterRow.\(row) (\(row.rawValue)).") + } + for slot in MToonTextureSlot.allCases { + let name = shaderConstantName(prefix: "mtoonSamplerSlot", case: slot) + #expect(shaderConstants[name] == Float(slot.rawValue), + "Shader constant \(name) does not match MToonTextureSlot.\(slot) (\(slot.rawValue)).") + } + } + + @Test + func testMToonNormalScaleIsPassedToShaderParameters() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanMaterial(name: "normal-scale") { material in + material["normalTexture"] = ["index": 0, "scale": 0.35] + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + let parameters = try mtoonParameters(in: vrmEntity, materialIndex: 0) + + #expect(parameters.normalParameters.x.isApproximatelyEqual(to: 0.35)) + } + + @Test + func testMToonSkipsWorkThatCannotContribute() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // Seed-san material 0 ("hair") has an outlineWidthMultiplyTexture; 1 + // ("huku_bake") does not, so the flag must differ between them. + let entity = try VRMEntityLoader(withData: TestSupport.seedSanData).loadEntity() + #expect(try mtoonParameters(in: entity, materialIndex: 0).normalParameters.y == 1) + #expect(try mtoonParameters(in: entity, materialIndex: 1).normalParameters.y == 0) + } + + @Test + func testMToonMaskTextureSlotsUseRawSemantic() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let textureIndex = 0 + let modified = try TestSupport.modifiedSeedSanMToonExtension(name: "raw-mask-textures") { mtoon in + mtoon["shadingShiftTexture"] = ["index": textureIndex] + mtoon["outlineWidthMultiplyTexture"] = ["index": textureIndex] + mtoon["uvAnimationMaskTexture"] = ["index": textureIndex] + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let material = try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial) + let rawTexture = try loader.texture(withTextureIndex: textureIndex, semantic: .raw) + let colorTexture = try loader.texture(withTextureIndex: textureIndex, semantic: .color) + + #expect(material.specular.texture != nil) + #expect(material.clearcoat.texture != nil) + #expect(material.ambientOcclusion.texture != nil) + #expect(MToonTextureSlot.shadingShift.semantic == .raw) + #expect(MToonTextureSlot.outlineWidth.semantic == .raw) + #expect(MToonTextureSlot.uvAnimationMask.semantic == .raw) + #expect(rawTexture !== colorTexture) + } + + @Test + func testMToonEmissiveFlagFactorAndEmissionColorBind() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let vrmEntity = try vrmLoader.loadEntity() + var parameters = try firstMToonParameters(in: vrmEntity) + let boundColor = SIMD4(0.25, 0.5, 0.75, 0.2) + + #expect(parameters.extraFlags.z == 0 || parameters.extraFlags.z == 1) + #expect(parameters.color(for: .emissionColor).isApproximatelyEqual(to: parameters.emissiveFactor)) + parameters.setColor(boundColor, for: .emissionColor) + #expect(parameters.emissiveFactor.isApproximatelyEqual(to: SIMD4(0.25, 0.5, 0.75, 1))) + #expect(parameters.color(for: .emissionColor).isApproximatelyEqual(to: parameters.emissiveFactor)) + } + + @Test + func testMToonShadeMultiplyTextureFallsBackToWhite() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try seedSanDataWithNonDefaultEyeSampler() + let vrmLoader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let vrmEntity = try vrmLoader.loadEntity() + let eyeTransparentParameters = try mtoonParameters(in: vrmEntity, materialIndex: 4) + + #expect(eyeTransparentParameters.samplers[MToonTextureSlot.base.rawValue] != MToonMaterialParameters.defaultSampler) + #expect(eyeTransparentParameters.samplers[MToonTextureSlot.shade.rawValue] == MToonMaterialParameters.defaultSampler) + } + + @Test + func testMToonRespectsDoubleSidedMaterialFlag() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let singleSidedLoader = try VRMEntityLoader(withData: seedSan, isOutlineEnabled: false) + let singleSided = try #require(singleSidedLoader.material(withMaterialIndex: 0) as? CustomMaterial) + #expect(singleSided.faceCulling == .back) + + let doubleSidedData = try TestSupport.modifiedSeedSanMaterial(name: "double-sided") { material in + material["doubleSided"] = true + } + + let doubleSidedLoader = try VRMEntityLoader(withData: doubleSidedData, isOutlineEnabled: false) + let doubleSided = try #require(doubleSidedLoader.material(withMaterialIndex: 0) as? CustomMaterial) + #expect(doubleSided.faceCulling == .none) + } + + @Test + func testTransparentOutlinePreservesBaseTextureAlpha() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanMaterial(name: "transparent-outline") { material in + material["alphaMode"] = "BLEND" + var pbr = material["pbrMetallicRoughness"] as? [String: Any] ?? [:] + pbr["baseColorFactor"] = [1.0, 1.0, 1.0, 0.5] + material["pbrMetallicRoughness"] = pbr + } + + let loader = try VRMEntityLoader(withData: modified) + let vrmEntity = try loader.loadEntity() + let outline = try customMaterial(in: vrmEntity, + materialIndex: 0, + faceCulling: .front) + #expect(TestSupport.isTransparent(outline.blending)) + #expect(outline.baseColor.texture != nil) + + } + + @Test + func testMToonUsesFirstUVTransformWhenTextureSlotsDiffer() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanMaterial(name: "different-uv-transforms") { material in + guard var pbr = material["pbrMetallicRoughness"] as? [String: Any], + var baseTexture = pbr["baseColorTexture"] as? [String: Any], + var extensions = material["extensions"] as? [String: Any], + var mtoon = extensions["VRMC_materials_mtoon"] as? [String: Any], + var shadeTexture = mtoon["shadeMultiplyTexture"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san MToon texture fixture data") + } + baseTexture["texCoord"] = 1 + baseTexture["extensions"] = [ + "KHR_texture_transform": [ + "offset": [0.25, 0.5], + "scale": [0.75, 0.5], + "rotation": 0.2, + "texCoord": 1 + ] + ] + pbr["baseColorTexture"] = baseTexture + material["pbrMetallicRoughness"] = pbr + + shadeTexture["extensions"] = [ + "KHR_texture_transform": [ + "offset": [0.9, 0.8], + "scale": [0.4, 0.3] + ] + ] + mtoon["shadeMultiplyTexture"] = shadeTexture + extensions["VRMC_materials_mtoon"] = mtoon + material["extensions"] = extensions + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let material = try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial) + let transform = try loader.currentTextureTransform(withMaterialIndex: 0) + #expect(transform.offset.isApproximatelyEqual(to: SIMD2(0.25, 0.5))) + #expect(transform.scale.isApproximatelyEqual(to: SIMD2(0.75, 0.5))) + #expect(transform.rotation.isApproximatelyEqual(to: 0.2)) + // MToon.metal applies the transform from the parameter rows, so the + // material-level transform must stay identity — otherwise RealityKit + // would transform the primary UV a second time. + #expect(material.textureCoordinateTransform.offset == SIMD2(0, 0)) + #expect(material.textureCoordinateTransform.scale == SIMD2(1, 1)) + #expect(material.textureCoordinateTransform.rotation == 0) + } + + @Test + func testMToonKeepsAnIdentityTransformWhenOnlyALaterSlotHasOne() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanMToonExtension(name: "shade-only-uv-transform") { mtoon in + guard var shadeTexture = mtoon["shadeMultiplyTexture"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san MToon texture fixture data") + } + shadeTexture["extensions"] = [ + "KHR_texture_transform": [ + "offset": [0.9, 0.8], + "scale": [0.4, 0.3] + ] + ] + mtoon["shadeMultiplyTexture"] = shadeTexture + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + _ = try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial) + + // Base color is the first UV-accessed slot, so its identity transform + // wins; taking the first non-nil transform instead would apply the + // shade slot's transform to the base color texture. + let transform = try loader.currentTextureTransform(withMaterialIndex: 0) + #expect(transform.offset == SIMD2(0, 0)) + #expect(transform.scale == SIMD2(1, 1)) + #expect(transform.rotation == 0) + } + + @Test + func testNormalMappedMeshesCarryACompleteTangentBasis() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + var checkedParts = 0 + for modelEntity in TestSupport.modelEntities(in: vrmEntity) { + guard let mesh = modelEntity.components[ModelComponent.self]?.mesh else { continue } + for part in mesh.contents.models.flatMap(\.parts) { + guard let tangents = part.tangents?.elements, !tangents.isEmpty else { continue } + // MToon.metal falls back to the geometry normal unless both + // buffers are present and non-degenerate, and RealityKit derives + // neither buffer from the other. + let bitangents = try #require(part.bitangents?.elements) + #expect(tangents.count == part.positions.count) + #expect(bitangents.count == tangents.count) + #expect(tangents.allSatisfy { simd_length_squared($0) > 0.5 }) + #expect(bitangents.allSatisfy { simd_length_squared($0) > 0.5 }) + checkedParts += 1 + } + } + // Seed-san's normal-mapped materials have no glTF TANGENT attribute, so + // reaching this count also proves the generated basis is used. + #expect(checkedParts > 0) + } + + /// MToon samples the normal map in glTF UV space, where v points down, while + /// the mesh stores UVs with v up. A basis generated from the stored UVs would + /// have the opposite handedness to the one a TANGENT accessor supplies, so + /// the generated bitangent has to follow glTF +v. + @Test + func testGeneratedBitangentsFollowTheGLTFUVOrientation() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + var checkedTriangles = 0 + var agreeingTriangles = 0 + for modelEntity in TestSupport.modelEntities(in: vrmEntity) { + guard let mesh = modelEntity.components[ModelComponent.self]?.mesh else { continue } + for part in mesh.contents.models.flatMap(\.parts) { + guard let bitangents = part.bitangents?.elements, !bitangents.isEmpty, + let texcoords = part.textureCoordinates?.elements, + let indices = part.triangleIndices?.elements else { continue } + let positions = part.positions.elements + for triangle in stride(from: 0, to: indices.count - 2, by: 3) { + let i0 = Int(indices[triangle]) + let i1 = Int(indices[triangle + 1]) + let i2 = Int(indices[triangle + 2]) + let deltaUV1 = texcoords[i1] - texcoords[i0] + let deltaUV2 = texcoords[i2] - texcoords[i0] + // The stored v runs the other way, so the glTF-space + // determinant is the stored one negated. + let determinant = deltaUV2.x * deltaUV1.y - deltaUV1.x * deltaUV2.y + guard abs(determinant) > 1e-9 else { continue } + let edge1 = positions[i1] - positions[i0] + let edge2 = positions[i2] - positions[i0] + let expected = (edge2 * deltaUV1.x - edge1 * deltaUV2.x) / determinant + guard simd_length_squared(expected) > 1e-10 else { continue } + checkedTriangles += 1 + if simd_dot(simd_normalize(expected), bitangents[i0]) > 0 { + agreeingTriangles += 1 + } + } + } + } + #expect(checkedTriangles > 0) + // Vertices shared by triangles with opposing UV gradients average out, so + // a few legitimately disagree; a flipped basis inverts the ratio entirely. + #expect(Float(agreeingTriangles) > Float(checkedTriangles) * 0.9) + } + + /// glTF requires every vertex attribute to match POSITION in length, so a + /// short NORMAL accessor has to fail the load rather than be indexed per + /// vertex while the tangent basis is built. + @Test + func testVertexAttributeShorterThanPositionFailsTheLoad() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let data = try TestSupport.modifiedSeedSanData(name: "short NORMAL") { json in + guard var accessors = json["accessors"] as? [[String: Any]], + let meshes = json["meshes"] as? [[String: Any]], + let primitives = meshes.first?["primitives"] as? [[String: Any]], + let attributes = primitives.first?["attributes"] as? [String: Any], + let normalIndex = attributes["NORMAL"] as? Int, + accessors.indices.contains(normalIndex), + let count = accessors[normalIndex]["count"] as? Int, count > 1 else { + throw VRMError.dataInconsistent("Missing Seed-san NORMAL accessor") + } + accessors[normalIndex]["count"] = count - 1 + json["accessors"] = accessors + } + + let loader = try VRMEntityLoader(withData: data) + #expect(throws: VRMError.self) { + try loader.loadEntity() + } + } + + /// glTF defines `JOINTS_n` as unsigned integer indices. A signed or floating + /// point component type is a malformed file, and converting one to `UInt32` + /// traps for a negative or out-of-range value, so the load has to fail. + @Test + func testSignedJointIndicesFailTheLoad() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let data = try TestSupport.modifiedSeedSanData(name: "signed JOINTS_0") { json in + guard var accessors = json["accessors"] as? [[String: Any]], + let meshes = json["meshes"] as? [[String: Any]], + let primitives = meshes.first?["primitives"] as? [[String: Any]], + let attributes = primitives.first?["attributes"] as? [String: Any], + let jointsIndex = attributes["JOINTS_0"] as? Int, + accessors.indices.contains(jointsIndex), + // The signed counterpart of the same width, so the accessor + // still fits its buffer view and the component type is what + // fails the load. + let signed = [5121: 5120, 5123: 5122][accessors[jointsIndex]["componentType"] as? Int ?? 0] else { + throw VRMError.dataInconsistent("Missing Seed-san JOINTS_0 accessor") + } + accessors[jointsIndex]["componentType"] = signed + json["accessors"] = accessors + } + + let loader = try VRMEntityLoader(withData: data) + #expect(throws: VRMError.self) { + try loader.loadEntity() + } + } + + /// glTF defines `inverseBindMatrices` as one matrix per skin joint. An + /// accessor covering fewer of them is a broken file, and quietly padding it + /// with identities would bind those joints to the wrong rest pose. + @Test + func testShortInverseBindMatricesFailTheLoad() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let data = try TestSupport.modifiedSeedSanData(name: "short inverseBindMatrices") { json in + guard var accessors = json["accessors"] as? [[String: Any]], + let skins = json["skins"] as? [[String: Any]], + let matricesIndex = skins.first?["inverseBindMatrices"] as? Int, + accessors.indices.contains(matricesIndex), + let count = accessors[matricesIndex]["count"] as? Int, count > 1 else { + throw VRMError.dataInconsistent("Missing Seed-san inverseBindMatrices accessor") + } + accessors[matricesIndex]["count"] = count - 1 + json["accessors"] = accessors + } + + let loader = try VRMEntityLoader(withData: data) + #expect { + try loader.loadEntity() + } throws: { error in + isDataInconsistent(error, containing: "inverseBindMatrices") + } + } + + /// A TRIANGLES primitive holds a multiple of three indices. Trimming the + /// remainder away would load a file whose triangle list is not the one it + /// describes, so the load fails instead. + @Test + func testTriangleIndexCountThatIsNotAMultipleOfThreeFailsTheLoad() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let data = try TestSupport.modifiedSeedSanData(name: "partial triangle") { json in + guard var accessors = json["accessors"] as? [[String: Any]], + let meshes = json["meshes"] as? [[String: Any]], + let primitives = meshes.first?["primitives"] as? [[String: Any]], + let indicesIndex = primitives.first?["indices"] as? Int, + accessors.indices.contains(indicesIndex), + let count = accessors[indicesIndex]["count"] as? Int, count > 3 else { + throw VRMError.dataInconsistent("Missing Seed-san indices accessor") + } + accessors[indicesIndex]["count"] = count - 1 + json["accessors"] = accessors + } + + let loader = try VRMEntityLoader(withData: data) + #expect { + try loader.loadEntity() + } throws: { error in + isDataInconsistent(error, containing: "TRIANGLES") + } + } + + /// glTF stores `WEIGHTS_n` as floats or as normalized integers. An + /// unnormalized integer accessor holds raw counts, not the 0...1 weights the + /// joint influences are built from. + @Test + func testUnnormalizedIntegerJointWeightsFailTheLoad() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let data = try TestSupport.modifiedSeedSanData(name: "unnormalized WEIGHTS_0") { json in + guard var accessors = json["accessors"] as? [[String: Any]], + let meshes = json["meshes"] as? [[String: Any]], + let primitives = meshes.first?["primitives"] as? [[String: Any]], + let attributes = primitives.first?["attributes"] as? [String: Any], + let weightsIndex = attributes["WEIGHTS_0"] as? Int, + accessors.indices.contains(weightsIndex) else { + throw VRMError.dataInconsistent("Missing Seed-san WEIGHTS_0 accessor") + } + // Narrower than the float components the fixture ships, so the + // accessor still fits its buffer view and the missing `normalized` + // flag is what fails the load. + accessors[weightsIndex]["componentType"] = 5121 + accessors[weightsIndex]["normalized"] = false + json["accessors"] = accessors + } + + let loader = try VRMEntityLoader(withData: data) + #expect { + try loader.loadEntity() + } throws: { error in + isDataInconsistent(error, containing: "WEIGHTS_0") + } + } + + /// The light direction is carried in `custom.value` and the light color in a + /// parameter row, so updating one must not drop the other. + @Test + func testMToonLightColorUpdateKeepsTheCustomLightDirection() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let vrmLoader = try VRMEntityLoader(withData: TestSupport.seedSanData) + let vrmEntity = try vrmLoader.loadEntity() + + vrmEntity.setMToonLightDirection(SIMD3(0, 0, -2)) + vrmEntity.setMToonLightColor(SIMD3(0.8, 0.7, 0.6)) + + let parameters = try firstMToonParameters(in: vrmEntity) + #expect(parameters.lightColor.isApproximatelyEqual(to: SIMD4(0.8, 0.7, 0.6, 1))) + #expect(parameters.lightDirection.isApproximatelyEqual(to: SIMD3(0, 0, -1))) +#if !os(visionOS) + let material = try firstCustomMaterial(in: vrmEntity) + #expect(material.custom.value.isApproximatelyEqual(to: SIMD4(0, 0, -1, 0))) + #expect(material.custom.texture != nil) +#endif + } + + private func isDataInconsistent(_ error: any Error, containing fragment: String) -> Bool { + guard case VRMError.dataInconsistent(let message) = error else { return false } + return message.contains(fragment) + } + + /// VRM 0.x keeps its normal map in Unity's `_BumpMap`, which the migration + /// surfaces on the MToon descriptor and not on the glTF material. A + /// primitive without a TANGENT accessor still needs a generated basis for + /// it, or MToon.metal falls back to the geometry normal and the map does + /// nothing. + @Test + func testVRM0BumpMapGeneratesATangentBasisWithoutTANGENT() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let materialIndex = 0 + let data = try TestSupport.modifiedAliciaSolidData(name: "VRM0 _BumpMap without TANGENT") { json in + guard var extensions = json["extensions"] as? [String: Any], + var vrm = extensions["VRM"] as? [String: Any], + var properties = vrm["materialProperties"] as? [[String: Any]], + properties.indices.contains(materialIndex), + var textures = properties[materialIndex]["textureProperties"] as? [String: Any], + let mainTexture = textures["_MainTex"], + var meshes = json["meshes"] as? [[String: Any]] else { + throw VRMError.dataInconsistent("Missing AliciaSolid material properties") + } + // The fixture ships unlit materials with a TANGENT accessor, which + // is the opposite of what this exercises. + properties[materialIndex]["shader"] = "VRM/MToon" + textures["_BumpMap"] = mainTexture + properties[materialIndex]["textureProperties"] = textures + vrm["materialProperties"] = properties + extensions["VRM"] = vrm + json["extensions"] = extensions + + for meshIndex in meshes.indices { + guard var primitives = meshes[meshIndex]["primitives"] as? [[String: Any]] else { continue } + for primitiveIndex in primitives.indices { + guard primitives[primitiveIndex]["material"] as? Int == materialIndex, + var attributes = primitives[primitiveIndex]["attributes"] as? [String: Any] else { continue } + attributes.removeValue(forKey: "TANGENT") + primitives[primitiveIndex]["attributes"] = attributes + } + meshes[meshIndex]["primitives"] = primitives + } + json["meshes"] = meshes + } + + let loader = try VRMEntityLoader(withData: data, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + var checkedParts = 0 + for modelEntity in TestSupport.modelEntities(in: vrmEntity) + where modelEntity.components[VRMMaterialIndexComponent.self]?.materialIndex == materialIndex { + guard let mesh = modelEntity.components[ModelComponent.self]?.mesh else { continue } + for part in mesh.contents.models.flatMap(\.parts) { + let tangents = try #require(part.tangents?.elements) + let bitangents = try #require(part.bitangents?.elements) + #expect(tangents.count == part.positions.count) + #expect(bitangents.count == tangents.count) + #expect(tangents.contains { simd_length_squared($0) > 0.5 }) + checkedParts += 1 + } + } + #expect(checkedParts > 0) + } + + @Test + func testSetMToonLightAndAmbientColorUpdateParameterRows() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let vrmEntity = try vrmLoader.loadEntity() + let lightColor = SIMD3(0.8, 0.7, 0.6) + let ambientColor = SIMD3(0.05, 0.1, 0.15) + + vrmEntity.setMToonLightColor(lightColor) + vrmEntity.setMToonAmbientColor(ambientColor) + + let parameters = try firstMToonParameters(in: vrmEntity) + #expect(parameters.lightColor.isApproximatelyEqual(to: SIMD4(0.8, 0.7, 0.6, 1))) + #expect(parameters.ambientColor.isApproximatelyEqual(to: SIMD4(0.05, 0.1, 0.15, 1))) + let material = try firstCustomMaterial(in: vrmEntity) + #expect(material.custom.texture != nil) + } + + @Test + func testMToonTextureTransformBindUpdatesParameterTexture() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan, isOutlineEnabled: false) + let vrmEntity = try vrmLoader.loadEntity() + + vrmEntity.setExpression(value: 1, for: .preset(.happy)) + + let parameters = try mtoonParameters(in: vrmEntity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.25, 0))) + #expect(parameters.uvTransformRotation.isApproximatelyEqual(to: SIMD4(1, 0, 0, 0))) + // The parameter rows are the only UV-transform source for MToon; the + // material-level transform stays identity so the shader applies it once. + let material = try customMaterial(in: vrmEntity, materialIndex: 11) + #expect(material.textureCoordinateTransform.offset == SIMD2(0, 0)) + #expect(material.textureCoordinateTransform.scale == SIMD2(1, 1)) + } + + @Test + func testExpressionTextureTransformsAccumulateAndResetIndependently() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let loader = try VRMEntityLoader(withData: seedSan, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + vrmEntity.setExpression(value: 1, for: .preset(.happy)) + vrmEntity.setExpression(value: 1, for: .preset(.angry)) + var parameters = try mtoonParameters(in: vrmEntity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.75, 0))) + #expect(vrmEntity.expression(for: .preset(.happy)) == 1) + #expect(vrmEntity.expression(for: .preset(.angry)) == 1) + + vrmEntity.setExpression(value: 0, for: .preset(.happy)) + parameters = try mtoonParameters(in: vrmEntity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.5, 0))) + #expect(vrmEntity.expression(for: .preset(.happy)) == 0) + #expect(vrmEntity.expression(for: .preset(.angry)) == 1) + } + + @Test + func testExpressionMaterialColorsAccumulateAndResetIndependently() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanData(name: "accumulated-material-colors") { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(0), + var pbr = materials[0]["pbrMetallicRoughness"] as? [String: Any], + var extensions = json["extensions"] as? [String: Any], + var vrm = extensions["VRMC_vrm"] as? [String: Any], + var expressions = vrm["expressions"] as? [String: Any], + var preset = expressions["preset"] as? [String: Any], + var happy = preset["happy"] as? [String: Any], + var angry = preset["angry"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san expression fixture data") + } + pbr["baseColorFactor"] = [1.0, 1.0, 1.0, 1.0] + materials[0]["pbrMetallicRoughness"] = pbr + happy["materialColorBinds"] = [[ + "material": 0, + "type": "color", + "targetValue": [0.8, 1.0, 1.0, 1.0] + ]] + angry["materialColorBinds"] = [[ + "material": 0, + "type": "color", + "targetValue": [1.0, 0.6, 1.0, 1.0] + ]] + preset["happy"] = happy + preset["angry"] = angry + expressions["preset"] = preset + vrm["expressions"] = expressions + extensions["VRMC_vrm"] = vrm + json["extensions"] = extensions + json["materials"] = materials + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + vrmEntity.setExpression(value: 1, for: .preset(.happy)) + vrmEntity.setExpression(value: 1, for: .preset(.angry)) + var parameters = try mtoonParameters(in: vrmEntity, materialIndex: 0) + #expect(parameters.baseColor.isApproximatelyEqual(to: SIMD4(0.8, 0.6, 1, 1))) + + vrmEntity.setExpression(value: 0, for: .preset(.happy)) + parameters = try mtoonParameters(in: vrmEntity, materialIndex: 0) + #expect(parameters.baseColor.isApproximatelyEqual(to: SIMD4(1, 0.6, 1, 1))) + } + + /// Metallib freshness is checked by scripts/build-mtoon-metallibs.sh --check + /// (which CI runs), so this only covers what the bundle itself must contain. + @Test + func testBundledMToonMetallibsArePackagedWithoutShaderSource() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // The same bundle the loader itself reads from. + let bundle = MToonShaderLibraryLoader.resourceBundle + + // Shader source must never ship as a bundle resource (App Store safety). + #expect(bundle.url(forResource: "MToon", withExtension: "metal") == nil) + #expect(bundle.url(forResource: "MToonCore", withExtension: "h") == nil) + + for resourceName in ["MToon-macos", "MToon-ios", "MToon-iossim"] { + #expect(bundle.url(forResource: resourceName, withExtension: "metallib") != nil, + "Missing bundled metallib: \(resourceName).metallib. Run scripts/build-mtoon-metallibs.sh.") + } + } + + @Test + func testBundledMToonLibraryCanBeLoadedOnSupportedPlatforms() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } +#if targetEnvironment(macCatalyst) + // Mac Catalyst intentionally bundles no metallib. + #expect(MToonShaderLibraryLoader.resourceName == nil) +#else + // On supported platforms this must hard-fail instead of skipping when + // the bundled metallib cannot be loaded. + let device = try #require(MTLCreateSystemDefaultDevice(), "A Metal device is required to run this test.") + let library = try MToonShaderLibraryLoader.load(device: device) + let functions = Set(library.functionNames) + + #expect(functions.contains("mtoonSurface")) + #expect(functions.contains("mtoonOutlineSurface")) + #expect(functions.contains("mtoonOutlineGeometry")) +#endif + } + + @Test + func testMToonShadeColorBindDoesNotOverwriteCustomLightDirection() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let material = try vrmLoader.material(withMaterialIndex: 0) + let customMaterial = try #require(material as? CustomMaterial, + TestSupport.expectedCustomMaterialMessage) + let initialValue = customMaterial.custom.value + + let updatedMaterial = customMaterial.settingColor(VRMColor(red: 0.2, green: 0.3, blue: 0.4, alpha: 1), + for: .shadeColor) + let updatedCustomMaterial = try #require(updatedMaterial as? CustomMaterial) + + #expect(updatedCustomMaterial.custom.value == initialValue) + } +#endif + + @Test + func testFallbackShadeAndOutlineColorBindsDoNotOverwriteBaseColor() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let baseColor = VRMColor(red: 0.1, green: 0.2, blue: 0.3, alpha: 1) + let boundColor = VRMColor(red: 0.8, green: 0.7, blue: 0.6, alpha: 1) + + var pbr = PhysicallyBasedMaterial() + pbr.baseColor.tint = baseColor + let shadeUpdatedPBR = try #require(pbr.settingColor(boundColor, for: .shadeColor) as? PhysicallyBasedMaterial) + let outlineUpdatedPBR = try #require(pbr.settingColor(boundColor, for: .outlineColor) as? PhysicallyBasedMaterial) + let colorUpdatedPBR = try #require(pbr.settingColor(boundColor, for: .color) as? PhysicallyBasedMaterial) + + #expect(shadeUpdatedPBR.baseColor.tint.isApproximatelyEqual(to: baseColor)) + #expect(outlineUpdatedPBR.baseColor.tint.isApproximatelyEqual(to: baseColor)) + #expect(colorUpdatedPBR.baseColor.tint.isApproximatelyEqual(to: boundColor)) + #expect(pbr.currentColor(for: .shadeColor).isApproximatelyEqual(to: SIMD4(1, 1, 1, 1))) + #expect(pbr.currentColor(for: .outlineColor).isApproximatelyEqual(to: SIMD4(1, 1, 1, 1))) + + // matcapColor / rimColor are MToon-only, so on the PBR fallback they are + // a no-op rather than being redirected onto the emissive channel. + let emissive = VRMColor(red: 0.4, green: 0.5, blue: 0.6, alpha: 1) + var emissivePBR = pbr + emissivePBR.emissiveColor = .init(color: emissive) + for type in [VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType.matcapColor, .rimColor] { + let updated = try #require(emissivePBR.settingColor(boundColor, for: type) as? PhysicallyBasedMaterial) + #expect(updated.emissiveColor.color.isApproximatelyEqual(to: emissive)) + #expect(emissivePBR.currentColor(for: type).isApproximatelyEqual(to: SIMD4(1, 1, 1, 1))) + } + let emissionUpdatedPBR = try #require(emissivePBR.settingColor(boundColor, for: .emissionColor) as? PhysicallyBasedMaterial) + #expect(emissionUpdatedPBR.emissiveColor.color.isApproximatelyEqual(to: boundColor)) + + var unlit = UnlitMaterial() + unlit.color.tint = baseColor + let shadeUpdatedUnlit = try #require(unlit.settingColor(boundColor, for: .shadeColor) as? UnlitMaterial) + let colorUpdatedUnlit = try #require(unlit.settingColor(boundColor, for: .color) as? UnlitMaterial) + + #expect(shadeUpdatedUnlit.color.tint.isApproximatelyEqual(to: baseColor)) + #expect(colorUpdatedUnlit.color.tint.isApproximatelyEqual(to: boundColor)) + #expect(unlit.currentColor(for: .shadeColor).isApproximatelyEqual(to: SIMD4(1, 1, 1, 1))) + } + + @Test + func testBlockingExpressionOverrideSuppressesBlinkAndLookAtWeights() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // Seed-san's `relaxed` declares overrideBlink / overrideLookAt = block. + let vrmEntity = try VRMEntityLoader(withData: TestSupport.seedSanData, + isOutlineEnabled: false).loadEntity() + + vrmEntity.setExpression(value: 1, for: .preset(.blink)) + vrmEntity.setExpression(value: 1, for: .preset(.lookUp)) + vrmEntity.setExpression(value: 1, for: .preset(.aa)) + #expect(morphWeight(in: vrmEntity, targetIndex: 1) == 1) + #expect(morphWeight(in: vrmEntity, targetIndex: 39) == 1) + #expect(morphWeight(in: vrmEntity, targetIndex: 25) == 1) + + vrmEntity.setExpression(value: 1, for: .preset(.relaxed)) + #expect(morphWeight(in: vrmEntity, targetIndex: 1) == 0) + #expect(morphWeight(in: vrmEntity, targetIndex: 39) == 0) + // overrideMouth is `none`, so mouth expressions keep their weight. + #expect(morphWeight(in: vrmEntity, targetIndex: 25) == 1) + // The input weights themselves are untouched by the override. + #expect(vrmEntity.expression(for: .preset(.blink)) == 1) + + vrmEntity.setExpression(value: 0, for: .preset(.relaxed)) + #expect(morphWeight(in: vrmEntity, targetIndex: 1) == 1) + #expect(morphWeight(in: vrmEntity, targetIndex: 39) == 1) + } + + @Test + func testBlendingExpressionOverrideScalesBlinkWeights() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // Seed-san's `happy` declares overrideBlink = blend, but is binary; a + // non-binary variant makes the partial blend observable. + let modified = try TestSupport.modifiedSeedSanExpressions(name: "non-binary-happy") { preset in + guard var happy = preset["happy"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san happy expression") + } + happy["isBinary"] = false + preset["happy"] = happy + } + let vrmEntity = try VRMEntityLoader(withData: modified, isOutlineEnabled: false).loadEntity() + + vrmEntity.setExpressions([.preset(.blink): 1, .preset(.happy): 0.25]) + + let blinkWeight = try #require(morphWeight(in: vrmEntity, targetIndex: 1)) + #expect(blinkWeight.isApproximatelyEqual(to: 0.75)) + } + + @Test + func testSimultaneousBlendOverridesAccumulateBeforeSaturating() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // Two non-binary expressions that each blend-override blink. VRM sums + // their weights and saturates, so 0.5 + 0.5 fully suppresses blink; + // composing the factors multiplicatively would leave 0.25 behind. + let modified = try TestSupport.modifiedSeedSanExpressions(name: "two-blend-overrides") { preset in + for name in ["happy", "sad"] { + guard var expression = preset[name] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san \(name) expression") + } + expression["isBinary"] = false + expression["overrideBlink"] = "blend" + preset[name] = expression + } + } + let vrmEntity = try VRMEntityLoader(withData: modified, isOutlineEnabled: false).loadEntity() + + vrmEntity.setExpressions([.preset(.blink): 1, .preset(.happy): 0.5]) + #expect(try #require(morphWeight(in: vrmEntity, targetIndex: 1)).isApproximatelyEqual(to: 0.5)) + + vrmEntity.setExpression(value: 0.5, for: .preset(.sad)) + #expect(morphWeight(in: vrmEntity, targetIndex: 1) == 0) + + // Past saturation the weight stays at 0 rather than going negative. + vrmEntity.setExpressions([.preset(.happy): 1, .preset(.sad): 1]) + #expect(morphWeight(in: vrmEntity, targetIndex: 1) == 0) + } + + @Test + func testOverriddenBinaryExpressionIsSuppressedEntirely() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // A binary expression has no partial state, so *any* override effect + // must zero it rather than scale it. + let modified = try TestSupport.modifiedSeedSanExpressions(name: "binary-blink") { preset in + guard var blink = preset["blink"] as? [String: Any], + var happy = preset["happy"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san blink/happy expressions") + } + blink["isBinary"] = true + happy["isBinary"] = false + happy["overrideBlink"] = "blend" + preset["blink"] = blink + preset["happy"] = happy + } + let vrmEntity = try VRMEntityLoader(withData: modified, isOutlineEnabled: false).loadEntity() + + vrmEntity.setExpression(value: 1, for: .preset(.blink)) + #expect(morphWeight(in: vrmEntity, targetIndex: 1) == 1) + + // A 0.25 blend would leave 0.75 on a non-binary expression. + vrmEntity.setExpression(value: 0.25, for: .preset(.happy)) + #expect(morphWeight(in: vrmEntity, targetIndex: 1) == 0) + } + + @Test + func testExpressionDoesNotOverrideItsOwnKind() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // "Like overrideBlink for blink, settings for the same kind are treated + // as invalid" — blink must not suppress itself or its own group. + let modified = try TestSupport.modifiedSeedSanExpressions(name: "self-overriding-blink") { preset in + guard var blink = preset["blink"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san blink expression") + } + blink["overrideBlink"] = "block" + preset["blink"] = blink + } + let vrmEntity = try VRMEntityLoader(withData: modified, isOutlineEnabled: false).loadEntity() + + vrmEntity.setExpressions([.preset(.blink): 1, .preset(.blinkLeft): 1]) + + #expect(morphWeight(in: vrmEntity, targetIndex: 2) == 1) + } + +// These tests observe MToon runtime state, which visionOS never produces: +// there is no `CustomMaterial`, so MToon falls back to Unlit / PBR materials. +#if !os(visionOS) + @Test + func testBinaryExpressionIsOnlyActiveAboveHalf() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // `angry` is binary and carries a textureTransformBind on material 11. + let vrmEntity = try VRMEntityLoader(withData: TestSupport.seedSanData, + isOutlineEnabled: false).loadEntity() + + vrmEntity.setExpression(value: 0.5, for: .preset(.angry)) + var parameters = try mtoonParameters(in: vrmEntity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0, 0))) + #expect(vrmEntity.expression(for: .preset(.angry)) == 0) + + vrmEntity.setExpression(value: 0.51, for: .preset(.angry)) + parameters = try mtoonParameters(in: vrmEntity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.5, 0))) + #expect(vrmEntity.expression(for: .preset(.angry)) == 1) + } + + @Test + func testSetExpressionsAppliesEveryWeightAtOnce() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let vrmEntity = try VRMEntityLoader(withData: TestSupport.seedSanData, + isOutlineEnabled: false).loadEntity() + + vrmEntity.setExpressions([.preset(.happy): 1, .preset(.angry): 1]) + + // Same accumulated result as setting each expression on its own. + let parameters = try mtoonParameters(in: vrmEntity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.75, 0))) + #expect(vrmEntity.expression(for: .preset(.happy)) == 1) + #expect(vrmEntity.expression(for: .preset(.angry)) == 1) + + vrmEntity.setExpressions([.preset(.happy): 0, .preset(.angry): 0]) + #expect(try mtoonParameters(in: vrmEntity, materialIndex: 11) + .uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0, 0))) + } + + @Test + func testMToonSamplerKeepsMagAndMinFiltersIndependent() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // glTF magFilter and minFilter are independent, so a LINEAR magFilter + // must stay linear next to a NEAREST minFilter. + // Material 0's base color texture uses sampler 0. + let modified = try TestSupport.modifiedSeedSanData(name: "mixed-filter-sampler") { json in + guard var samplers = json["samplers"] as? [[String: Any]], !samplers.isEmpty else { + throw VRMError.dataInconsistent("Missing Seed-san sampler fixture data") + } + samplers[0]["magFilter"] = 9729 // LINEAR + samplers[0]["minFilter"] = 9728 // NEAREST + samplers[0]["wrapS"] = 33071 // CLAMP_TO_EDGE + samplers[0]["wrapT"] = 33648 // MIRRORED_REPEAT + json["samplers"] = samplers + } + + let vrmEntity = try VRMEntityLoader(withData: modified, isOutlineEnabled: false).loadEntity() + let parameters = try mtoonParameters(in: vrmEntity, materialIndex: 0) + + // (wrapS, wrapT, filterIndex, 0): clamp / mirrored repeat, and a + // magnification filter that stays linear while minification is nearest. + let filter = MToonSamplerFilter(magnification: .linear, minification: .nearest, mip: .none) + #expect(parameters.samplers[MToonTextureSlot.base.rawValue] + .isApproximatelyEqual(to: SIMD4(1, 2, Float(filter.index), 0))) + // Untouched slots keep the glTF defaults: repeat/repeat, linear + // magnification, trilinear minification. + #expect(MToonMaterialParameters.defaultSampler + == SIMD4(0, 0, Float(MToonSamplerFilter.default.index), 0)) + #expect(MToonSamplerFilter.default.magnification == .linear) + #expect(MToonSamplerFilter.default.minification == .linear) + #expect(MToonSamplerFilter.default.mip == .linear) + } + + @Test + func testEveryGLTFMinFilterMapsToADistinctSampler() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // glTF's six minFilter values are two independent choices — the + // minification texel filter and the filter between mip levels — so each + // has to encode to its own filter index. + let minFilters: [(raw: Int, minification: MToonSamplerFilter.TexelFilter, mip: MToonSamplerFilter.MipFilter)] = [ + (9728, .nearest, .none), // NEAREST + (9729, .linear, .none), // LINEAR + (9984, .nearest, .nearest), // NEAREST_MIPMAP_NEAREST + (9985, .linear, .nearest), // LINEAR_MIPMAP_NEAREST + (9986, .nearest, .linear), // NEAREST_MIPMAP_LINEAR + (9987, .linear, .linear) // LINEAR_MIPMAP_LINEAR + ] + + var seenIndexes: Set = [] + for entry in minFilters { + let modified = try TestSupport.modifiedSeedSanData(name: "min-filter-\(entry.raw)") { json in + guard var samplers = json["samplers"] as? [[String: Any]], !samplers.isEmpty else { + throw VRMError.dataInconsistent("Missing Seed-san sampler fixture data") + } + samplers[0]["minFilter"] = entry.raw + json["samplers"] = samplers + } + let vrmEntity = try VRMEntityLoader(withData: modified, isOutlineEnabled: false).loadEntity() + let parameters = try mtoonParameters(in: vrmEntity, materialIndex: 0) + let expected = MToonSamplerFilter(magnification: .linear, + minification: entry.minification, + mip: entry.mip) + #expect(parameters.samplers[MToonTextureSlot.base.rawValue].z + .isApproximatelyEqual(to: Float(expected.index)), + "glTF minFilter \(entry.raw) mapped to the wrong sampler") + seenIndexes.insert(expected.index) + } + #expect(seenIndexes.count == minFilters.count) + #expect(MToonSamplerFilter.count == 12) + } + + @Test + func testTransparentWithZWriteControlsDepthWriting() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + func customMaterial(alphaMode: String, transparentWithZWrite: Bool) throws -> CustomMaterial { + let modified = try TestSupport.modifiedSeedSanMaterial(name: "z-write-\(alphaMode)-\(transparentWithZWrite)") { material in + material["alphaMode"] = alphaMode + guard var extensions = material["extensions"] as? [String: Any], + var mtoon = extensions["VRMC_materials_mtoon"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san MToon extension") + } + mtoon["transparentWithZWrite"] = transparentWithZWrite + extensions["VRMC_materials_mtoon"] = mtoon + material["extensions"] = extensions + } + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + return try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial, + TestSupport.expectedCustomMaterialMessage) + } + + // Only a blended material may stop writing depth. + #expect(try !customMaterial(alphaMode: "BLEND", transparentWithZWrite: false).writesDepth) + #expect(try customMaterial(alphaMode: "BLEND", transparentWithZWrite: true).writesDepth) + #expect(try customMaterial(alphaMode: "OPAQUE", transparentWithZWrite: false).writesDepth) + #expect(try customMaterial(alphaMode: "MASK", transparentWithZWrite: false).writesDepth) + } +#endif + + @Test + func testUpdateAppliesSpringBonePosesWithoutAFrameOfLag() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let vrmEntity = try VRMEntityLoader(withData: TestSupport.seedSanData, + isMToonEnabled: false, + isOutlineEnabled: false).loadEntity() + + // Rotating a parent bone drags the spring chains, so spring bones write + // new joint transforms during update(). + let head = try #require(vrmEntity.humanoid.node(for: .head)) + head.transform.rotation = simd_quatf(angle: .pi / 3, axis: SIMD3(1, 0, 0)) + vrmEntity.update(deltaTime: 1.0 / 60.0) + + var checkedJoints = 0 + for modelEntity in TestSupport.modelEntities(in: vrmEntity) { + guard let model = modelEntity.components[ModelComponent.self], + let skeleton = model.mesh.contents.skeletons.first, + let pose = modelEntity.components[SkeletalPosesComponent.self]?.poses.default, + pose.jointTransforms.count == skeleton.joints.count else { + continue + } + let jointEntities = skeleton.joints.map { vrmEntity.findEntity(named: $0.name) } + let jointWorlds = jointEntities.map { $0?.transformMatrix(relativeTo: nil) } + let modelWorldInverse = modelEntity.transformMatrix(relativeTo: nil).inverse + + for index in skeleton.joints.indices { + guard let jointWorld = jointWorlds[index] else { continue } + let expected: simd_float4x4 + if let parentIndex = skeleton.joints[index].parentIndex, + let parentWorld = jointWorlds[parentIndex] { + expected = parentWorld.inverse * jointWorld + } else { + expected = modelWorldInverse * jointWorld + } + // The pose must describe the hierarchy as it stands after + // update(), not as it stood before the spring bones ran. + #expect(pose.jointTransforms[index].matrix.isApproximatelyEqual(to: expected, tolerance: 0.0005)) + checkedJoints += 1 + } + } + #expect(checkedJoints > 0) + } + + @Test + func testScenesSharingNodesLoadIndependentEntityGraphs() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // A second scene referencing the same root nodes. Entity instances + // cannot be shared between scenes, so each load must build its own. + let modified = try TestSupport.modifiedSeedSanData(name: "duplicated-scene") { json in + guard var scenes = json["scenes"] as? [[String: Any]], let first = scenes.first else { + throw VRMError.dataInconsistent("Missing Seed-san scene fixture data") + } + scenes.append(first) + json["scenes"] = scenes + } + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + + let firstScene = try loader.loadEntity(withSceneIndex: 0) + let secondScene = try loader.loadEntity(withSceneIndex: 1) + + #expect(firstScene !== secondScene) + // Neither scene may have had its nodes stolen by the other: both keep a + // full hierarchy, and no entity appears in both. + #expect(!firstScene.children.isEmpty) + #expect(firstScene.children.count == secondScene.children.count) + let firstModels = Set(TestSupport.modelEntities(in: firstScene).map(ObjectIdentifier.init)) + let secondModels = Set(TestSupport.modelEntities(in: secondScene).map(ObjectIdentifier.init)) + #expect(!firstModels.isEmpty) + #expect(firstModels.count == secondModels.count) + #expect(firstModels.isDisjoint(with: secondModels)) + + // Each scene drives its own runtime state. + firstScene.setExpression(value: 1, for: .preset(.happy)) + #expect(morphWeight(in: firstScene, targetIndex: 33) == 1) + #expect(morphWeight(in: secondScene, targetIndex: 33) == 0) + + // Re-requesting a scene returns the same instance rather than rebuilding. + #expect(try loader.loadEntity(withSceneIndex: 0) === firstScene) + } + + @Test + func testVRM1FirstPersonAutoHidesHeadDescendants() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let vrmEntity = try vrmLoader.loadEntity() + let annotatedEntity = try vrmLoader.node(withNodeIndex: 0) + + #expect(annotatedEntity.isEnabled == true) + vrmEntity.setFirstPersonRenderMode(.firstPerson) + #expect(annotatedEntity.isEnabled == false) + vrmEntity.setFirstPersonRenderMode(.thirdPerson) + #expect(annotatedEntity.isEnabled == true) + } + +#if !os(visionOS) + @Test + func testUpdateDoesNotMutateMToonMaterialsPerFrame() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let vrmEntity = try vrmLoader.loadEntity() + let initialValue = try firstCustomMaterial(in: vrmEntity).custom.value + + vrmEntity.update(deltaTime: 0.25) + vrmEntity.update(deltaTime: 0.5) + + // UV animation time comes from params.uniforms().time() on the GPU; + // update() must not touch MToon materials at all. + let afterUpdateValue = try firstCustomMaterial(in: vrmEntity).custom.value + #expect(afterUpdateValue == initialValue) + } + + @Test + func testSetMToonLightDirectionUpdatesRegisteredMaterials() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let vrmLoader = try VRMEntityLoader(withData: seedSan) + let vrmEntity = try vrmLoader.loadEntity() + + // The direction is normalized before it reaches the parameter rows. + vrmEntity.setMToonLightDirection(SIMD3(0, 0, -2)) + + // Every MToon material is rebound, not just the first one found, and + // each keeps the parameter texture the shader samples. + var checkedMaterials = 0 + for modelEntity in TestSupport.modelEntities(in: vrmEntity) { + guard let model = modelEntity.components[ModelComponent.self] else { continue } + for material in model.materials.compactMap({ $0 as? CustomMaterial }) { + #expect(material.custom.value.isApproximatelyEqual(to: SIMD4(0, 0, -1, 0))) + #expect(material.custom.texture != nil) + checkedMaterials += 1 + } + } + #expect(checkedMaterials > 0) + } + + @Test + func testMalformedMaterialColorBindDoesNotFailModelLoad() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanData(name: "malformed-color-bind") { json in + guard var extensions = json["extensions"] as? [String: Any], + var vrm = extensions["VRMC_vrm"] as? [String: Any], + var expressions = vrm["expressions"] as? [String: Any], + var preset = expressions["preset"] as? [String: Any], + var happy = preset["happy"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san expression fixture data") + } + happy["materialColorBinds"] = [ + [ + "material": 9999, + "type": "color", + "targetValue": [1.0, 0.0, 0.0, 1.0] + ], + [ + "material": 0, + "type": "color", + "targetValue": [0.5, 1.0, 1.0, 1.0] + ] + ] + preset["happy"] = happy + expressions["preset"] = preset + vrm["expressions"] = expressions + extensions["VRMC_vrm"] = vrm + json["extensions"] = extensions + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + // The invalid bind is skipped while the valid one keeps working. + vrmEntity.setExpression(value: 1, for: .preset(.happy)) + let parameters = try mtoonParameters(in: vrmEntity, materialIndex: 0) + #expect(parameters.baseColor.x.isApproximatelyEqual(to: 0.5)) + } + + /// A malformed MToon extension is a rendering limitation, not a broken file: + /// the material falls back to Unlit / PBR and the model still loads. + @Test + func testMToonExtensionWithAnInvalidTextureFallsBackInsteadOfFailingTheLoad() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanMToonExtension(name: "mtoon-texture-out-of-range") { mtoon in + mtoon["shadeMultiplyTexture"] = ["index": 9999] + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + // The state goes with the material, so no expression bind keeps writing + // parameter rows no shader reads. + #expect(vrmEntity.mtoonParameters(forMaterialIndex: 0) == nil) + #expect(!TestSupport.modelEntities(in: vrmEntity).isEmpty) + } + + /// One unbuildable material must cost that primitive its material, not the + /// whole model. + @Test + func testMaterialThatCannotBeBuiltFallsBackToTheDefaultMaterial() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let modified = try TestSupport.modifiedSeedSanMaterial(name: "base-texture-out-of-range") { material in + // Without the MToon extension the material takes the Unlit / PBR + // path, where an out-of-range base color texture throws. + material["extensions"] = [String: Any]() + material["pbrMetallicRoughness"] = ["baseColorTexture": ["index": 9999]] + } + + let loader = try VRMEntityLoader(withData: modified, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + #expect(TestSupport.materialIndexes(in: vrmEntity).contains(0)) + #expect(!TestSupport.modelEntities(in: vrmEntity).isEmpty) + } + + @Test + func testFallbackMaterialsDoNotCarryMToonRuntimeState() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let loader = try VRMEntityLoader(withData: seedSan, isMToonEnabled: false) + let vrmEntity = try loader.loadEntity() + + // With MToon disabled, no entity carries MToon runtime state, and + // expression color binds resolve through the fallback material path. + #expect(!TestSupport.hasMToonParameters(in: vrmEntity)) + let fallbackColor = try loader.currentMaterialColor(withMaterialIndex: 0, type: .color) + let fallbackMaterial = try loader.material(withMaterialIndex: 0) + #expect(fallbackColor.isApproximatelyEqual(to: fallbackMaterial.currentColor(for: .color))) + } + + @Test + func testVRM1MToonOutlineEntitiesFollowTheOutlineOption() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let seedSan = TestSupport.seedSanData + let outlineEntity = try VRMEntityLoader(withData: seedSan).loadEntity() + #expect(hasOutlineEntities(in: outlineEntity)) + + // Outlines are the inverted hull only; disabling them must not take the + // MToon surface shader with them. + let noOutlineEntity = try VRMEntityLoader(withData: seedSan, isOutlineEnabled: false).loadEntity() + #expect(!hasOutlineEntities(in: noOutlineEntity)) + #expect(TestSupport.hasCustomMaterial(in: noOutlineEntity)) + } + + @Test + func testMToonMaterialsPreserveTheFixtureAlphaModes() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData, isOutlineEnabled: false) + let opaqueMaterial = try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial) + let blendMaterial = try #require(loader.material(withMaterialIndex: 4) as? CustomMaterial) + + #expect(TestSupport.isOpaque(opaqueMaterial.blending)) + #expect(TestSupport.isTransparent(blendMaterial.blending)) + } + + /// MToon outlines are the inverted-hull entities, identified by their + /// front-face culling rather than by an entity name. + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func hasOutlineEntities(in entity: Entity) -> Bool { + TestSupport.modelEntities(in: entity).contains { modelEntity in + guard let model = modelEntity.components[ModelComponent.self] else { return false } + return model.materials.contains { ($0 as? CustomMaterial)?.faceCulling == .front } + } + } +#endif + +#if os(visionOS) + /// visionOS has no `CustomMaterial`, so MToon always resolves to the + /// Unlit / PBR conversion even when the loader is asked for it. + @Test + func testVisionOSUsesFallbackMaterialWhenMToonIsRequested() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData) + let material = try loader.material(withMaterialIndex: 0) + + #expect(loader.isMToonEnabled) + #expect(loader.isOutlineEnabled) + #expect(material is UnlitMaterial || material is PhysicallyBasedMaterial) + } +#endif + + /// The MToon parameters of the lowest material index that renders as MToon. + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func firstMToonParameters(in vrmEntity: VRMEntity) throws -> MToonMaterialParameters { + for materialIndex in TestSupport.materialIndexes(in: vrmEntity) { + if let parameters = vrmEntity.mtoonParameters(forMaterialIndex: materialIndex) { + return parameters + } + } + throw VRMError.dataInconsistent("Expected at least one MToon material") + } + + /// The blend-shape weight currently applied for a glTF morph target index, + /// read back from the model entities the way RealityKit renders it. + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func morphWeight(in root: Entity, targetIndex: Int) -> Float? { + let targetName = "blendShape_\(targetIndex)" + for modelEntity in TestSupport.modelEntities(in: root) { + let weights = modelEntity.blendWeights + let names = modelEntity.blendWeightNames + for setIndex in names.indices where setIndex < weights.count { + guard let nameIndex = names[setIndex].firstIndex(of: targetName), + nameIndex < weights[setIndex].count else { continue } + return weights[setIndex][nameIndex] + } + } + return nil + } + + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func mtoonParameters(in vrmEntity: VRMEntity, materialIndex: Int) throws -> MToonMaterialParameters { + guard let parameters = vrmEntity.mtoonParameters(forMaterialIndex: materialIndex) else { + throw VRMError.dataInconsistent("Expected MToon parameters for material \(materialIndex)") + } + return parameters + } + +#if !os(visionOS) + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func customMaterial(in root: Entity, + materialIndex: Int, + faceCulling: CustomMaterial.FaceCulling? = nil) throws -> CustomMaterial { + for modelEntity in TestSupport.modelEntities(in: root) { + guard modelEntity.components[VRMMaterialIndexComponent.self]?.materialIndex == materialIndex, + let model = modelEntity.components[ModelComponent.self], + let material = model.materials.first as? CustomMaterial else { + continue + } + if let faceCulling, material.faceCulling != faceCulling { + continue + } + return material + } + throw VRMError.dataInconsistent("Expected CustomMaterial for material \(materialIndex)") + } +#endif + +#if !os(visionOS) + + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func firstCustomMaterial(in root: Entity) throws -> CustomMaterial { + for modelEntity in TestSupport.modelEntities(in: root) { + guard let model = modelEntity.components[ModelComponent.self] else { continue } + if let material = model.materials.first(where: { $0 is CustomMaterial }) as? CustomMaterial { + return material + } + } + throw VRMError.dataInconsistent("Expected at least one CustomMaterial") + } +#endif + + private func shaderConstantName(prefix: String, case value: Case) -> String { + let name = String(describing: value) + return prefix + name.prefix(1).uppercased() + name.dropFirst() + } + + /// Parses `constant float = ;` declarations out of the shader. + private func shaderFloatConstants(in shader: String) -> [String: Float] { + var constants: [String: Float] = [:] + let pattern = #"constant\s+float\s+(\w+)\s*=\s*([0-9.]+)\s*;"# + let regex = try? NSRegularExpression(pattern: pattern) + let range = NSRange(shader.startIndex.. Data { + try TestSupport.modifiedSeedSanData(name: "nondefault-eye-sampler") { json in + guard var samplers = json["samplers"] as? [[String: Any]], + samplers.indices.contains(7) else { + throw VRMError.dataInconsistent("Missing Seed-san sampler fixture data") + } + samplers[7]["magFilter"] = 9728 + samplers[7]["minFilter"] = 9728 + samplers[7]["wrapS"] = 33071 + samplers[7]["wrapT"] = 33071 + json["samplers"] = samplers + } + } +} + +private extension VRMColor { + func isApproximatelyEqual(to other: VRMColor, tolerance: Float = 0.0001) -> Bool { + simd.isApproximatelyEqual(to: other.simd, tolerance: tolerance) + } +} + +#endif diff --git a/Tests/VRMRealityKitTests/VRMUpdateSystemTests.swift b/Tests/VRMRealityKitTests/VRMUpdateSystemTests.swift new file mode 100644 index 00000000..827c8343 --- /dev/null +++ b/Tests/VRMRealityKitTests/VRMUpdateSystemTests.swift @@ -0,0 +1,73 @@ +#if canImport(RealityKit) +import Foundation +import RealityKit +import Testing +import VRMKit +@testable import VRMRealityKit + +@Suite +@MainActor +struct VRMUpdateSystemTests { + @Test + func testAutomaticUpdateIsOnByDefaultAndCanBeToggled() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData) + let vrmEntity = try loader.loadEntity() + #expect(vrmEntity.isAutomaticUpdateEnabled) + + vrmEntity.isAutomaticUpdateEnabled = false + #expect(!vrmEntity.isAutomaticUpdateEnabled) + + vrmEntity.isAutomaticUpdateEnabled = true + #expect(vrmEntity.isAutomaticUpdateEnabled) + } + + @Test + func testParentKeepsTheLoadedEntityAlive() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // Adding the model to a scene is all its lifetime needs: nothing outside + // the entity graph has to hold on to it for the system to keep animating. + let parent = Entity() + weak var loaded: VRMEntity? + do { + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData) + let vrmEntity = try loader.loadEntity() + loaded = vrmEntity + parent.addChild(vrmEntity) + } + + #expect(loaded != nil) + #expect(parent.children.first === loaded) + } + + @Test + func testClonedEntityIsInert() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + // RealityKit builds clones through `init()`, so a clone is a copy of the + // rendered hierarchy with no runtime behind it. + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData) + let vrmEntity = try loader.loadEntity() + let clone = vrmEntity.clone(recursive: true) + + #expect(vrmEntity.humanoid.node(for: .neck) != nil) + #expect(clone.humanoid.node(for: .neck) == nil) + // The system reaches the clone too, so updating it has to stay harmless. + clone.update(deltaTime: 1.0 / 60.0) + } + + /// The VRM rides on the entity as a component, so a clone can still be asked + /// what model it came from instead of trapping. + @Test + func testClonedEntityStillCarriesItsVRM() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let loader = try VRMEntityLoader(withData: TestSupport.seedSanData) + let vrmEntity = try loader.loadEntity() + let clone = vrmEntity.clone(recursive: true) + + guard case .v1 = clone.vrm else { + Issue.record("The clone lost the VRM it was copied from") + return + } + } +} +#endif diff --git a/Tests/VRMTestSupport/GLBRewriter.swift b/Tests/VRMTestSupport/GLBRewriter.swift new file mode 100644 index 00000000..88b1d0ac --- /dev/null +++ b/Tests/VRMTestSupport/GLBRewriter.swift @@ -0,0 +1,77 @@ +import Foundation + +/// Rewrites the JSON chunk of a GLB in memory. +/// +/// Every test target needs the same thing — take a bundled `.vrm`, change a few +/// fields, and hand the result straight to a loader — so the chunk walking and +/// header rebuilding live here rather than once per target. +public enum GLBRewriter { + public enum Error: Swift.Error { + case notGLB + case invalidChunk + case missingJSONChunk + case invalidJSON + } + + private static let magic: [UInt8] = [0x67, 0x6c, 0x54, 0x46] // "glTF" + private static let jsonChunkType: UInt32 = 0x4e4f534a // "JSON" + + /// Returns `data` with its glTF JSON replaced by whatever `modify` produces. + public static func rewritingJSON(of data: Data, + _ modify: (inout [String: Any]) throws -> Void) throws -> Data { + guard data.count >= 20, Array(data.prefix(4)) == magic else { + throw Error.notGLB + } + + var chunks: [(type: UInt32, data: Data)] = [] + var offset = 12 + while offset + 8 <= data.count { + let length = Int(data.uint32LE(at: offset)) + let type = data.uint32LE(at: offset + 4) + offset += 8 + guard offset + length <= data.count else { throw Error.invalidChunk } + chunks.append((type, Data(data[offset ..< offset + length]))) + offset += length + } + + guard let jsonIndex = chunks.firstIndex(where: { $0.type == jsonChunkType }) else { + throw Error.missingJSONChunk + } + var jsonData = chunks[jsonIndex].data + while jsonData.last == 0x20 || jsonData.last == 0x00 { jsonData.removeLast() } + guard var json = try JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else { + throw Error.invalidJSON + } + try modify(&json) + + // GLB chunks are 4-byte aligned; JSON pads with spaces. + var rewritten = try JSONSerialization.data(withJSONObject: json) + while rewritten.count % 4 != 0 { rewritten.append(0x20) } + chunks[jsonIndex].data = rewritten + + var output = Data(magic) + output.appendUInt32LE(data.uint32LE(at: 4)) // version + output.appendUInt32LE(0) // total length, patched below + for chunk in chunks { + output.appendUInt32LE(UInt32(chunk.data.count)) + output.appendUInt32LE(chunk.type) + output.append(chunk.data) + } + output.writeUInt32LE(UInt32(output.count), at: 8) + return output + } +} + +public extension Data { + func uint32LE(at offset: Int) -> UInt32 { + withUnsafeBytes { UInt32(littleEndian: $0.loadUnaligned(fromByteOffset: offset, as: UInt32.self)) } + } + + mutating func appendUInt32LE(_ value: UInt32) { + Swift.withUnsafeBytes(of: value.littleEndian) { append(contentsOf: $0) } + } + + mutating func writeUInt32LE(_ value: UInt32, at offset: Int) { + Swift.withUnsafeBytes(of: value.littleEndian) { replaceSubrange(offset ..< offset + 4, with: $0) } + } +} diff --git a/scripts/build-mtoon-metallibs.sh b/scripts/build-mtoon-metallibs.sh new file mode 100755 index 00000000..b6c59ea3 --- /dev/null +++ b/scripts/build-mtoon-metallibs.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Precompiles the RealityKit MToon shader into per-platform Metal libraries. +# +# CustomMaterial shaders must be compiled offline (TN3133): SwiftPM/Xcode Metal +# compilation of package targets is not reliable across build environments +# (swift build does not compile .metal, and Xcode 26+ requires a separately +# installed Metal Toolchain). The resulting .metallib files are committed to +# the repository and loaded at runtime with MTLDevice.makeLibrary(URL:). +# +# Run this script whenever anything the metallibs are built from changes -- +# the shader sources, the compile settings below, or this script: +# ./scripts/build-mtoon-metallibs.sh +# +# Pass --check to only verify that the recorded build inputs match the current +# ones (no Metal toolchain required), which is what CI runs: +# ./scripts/build-mtoon-metallibs.sh --check +set -euo pipefail + +cd "$(dirname "$0")/.." + +SHADERS="Sources/VRMRealityKit/Shaders" +SOURCE="$SHADERS/MToon.metal" +CORE_HEADER="$SHADERS/MToonCore.h" +RESOURCES="Sources/VRMRealityKit/Resources" +# Spelled out rather than taken from $0, which varies with how the script is invoked. +SCRIPT="scripts/build-mtoon-metallibs.sh" + +# Pin the Metal Shading Language version instead of relying on the compiler +# default (which advances with new toolchains). MSL 2.4 matches the minimum +# deployment targets below (macOS 12 / iOS 15) and the RealityKit shader API. +# Override with MSL_STD after verifying compatibility on the oldest targets. +MSL_STD="${MSL_STD:-metal2.4}" +COMPILE_FLAGS=(-Wall -Wextra -Werror) + +# sdk | -std= prefix | deployment target flag | output metallib. +# Minimum OS versions match the CustomMaterial availability used by VRMEntityLoader. +TARGETS=( + "macosx|macos-|-mmacosx-version-min=12.0|MToon-macos.metallib" + "iphoneos|ios-|-mios-version-min=15.0|MToon-ios.metallib" + "iphonesimulator|ios-|-miphonesimulator-version-min=15.0|MToon-iossim.metallib" +) + +# Everything the compiled metallibs depend on. Hashing only the shader sources +# would report "up to date" after a change to the language version, a +# deployment target, a compile flag or this script. +MANIFEST_FILE="$SHADERS/MToonMetallibInputs.txt" + +build_inputs() { + echo "msl-std=$MSL_STD" + echo "flags=${COMPILE_FLAGS[*]}" + printf 'target=%s\n' "${TARGETS[@]}" + shasum -a 256 "$CORE_HEADER" "$SOURCE" "$SCRIPT" +} + +if [ "${1:-}" = "--check" ]; then + if [ ! -f "$MANIFEST_FILE" ]; then + echo "Missing $MANIFEST_FILE. Run ./$SCRIPT." >&2 + exit 1 + fi + if ! diff -u "$MANIFEST_FILE" <(build_inputs); then + echo "MToon metallib build inputs changed without regenerating the metallibs." >&2 + echo "Run ./$SCRIPT and commit the regenerated resources." >&2 + exit 1 + fi + echo "Bundled MToon metallibs are up to date." + exit 0 +fi + +# The MToon entry points are [[visible]] functions, which are not entry points +# as far as the Metal compiler is concerned, so the per-entry-point limits are +# never checked when they are compiled on their own. RealityKit links them into +# the shaders it generates at runtime, where exceeding a limit fails the +# pipeline silently and the mesh simply stops drawing -- so the limit that the +# sampler table is up against is checked here, by compiling the shader's +# sampling code as a real fragment function. +PROBE_SOURCE="$(mktemp -t MToonEntryPointProbe).metal" +trap 'rm -f "$PROBE_SOURCE"' EXIT +cat > "$PROBE_SOURCE" < texture [[texture(0)]], + constant float4 &samplerParameters [[buffer(0)]]) +{ + return mtoonSample(texture, position.xy, half4(samplerParameters)) + + texture.sample(mtoonParameterSampler, position.xy); +} +PROBE + +verify_entry_point_limits() { + local sdk="$1" std_prefix="$2" min_flag="$3" + echo "Verifying entry-point limits for $sdk" + if ! xcrun -sdk "$sdk" metal \ + "${COMPILE_FLAGS[@]}" \ + "-std=${std_prefix}${MSL_STD}" \ + "$min_flag" \ + -I "$SHADERS" \ + -o /dev/null \ + -c "$PROBE_SOURCE"; then + echo "$SOURCE exceeds a Metal entry-point limit on $sdk; RealityKit would fail to build the pipeline at runtime." >&2 + exit 1 + fi +} + +mkdir -p "$RESOURCES" + +for target in "${TARGETS[@]}"; do + IFS='|' read -r sdk std_prefix min_flag output <<< "$target" + verify_entry_point_limits "$sdk" "$std_prefix" "$min_flag" + echo "Compiling $SOURCE for $sdk -> $output (-std=${std_prefix}${MSL_STD})" + xcrun -sdk "$sdk" metal \ + "${COMPILE_FLAGS[@]}" \ + "-std=${std_prefix}${MSL_STD}" \ + "$min_flag" \ + -o "$RESOURCES/$output" \ + "$SOURCE" +done + +# Record the build inputs so CI can detect stale metallibs. +build_inputs > "$MANIFEST_FILE" + +echo "Done. Regenerated metallibs:" +ls -la "$RESOURCES"/MToon-*.metallib "$MANIFEST_FILE"