374 lines
19 KiB
Python
374 lines
19 KiB
Python
"""LittleBear fixed Mixamo-41 fitting, TokenRig weight restoration and FBX export.
|
|
|
|
Blender: --python this_file -- inspect|prepare|restore|render
|
|
The fitted skeleton is frozen before inference. No reference weights are used.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import bpy
|
|
import numpy as np
|
|
from mathutils import Matrix, Vector
|
|
from mathutils.kdtree import KDTree
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
BASE = ROOT / "Tools/output/littlebear_studio"
|
|
OUT = BASE / "rig_mixamo41"
|
|
STATIC = BASE / "static/LittleBear.fbx"
|
|
REFERENCE = ROOT / "AIClient/Assets/Bundles/Characters/Human/test/Boy_mixamo/CampGuideStudio.fbx"
|
|
HEIGHT = 1.2
|
|
|
|
|
|
def write(name, value):
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
(OUT / name).write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
|
|
def points(mesh):
|
|
return np.array([mesh.matrix_world @ v.co for v in mesh.data.vertices])
|
|
|
|
|
|
def activate(obj):
|
|
bpy.ops.object.select_all(action="DESELECT")
|
|
obj.select_set(True)
|
|
bpy.context.view_layer.objects.active = obj
|
|
|
|
|
|
def load_static():
|
|
bpy.ops.wm.read_factory_settings(use_empty=True)
|
|
bpy.ops.import_scene.fbx(filepath=str(STATIC), use_anim=False)
|
|
mesh = next(o for o in bpy.context.scene.objects if o.type == "MESH")
|
|
activate(mesh)
|
|
bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
|
|
p = points(mesh)
|
|
if int(np.argmax(np.ptp(p, axis=0))) != 2:
|
|
raise ValueError("Static FBX must reimport upright in Blender Z-up")
|
|
mesh.scale = (HEIGHT / np.ptp(p[:, 2]),) * 3
|
|
bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
|
|
p = points(mesh)
|
|
mesh.location -= Vector(((p[:, 0].min() + p[:, 0].max()) / 2,
|
|
(p[:, 1].min() + p[:, 1].max()) / 2, p[:, 2].min()))
|
|
bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
|
|
return mesh
|
|
|
|
|
|
def material(mesh):
|
|
mat = bpy.data.materials.new("LittleBear_Body")
|
|
mat.use_nodes = True
|
|
bsdf = mat.node_tree.nodes.get("Principled BSDF")
|
|
for suffix, socket in [("color", "Base Color"), ("roughness", "Roughness"), ("metallic", "Metallic"), ("normal", None)]:
|
|
image = bpy.data.images.load(str(BASE / "static" / ("LittleBear_" + suffix + ".png")), check_existing=True)
|
|
if suffix != "color":
|
|
image.colorspace_settings.name = "Non-Color"
|
|
node = mat.node_tree.nodes.new("ShaderNodeTexImage")
|
|
node.image = image
|
|
if socket:
|
|
mat.node_tree.links.new(node.outputs["Color"], bsdf.inputs[socket])
|
|
else:
|
|
normal = mat.node_tree.nodes.new("ShaderNodeNormalMap")
|
|
mat.node_tree.links.new(node.outputs["Color"], normal.inputs["Color"])
|
|
mat.node_tree.links.new(normal.outputs["Normal"], bsdf.inputs["Normal"])
|
|
if suffix == "color":
|
|
mat.node_tree.nodes.active = node
|
|
mesh.data.materials.clear()
|
|
mesh.data.materials.append(mat)
|
|
|
|
|
|
def render(mesh, tag, assign_material=True):
|
|
if assign_material:
|
|
material(mesh)
|
|
p = points(mesh)
|
|
center = Vector(((p[:, 0].min() + p[:, 0].max()) / 2,
|
|
(p[:, 1].min() + p[:, 1].max()) / 2,
|
|
(p[:, 2].min() + p[:, 2].max()) / 2))
|
|
scene = bpy.context.scene
|
|
scene.render.engine = "BLENDER_WORKBENCH"
|
|
scene.render.resolution_x = 900
|
|
scene.render.resolution_y = 1000
|
|
scene.render.resolution_percentage = 100
|
|
scene.render.image_settings.file_format = "PNG"
|
|
shading = scene.display.shading
|
|
shading.light = "STUDIO"
|
|
shading.color_type = "TEXTURE"
|
|
shading.show_shadows = True
|
|
shading.show_cavity = True
|
|
shading.background_type = "WORLD"
|
|
scene.world = bpy.data.worlds.new("BearReviewWorld")
|
|
scene.world.color = (.78, .78, .78)
|
|
camera = bpy.data.objects.new("ReviewCamera", bpy.data.cameras.new("ReviewCamera"))
|
|
scene.collection.objects.link(camera)
|
|
scene.camera = camera
|
|
camera.data.type = "ORTHO"
|
|
camera.data.ortho_scale = max(HEIGHT * 1.15, np.ptp(p[:, 0]) * 1000 / 900 * 1.1)
|
|
for label, offset in [("front", (0, -3, 0)), ("back", (0, 3, 0)), ("three_quarter", (2, -3, .4))]:
|
|
camera.location = center + Vector(offset)
|
|
camera.rotation_euler = (center - camera.location).to_track_quat("-Z", "Y").to_euler()
|
|
scene.render.filepath = str(OUT / (tag + "_" + label + ".png"))
|
|
bpy.ops.render.render(write_still=True)
|
|
|
|
|
|
def inspect():
|
|
mesh = load_static()
|
|
p = points(mesh)
|
|
bands = []
|
|
for z in np.linspace(.02, .98, 49):
|
|
a = p[abs(p[:, 2] / HEIGHT - z) < .012]
|
|
if len(a):
|
|
bands.append(dict(z=float(z), count=len(a), min=a.min(0).tolist(), max=a.max(0).tolist()))
|
|
write("mesh_inspection.json", dict(bounds=[p.min(0).tolist(), p.max(0).tolist()], bands=bands))
|
|
render(mesh, "static")
|
|
|
|
|
|
def inspect_high():
|
|
bpy.ops.wm.read_factory_settings(use_empty=True)
|
|
bpy.ops.import_scene.gltf(filepath=str(ROOT / "Tools/output/littlebear_3d/littlebear.glb"))
|
|
mesh = next(o for o in bpy.context.scene.objects if o.type == "MESH")
|
|
activate(mesh)
|
|
bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
|
|
p = points(mesh)
|
|
mesh.scale = (HEIGHT / np.ptp(p[:, 2]),) * 3
|
|
bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
|
|
for slot in mesh.material_slots:
|
|
mat = slot.material
|
|
if mat and mat.use_nodes:
|
|
for node in mat.node_tree.nodes:
|
|
if node.type == "TEX_IMAGE" and any(link.to_socket.name == "Base Color" for link in node.outputs["Color"].links):
|
|
mat.node_tree.nodes.active = node
|
|
break
|
|
render(mesh, "high", assign_material=False)
|
|
|
|
|
|
def prepare():
|
|
mesh = load_static()
|
|
p = points(mesh)
|
|
before = set(bpy.data.objects)
|
|
bpy.ops.import_scene.fbx(filepath=str(REFERENCE), use_anim=False)
|
|
arm = next(o for o in bpy.context.scene.objects if o not in before and o.type == "ARMATURE")
|
|
for o in list(bpy.data.objects):
|
|
if o not in (mesh, arm):
|
|
bpy.data.objects.remove(o, do_unlink=True)
|
|
arm.animation_data_clear()
|
|
d = arm.matrix_world @ arm.data.bones["mixamorig:Head"].head_local - arm.matrix_world @ arm.data.bones["mixamorig:Hips"].head_local
|
|
if abs(d.y) > abs(d.z):
|
|
arm.matrix_world = Matrix.Rotation(math.pi / 2 if d.y > 0 else -math.pi / 2, 4, "X") @ arm.matrix_world
|
|
activate(arm)
|
|
bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
|
|
toe = arm.data.bones["mixamorig:LeftToeBase"].head_local - arm.data.bones["mixamorig:LeftFoot"].head_local
|
|
if toe.y > 0:
|
|
arm.matrix_world = Matrix.Rotation(math.pi, 4, "Z") @ arm.matrix_world
|
|
bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
|
|
assert len(arm.data.bones) == 41
|
|
# Fractions reflect this bear's inspected T-pose silhouette, not human landmarks.
|
|
positions = {"Hips": (0, 0, .385), "Spine": (0, 0, .440), "Spine1": (0, 0, .510),
|
|
"Spine2": (0, 0, .595), "Neck": (0, 0, .677), "Head": (0, 0, .723),
|
|
"HeadTop_End": (0, 0, .952)}
|
|
reach = max(abs(p[:, 0].min()), abs(p[:, 0].max())) / HEIGHT
|
|
arm_z = float(np.median(p[(abs(p[:, 0]) > HEIGHT * .28) & (abs(p[:, 0]) < HEIGHT * .36), 2])) / HEIGHT
|
|
if not .54 < arm_z < .76:
|
|
raise ValueError("Inspect bear shoulder height before fitting: " + str(arm_z))
|
|
for side, sign in [("Left", 1), ("Right", -1)]:
|
|
local = {"Shoulder": (.064, 0, arm_z), "Arm": (.146, 0, arm_z),
|
|
"ForeArm": (reach * .57, 0, arm_z), "Hand": (reach * .82, 0, arm_z),
|
|
"UpLeg": (.079, 0, .383), "Leg": (.095, -.006, .213),
|
|
"Foot": (.099, .004, .065), "ToeBase": (.099, -.070, .024),
|
|
"Toe_End": (.099, -.105, .024)}
|
|
for name, v in local.items():
|
|
positions[side + name] = (v[0] * sign, v[1], v[2])
|
|
old = {b.name: b.matrix_local.copy() for b in arm.data.bones}
|
|
template_height = old["mixamorig:HeadTop_End"].translation.z - old["mixamorig:LeftToe_End"].translation.z
|
|
for side in ["Left", "Right"]:
|
|
hand = old["mixamorig:" + side + "Hand"].translation
|
|
for name, m in old.items():
|
|
short = name.split(":")[-1]
|
|
if short.startswith(side + "Hand") and short != side + "Hand":
|
|
positions[short] = (Vector(positions[side + "Hand"]) + (m.translation - hand) * (.75 / template_height))[:]
|
|
bpy.ops.object.mode_set(mode="EDIT")
|
|
for b in arm.data.edit_bones:
|
|
b.use_connect = False
|
|
for b in arm.data.edit_bones:
|
|
delta, roll = b.tail - b.head, b.roll
|
|
b.head = Vector(positions[b.name.split(":")[-1]]) * HEIGHT
|
|
b.tail = b.head + delta
|
|
b.roll = roll
|
|
bpy.ops.object.mode_set(mode="OBJECT")
|
|
bpy.context.view_layer.update()
|
|
drift = max(old[b.name].to_quaternion().rotation_difference(b.matrix_local.to_quaternion()).angle for b in arm.data.bones)
|
|
assert drift < .001, drift
|
|
mesh.vertex_groups.clear()
|
|
for b in arm.data.bones:
|
|
mesh.vertex_groups.new(name=b.name)
|
|
mesh.vertex_groups["mixamorig:Hips"].add(list(range(len(p))), 1, "REPLACE")
|
|
for i, b in enumerate(arm.data.bones):
|
|
if b.name != "mixamorig:Hips":
|
|
mesh.vertex_groups["mixamorig:Hips"].remove([i])
|
|
mesh.vertex_groups[b.name].add([i], 1, "REPLACE")
|
|
mesh.parent = arm
|
|
mod = mesh.modifiers.new("FixedMixamoInput", "ARMATURE")
|
|
mod.object = arm
|
|
arm.data.pose_position = "REST"
|
|
mesh.data.materials.clear()
|
|
manifest = dict(character="LittleBear", height=HEIGHT, vertices=len(p),
|
|
skeleton_source=str(REFERENCE.relative_to(ROOT)),
|
|
input_skin="Non-anatomical keepalive placeholders only; all reference weights discarded",
|
|
landmarks={n: list(v) for n, v in positions.items()},
|
|
max_axis_rotation_drift_rad=drift,
|
|
bones={b.name: dict(parent=b.parent.name if b.parent else None,
|
|
head=list(arm.matrix_world @ b.head_local),
|
|
world_matrix=[list(r) for r in arm.matrix_world @ b.matrix_local]) for b in arm.data.bones})
|
|
write("input_manifest.json", manifest)
|
|
bpy.ops.wm.save_as_mainfile(filepath=str(OUT / "prepared.blend"), compress=True)
|
|
activate(mesh)
|
|
arm.select_set(True)
|
|
bpy.ops.export_scene.gltf(filepath=str(OUT / "source.glb"), export_format="GLB", export_yup=True,
|
|
use_selection=True, export_skins=True, export_animations=False, export_def_bones=False)
|
|
print("PREPARED", len(arm.data.bones), len(p), "shoulder fraction", arm_z)
|
|
|
|
|
|
def restore():
|
|
bpy.ops.wm.open_mainfile(filepath=str(OUT / "prepared.blend"))
|
|
arm = next(o for o in bpy.data.objects if o.type == "ARMATURE")
|
|
mesh = next(o for o in bpy.data.objects if o.type == "MESH")
|
|
arm.data.pose_position = "POSE"
|
|
names = [b.name for b in arm.data.bones]
|
|
p = points(mesh)
|
|
decoded = json.loads((OUT / "decoded_weights.json").read_text())
|
|
tree = KDTree(len(decoded))
|
|
for i, row in enumerate(decoded):
|
|
x, y, z = row["position_y_up"]
|
|
tree.insert(Vector((x, -z, y)), i)
|
|
tree.balance()
|
|
weights = np.zeros((len(p), len(names)))
|
|
distances = []
|
|
for i, pos in enumerate(p):
|
|
_, j, dist = tree.find(Vector(pos))
|
|
distances.append(dist)
|
|
for n, w in decoded[j]["weights"].items():
|
|
weights[i, names.index(n)] += w
|
|
assert max(distances) < 1e-5, max(distances)
|
|
weights /= weights.sum(1, keepdims=True)
|
|
raw = weights.copy()
|
|
fit = json.loads((OUT / "input_manifest.json").read_text())
|
|
marks = fit["landmarks"]
|
|
wrist_x = abs(marks["LeftHand"][0]) * HEIGHT
|
|
head_z = marks["Head"][2] * HEIGHT
|
|
for i, pos in enumerate(p):
|
|
side = "Left" if pos[0] > 0 else "Right"
|
|
if pos[2] > head_z:
|
|
allowed = ["mixamorig:Head"]
|
|
elif abs(pos[0]) > wrist_x:
|
|
allowed = ["mixamorig:" + side + "Hand"]
|
|
elif abs(pos[0]) > .16 * HEIGHT and abs(pos[2] / HEIGHT - marks["LeftArm"][2]) < .085:
|
|
allowed = ["mixamorig:" + side + part for part in ["Arm", "ForeArm", "Hand"]]
|
|
elif pos[2] < .35 * HEIGHT:
|
|
allowed = ["mixamorig:Hips"] + ["mixamorig:" + side + part for part in ["UpLeg", "Leg", "Foot", "ToeBase", "Toe_End"]]
|
|
if pos[2] > .24 * HEIGHT and abs(pos[0]) < .06 * HEIGHT:
|
|
allowed += ["mixamorig:LeftUpLeg", "mixamorig:RightUpLeg"]
|
|
else:
|
|
allowed = ["mixamorig:" + part for part in ["Hips", "Spine", "Spine1", "Spine2", "Neck", "Head"]]
|
|
allowed += ["mixamorig:" + s + part for s in ["Left", "Right"] for part in ["Shoulder", "Arm", "UpLeg"]]
|
|
keep = np.array([n in allowed for n in names])
|
|
# Preserve mitten hand contribution at the wrist instead of tiny finger joints.
|
|
if "mixamorig:" + side + "Hand" in allowed:
|
|
hand = names.index("mixamorig:" + side + "Hand")
|
|
weights[i, hand] = sum(weights[i, j] for j, n in enumerate(names) if n.startswith("mixamorig:" + side + "Hand"))
|
|
weights[i, ~keep] = 0
|
|
if weights[i].sum() < 1e-8:
|
|
if len(allowed) == 1:
|
|
weights[i, names.index(allowed[0])] = 1
|
|
else:
|
|
near = min(allowed, key=lambda n: (arm.data.bones[n].head_local - Vector(pos)).length)
|
|
weights[i, names.index(near)] = 1
|
|
weights[i] /= weights[i].sum()
|
|
# The connected crotch must blend continuously across the left/right
|
|
# boundary. Pin its center softly to Hips rather than switching legs.
|
|
if .24 * HEIGHT < pos[2] < .40 * HEIGHT:
|
|
t = min(abs(pos[0]) / (.055 * HEIGHT), 1.)
|
|
vertical = min(max((pos[2] / HEIGHT - .24) / .09, 0.), 1.)
|
|
center = (1 - t * t * (3 - 2 * t)) * vertical * vertical * (3 - 2 * vertical)
|
|
weights[i] *= 1 - center
|
|
weights[i, names.index("mixamorig:Hips")] += center
|
|
ids = np.argsort(-weights[i])[4:]
|
|
weights[i, ids] = 0
|
|
weights[i, weights[i] < .005] = 0
|
|
weights[i] /= weights[i].sum()
|
|
mesh.vertex_groups.clear()
|
|
for n in names:
|
|
mesh.vertex_groups.new(name=n)
|
|
for i, row in enumerate(weights):
|
|
for j in np.flatnonzero(row):
|
|
mesh.vertex_groups[names[j]].add([i], float(row[j]), "REPLACE")
|
|
for b in arm.data.bones:
|
|
assert (b.parent.name if b.parent else None) == fit["bones"][b.name]["parent"]
|
|
expected = np.array(fit["bones"][b.name]["world_matrix"])
|
|
assert np.max(abs(np.array(arm.matrix_world @ b.matrix_local) - expected)) < 1e-6
|
|
material(mesh)
|
|
bpy.ops.wm.save_as_mainfile(filepath=str(OUT / "LittleBear_Rigged.blend"), compress=True)
|
|
activate(mesh)
|
|
arm.select_set(True)
|
|
bpy.ops.export_scene.gltf(filepath=str(OUT / "LittleBear_Rigged.glb"), export_format="GLB", export_yup=True,
|
|
use_selection=True, export_skins=True, export_animations=False, export_def_bones=False)
|
|
target = OUT / "LittleBear_Rigged.fbx"
|
|
bpy.ops.export_scene.fbx(filepath=str(target), use_selection=True, object_types={"ARMATURE", "MESH"},
|
|
axis_forward="-Z", axis_up="Y", global_scale=1, apply_unit_scale=True,
|
|
apply_scale_options="FBX_SCALE_ALL", bake_space_transform=False, add_leaf_bones=False,
|
|
use_armature_deform_only=False, armature_nodetype="NULL", bake_anim=False,
|
|
mesh_smooth_type="OFF", use_mesh_modifiers=False, path_mode="COPY", embed_textures=True)
|
|
write("skin_receipt.json", dict(bones=len(names), vertices=len(p),
|
|
triangles=sum(len(poly.vertices) - 2 for poly in mesh.data.polygons), height=HEIGHT,
|
|
maxInfluences=int(np.count_nonzero(weights, axis=1).max()),
|
|
localCleanupChangedVertices=int(np.any(abs(weights - raw) > .001, axis=1).sum()),
|
|
weightsSource="TokenRig prediction, supplied fixed Mixamo skeleton; no website weights",
|
|
maxPredictionMappingDistanceM=max(distances),
|
|
fbxSha256=hashlib.sha256(target.read_bytes()).hexdigest()))
|
|
render(mesh, "rigged_rest")
|
|
print("RIGGED_EXPORT", target)
|
|
|
|
|
|
def verify_export():
|
|
def snapshot():
|
|
arm = next(o for o in bpy.data.objects if o.type == "ARMATURE")
|
|
mesh = next(o for o in bpy.data.objects if o.type == "MESH")
|
|
return dict(points=points(mesh),
|
|
bones={b.name: dict(parent=b.parent.name if b.parent else None,
|
|
head=list(arm.matrix_world @ b.head_local)) for b in arm.data.bones},
|
|
weights=[{mesh.vertex_groups[g.group].name: g.weight for g in v.groups if g.weight > 1e-8} for v in mesh.data.vertices],
|
|
uv=sorted(map(tuple, np.round(np.array([tuple(v.uv) for v in mesh.data.uv_layers.active.data]), 5))),
|
|
triangles=sum(len(p.vertices) - 2 for p in mesh.data.polygons))
|
|
bpy.ops.wm.open_mainfile(filepath=str(OUT / "LittleBear_Rigged.blend"))
|
|
expected = snapshot()
|
|
bpy.ops.wm.read_factory_settings(use_empty=True)
|
|
bpy.ops.import_scene.fbx(filepath=str(OUT / "LittleBear_Rigged.fbx"), use_anim=False)
|
|
actual = snapshot()
|
|
tree = KDTree(len(expected["points"]))
|
|
for i, p in enumerate(expected["points"]):
|
|
tree.insert(Vector(p), i)
|
|
tree.balance()
|
|
position_error = weight_error = 0.
|
|
for p, row in zip(actual["points"], actual["weights"]):
|
|
_, i, distance = tree.find(Vector(p))
|
|
position_error = max(position_error, distance)
|
|
ref = expected["weights"][i]
|
|
weight_error = max(weight_error, max(abs(row.get(n, 0) - ref.get(n, 0)) for n in set(row) | set(ref)))
|
|
assert set(actual["bones"]) == set(expected["bones"])
|
|
assert all(actual["bones"][n]["parent"] == expected["bones"][n]["parent"] for n in actual["bones"])
|
|
bone_error = max(float(np.linalg.norm(np.array(actual["bones"][n]["head"]) - expected["bones"][n]["head"])) for n in actual["bones"])
|
|
assert position_error < 1e-5 and weight_error < 1e-5 and bone_error < 1e-5
|
|
assert actual["uv"] == expected["uv"]
|
|
assert actual["triangles"] == expected["triangles"] == 3000
|
|
assert max(len(row) for row in actual["weights"]) <= 4
|
|
report = dict(bones=len(actual["bones"]), triangles=actual["triangles"],
|
|
maxPositionErrorM=position_error, maxWeightError=weight_error, maxBoneHeadErrorM=bone_error,
|
|
uvPreserved=True, parentHierarchyPreserved=True, sourceVertices=len(expected["points"]),
|
|
roundtripVertices=len(actual["points"]), valid=True)
|
|
write("export_verification.json", report)
|
|
print("ROUNDTRIP_VERIFIED", json.dumps(report))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
action = sys.argv[sys.argv.index("--") + 1]
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
{"inspect": inspect, "inspect_high": inspect_high, "prepare": prepare, "restore": restore, "verify_export": verify_export}[action]()
|