Make imported asset fingerprints deterministic
This commit is contained in:
1 parent
a048233fce
commit
e1b2f86a49
27 files changed
+798
-135
No files matched your search
@@ -9,12 +9,13 @@ use shared::{
|
||||
animation_clip_source_index, animation_clip_sub_asset_id, animation_skeleton_source_index,
|
||||
animation_skeleton_sub_asset_id,
|
||||
brush_math::{validate_brush, BrushDiagnosticSeverity},
|
||||
AnimationControllerDesc, AnimationDiagnosticSeverity, AnimationManifest, AudioListenerDesc,
|
||||
AudioSourceDesc, BrushDesc, ColliderDesc, ColliderShapeDesc, EditorAssetRef, MaterialAsset,
|
||||
MaterialDesc, MaterialInstanceAsset, MaterialOverride, ModelRef, PostProcessEffectAsset,
|
||||
PostProcessVolumeDesc, PrefabInstance, PrefabRef, RendererMaterialSet, ShaderSchemaAsset,
|
||||
SkinnedMeshRenderer, StaticMeshRenderer, TerrainDesc, ANIMATION_MANIFEST_SCHEMA_VERSION,
|
||||
AUDIO_CLIP_SUB_ASSET_ID, COMPONENT_ANIMATION_CONTROLLER_DESC, COMPONENT_SKINNED_MESH_RENDERER,
|
||||
AnimationControllerDesc, AnimationDiagnosticSeverity, AnimationManifest,
|
||||
AssetSourceFingerprint, AudioListenerDesc, AudioSourceDesc, BrushDesc, ColliderDesc,
|
||||
ColliderShapeDesc, EditorAssetRef, MaterialAsset, MaterialDesc, MaterialInstanceAsset,
|
||||
MaterialOverride, ModelRef, PostProcessEffectAsset, PostProcessVolumeDesc, PrefabInstance,
|
||||
PrefabRef, RendererMaterialSet, ShaderSchemaAsset, SkinnedMeshRenderer, StaticMeshRenderer,
|
||||
TerrainDesc, ANIMATION_MANIFEST_SCHEMA_VERSION, AUDIO_CLIP_SUB_ASSET_ID,
|
||||
COMPONENT_ANIMATION_CONTROLLER_DESC, COMPONENT_SKINNED_MESH_RENDERER,
|
||||
NAVIGATION_GENERATED_ARTIFACT_DIRECTORY,
|
||||
};
|
||||
|
||||
@@ -322,6 +323,8 @@ struct RegistryAssetRecord {
|
||||
path: String,
|
||||
kind_tag: String,
|
||||
#[serde(default)]
|
||||
source_fingerprint: Option<AssetSourceFingerprint>,
|
||||
#[serde(default)]
|
||||
import_settings: RegistryImportSettings,
|
||||
#[serde(default)]
|
||||
dependencies: Vec<String>,
|
||||
@@ -482,10 +485,96 @@ fn validate_asset_registry(
|
||||
repair: "Convert the clip to Ogg Vorbis, WAV, MP3, or FLAC and reimport it.".into(),
|
||||
});
|
||||
}
|
||||
if matches!(record.kind_tag.as_str(), "Model" | "Texture" | "AudioClip") {
|
||||
validate_content_fingerprint(
|
||||
project_root,
|
||||
report,
|
||||
"assets/.index/registry.ron",
|
||||
&record.path,
|
||||
record.source_fingerprint.as_ref(),
|
||||
"registry.source_fingerprint_missing",
|
||||
"registry.source_fingerprint_stale",
|
||||
"registered imported source",
|
||||
);
|
||||
}
|
||||
}
|
||||
animation_catalog
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn validate_content_fingerprint(
|
||||
project_root: &Path,
|
||||
report: &mut ProjectValidationReport,
|
||||
owner_path: &str,
|
||||
source_path: &str,
|
||||
fingerprint: Option<&AssetSourceFingerprint>,
|
||||
missing_code: &str,
|
||||
stale_code: &str,
|
||||
source_label: &str,
|
||||
) {
|
||||
let Some(fingerprint) = fingerprint.filter(|fingerprint| fingerprint.is_complete()) else {
|
||||
report.findings.push(ProjectValidationFinding {
|
||||
severity: ValidationSeverity::Error,
|
||||
code: missing_code.into(),
|
||||
source_path: owner_path.into(),
|
||||
owner_actor_id: None,
|
||||
reference: Some(source_path.into()),
|
||||
message: format!("{source_label} has no complete BLAKE3 content fingerprint"),
|
||||
repair: "Refresh the asset registry or reimport the source with this editor version."
|
||||
.into(),
|
||||
});
|
||||
return;
|
||||
};
|
||||
let Some(path) = resolve_reference(project_root, source_path) else {
|
||||
return;
|
||||
};
|
||||
let Ok(metadata) = std::fs::metadata(&path) else {
|
||||
return;
|
||||
};
|
||||
if metadata.len() != fingerprint.byte_len {
|
||||
report.findings.push(ProjectValidationFinding {
|
||||
severity: ValidationSeverity::Error,
|
||||
code: stale_code.into(),
|
||||
source_path: owner_path.into(),
|
||||
owner_actor_id: None,
|
||||
reference: Some(source_path.into()),
|
||||
message: format!(
|
||||
"{source_label} size changed from {} to {} bytes",
|
||||
fingerprint.byte_len,
|
||||
metadata.len()
|
||||
),
|
||||
repair: "Refresh or reimport the changed source before validation or packaging.".into(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
match blake3_file_hash(&path) {
|
||||
Ok(content_hash) if content_hash != fingerprint.content_hash => {
|
||||
report.findings.push(ProjectValidationFinding {
|
||||
severity: ValidationSeverity::Error,
|
||||
code: stale_code.into(),
|
||||
source_path: owner_path.into(),
|
||||
owner_actor_id: None,
|
||||
reference: Some(source_path.into()),
|
||||
message: format!(
|
||||
"{source_label} content changed even though its byte length is unchanged"
|
||||
),
|
||||
repair: "Refresh or reimport the changed source before validation or packaging."
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => report.findings.push(ProjectValidationFinding {
|
||||
severity: ValidationSeverity::Error,
|
||||
code: stale_code.into(),
|
||||
source_path: owner_path.into(),
|
||||
owner_actor_id: None,
|
||||
reference: Some(source_path.into()),
|
||||
message: format!("could not hash {source_label}: {error}"),
|
||||
repair: "Restore readable source content and rerun validation.".into(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn reference_is_runtime_package_asset(reference: &str) -> bool {
|
||||
let relative = Path::new(reference.split('#').next().unwrap_or(reference));
|
||||
if relative.is_absolute()
|
||||
@@ -709,10 +798,12 @@ struct StaticMeshManifestView {
|
||||
struct StaticMeshSourceView {
|
||||
path: String,
|
||||
#[serde(default)]
|
||||
fingerprint: AssetSourceFingerprint,
|
||||
#[serde(default)]
|
||||
dependencies: Vec<String>,
|
||||
}
|
||||
|
||||
const STATIC_MESH_MANIFEST_SCHEMA_VERSION: u32 = 3;
|
||||
const STATIC_MESH_MANIFEST_SCHEMA_VERSION: u32 = 4;
|
||||
|
||||
fn validate_static_mesh_manifests(project_root: &Path, report: &mut ProjectValidationReport) {
|
||||
for path in ron_files(
|
||||
@@ -754,6 +845,16 @@ fn validate_static_mesh_manifests(project_root: &Path, report: &mut ProjectValid
|
||||
"model",
|
||||
&manifest.source.path,
|
||||
);
|
||||
validate_content_fingerprint(
|
||||
project_root,
|
||||
report,
|
||||
&source,
|
||||
&manifest.source.path,
|
||||
Some(&manifest.source.fingerprint),
|
||||
"static_mesh.manifest_fingerprint_missing",
|
||||
"static_mesh.manifest_stale",
|
||||
"static mesh source",
|
||||
);
|
||||
for dependency in manifest.source.dependencies {
|
||||
add_reference(
|
||||
project_root,
|
||||
@@ -975,12 +1076,7 @@ fn validate_animation_manifest_contract(
|
||||
}
|
||||
|
||||
let fingerprint = &manifest.source.fingerprint;
|
||||
let hash_is_present = fingerprint.content_hash.len() == 64
|
||||
&& fingerprint
|
||||
.content_hash
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit());
|
||||
if fingerprint.byte_len == 0 || fingerprint.modified_unix_secs == 0 || !hash_is_present {
|
||||
if fingerprint.byte_len == 0 || !fingerprint.is_complete() {
|
||||
report.findings.push(ProjectValidationFinding {
|
||||
severity: ValidationSeverity::Error,
|
||||
code: "animation.manifest_fingerprint_missing".into(),
|
||||
@@ -3071,7 +3167,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn static_mesh_manifest_schema_v3_is_current() {
|
||||
fn static_mesh_manifest_schema_v4_is_current() {
|
||||
let root = fixture_root();
|
||||
let source_path = "assets/models/static.glb";
|
||||
std::fs::create_dir_all(root.join("assets/models")).unwrap();
|
||||
@@ -3081,7 +3177,8 @@ mod tests {
|
||||
std::fs::write(
|
||||
artifact_dir.join("static-id.static_mesh.ron"),
|
||||
format!(
|
||||
"(schema_version:{STATIC_MESH_MANIFEST_SCHEMA_VERSION},asset_id:\"static-id\",source:(path:\"{source_path}\",dependencies:[]),warnings:[])"
|
||||
"(schema_version:{STATIC_MESH_MANIFEST_SCHEMA_VERSION},asset_id:\"static-id\",source:(path:\"{source_path}\",fingerprint:(byte_len:5,content_hash:\"{}\"),dependencies:[]),warnings:[])",
|
||||
blake3::hash(b"model").to_hex()
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3095,6 +3192,70 @@ mod tests {
|
||||
std::fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_size_static_mesh_source_change_invalidates_content_hash() {
|
||||
let root = fixture_root();
|
||||
let source_path = "assets/models/static.glb";
|
||||
std::fs::create_dir_all(root.join("assets/models")).unwrap();
|
||||
std::fs::write(root.join(source_path), b"model").unwrap();
|
||||
let fingerprint = AssetSourceFingerprint::from_bytes(b"model");
|
||||
let artifact = root.join("assets/meshes/generated/static-id.static_mesh.ron");
|
||||
std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
|
||||
std::fs::write(
|
||||
&artifact,
|
||||
format!(
|
||||
"(schema_version:{STATIC_MESH_MANIFEST_SCHEMA_VERSION},asset_id:\"static-id\",source:(path:\"{source_path}\",fingerprint:(byte_len:{},content_hash:\"{}\"),dependencies:[]),warnings:[])",
|
||||
fingerprint.byte_len, fingerprint.content_hash
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(root.join(source_path), b"modem").unwrap();
|
||||
|
||||
let report = validate_project(&root);
|
||||
|
||||
let finding = report
|
||||
.findings
|
||||
.iter()
|
||||
.find(|finding| finding.code == "static_mesh.manifest_stale")
|
||||
.expect("same-size model edit must stale the static mesh manifest");
|
||||
assert!(finding.message.contains("byte length is unchanged"));
|
||||
assert!(finding.repair.contains("Refresh") || finding.repair.contains("reimport"));
|
||||
std::fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_detects_same_size_changes_for_all_imported_source_kinds() {
|
||||
for (kind, relative) in [
|
||||
("Model", "assets/models/source.glb"),
|
||||
("Texture", "assets/textures/source.png"),
|
||||
("AudioClip", "assets/audio/source.ogg"),
|
||||
] {
|
||||
let root = fixture_root();
|
||||
let path = root.join(relative);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(&path, b"source-a").unwrap();
|
||||
let fingerprint = AssetSourceFingerprint::from_bytes(b"source-a");
|
||||
std::fs::write(
|
||||
root.join("assets/.index/registry.ron"),
|
||||
format!(
|
||||
"[(id:(\"00000000-0000-0000-0000-000000000001\"),path:\"{relative}\",kind_tag:\"{kind}\",source_fingerprint:Some((byte_len:{},content_hash:\"{}\")),dependencies:[])]",
|
||||
fingerprint.byte_len, fingerprint.content_hash
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(&path, b"source-b").unwrap();
|
||||
|
||||
let report = validate_project(&root);
|
||||
|
||||
assert!(report.findings.iter().any(|finding| {
|
||||
finding.code == "registry.source_fingerprint_stale"
|
||||
&& finding.reference.as_deref() == Some(relative)
|
||||
&& finding.message.contains("byte length is unchanged")
|
||||
}));
|
||||
std::fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn write_manifest(root: &Path, version: u32) {
|
||||
std::fs::write(
|
||||
root.join("assets/project.ron"),
|
||||
@@ -3128,7 +3289,6 @@ mod tests {
|
||||
format: source_format.into(),
|
||||
fingerprint: AnimationSourceFingerprint {
|
||||
byte_len: 5,
|
||||
modified_unix_secs: 1,
|
||||
content_hash: blake3::hash(b"model").to_hex().to_string(),
|
||||
},
|
||||
dependencies: Vec::new(),
|
||||
@@ -3188,10 +3348,12 @@ mod tests {
|
||||
let manifest = animation_manifest_path
|
||||
.map(|path| format!("Some({path:?})"))
|
||||
.unwrap_or_else(|| "None".into());
|
||||
let fingerprint = AssetSourceFingerprint::from_bytes(b"model");
|
||||
std::fs::write(
|
||||
root.join("assets/.index/registry.ron"),
|
||||
format!(
|
||||
"[(id:(\"{ANIMATION_ASSET_ID}\"),path:\"{source_path}\",kind_tag:\"Model\",import_settings:(static_mesh_manifest_path:None,animation_manifest_path:{manifest}),dependencies:[])]"
|
||||
"[(id:(\"{ANIMATION_ASSET_ID}\"),path:\"{source_path}\",kind_tag:\"Model\",source_fingerprint:Some((byte_len:{},content_hash:\"{}\")),import_settings:(static_mesh_manifest_path:None,animation_manifest_path:{manifest}),dependencies:[])]",
|
||||
fingerprint.byte_len, fingerprint.content_hash
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3205,10 +3367,12 @@ mod tests {
|
||||
let source = root.join(source_path);
|
||||
std::fs::create_dir_all(source.parent().unwrap()).unwrap();
|
||||
std::fs::write(source, b"model").unwrap();
|
||||
let fingerprint = AssetSourceFingerprint::from_bytes(b"model");
|
||||
std::fs::write(
|
||||
root.join("assets/.index/registry.ron"),
|
||||
format!(
|
||||
"[(id:(\"{ANIMATION_ASSET_ID}\"),path:\"{source_path}\",kind_tag:\"Model\",import_settings:(static_mesh_manifest_path:None,animation_manifest_path:Some(\"{ANIMATION_MANIFEST_PATH}\"),default_animation_clip_id:Some(\"{default_clip_id}\")),dependencies:[])]"
|
||||
"[(id:(\"{ANIMATION_ASSET_ID}\"),path:\"{source_path}\",kind_tag:\"Model\",source_fingerprint:Some((byte_len:{},content_hash:\"{}\")),import_settings:(static_mesh_manifest_path:None,animation_manifest_path:Some(\"{ANIMATION_MANIFEST_PATH}\"),default_animation_clip_id:Some(\"{default_clip_id}\")),dependencies:[])]",
|
||||
fingerprint.byte_len, fingerprint.content_hash
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3632,6 +3796,49 @@ mod tests {
|
||||
std::fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn project_validation_preserves_registry_and_generated_artifact_bytes() {
|
||||
let root = fixture_root();
|
||||
let source_path = "assets/models/hero.glb";
|
||||
let manifest = animation_fixture_manifest(source_path, "glb", "rig-signature", true);
|
||||
write_animation_source_and_registry(&root, source_path, Some(ANIMATION_MANIFEST_PATH));
|
||||
write_animation_manifest(&root, &manifest);
|
||||
write_animation_scene(&root, &animation_fixture_controller(&manifest));
|
||||
|
||||
let static_path = root.join("assets/meshes/generated/model-id.static_mesh.ron");
|
||||
std::fs::create_dir_all(static_path.parent().unwrap()).unwrap();
|
||||
std::fs::write(
|
||||
&static_path,
|
||||
format!(
|
||||
"( schema_version: {STATIC_MESH_MANIFEST_SCHEMA_VERSION}, asset_id: \"model-id\", source: ( path: \"{source_path}\", fingerprint: ( byte_len: {}, content_hash: \"{}\", ), dependencies: [], ), warnings: [], )\n\n",
|
||||
manifest.source.fingerprint.byte_len,
|
||||
manifest.source.fingerprint.content_hash
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let paths = [
|
||||
root.join("assets/.index/registry.ron"),
|
||||
root.join(ANIMATION_MANIFEST_PATH),
|
||||
static_path,
|
||||
];
|
||||
let before = paths
|
||||
.iter()
|
||||
.map(std::fs::read)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
|
||||
let _ = validate_project(&root);
|
||||
let _ = validate_project(&root);
|
||||
|
||||
let after = paths
|
||||
.iter()
|
||||
.map(std::fs::read)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(after, before);
|
||||
std::fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_animation_manifest_is_reported_globally_and_on_the_actor_property() {
|
||||
let root = fixture_root();
|
||||
|
||||
Reference in new issue
Block a user