Registry & reference

worldfoundry-eval metric list
worldfoundry-eval metric list --json

worldfoundry-eval metric show fid
worldfoundry-eval metric show clip-score --json

worldfoundry-eval metric validate fid clip_score has_artifact:generated_artifact

from worldfoundry.evaluation.tasks.metrics.registry import (
    default_metric_registry,
    list_metric_registry_entries,
    validate_metric_ids,
    create_existing_results_metric,
)

# List all registered entries
for entry in list_metric_registry_entries():
    print(entry.id, entry.family, entry.aliases)

# Resolve alias → canonical id
registry = default_metric_registry()
entry = registry.get("clip-score")
canonical = registry.canonical_metric_id("numeric:reward")

# Bulk validation
payload = validate_metric_ids(["fid", "unknown_metric"])
assert payload["ok"] is False

# Build offline evaluator for scorecard runs
metric = create_existing_results_metric(
    metrics=["artifact_count", "has_artifact:video"],
    required_artifacts=["generated_artifact"],
)

from worldfoundry.evaluation.tasks.metrics import (
    compute_distribution_metrics,
    compute_fid,
    compute_inception_score,
    compute_clip_fid,
    compute_cmmd,
    compute_clean_fid,
    compute_fwd,
    compute_improved_precision_recall,
    compute_vendi_score,
    compute_mean_rarity_score,
)

# Batch via torch-fidelity flags: isc, fid, kid, prc
scores = compute_distribution_metrics(
    "/path/to/reference",
    "/path/to/generated",
    metrics=("fid", "kid", "prc"),
)

fid = compute_fid("/path/to/reference", "/path/to/generated")
clip_fid = compute_fid("/path/to/reference", "/path/to/generated", feature_extractor="clip-vit-b-32")
swav_fid = compute_fid("/path/to/reference", "/path/to/generated", feature_extractor="swav-resnet50")
scene_fid = compute_scene_fid("/path/ref", "/path/gen", reference_bboxes_json="...")
is_scores = compute_inception_score("/path/to/generated")
cmmd = compute_cmmd("/path/to/reference", "/path/to/generated")
clean_fid = compute_clean_fid("/path/to/reference", "/path/to/generated")
fwd = compute_fwd("/path/to/reference", "/path/to/generated")
ipr = compute_improved_precision_recall("/path/to/reference", "/path/to/generated")
vendi = compute_vendi_score(feature_matrix)

import numpy as np
from worldfoundry.evaluation.tasks.metrics import (
    compute_fvd_from_numpy,
    compute_fvd_from_frame_dirs,
    compute_fid_vid,
    compute_fvmd,
    compute_jedi_from_features,
)

# uint8 arrays: (N, T, H, W, C)
fvd = compute_fvd_from_numpy(real_videos, gen_videos, device="cuda")

fvd = compute_fvd_from_frame_dirs(
    reference_frame_dirs=["/path/to/ref_frames/video_0"],
    generated_frame_dirs=["/path/to/gen_frames/video_0"],
)

fid_vid = compute_fid("/path/to/ref_frames", "/path/to/gen_frames")  # alias: fid_vid
fvmd = compute_fvmd("/path/to/reference_videos", "/path/to/generated_videos")

jedi = compute_jedi_from_features(train_features, test_features)

import numpy as np
from worldfoundry.evaluation.tasks.metrics import (
    compute_lpips,
    compute_ssim,
    compute_ms_ssim,
    compute_psnr,
    compute_perceptual_bundle,
    compute_dino_similarity,
    compute_dreamsim,
)

ref = np.load("reference.npy")  # HxWxC uint8 or float
gen = np.load("generated.npy")

bundle = compute_perceptual_bundle(ref, gen)  # lpips, ssim, ms_ssim, psnr
dino_sim = compute_dino_similarity(ref, gen)
dreamsim_dist = compute_dreamsim(ref, gen)

pip install -e ".[distribution_metrics]"

worldfoundry/evaluation/tasks/metrics/
├── _base.py             # MetricModuleSpec protocol + registry-entry factory
├── _discover.py         # Auto-discover METRIC_MODULE exports from per-metric folders
├── _shared/             # Cross-metric infra (distribution helpers, torch-fidelity loader)
│   └── vendor/torch_fidelity/  # Vendored torch-fidelity (Apache-2.0)
├── registry.py          # MetricRegistryEntry, MetricRegistry, legacy + discovered entries
├── builtins.py          # BuiltinExistingResultsMetric (offline result scoring)
├── fid/ kid/ …          # One folder per distribution metric (METRIC_ID, compute, METRIC_MODULE)
├── lpips/ ssim/ …       # One folder per pairwise perceptual metric
├── fvd/ fvmd/ opens2v/ sadpad/  # Video distribution + OpenS2V + SadPaD attributes
├── cmmd/ clean_fid/ vendi_score/ …      # Standalone distribution helpers
└── jedi/                # Video JEPA distance (wrapper.py + vendored JEDi)