{"passport":{"unfragile":{"@version":"1.0","version":"2026-05","artifact":{"id":"wmdp","slug":"wmdp","name":"WMDP","type":"benchmark","url":"https://www.wmdp.ai","page_url":"https://unfragile.ai/wmdp","categories":["testing-quality","code-review-security"],"tags":[],"pricing":{"model":"free","free":true,"starting_price":null},"status":"active","verified":false},"capabilities":[{"id":"wmdp__cap_0","uri":"capability://safety.moderation.multi.domain.dangerous.knowledge.assessment.across.biosecurity.cybersecurity.and.chemical.security","name":"multi-domain dangerous knowledge assessment across biosecurity, cybersecurity, and chemical security","description":"Evaluates LLM outputs against curated question sets spanning three distinct hazard domains (biosecurity, cybersecurity, chemical security) using domain-expert-validated benchmarks. The assessment framework maps model responses to risk levels within each domain, enabling quantitative measurement of dangerous capability presence. Responses are scored against rubrics developed by security domain experts to identify whether models can produce actionable harmful information.","intents":["measure whether an LLM has learned dangerous knowledge across multiple security domains","identify which hazard categories a model is most vulnerable to","track dangerous capability regression across model versions and training approaches","benchmark unlearning method effectiveness by comparing pre/post-intervention scores"],"best_for":["AI safety researchers developing unlearning techniques","model developers evaluating safety before deployment","red-teamers assessing LLM vulnerability to misuse","policy makers needing quantitative dangerous capability metrics"],"limitations":["benchmark questions may become stale as adversarial techniques evolve; requires periodic expert review and updates","scoring relies on human expert annotation which introduces subjectivity; inter-rater reliability not fully documented","covers only three domains; emerging hazard categories (e.g., autonomous systems, synthetic biology) may not be represented","static question set may not capture all ways a model could produce dangerous outputs; adversaries may find novel phrasings"],"requires":["access to model inference API or local model weights","ability to run model inference at scale (hundreds to thousands of prompts)","Python 3.8+ for benchmark harness","computational resources for batch evaluation (GPU recommended for large models)"],"input_types":["text prompts (natural language questions designed to elicit dangerous knowledge)","model identifiers (to specify which LLM to evaluate)"],"output_types":["structured JSON with per-domain scores (0-100 scale)","per-question response transcripts with risk annotations","aggregate statistics (mean, std dev, percentile distributions across domains)"],"categories":["safety-moderation","benchmark-evaluation"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__cap_1","uri":"capability://safety.moderation.unlearning.method.evaluation.and.comparison.framework","name":"unlearning method evaluation and comparison framework","description":"Provides standardized evaluation infrastructure to measure the effectiveness of unlearning techniques (methods that remove dangerous capabilities from trained models) by comparing model performance before and after unlearning interventions. The framework isolates the impact of unlearning by holding the benchmark constant while varying the model state, enabling quantitative assessment of whether dangerous knowledge has been successfully suppressed.","intents":["measure whether an unlearning technique successfully removes dangerous capabilities","compare effectiveness across different unlearning methods (e.g., gradient ascent vs. data deletion)","detect capability degradation or side effects from unlearning (e.g., loss of benign knowledge)","establish baseline metrics for publishing unlearning research"],"best_for":["researchers developing new unlearning algorithms","safety teams validating unlearning before model release","organizations comparing commercial unlearning services","academic groups publishing safety research"],"limitations":["benchmark measures only what is explicitly tested; unlearning may fail on out-of-distribution dangerous queries not in the benchmark","does not measure capability recovery through fine-tuning or prompt engineering; adversaries may circumvent unlearning","requires access to model weights or fine-tuning APIs; cannot evaluate proprietary models without vendor cooperation","no standardized metrics for acceptable residual dangerous capability; interpretation of scores requires domain expertise"],"requires":["two versions of the same model (pre-unlearning and post-unlearning)","ability to run inference on both model versions","computational budget for full benchmark evaluation (typically 1-4 hours per model on GPU)","Python 3.8+ and PyTorch or TensorFlow for model manipulation"],"input_types":["model checkpoint (pre-unlearning state)","unlearning intervention (code, weights, or configuration)","model checkpoint (post-unlearning state)"],"output_types":["delta scores (difference in dangerous knowledge before/after unlearning)","per-domain effectiveness metrics","side-effect analysis (benign capability retention scores)","statistical significance tests comparing pre/post distributions"],"categories":["safety-moderation","benchmark-evaluation"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__cap_2","uri":"capability://safety.moderation.expert.annotated.hazard.rubric.scoring.system","name":"expert-annotated hazard rubric scoring system","description":"Implements a structured scoring framework where model responses to dangerous knowledge questions are evaluated against expert-developed rubrics that assess the degree of hazard (e.g., specificity, actionability, completeness of harmful information). Responses are scored on multi-point scales (typically 0-4 or 0-5) rather than binary pass/fail, capturing nuance in how dangerous a model's output actually is. Rubrics are domain-specific (biosecurity, cybersecurity, chemical) and developed by subject matter experts to ensure validity.","intents":["quantify the severity of dangerous knowledge in model outputs, not just presence/absence","enable fine-grained comparison of models with similar overall safety profiles","identify which types of dangerous information a model is most prone to generating","provide interpretable scores for stakeholders (e.g., 'model scores 2.3/5 on biosecurity, indicating moderate hazard')"],"best_for":["safety researchers needing nuanced capability measurement","model developers comparing safety across versions","auditors assessing model risk for deployment decisions","organizations documenting safety properties for compliance"],"limitations":["rubric development requires domain expertise; creating new rubrics for emerging hazards is time-consuming","inter-rater reliability depends on rubric clarity and annotator training; disagreement rates not always reported","rubrics may encode subjective judgments about what constitutes 'actionable' or 'specific' harmful information","scoring is not fully automated; requires human review or trained classifiers, limiting scalability"],"requires":["access to model outputs (text responses to benchmark questions)","trained annotators or automated scoring system calibrated to rubrics","rubric documentation (typically 5-20 pages per domain with examples)","inter-rater agreement validation (Cohen's kappa or similar metric)"],"input_types":["model response text (free-form LLM output)","question context (the prompt that elicited the response)","domain identifier (biosecurity, cybersecurity, or chemical)"],"output_types":["numeric score (e.g., 0-4 scale per response)","score distribution (histogram of scores across all responses in a domain)","aggregate metrics (mean, median, 95th percentile scores)","annotator confidence or uncertainty estimates"],"categories":["safety-moderation","data-processing-analysis"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__cap_3","uri":"capability://data.processing.analysis.cross.domain.dangerous.knowledge.correlation.analysis","name":"cross-domain dangerous knowledge correlation analysis","description":"Analyzes patterns in how dangerous knowledge correlates across the three benchmark domains (biosecurity, cybersecurity, chemical security), identifying whether models that excel at suppressing one type of hazard tend to suppress others. The analysis uses statistical correlation and clustering techniques to reveal whether dangerous capabilities are independent or coupled in model behavior. This enables understanding of whether unlearning interventions have domain-specific or global effects.","intents":["understand whether dangerous knowledge in different domains stems from shared model mechanisms","predict whether unlearning in one domain will affect capabilities in other domains","identify models with domain-specific vulnerabilities (e.g., strong on biosecurity but weak on cybersecurity)","design targeted unlearning interventions that address specific domain weaknesses"],"best_for":["safety researchers studying the structure of dangerous capabilities in LLMs","model developers optimizing safety across multiple hazard categories","organizations with domain-specific risk profiles (e.g., biotech companies prioritizing biosecurity)"],"limitations":["correlation analysis requires evaluation of many models; computationally expensive","correlations may be spurious if domains are confounded (e.g., both require technical knowledge)","small sample size of models in early benchmark versions limits statistical power","does not establish causality; cannot determine whether shared mechanisms cause correlation"],"requires":["scores from at least 10-20 models across all three domains","Python 3.8+ with scipy/numpy for statistical analysis","domain expertise to interpret correlations meaningfully"],"input_types":["per-model, per-domain aggregate scores (e.g., biosecurity score, cybersecurity score, chemical score for each model)"],"output_types":["correlation matrix (3x3 showing pairwise domain correlations)","clustering visualization (grouping models by domain vulnerability profiles)","statistical tests (Pearson/Spearman correlation with p-values)","domain-specific vulnerability profiles (e.g., 'high biosecurity risk, low cybersecurity risk')"],"categories":["data-processing-analysis","safety-moderation"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__cap_4","uri":"capability://data.processing.analysis.benchmark.dataset.versioning.and.curation.pipeline","name":"benchmark dataset versioning and curation pipeline","description":"Manages the creation, validation, and versioning of benchmark questions and rubrics through a structured curation pipeline involving domain experts, adversarial testing, and iterative refinement. The pipeline ensures questions are sufficiently difficult to elicit dangerous knowledge without being unrealistic, and rubrics are calibrated through inter-rater agreement studies. Version control enables tracking of benchmark evolution and ensures reproducibility across research papers.","intents":["maintain a high-quality, expert-validated benchmark that doesn't become stale","enable reproducible research by providing versioned benchmark datasets","incorporate new hazard categories and attack vectors as threats evolve","document benchmark design decisions and limitations for transparency"],"best_for":["benchmark maintainers ensuring quality and freshness","researchers needing reproducible benchmark versions for publications","organizations tracking benchmark evolution over time","safety teams validating that benchmark updates don't introduce bias"],"limitations":["curation is labor-intensive; requires ongoing expert involvement","versioning can fragment research community if different papers use different benchmark versions","adversarial testing may not catch all ways models can produce dangerous outputs","rubric updates may make historical comparisons difficult (e.g., scores from v1.0 not directly comparable to v2.0)"],"requires":["domain expert panel (typically 3-5 experts per domain)","version control system (Git or similar)","inter-rater agreement validation infrastructure","adversarial testing process (red-teaming or automated attack generation)"],"input_types":["candidate benchmark questions (text)","proposed rubric updates (rubric documents)","model responses for inter-rater validation (text)"],"output_types":["versioned benchmark dataset (JSON with questions, rubrics, metadata)","inter-rater agreement reports (Cohen's kappa, Fleiss' kappa)","changelog documenting updates between versions","benchmark documentation (design rationale, limitations, usage guide)"],"categories":["data-processing-analysis","safety-moderation"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__cap_5","uri":"capability://tool.use.integration.model.agnostic.inference.abstraction.for.diverse.llm.architectures","name":"model-agnostic inference abstraction for diverse llm architectures","description":"Provides a unified interface for evaluating diverse LLM architectures (open-source models, API-based models, fine-tuned variants) by abstracting away implementation differences. The abstraction handles API calls (OpenAI, Anthropic, etc.), local inference (Hugging Face, Ollama), and custom model serving, enabling consistent benchmark administration across heterogeneous model types. This enables fair comparison between models with different deployment modalities.","intents":["evaluate both open-source and proprietary models using the same benchmark","compare models deployed via APIs vs. locally without reimplementing benchmark logic","support emerging model architectures without modifying benchmark code","enable researchers to test their own fine-tuned models alongside baselines"],"best_for":["researchers comparing diverse model families (open vs. closed, different scales)","organizations evaluating both internal and external models","benchmark maintainers supporting multiple model deployment options"],"limitations":["abstraction adds latency (typically 50-200ms per request) due to interface overhead","API-based models may have rate limits or cost implications for large-scale evaluation","local inference requires sufficient computational resources; not all models fit on consumer hardware","proprietary models may have usage restrictions preventing certain types of analysis (e.g., response logging)"],"requires":["Python 3.8+ with model-specific SDKs (transformers, openai, anthropic, etc.)","API keys for proprietary models (OpenAI, Anthropic, etc.)","GPU or TPU for local inference (optional but recommended)","model weights or API access for each model being evaluated"],"input_types":["model identifier (e.g., 'gpt-4', 'meta-llama/Llama-2-7b', 'claude-3-opus')","prompt text (benchmark question)","model configuration (temperature, max_tokens, etc.)"],"output_types":["model response text","metadata (latency, token count, API cost if applicable)","error handling (graceful fallback for rate limits or API failures)"],"categories":["tool-use-integration","automation-workflow"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__cap_6","uri":"capability://data.processing.analysis.statistical.significance.testing.and.confidence.interval.estimation","name":"statistical significance testing and confidence interval estimation","description":"Implements rigorous statistical testing to determine whether differences in dangerous knowledge scores between models or unlearning methods are statistically significant or due to random variation. Uses techniques like bootstrap confidence intervals, permutation tests, and effect size estimation to quantify uncertainty in benchmark results. This prevents overconfident claims about safety improvements that may not be robust.","intents":["determine whether an unlearning method genuinely reduces dangerous knowledge or just got lucky","compare two models and quantify confidence in the comparison","estimate how much dangerous knowledge remains with confidence bounds","detect when benchmark sample size is insufficient for reliable conclusions"],"best_for":["researchers publishing safety claims that require statistical rigor","organizations making deployment decisions based on safety metrics","peer reviewers validating safety research","safety teams documenting confidence in safety properties"],"limitations":["statistical testing assumes benchmark questions are representative; biased question set invalidates tests","multiple comparisons (e.g., comparing many models) require correction (Bonferroni, FDR) which reduces power","small sample sizes (few models or few questions) limit statistical power; may fail to detect real differences","assumes independence of questions; if questions are correlated, effective sample size is smaller"],"requires":["Python 3.8+ with scipy.stats for statistical tests","at least 20-30 benchmark questions per domain for reliable estimation","understanding of statistical concepts (p-values, confidence intervals, effect sizes)"],"input_types":["per-question scores (numeric values for each benchmark question)","model identifiers or intervention labels (for grouping comparisons)"],"output_types":["p-values (significance tests)","confidence intervals (e.g., 95% CI for mean score)","effect sizes (Cohen's d, Hedges' g)","power analysis (sample size needed for desired statistical power)"],"categories":["data-processing-analysis","safety-moderation"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__cap_7","uri":"capability://safety.moderation.red.teaming.and.adversarial.prompt.generation.for.benchmark.validation","name":"red-teaming and adversarial prompt generation for benchmark validation","description":"Employs adversarial testing techniques to validate that benchmark questions reliably elicit dangerous knowledge and cannot be easily circumvented by prompt engineering. Red-teamers attempt to find questions that fail to elicit dangerous knowledge or rubric edge cases, and the benchmark is iteratively refined based on findings. This ensures the benchmark is robust to adversarial adaptation and captures genuine dangerous capabilities rather than surface-level patterns.","intents":["identify benchmark questions that are too easy or too hard to elicit dangerous knowledge","discover prompt engineering techniques that allow models to evade benchmark questions","validate that rubric scores are robust to paraphrasing or obfuscation","ensure benchmark remains effective as adversaries develop new attack techniques"],"best_for":["benchmark maintainers ensuring robustness to adversarial adaptation","safety researchers studying prompt engineering defenses","organizations validating that safety metrics cannot be gamed"],"limitations":["red-teaming is labor-intensive; requires skilled adversarial testers","adversarial testing may not discover all possible evasion techniques","benchmark updates based on red-teaming may introduce new biases","arms race dynamic: as benchmark improves, adversaries develop more sophisticated attacks"],"requires":["red-teaming team (typically 2-5 skilled adversarial testers)","access to models being tested","documentation of red-teaming methodology and findings","iterative refinement process (multiple rounds of testing and updates)"],"input_types":["benchmark questions (to be tested for robustness)","model responses (to identify evasion patterns)","rubric definitions (to find edge cases)"],"output_types":["red-teaming report (findings, evasion techniques discovered, recommendations)","updated benchmark questions (refined based on red-teaming findings)","rubric clarifications (addressing edge cases found during testing)","adversarial prompt examples (demonstrating evasion techniques)"],"categories":["safety-moderation","planning-reasoning"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"wmdp__headline","uri":"capability://testing.quality.benchmark.for.evaluating.dangerous.knowledge.in.llms","name":"benchmark for evaluating dangerous knowledge in llms","description":"WMDP is a benchmark designed to measure and evaluate dangerous knowledge in large language models across biosecurity, cybersecurity, and chemical security domains, helping to develop unlearning methods for hazardous capabilities.","intents":["best benchmark for LLM safety","benchmark for evaluating AI security risks","tools for measuring dangerous knowledge in AI","how to assess hazardous capabilities in LLMs","top tools for AI safety testing"],"best_for":["researchers in AI safety","developers working on LLMs"],"limitations":[],"requires":[],"input_types":[],"output_types":[],"categories":["testing-quality"],"confidence":0.5,"matches":0,"success_rate":0}],"trust":{"score":62,"verified":false,"data_access_risk":"low","permissions":["access to model inference API or local model weights","ability to run model inference at scale (hundreds to thousands of prompts)","Python 3.8+ for benchmark harness","computational resources for batch evaluation (GPU recommended for large models)","two versions of the same model (pre-unlearning and post-unlearning)","ability to run inference on both model versions","computational budget for full benchmark evaluation (typically 1-4 hours per model on GPU)","Python 3.8+ and PyTorch or TensorFlow for model manipulation","access to model outputs (text responses to benchmark questions)","trained annotators or automated scoring system calibrated to rubrics"],"failure_modes":["benchmark questions may become stale as adversarial techniques evolve; requires periodic expert review and updates","scoring relies on human expert annotation which introduces subjectivity; inter-rater reliability not fully documented","covers only three domains; emerging hazard categories (e.g., autonomous systems, synthetic biology) may not be represented","static question set may not capture all ways a model could produce dangerous outputs; adversaries may find novel phrasings","benchmark measures only what is explicitly tested; unlearning may fail on out-of-distribution dangerous queries not in the benchmark","does not measure capability recovery through fine-tuning or prompt engineering; adversaries may circumvent unlearning","requires access to model weights or fine-tuning APIs; cannot evaluate proprietary models without vendor cooperation","no standardized metrics for acceptable residual dangerous capability; interpretation of scores requires domain expertise","rubric development requires domain expertise; creating new rubrics for emerging hazards is time-consuming","inter-rater reliability depends on rubric clarity and annotator training; disagreement rates not always reported","builder identity is not verified yet","no observed match outcomes yet"],"rank_breakdown":{"adoption":0.7,"quality":0.8500000000000001,"ecosystem":0.39999999999999997,"match_graph":0.25,"freshness":0.75,"weights":{"adoption":0.25,"quality":0.35,"ecosystem":0.15,"match_graph":0.2,"freshness":0.05}},"observed_outcomes":{"matches":0,"success_rate":0,"avg_confidence":0,"top_intents":[],"last_matched_at":null},"maintenance":{"status":"active","updated_at":"2026-05-24T12:16:34.803Z","last_scraped_at":null,"last_commit":null},"community":{"stars":null,"forks":null,"weekly_downloads":null,"model_downloads":null,"model_likes":null}},"distribution":{"claim_url":"https://unfragile.ai/submit?claim=wmdp","compare_url":"https://unfragile.ai/compare?artifact=wmdp"}},"signature":"98qepPfGcnejqRUZWxEBQrfyJ4eGUSvm9huwhGIbRtPnjwp9J0Dg8rqxcTl7yEMHWwAXe8lB2yUaiowaUFFyDg==","signedAt":"2026-06-21T21:01:20.028Z","signedBy":"unfragile.ai","version":1},"_links":{"self":"https://unfragile.ai/api/v1/passport/wmdp","artifact":"https://unfragile.ai/wmdp","verify":"https://unfragile.ai/api/v1/verify?slug=wmdp","publicKey":"https://unfragile.ai/api/v1/trust-passport-public-key","spec":"https://unfragile.ai/trust","schema":"https://unfragile.ai/schema.json","docs":"https://unfragile.ai/docs"}}