{"entity":{"id":"idea-tr2-open-cdx-validation-sets","kind":"idea","name":"Public gold-standard datasets for validating every cancer biomarker test","aka":[],"tldr":"Anyone building a new test for HER2, PD-L1 or tumour DNA should be able to check it against the same public reference set. Today each developer validates on private data nobody can inspect.","summary":"Analytical validation of biomarker tests uses proprietary sample sets, making performance claims incomparable. A public repository of consensus-annotated cases per biomarker (whole slide images with multi-pathologist scores, sequencing data with orthogonally confirmed variants, plasma samples with defined variant allele fractions) would allow head-to-head benchmarking and independent verification of any test or algorithm, including AI-based ones.","asOf":"2026-09-08","links":[{"label":"Genome in a Bottle","url":"https://www.nist.gov/programs-projects/genome-bottle"}],"tags":[],"related":["idea-tr2-liquid-biopsy-challenge","idea-tr2-pdl1-digital-calibration"],"cancers":[],"sections":[],"technologies":["digital-pathology-ai","liquid-biopsy","ngs"],"targets":[],"drugs":[],"companies":[],"institutions":[],"pathways":[],"terms":[],"trials":[],"people":[],"bottlenecks":["b-biomarker-validation","b-ai-validation"],"keyPapers":[],"journals":[],"dependsOn":[],"notes":[],"hypothesis":"Tests benchmarked on the public sets will show performance differences hidden by developer-reported validations, and regulators will begin requiring performance on the reference sets within three years.","rationale":"ImageNet-style benchmarks drove progress and honesty in machine learning; the Genome in a Bottle reference genomes did the same for variant calling.","test":"Release reference sets for HER2, PD-L1 and ctDNA; invite all commercial and academic tests to report; publish a leaderboard.","maturity":"early-clinical","actor":"data","cost":"medium","horizonYears":2},"route":"/ideas/idea-tr2-open-cdx-validation-sets/","neighbours":{"idea":[{"id":"idea-tr2-ai-external-validation-registry","kind":"idea","name":"A registry of external validation datasets for cancer AI models, with mandatory reporting","route":"/ideas/idea-tr2-ai-external-validation-registry/"},{"id":"idea-tr2-liquid-biopsy-challenge","kind":"idea","name":"An annual blinded shoot-out for liquid biopsy tests","route":"/ideas/idea-tr2-liquid-biopsy-challenge/"},{"id":"idea-tr2-pdl1-digital-calibration","kind":"idea","name":"One digital PD-L1 scale that maps across all the competing assays","route":"/ideas/idea-tr2-pdl1-digital-calibration/"},{"id":"idea-tr2-ai-cdx-change-control","kind":"idea","name":"Version control and locked reference sets for AI algorithms used as companion diagnostics","route":"/ideas/idea-tr2-ai-cdx-change-control/"}],"technology":[{"id":"digital-pathology-ai","kind":"technology","name":"Digital pathology & AI","route":"/technologies/digital-pathology-ai/"},{"id":"liquid-biopsy","kind":"technology","name":"Liquid biopsy (ctDNA)","route":"/technologies/liquid-biopsy/"}],"term":[{"id":"ngs","kind":"term","name":"Next-generation sequencing (NGS)","route":"/terms/ngs/"}],"bottleneck":[{"id":"b-ai-validation","kind":"bottleneck","name":"AI that is built but not validated or deployed","route":"/bottlenecks/b-ai-validation/"},{"id":"b-biomarker-validation","kind":"bottleneck","name":"Biomarkers are not validated or standardised","route":"/bottlenecks/b-biomarker-validation/"}]}}