{"entity":{"id":"idea-multimodal-foundation-model","kind":"idea","name":"Patient-level multimodal foundation models for treatment selection","aka":[],"tldr":"Train one AI on scans, pathology slides, genomics and treatment outcomes pooled across patients, including completed phase 3 trials, so it can predict which treatment will work for a new patient. Pathology and radiology models already exist separately; combining them with genomic and trial outcome data is the untested step.","summary":"Train one AI on scans, slides, genomics and outcomes pooled across patients, including completed phase 3 trials, so that it can predict, for a new patient, which treatment will work. Pathology and radiology foundation models exist separately; combining them with genomic and clinical data, as Tempus AI, Owkin and CanSim-style efforts are doing, is the next step, and ArteraAI showed that single-modality models can be predictive. Adding outcomes from randomised trials would allow causal treatment-effect estimation, for example TROP2 ADC versus chemotherapy. The test is to train on completed phase 3 datasets with sponsors, validate on held-out trials and then run a prospective biomarker-stratified trial. With preclinical evidence, it addresses the bottlenecks Data silos and AI that is built but not validated or deployed.","asOf":"2026-09-04","links":[{"label":"Chen et al., Towards a general-purpose foundation model for computational pathology (Nature Medicine 2024)","url":"https://doi.org/10.1038/s41591-024-02857-3"}],"tags":[],"related":[],"cancers":[],"sections":[],"technologies":["pathology-foundation-model","digital-pathology-ai","ai-trial-matching"],"targets":[],"drugs":["artera-ai-breast"],"companies":["artera","tempus","owkin"],"institutions":[],"pathways":[],"terms":[],"trials":[],"people":[],"bottlenecks":[],"keyPapers":["paper-chen-nat-med"],"journals":[],"dependsOn":[],"notes":[],"hypothesis":"A multimodal model trained on trial cohorts predicts benefit from a specific therapy (e.g., TROP2 ADC vs chemotherapy) with clinically useful discrimination beyond current biomarkers.","rationale":"ArteraAI showed single-modality models can be predictive; adding modalities and outcomes from randomised trials allows causal treatment-effect estimation.","test":"Train on completed phase 3 datasets (with sponsors), validate on held-out trials; prospective biomarker-stratified trial.","maturity":"preclinical-evidence"},"route":"/ideas/idea-multimodal-foundation-model/","neighbours":{"technology":[{"id":"ai-trial-matching","kind":"technology","name":"AI trial matching & clinical decision support","route":"/technologies/ai-trial-matching/"},{"id":"digital-pathology-ai","kind":"technology","name":"Digital pathology & AI","route":"/technologies/digital-pathology-ai/"},{"id":"pathology-foundation-model","kind":"technology","name":"Pathology & radiology foundation models","route":"/technologies/pathology-foundation-model/"}],"drug":[{"id":"artera-ai-breast","kind":"drug","name":"ArteraAI Breast","route":"/drugs/artera-ai-breast/"}],"company":[{"id":"artera","kind":"company","name":"Artera","route":"/companies/artera/"},{"id":"owkin","kind":"company","name":"Owkin","route":"/companies/owkin/"},{"id":"tempus","kind":"company","name":"Tempus AI","route":"/companies/tempus/"}],"paper":[{"id":"paper-alphafold2-jumper-nature-2021","kind":"paper","name":"AlphaFold 2: predicting protein structures to near-experimental accuracy","route":"/key-papers/paper-alphafold2-jumper-nature-2021/"},{"id":"paper-chen-nat-med","kind":"paper","name":"Towards a general-purpose foundation model for computational pathology","route":"/key-papers/paper-chen-nat-med/"}],"bottleneck":[{"id":"b-ai-validation","kind":"bottleneck","name":"AI that is built but not validated or deployed","route":"/bottlenecks/b-ai-validation/"},{"id":"b-data-silos","kind":"bottleneck","name":"Data silos","route":"/bottlenecks/b-data-silos/"},{"id":"b-immunotherapy-response","kind":"bottleneck","name":"No one can predict who responds to immunotherapy","route":"/bottlenecks/b-immunotherapy-response/"}],"idea":[{"id":"idea-fund-federated-learning-consortium","kind":"idea","name":"A federated learning consortium of cancer centres that jointly own the models","route":"/ideas/idea-fund-federated-learning-consortium/"},{"id":"idea-data-precompetitive-cancer-foundation-model","kind":"idea","name":"A pre-competitive consortium to train a shared multimodal cancer foundation model","route":"/ideas/idea-data-precompetitive-cancer-foundation-model/"},{"id":"idea-tr2-ai-external-validation-registry","kind":"idea","name":"A registry of external validation datasets for cancer AI models, with mandatory reporting","route":"/ideas/idea-tr2-ai-external-validation-registry/"},{"id":"idea-data-digital-twin-predict-then-observe","kind":"idea","name":"Digital twins for treatment selection, validated by predicting before observing","route":"/ideas/idea-data-digital-twin-predict-then-observe/"},{"id":"idea-bio2-io-biomarker-data-commons","kind":"idea","name":"Pool every immunotherapy trial's biomarker data into one commons","route":"/ideas/idea-bio2-io-biomarker-data-commons/"},{"id":"idea-moon-validated-digital-twins","kind":"idea","name":"Whole-patient digital twins validated in prospective randomised trials","route":"/ideas/idea-moon-validated-digital-twins/"}],"roadmap":[{"id":"ai-oncology-roadmap","kind":"roadmap","name":"AI in oncology roadmap: pattern readers → foundation models → agents in the workflow","route":"/roadmaps/ai-oncology-roadmap/"},{"id":"ai-oncology-clinic","kind":"roadmap","name":"AI in the oncology clinic: from narrow cleared tools to multimodal decision support","route":"/roadmaps/ai-oncology-clinic/"},{"id":"virtual-cell","kind":"roadmap","name":"Virtual cell roadmap: from bulk omics to a predictive model of a cancer cell","route":"/roadmaps/virtual-cell/"}],"term":[{"id":"cancer-ai-vocabulary","kind":"term","name":"Cancer AI vocabulary (CanSim terms map)","route":"/terms/cancer-ai-vocabulary/"},{"id":"foundation-model","kind":"term","name":"Foundation model","route":"/terms/foundation-model/"}]}}