xml: <clinical_study>
    <!-- This xml conforms to an XML Schema at:
      https://clinicaltrials.gov/ct2/html/images/info/public.xsd -->
    <required_header>
      <download_date>ClinicalTrials.gov processed this data on September 20, 2023</download_date>
      <link_text>Link to the current ClinicalTrials.gov record.</link_text>
      <url>https://clinicaltrials.gov/ct2/show/NCT06018545</url>
    </required_header>
    <id_info>
      <org_study_id>310995 - A</org_study_id>
      <nct_id>NCT06018545</nct_id>
    </id_info>
    <brief_title>AI Assisted Reader Evaluation in Acute Computed Tomography (CT) Head Interpretation</brief_title>
    <acronym>AI-REACT</acronym>
    <official_title>AI Assisted Reader Evaluation in Acute CT Head Interpretation</official_title>
    <sponsors>
      <lead_sponsor>
        <agency>Oxford University Hospitals NHS Trust</agency>
        <agency_class>Other</agency_class>
      </lead_sponsor>
    </sponsors>
    <source>Oxford University Hospitals NHS Trust</source>
    <oversight_info>
      <has_dmc>No</has_dmc>
      <is_fda_regulated_drug>No</is_fda_regulated_drug>
      <is_fda_regulated_device>No</is_fda_regulated_device>
    </oversight_info>
    <brief_summary>
      <textblock>
        This study has been added as a sub study to the Simulation Training for Emergency Department&#xD;
        Imaging 2 study (ClinicalTrials.gov ID NCT05427838).&#xD;
  &#xD;
        The purpose of the study is to assess the impact of an Artificial Intelligence (AI) tool&#xD;
        called qER 2.0 EU on the performance of readers, including general radiologists, emergency&#xD;
        medicine clinicians, and radiographers, in interpreting non-contrast CT head scans. The study&#xD;
        aims to evaluate the changes in accuracy, review time, and diagnostic confidence when using&#xD;
        the AI tool. It also seeks to provide evidence on the diagnostic performance of the AI tool&#xD;
        and its potential to improve efficiency and patient care in the context of the National&#xD;
        Health Service (NHS). The study will use a dataset of 150 CT head scans, including both&#xD;
        control cases and abnormal cases with specific abnormalities. The results of this study will&#xD;
        inform larger follow-up studies in real-life Emergency Department (ED) settings.&#xD;
      </textblock>
    </brief_summary>
    <overall_status>Recruiting</overall_status>
    <start_date type="Actual">June 1, 2023</start_date>
    <completion_date type="Anticipated">September 1, 2024</completion_date>
    <primary_completion_date type="Anticipated">September 29, 2023</primary_completion_date>
    <study_type>Observational</study_type>
    <has_expanded_access>No</has_expanded_access>
    <study_design_info>
      <observational_model>Cohort</observational_model>
      <time_perspective>Retrospective</time_perspective>
    </study_design_info>
    <primary_outcome>
      <measure>Reader performance: Sensitivity, specificity, comparative between with and without AI assistance.</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>Reader performance will be evaluated as sensitivity, specificity, with and without AI assistance.</description>
    </primary_outcome>
    <primary_outcome>
      <measure>Reader performance: Positive and negative predictive value, comparative between with and without AI assistance.</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>Reader performance will be evaluated as Positive Predictive Value (PPV) and negative predictive value (NPV), with and without AI assistance.</description>
    </primary_outcome>
    <primary_outcome>
      <measure>Reader performance: Area Under Receiver Operating Characteristic Curve (AUROC), comparative between with and without AI assistance.</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>Reader performance will be evaluated as Area Under Receiver Operating Characteristic Curve (AUROC), with and without AI assistance.</description>
    </primary_outcome>
    <primary_outcome>
      <measure>Reader speed: Mean time taken to review a scan, with versus without AI assistance.</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>Reader speed will be evaluated as the man time taken to review a scan, using time unite of seconds.</description>
    </primary_outcome>
    <primary_outcome>
      <measure>Reader confidence: Self-reported diagnostic confidence on a 10 point visual analogue scale, with vs without AI assistance.</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>On the reading platform (RAIQC), one of the questions asks the level of confidence that the participant has in their diagnostic opinion. The question offers a scale of 1 to 10, where 1 is not confident, and 10 is highly confident.</description>
    </primary_outcome>
    <primary_outcome>
      <measure>qER (AI algorithm) performance: Sensitivity and specificity</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>qER performance will be evaluated as sensitivity, specificity.</description>
    </primary_outcome>
    <primary_outcome>
      <measure>qER (AI algorithm) performance: Positive and negative predictive value.</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>qER performance will be evaluated as Positive Predictive Value (PPV) and negative predictive value (NPV).</description>
    </primary_outcome>
    <primary_outcome>
      <measure>qER (AI algorithm) performance: Area Under Receiver Operating Characteristic Curve (AUROC).</measure>
      <time_frame>During 6 weeks, which is the period for reading or reviewing the cases/scans.</time_frame>
      <description>qER performance will be evaluated as Area Under Receiver Operating Characteristic Curve (AUROC)</description>
    </primary_outcome>
    <number_of_groups>2</number_of_groups>
    <enrollment type="Anticipated">30</enrollment>
    <condition>Intracranial Hemorrhages</condition>
    <condition>Acute Ischemic Stroke</condition>
    <condition>Hydrocephalus</condition>
    <condition>Cerebral Infarction</condition>
    <condition>Cerebral Edema</condition>
    <condition>Cerebral Injury</condition>
    <arm_group>
      <arm_group_label>Readers</arm_group_label>
      <description>30 readers will be recruited across four NHS trusts including ten general radiologists, fifteen emergency medicine clinicians, and five CT radiographers of varying seniority. Readers will interpret each scan first without, then with, the assistance of the AI tool, with an intervening 4-week washout period. Using a panel of neuroradiologists as ground truth, the stand-alone performance of qER will be assessed, and its impact on the readers' performance will be analysed as change in accuracy, mean review time per scan, and self-reported diagnostic confidence. Subgroup analyses will be performed by reader professional group, reader seniority, pathological finding, and neuroradiologist-rated difficulty.</description>
    </arm_group>
    <arm_group>
      <arm_group_label>Ground truthers</arm_group_label>
      <description>Two Consultant neuroradiologists will independently review the images to establish the 'ground truth' findings on the CT scans which will be used as the reference standard. In the case of disagreement, a third senior neuroradiologist's opinion will be sought for arbitration. A difficulty score will be assigned to each scan by the ground truthers using a 5-point Likert scale.</description>
    </arm_group>
    <intervention>
      <intervention_type>Other</intervention_type>
      <intervention_name>Ground truthing</intervention_name>
      <description>Two Consultant neuroradiologists will independently review the images to establish the 'ground truth' findings on the CT scans which will be used as the reference standard. In the case of disagreement, a third senior neuroradiologist's opinion will be sought for arbitration.</description>
      <arm_group_label>Ground truthers</arm_group_label>
    </intervention>
    <intervention>
      <intervention_type>Other</intervention_type>
      <intervention_name>Reading</intervention_name>
      <description>All 30 readers will review all 150 cases, in each of two study phases. The readers will provide their opinion on the presence or absence of some acute abnormalities, including intracranial haemorrhage, infarct, midline shift and fracture. They will provide a confidence in their diagnosis (10-point visual analogue scale), and a single click point to mark the location of each abnormality that they consider as being present. The time taken for each scan will be automatically recorded.</description>
      <arm_group_label>Readers</arm_group_label>
    </intervention>
    <eligibility>
      <study_pop>
        <textblock>
          Setting:&#xD;
  &#xD;
          Readers will be recruited from the following four hospital Trusts (secondary and tertiary&#xD;
          level:&#xD;
  &#xD;
            -  Guy's &amp; St Thomas NHS Foundation Trust&#xD;
  &#xD;
            -  Northumbria Healthcare NHS Foundation Trust&#xD;
  &#xD;
            -  NHS Greater Glasgow and Clyde&#xD;
  &#xD;
            -  Oxford University Hospitals NHS Foundation Trust&#xD;
  &#xD;
          Participants:&#xD;
  &#xD;
          30 volunteer participant readers will be selected from the following groups:&#xD;
  &#xD;
            -  Emergency Medicine Consultants and Registrars (5 Consultant, 5 Registrar (ST3-6), 5&#xD;
               junior (F1-ST2)&#xD;
  &#xD;
            -  General Radiologist Consultants and Registrars (5 Consultant, 5 Registrar)&#xD;
  &#xD;
            -  5 CT Radiographers&#xD;
        </textblock>
      </study_pop>
      <sampling_method>Non-Probability Sample</sampling_method>
      <criteria>
        <textblock>
          Inclusion Criteria:&#xD;
  &#xD;
            -  Radiologists/Radiographers/ED clinicians who review CT head scans as part of their&#xD;
               clinical practice&#xD;
  &#xD;
          Exclusion Criteria:&#xD;
  &#xD;
            -  Neuroradiologists.&#xD;
  &#xD;
            -  Non-radiologist groups: Clinicians with previous formal postgraduate CT reporting&#xD;
               training&#xD;
  &#xD;
            -  Emergency Medicine group: Clinicians with previous career in radiology/neurosurgery to&#xD;
               registrar level&#xD;
        </textblock>
      </criteria>
      <gender>All</gender>
      <minimum_age>N/A</minimum_age>
      <maximum_age>N/A</maximum_age>
      <healthy_volunteers>Accepts Healthy Volunteers</healthy_volunteers>
    </eligibility>
    <overall_official>
      <last_name>Alex Novak, MSc</last_name>
      <role>Principal Investigator</role>
      <affiliation>National Health Services in the United Kingdom (NHS UK)</affiliation>
    </overall_official>
    <overall_official>
      <last_name>Sarim Ather, PhD</last_name>
      <role>Principal Investigator</role>
      <affiliation>National Health Services in the United Kingdom (NHS UK)</affiliation>
    </overall_official>
    <overall_contact>
      <last_name>Alex Novak, MSc</last_name>
      <phone>+44 7944653970</phone>
      <email>alex.novak@ouh.nhs.uk</email>
    </overall_contact>
    <overall_contact_backup>
      <last_name>Sarim Ather, PhD</last_name>
      <phone>+44 7877724961</phone>
      <email>sarim.ather@ouh.nhs.uk</email>
    </overall_contact_backup>
    <location>
      <facility>
        <name>Oxford University Hospitals NHS Foundation Trust</name>
        <address>
          <city>Oxford</city>
          <state>Oxfordshire</state>
          <zip>OX3 9DU</zip>
          <country>United Kingdom</country>
        </address>
      </facility>
      <status>Completed</status>
    </location>
    <location>
      <facility>
        <name>NHS Greater Glasgow and Clyde</name>
        <address>
          <city>Glasgow</city>
          <zip>G12 0XH</zip>
          <country>United Kingdom</country>
        </address>
      </facility>
      <status>Completed</status>
    </location>
    <location>
      <facility>
        <name>Guy's &amp; St Thomas NHS Foundation Trust</name>
        <address>
          <city>London</city>
          <zip>SE1 7EH</zip>
          <country>United Kingdom</country>
        </address>
      </facility>
      <status>Completed</status>
    </location>
    <location>
      <facility>
        <name>Northumbria Healthcare NHS Foundation Trust</name>
        <address>
          <city>Newcastle Upon Tyne</city>
          <zip>NE27 0QJ</zip>
          <country>United Kingdom</country>
        </address>
      </facility>
      <status>Recruiting</status>
      <contact>
        <last_name>Rahul Dharmadhikari</last_name>
        <email>rahul.dharmadhikari@northumbria-healthcare.nhs.uk</email>
      </contact>
    </location>
    <location_countries>
      <country>United Kingdom</country>
    </location_countries>
    <link>
      <url>https://www.england.nhs.uk/publication/diagnostics-recovery-and-renewal-report-of-the-independent-review-of-diagnostic-services-for-nhs-england/</url>
      <description>Richards M. Diagnostics: Recovery and Renewal - Report of the Independent Review of Diagnostic Services for NHS England. NHS England 2022.</description>
    </link>
    <link>
      <url>https://www.rcr.ac.uk/publication/clinical-radiology-uk-workforce-census-2019-report</url>
      <description>Royal College of Radiologists. Clinical radiology UK workforce census 2019 report. Royal College of Radiologists 2020.</description>
    </link>
    <link>
      <url>https://www.nice.org.uk/advice/mib207</url>
      <description>National Institute for Health and Care Excellence. Artificial intelligence for analysing CT brain scans. 2020.</description>
    </link>
    <reference>
      <citation>Juszczyk K, Ireland K, Thomas B, Kroon HM, Hollington P. Reduction in hospital admissions with an early computed tomography scan: results of an outpatient management protocol for uncomplicated acute diverticulitis. ANZ J Surg. 2019 Sep;89(9):1085-1090. doi: 10.1111/ans.15285. Epub 2019 Jun 17.</citation>
      <PMID>31206250</PMID>
    </reference>
    <reference>
      <citation>Chan J, Fan KS, Mak TLA, Loh SY, Ng SWY, Adapala R. Pre-Operative Imaging can Reduce Negative Appendectomy Rate in Acute Appendicitis. Ulster Med J. 2020 Jan;89(1):25-28. Epub 2020 Feb 18.</citation>
      <PMID>32218624</PMID>
    </reference>
    <reference>
      <citation>Greenhalgh R, Howlett DC, Drinkwater KJ. Royal College of Radiologists national audit evaluating the provision of imaging in the severely injured patient and compliance with national guidelines. Clin Radiol. 2020 Mar;75(3):224-231. doi: 10.1016/j.crad.2019.10.025. Epub 2019 Dec 19.</citation>
      <PMID>31864722</PMID>
    </reference>
    <reference>
      <citation>Lin E, Yuh EL. Computational Approaches for Acute Traumatic Brain Injury Image Recognition. Front Neurol. 2022 Mar 9;13:791816. doi: 10.3389/fneur.2022.791816. eCollection 2022.</citation>
      <PMID>35370919</PMID>
    </reference>
    <reference>
      <citation>Sheth SA, Giancardo L, Colasurdo M, Srinivasan VM, Niktabe A, Kan P. Machine learning and acute stroke imaging. J Neurointerv Surg. 2023 Feb;15(2):195-199. doi: 10.1136/neurintsurg-2021-018142. Epub 2022 May 25.</citation>
      <PMID>35613840</PMID>
    </reference>
    <reference>
      <citation>Yeo M, Tahayori B, Kok HK, Maingard J, Kutaiba N, Russell J, Thijs V, Jhamb A, Chandra RV, Brooks M, Barras CD, Asadi H. Review of deep learning algorithms for the automatic detection of intracranial hemorrhages on computed tomography head imaging. J Neurointerv Surg. 2021 Apr;13(4):369-378. doi: 10.1136/neurintsurg-2020-017099. Epub 2021 Jan 21.</citation>
      <PMID>33479036</PMID>
    </reference>
    <reference>
      <citation>Chilamkurthy S, Ghosh R, Tanamala S, Biviji M, Campeau NG, Venugopal VK, Mahajan V, Rao P, Warier P. Deep learning algorithms for detection of critical findings in head CT scans: a retrospective study. Lancet. 2018 Dec 1;392(10162):2388-2396. doi: 10.1016/S0140-6736(18)31645-3. Epub 2018 Oct 11.</citation>
      <PMID>30318264</PMID>
    </reference>
    <reference>
      <citation>Guo Y, He Y, Lyu J, Zhou Z, Yang D, Ma L, Tan HT, Chen C, Zhang W, Hu J, Han D, Ding G, Liu S, Qiao H, Xu F, Lou X, Dai Q. Deep learning with weak annotation from diagnosis reports for detection of multiple head disorders: a prospective, multicentre study. Lancet Digit Health. 2022 Aug;4(8):e584-e593. doi: 10.1016/S2589-7500(22)00090-5. Epub 2022 Jun 17. Erratum In: Lancet Digit Health. 2022 Aug;4(8):e572.</citation>
      <PMID>35725824</PMID>
    </reference>
    <reference>
      <citation>Lee JY, Kim JS, Kim TY, Kim YS. Detection and classification of intracranial haemorrhage on CT images using a novel deep-learning algorithm. Sci Rep. 2020 Nov 25;10(1):20546. doi: 10.1038/s41598-020-77441-z.</citation>
      <PMID>33239711</PMID>
    </reference>
    <reference>
      <citation>Arbabshirani MR, Fornwalt BK, Mongelluzzo GJ, Suever JD, Geise BD, Patel AA, Moore GJ. Advanced machine learning in action: identification of intracranial hemorrhage on computed tomography scans of the head with clinical workflow integration. NPJ Digit Med. 2018 Apr 4;1:9. doi: 10.1038/s41746-017-0015-z. eCollection 2018.</citation>
      <PMID>31304294</PMID>
    </reference>
    <reference>
      <citation>Davis MA, Rao B, Cedeno PA, Saha A, Zohrabian VM. Machine Learning and Improved Quality Metrics in Acute Intracranial Hemorrhage by Noncontrast Computed Tomography. Curr Probl Diagn Radiol. 2022 Jul-Aug;51(4):556-561. doi: 10.1067/j.cpradiol.2020.10.007. Epub 2020 Nov 15.</citation>
      <PMID>33243455</PMID>
    </reference>
    <reference>
      <citation>Wardlaw JM, Mair G, von Kummer R, Williams MC, Li W, Storkey AJ, Trucco E, Liebeskind DS, Farrall A, Bath PM, White P. Accuracy of Automated Computer-Aided Diagnosis for Stroke Imaging: A Critical Evaluation of Current Evidence. Stroke. 2022 Jul;53(7):2393-2403. doi: 10.1161/STROKEAHA.121.036204. Epub 2022 Apr 20.</citation>
      <PMID>35440170</PMID>
    </reference>
    <reference>
      <citation>Finck T, Moosbauer J, Probst M, Schlaeger S, Schuberth M, Schinz D, Yigitsoy M, Byas S, Zimmer C, Pfister F, Wiestler B. Faster and Better: How Anomaly Detection Can Accelerate and Improve Reporting of Head Computed Tomography. Diagnostics (Basel). 2022 Feb 10;12(2):452. doi: 10.3390/diagnostics12020452.</citation>
      <PMID>35204543</PMID>
    </reference>
    <reference>
      <citation>Warman R, Warman A, Warman P, Degnan A, Blickman J, Chowdhary V, Dash D, Sangal R, Vadhan J, Bueso T, Windisch T, Neves G. Deep Learning System Boosts Radiologist Detection of Intracranial Hemorrhage. Cureus. 2022 Oct 13;14(10):e30264. doi: 10.7759/cureus.30264. eCollection 2022 Oct.</citation>
      <PMID>36381767</PMID>
    </reference>
    <reference>
      <citation>Dyer T, Chawda S, Alkilani R, Morgan TN, Hughes M, Rasalingham S. Validation of an artificial intelligence solution for acute triage and rule-out normal of non-contrast CT head scans. Neuroradiology. 2022 Apr;64(4):735-743. doi: 10.1007/s00234-021-02826-4. Epub 2021 Oct 8.</citation>
      <PMID>34623478</PMID>
    </reference>
    <reference>
      <citation>Mallon DH, Taylor EJR, Vittay OI, Sheeka A, Doig D, Lobotesis K. Comparison of automated ASPECTS, large vessel occlusion detection and CTP analysis provided by Brainomix and RapidAI in patients with suspected ischaemic stroke. J Stroke Cerebrovasc Dis. 2022 Oct;31(10):106702. doi: 10.1016/j.jstrokecerebrovasdis.2022.106702. Epub 2022 Aug 19.</citation>
      <PMID>35994882</PMID>
    </reference>
    <reference>
      <citation>Andralojc LE, Kim DH, Edwards AJ. Diagnostic accuracy of a decision-support software for the detection of intracranial large-vessel occlusion in CT angiography. Clin Radiol. 2023 Apr;78(4):e313-e318. doi: 10.1016/j.crad.2022.10.017. Epub 2023 Jan 11.</citation>
      <PMID>36754714</PMID>
    </reference>
    <reference>
      <citation>Zech JR, Badgeley MA, Liu M, Costa AB, Titano JJ, Oermann EK. Variable generalization performance of a deep learning model to detect pneumonia in chest radiographs: A cross-sectional study. PLoS Med. 2018 Nov 6;15(11):e1002683. doi: 10.1371/journal.pmed.1002683. eCollection 2018 Nov.</citation>
      <PMID>30399157</PMID>
    </reference>
    <reference>
      <citation>Huang SC, Pareek A, Jensen M, Lungren MP, Yeung S, Chaudhari AS. Self-supervised learning for medical image classification: a systematic review and implementation guidelines. NPJ Digit Med. 2023 Apr 26;6(1):74. doi: 10.1038/s41746-023-00811-0.</citation>
      <PMID>37100953</PMID>
    </reference>
    <reference>
      <citation>Hillis SL, Obuchowski NA, Schartz KM, Berbaum KS. A comparison of the Dorfman-Berbaum-Metz and Obuchowski-Rockette methods for receiver operating characteristic (ROC) data. Stat Med. 2005 May 30;24(10):1579-607. doi: 10.1002/sim.2024.</citation>
      <PMID>15685718</PMID>
    </reference>
    <reference>
      <citation>Obuchowski NA. Sample size tables for receiver operating characteristic studies. AJR Am J Roentgenol. 2000 Sep;175(3):603-8. doi: 10.2214/ajr.175.3.1750603.</citation>
      <PMID>10954438</PMID>
    </reference>
    <verification_date>August 2023</verification_date>
    <study_first_submitted>July 25, 2023</study_first_submitted>
    <study_first_submitted_qc>August 25, 2023</study_first_submitted_qc>
    <study_first_posted type="Actual">August 30, 2023</study_first_posted>
    <last_update_submitted>August 25, 2023</last_update_submitted>
    <last_update_submitted_qc>August 25, 2023</last_update_submitted_qc>
    <last_update_posted type="Actual">August 30, 2023</last_update_posted>
    <responsible_party>
      <responsible_party_type>Principal Investigator</responsible_party_type>
      <investigator_affiliation>Oxford University Hospitals NHS Trust</investigator_affiliation>
      <investigator_full_name>Alex Novak</investigator_full_name>
      <investigator_title>Primary Investigator</investigator_title>
    </responsible_party>
    <keyword>Radiology</keyword>
    <keyword>Head tomography</keyword>
    <keyword>Emergency medicine</keyword>
    <keyword>Radiographer</keyword>
    <keyword>Artificial intelligence</keyword>
    <condition_browse>
      <!-- CAUTION:  The following MeSH terms are assigned with an imperfect algorithm            -->
      <mesh_term>Ischemic Stroke</mesh_term>
      <mesh_term>Hydrocephalus</mesh_term>
      <mesh_term>Cerebral Infarction</mesh_term>
      <mesh_term>Intracranial Hemorrhages</mesh_term>
      <mesh_term>Brain Edema</mesh_term>
      <mesh_term>Brain Injuries</mesh_term>
      <mesh_term>Infarction</mesh_term>
      <mesh_term>Hemorrhage</mesh_term>
    </condition_browse>
    <!-- Results have not yet been posted for this study                                          -->
  </clinical_study>
  
  
text: download date: ClinicalTrials.gov processed this data on September 20, 2023
  link text: Link to the current ClinicalTrials.gov record.
  url: https://clinicaltrials.gov/ct2/show/NCT06018545
  org study id: 310995 - A
  nct id: NCT06018545
  lead sponsor: 
  has dmc: No
  is fda regulated drug: No
  is fda regulated device: No
  This study has been added as a sub study to the Simulation Training for Emergency Department
        Imaging 2 study (ClinicalTrials.gov ID NCT05427838).
  
        The purpose of the study is to assess the impact of an Artificial Intelligence (AI) tool
        called qER 2.0 EU on the performance of readers, including general radiologists, emergency
        medicine clinicians, and radiographers, in interpreting non-contrast CT head scans. The study
        aims to evaluate the changes in accuracy, review time, and diagnostic confidence when using
        the AI tool. It also seeks to provide evidence on the diagnostic performance of the AI tool
        and its potential to improve efficiency and patient care in the context of the National
        Health Service (NHS). The study will use a dataset of 150 CT head scans, including both
        control cases and abnormal cases with specific abnormalities. The results of this study will
        inform larger follow-up studies in real-life Emergency Department (ED) settings.
  observational model: Cohort
  time perspective: Retrospective
  measure: Reader performance: Sensitivity, specificity, comparative between with and without AI assistance.
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: Reader performance will be evaluated as sensitivity, specificity, with and without AI assistance.
  measure: Reader performance: Positive and negative predictive value, comparative between with and without AI assistance.
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: Reader performance will be evaluated as Positive Predictive Value (PPV) and negative predictive value (NPV), with and without AI assistance.
  measure: Reader performance: Area Under Receiver Operating Characteristic Curve (AUROC), comparative between with and without AI assistance.
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: Reader performance will be evaluated as Area Under Receiver Operating Characteristic Curve (AUROC), with and without AI assistance.
  measure: Reader speed: Mean time taken to review a scan, with versus without AI assistance.
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: Reader speed will be evaluated as the man time taken to review a scan, using time unite of seconds.
  measure: Reader confidence: Self-reported diagnostic confidence on a 10 point visual analogue scale, with vs without AI assistance.
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: On the reading platform (RAIQC), one of the questions asks the level of confidence that the participant has in their diagnostic opinion. The question offers a scale of 1 to 10, where 1 is not confident, and 10 is highly confident.
  measure: qER (AI algorithm) performance: Sensitivity and specificity
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: qER performance will be evaluated as sensitivity, specificity.
  measure: qER (AI algorithm) performance: Positive and negative predictive value.
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: qER performance will be evaluated as Positive Predictive Value (PPV) and negative predictive value (NPV).
  measure: qER (AI algorithm) performance: Area Under Receiver Operating Characteristic Curve (AUROC).
  time frame: During 6 weeks, which is the period for reading or reviewing the cases/scans.
  description: qER performance will be evaluated as Area Under Receiver Operating Characteristic Curve (AUROC)
  arm group label: Readers
  description: 30 readers will be recruited across four NHS trusts including ten general radiologists, fifteen emergency medicine clinicians, and five CT radiographers of varying seniority. Readers will interpret each scan first without, then with, the assistance of the AI tool, with an intervening 4-week washout period. Using a panel of neuroradiologists as ground truth, the stand-alone performance of qER will be assessed, and its impact on the readers' performance will be analysed as change in accuracy, mean review time per scan, and self-reported diagnostic confidence. Subgroup analyses will be performed by reader professional group, reader seniority, pathological finding, and neuroradiologist-rated difficulty.
  arm group label: Ground truthers
  description: Two Consultant neuroradiologists will independently review the images to establish the 'ground truth' findings on the CT scans which will be used as the reference standard. In the case of disagreement, a third senior neuroradiologist's opinion will be sought for arbitration. A difficulty score will be assigned to each scan by the ground truthers using a 5-point Likert scale.
  intervention type: Other
  intervention name: Ground truthing
  description: Two Consultant neuroradiologists will independently review the images to establish the 'ground truth' findings on the CT scans which will be used as the reference standard. In the case of disagreement, a third senior neuroradiologist's opinion will be sought for arbitration.
  arm group label: Ground truthers
  intervention type: Other
  intervention name: Reading
  description: All 30 readers will review all 150 cases, in each of two study phases. The readers will provide their opinion on the presence or absence of some acute abnormalities, including intracranial haemorrhage, infarct, midline shift and fracture. They will provide a confidence in their diagnosis (10-point visual analogue scale), and a single click point to mark the location of each abnormality that they consider as being present. The time taken for each scan will be automatically recorded.
  arm group label: Readers
  study pop: 
  sampling method: Non-Probability Sample
  criteria: 
  gender: All
  minimum age: N/A
  maximum age: N/A
  healthy volunteers: Accepts Healthy Volunteers
  last name: Alex Novak, MSc
  role: Principal Investigator
  affiliation: National Health Services in the United Kingdom (NHS UK)
  last name: Sarim Ather, PhD
  role: Principal Investigator
  affiliation: National Health Services in the United Kingdom (NHS UK)
  last name: Alex Novak, MSc
  phone: +44 7944653970
  email: alex.novak@ouh.nhs.uk
  last name: Sarim Ather, PhD
  phone: +44 7877724961
  email: sarim.ather@ouh.nhs.uk
  facility: 
  status: Completed
  facility: 
  status: Completed
  facility: 
  status: Completed
  facility: 
  status: Recruiting
  contact: 
  country: United Kingdom
  url: https://www.england.nhs.uk/publication/diagnostics-recovery-and-renewal-report-of-the-independent-review-of-diagnostic-services-for-nhs-england/
  description: Richards M. Diagnostics: Recovery and Renewal - Report of the Independent Review of Diagnostic Services for NHS England. NHS England 2022.
  url: https://www.rcr.ac.uk/publication/clinical-radiology-uk-workforce-census-2019-report
  description: Royal College of Radiologists. Clinical radiology UK workforce census 2019 report. Royal College of Radiologists 2020.
  url: https://www.nice.org.uk/advice/mib207
  description: National Institute for Health and Care Excellence. Artificial intelligence for analysing CT brain scans. 2020.
  citation: Juszczyk K, Ireland K, Thomas B, Kroon HM, Hollington P. Reduction in hospital admissions with an early computed tomography scan: results of an outpatient management protocol for uncomplicated acute diverticulitis. ANZ J Surg. 2019 Sep;89(9):1085-1090. doi: 10.1111/ans.15285. Epub 2019 Jun 17.
  PMID: 31206250
  citation: Chan J, Fan KS, Mak TLA, Loh SY, Ng SWY, Adapala R. Pre-Operative Imaging can Reduce Negative Appendectomy Rate in Acute Appendicitis. Ulster Med J. 2020 Jan;89(1):25-28. Epub 2020 Feb 18.
  PMID: 32218624
  citation: Greenhalgh R, Howlett DC, Drinkwater KJ. Royal College of Radiologists national audit evaluating the provision of imaging in the severely injured patient and compliance with national guidelines. Clin Radiol. 2020 Mar;75(3):224-231. doi: 10.1016/j.crad.2019.10.025. Epub 2019 Dec 19.
  PMID: 31864722
  citation: Lin E, Yuh EL. Computational Approaches for Acute Traumatic Brain Injury Image Recognition. Front Neurol. 2022 Mar 9;13:791816. doi: 10.3389/fneur.2022.791816. eCollection 2022.
  PMID: 35370919
  citation: Sheth SA, Giancardo L, Colasurdo M, Srinivasan VM, Niktabe A, Kan P. Machine learning and acute stroke imaging. J Neurointerv Surg. 2023 Feb;15(2):195-199. doi: 10.1136/neurintsurg-2021-018142. Epub 2022 May 25.
  PMID: 35613840
  citation: Yeo M, Tahayori B, Kok HK, Maingard J, Kutaiba N, Russell J, Thijs V, Jhamb A, Chandra RV, Brooks M, Barras CD, Asadi H. Review of deep learning algorithms for the automatic detection of intracranial hemorrhages on computed tomography head imaging. J Neurointerv Surg. 2021 Apr;13(4):369-378. doi: 10.1136/neurintsurg-2020-017099. Epub 2021 Jan 21.
  PMID: 33479036
  citation: Chilamkurthy S, Ghosh R, Tanamala S, Biviji M, Campeau NG, Venugopal VK, Mahajan V, Rao P, Warier P. Deep learning algorithms for detection of critical findings in head CT scans: a retrospective study. Lancet. 2018 Dec 1;392(10162):2388-2396. doi: 10.1016/S0140-6736(18)31645-3. Epub 2018 Oct 11.
  PMID: 30318264
  citation: Guo Y, He Y, Lyu J, Zhou Z, Yang D, Ma L, Tan HT, Chen C, Zhang W, Hu J, Han D, Ding G, Liu S, Qiao H, Xu F, Lou X, Dai Q. Deep learning with weak annotation from diagnosis reports for detection of multiple head disorders: a prospective, multicentre study. Lancet Digit Health. 2022 Aug;4(8):e584-e593. doi: 10.1016/S2589-7500(22)00090-5. Epub 2022 Jun 17. Erratum In: Lancet Digit Health. 2022 Aug;4(8):e572.
  PMID: 35725824
  citation: Lee JY, Kim JS, Kim TY, Kim YS. Detection and classification of intracranial haemorrhage on CT images using a novel deep-learning algorithm. Sci Rep. 2020 Nov 25;10(1):20546. doi: 10.1038/s41598-020-77441-z.
  PMID: 33239711
  citation: Arbabshirani MR, Fornwalt BK, Mongelluzzo GJ, Suever JD, Geise BD, Patel AA, Moore GJ. Advanced machine learning in action: identification of intracranial hemorrhage on computed tomography scans of the head with clinical workflow integration. NPJ Digit Med. 2018 Apr 4;1:9. doi: 10.1038/s41746-017-0015-z. eCollection 2018.
  PMID: 31304294
  citation: Davis MA, Rao B, Cedeno PA, Saha A, Zohrabian VM. Machine Learning and Improved Quality Metrics in Acute Intracranial Hemorrhage by Noncontrast Computed Tomography. Curr Probl Diagn Radiol. 2022 Jul-Aug;51(4):556-561. doi: 10.1067/j.cpradiol.2020.10.007. Epub 2020 Nov 15.
  PMID: 33243455
  citation: Wardlaw JM, Mair G, von Kummer R, Williams MC, Li W, Storkey AJ, Trucco E, Liebeskind DS, Farrall A, Bath PM, White P. Accuracy of Automated Computer-Aided Diagnosis for Stroke Imaging: A Critical Evaluation of Current Evidence. Stroke. 2022 Jul;53(7):2393-2403. doi: 10.1161/STROKEAHA.121.036204. Epub 2022 Apr 20.
  PMID: 35440170
  citation: Finck T, Moosbauer J, Probst M, Schlaeger S, Schuberth M, Schinz D, Yigitsoy M, Byas S, Zimmer C, Pfister F, Wiestler B. Faster and Better: How Anomaly Detection Can Accelerate and Improve Reporting of Head Computed Tomography. Diagnostics (Basel). 2022 Feb 10;12(2):452. doi: 10.3390/diagnostics12020452.
  PMID: 35204543
  citation: Warman R, Warman A, Warman P, Degnan A, Blickman J, Chowdhary V, Dash D, Sangal R, Vadhan J, Bueso T, Windisch T, Neves G. Deep Learning System Boosts Radiologist Detection of Intracranial Hemorrhage. Cureus. 2022 Oct 13;14(10):e30264. doi: 10.7759/cureus.30264. eCollection 2022 Oct.
  PMID: 36381767
  citation: Dyer T, Chawda S, Alkilani R, Morgan TN, Hughes M, Rasalingham S. Validation of an artificial intelligence solution for acute triage and rule-out normal of non-contrast CT head scans. Neuroradiology. 2022 Apr;64(4):735-743. doi: 10.1007/s00234-021-02826-4. Epub 2021 Oct 8.
  PMID: 34623478
  citation: Mallon DH, Taylor EJR, Vittay OI, Sheeka A, Doig D, Lobotesis K. Comparison of automated ASPECTS, large vessel occlusion detection and CTP analysis provided by Brainomix and RapidAI in patients with suspected ischaemic stroke. J Stroke Cerebrovasc Dis. 2022 Oct;31(10):106702. doi: 10.1016/j.jstrokecerebrovasdis.2022.106702. Epub 2022 Aug 19.
  PMID: 35994882
  citation: Andralojc LE, Kim DH, Edwards AJ. Diagnostic accuracy of a decision-support software for the detection of intracranial large-vessel occlusion in CT angiography. Clin Radiol. 2023 Apr;78(4):e313-e318. doi: 10.1016/j.crad.2022.10.017. Epub 2023 Jan 11.
  PMID: 36754714
  citation: Zech JR, Badgeley MA, Liu M, Costa AB, Titano JJ, Oermann EK. Variable generalization performance of a deep learning model to detect pneumonia in chest radiographs: A cross-sectional study. PLoS Med. 2018 Nov 6;15(11):e1002683. doi: 10.1371/journal.pmed.1002683. eCollection 2018 Nov.
  PMID: 30399157
  citation: Huang SC, Pareek A, Jensen M, Lungren MP, Yeung S, Chaudhari AS. Self-supervised learning for medical image classification: a systematic review and implementation guidelines. NPJ Digit Med. 2023 Apr 26;6(1):74. doi: 10.1038/s41746-023-00811-0.
  PMID: 37100953
  citation: Hillis SL, Obuchowski NA, Schartz KM, Berbaum KS. A comparison of the Dorfman-Berbaum-Metz and Obuchowski-Rockette methods for receiver operating characteristic (ROC) data. Stat Med. 2005 May 30;24(10):1579-607. doi: 10.1002/sim.2024.
  PMID: 15685718
  citation: Obuchowski NA. Sample size tables for receiver operating characteristic studies. AJR Am J Roentgenol. 2000 Sep;175(3):603-8. doi: 10.2214/ajr.175.3.1750603.
  PMID: 10954438
  responsible party type: Principal Investigator
  investigator affiliation: Oxford University Hospitals NHS Trust
  investigator full name: Alex Novak
  investigator title: Primary Investigator
  mesh term: Ischemic Stroke
  mesh term: Hydrocephalus
  mesh term: Cerebral Infarction
  mesh term: Intracranial Hemorrhages
  mesh term: Brain Edema
  mesh term: Brain Injuries
  mesh term: Infarction
  mesh term: Hemorrhage