@Article{info:doi/10.2196/48002,
author="Takagi, Soshi
and Watari, Takashi
and Erabi, Ayano
and Sakaguchi, Kota",
title="Performance of GPT-3.5 and GPT-4 on the Japanese Medical Licensing Examination: Comparison Study",
journal="JMIR Med Educ",
year="2023",
month="Jun",
day="29",
volume="9",
pages="e48002",
keywords="ChatGPT; Chat Generative Pre-trained Transformer; GPT-4; Generative Pre-trained Transformer 4; artificial intelligence; AI; medical education; Japanese Medical Licensing Examination; medical licensing; clinical support; learning model",
abstract="Background: The competence of ChatGPT (Chat Generative Pre-Trained Transformer) in non-English languages is not well studied. Objective: This study compared the performances of GPT-3.5 (Generative Pre-trained Transformer) and GPT-4 on the Japanese Medical Licensing Examination (JMLE) to evaluate the reliability of these models for clinical reasoning and medical knowledge in non-English languages. Methods: This study used the default mode of ChatGPT, which is based on GPT-3.5; the GPT-4 model of ChatGPT Plus; and the 117th JMLE in 2023. A total of 254 questions were included in the final analysis, which were categorized into 3 types, namely general, clinical, and clinical sentence questions. Results: The results indicated that GPT-4 outperformed GPT-3.5 in terms of accuracy, particularly for general, clinical, and clinical sentence questions. GPT-4 also performed better on difficult questions and specific disease questions. Furthermore, GPT-4 achieved the passing criteria for the JMLE, indicating its reliability for clinical reasoning and medical knowledge in non-English languages. Conclusions: GPT-4 could become a valuable tool for medical education and clinical support in non--English-speaking regions, such as Japan. ",
issn="2369-3762",
doi="10.2196/48002",
url="https://mededu.jmir.org/2023/1/e48002",
url="https://doi.org/10.2196/48002",
url="http://www.ncbi.nlm.nih.gov/pubmed/37384388"
}