<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JME</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Med Educ</journal-id>
      <journal-title>JMIR Medical Education</journal-title>
      <issn pub-type="epub">2369-3762</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v6i1e15877</article-id>
      <article-id pub-id-type="pmid">32329739</article-id>
      <article-id pub-id-type="doi">10.2196/15877</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Current and Future Trends in Life Sciences Training: Questionnaire Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Johnson</surname>
            <given-names>Andrew</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Aguirre</surname>
            <given-names>Katherine</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Magagna</surname>
            <given-names>William</given-names>
          </name>
          <degrees>MEd</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-4264-1135</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Wang</surname>
            <given-names>Nicole</given-names>
          </name>
          <degrees>MSED</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <address>
            <institution>Department of Learning Performance System</institution>
            <institution>Pennsylvania State University</institution>
            <addr-line>Keller Building</addr-line>
            <addr-line>University Park, PA</addr-line>
            <country>United States</country>
            <phone>1 2154072119</phone>
            <email>nwang@psu.edu</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-3195-4205</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Peck</surname>
            <given-names>Kyle</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-0981-7020</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Siemens Healthineers</institution>
        <addr-line>Newark, DE</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Learning Performance System</institution>
        <institution>Pennsylvania State University</institution>
        <addr-line>University Park, PA</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Nicole Wang <email>nwang@psu.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <season>Jan-Jun</season>
        <year>2020</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>24</day>
        <month>4</month>
        <year>2020</year>
      </pub-date>
      <volume>6</volume>
      <issue>1</issue>
      <elocation-id>e15877</elocation-id>
      <history>
        <date date-type="received">
          <day>14</day>
          <month>8</month>
          <year>2019</year>
        </date>
        <date date-type="rev-request">
          <day>28</day>
          <month>10</month>
          <year>2019</year>
        </date>
        <date date-type="rev-recd">
          <day>16</day>
          <month>12</month>
          <year>2019</year>
        </date>
        <date date-type="accepted">
          <day>10</day>
          <month>2</month>
          <year>2020</year>
        </date>
      </history>
      <copyright-statement>©William Magagna, Nicole Wang, Kyle Peck. Originally published in JMIR Medical Education (http://mededu.jmir.org), 24.04.2020.</copyright-statement>
      <copyright-year>2020</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Medical Education, is properly cited. The complete bibliographic information, a link to the original publication on http://mededu.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="http://mededu.jmir.org/2020/1/e15877/" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Every year, the life science field spends billions of dollars on educational activities worldwide. The continuing professional development of employees, especially in this field, encompasses great challenges. Emerging technologies appear to offer opportunity, but relatively little research has been done on the effectiveness of pedagogies and tools that have been used in the life sciences, and even less research has been devoted to understanding the potential power of emerging options that might determine the field’s future.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>In collaboration with the Life Sciences Trainers &#38; Educators Network (LTEN), this study investigated the current state of the pedagogies and tools currently adopted by corporate training professionals in the life sciences as well as the professionals’ perceptions of the impacts of emerging technologies on training.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>This study adopted a mixed methods approach that included a survey and a follow-up interview. The survey consists of 18 broad questions with 15 subquestions in each of the five specific sectors of the life sciences field. Interviews were conducted by phone and lasted approximately 40 minutes, covering 18 questions designed to follow-up on findings from the survey items.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>Both survey and interview results indicated that the professionals were not satisfied with the status quo and that training and education in this field need to change. Most of the techniques and tools currently used have been used for some time. The professionals surveyed were not satisfied with the current techniques and tools and did not find them cost-effective. In addition, the respondents pictured the future of training in this field to be more engaging and effective.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>This is the first study in a series designed to better understand education and training in the life sciences on a macro level, in order to build a foundation for progress and evolution of the future landscape. Next steps involve developing strategies for how to extend this vision throughout individual organizations.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>professional training</kwd>
        <kwd>training with technologies</kwd>
        <kwd>life sciences professionals</kwd>
        <kwd>mixed methods</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>The life sciences have been evolving at a staggering rate in every aspect. According to Kaufman [<xref ref-type="bibr" rid="ref1">1</xref>], “Medicine has gone through major changes over the last 50 years. Today it is recognized that medical knowledge doubles every 6-8 years, with new medical procedures emerging every day.” In gene ontology, there are about 250 ontologies accessible to professionals in the field [<xref ref-type="bibr" rid="ref2">2</xref>]. In addition, new services, technologies, and applications have emerged through the evolution of life sciences education [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>]. For instance, studies have suggested the use of artificial intelligence in improving medical imaging [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref6">6</xref>] and automating medical diagnosis and prediction [<xref ref-type="bibr" rid="ref7">7</xref>,<xref ref-type="bibr" rid="ref8">8</xref>].</p>
        <p>Recent advances in technology have also drastically changed how people teach and learn. As Bonk et al [<xref ref-type="bibr" rid="ref9">9</xref>] mentioned, “Recent technological developments have converged to dramatically alter conception of teaching and learning process.” These advancements offer innovative approaches to teaching and learning professionals within the life sciences field. For instance, Garcia-Pallares et al [<xref ref-type="bibr" rid="ref10">10</xref>] and Tymcznska [<xref ref-type="bibr" rid="ref11">11</xref>] discussed approaches in which instructors incorporate a course management system into life sciences professional training. In addition, Mantovani [<xref ref-type="bibr" rid="ref12">12</xref>], Stansfield et al [<xref ref-type="bibr" rid="ref13">13</xref>], and Barsom et al [<xref ref-type="bibr" rid="ref14">14</xref>] addressed the potential to integrate virtual reality (VR) into training to enhance and improve learning experiences. As the rapid development and application of artificial intelligence (AI) continues [<xref ref-type="bibr" rid="ref15">15</xref>], opportunities for adopting AI into training in the life sciences field will also become increasingly evident.</p>
        <p>Although the fields of life sciences, teaching, and learning are moving forward, professional development of people in the life sciences field seems to fall behind. Gorman et al [<xref ref-type="bibr" rid="ref2">2</xref>] pointed out that current and even future health care professionals are trained using the 100-year-old apprenticeship model, which is “see one, do one, and teach one.” In addition to the apprenticeship model, common strategies include lectures and films as the basis of a life sciences professional’s training, even though billions of dollars are dedicated to continuing educational activities in the life sciences worldwide [<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref17">17</xref>]. When it comes to professional training in the life sciences, studies [<xref ref-type="bibr" rid="ref18">18</xref>] indicate the similarities of the constant need for learning new knowledge, skills, and attitudes required due to the complexity of the field.</p>
        <p>At presently, literature exploring the current state of teaching and learning in the life sciences field is sparse, especially within the professional training realm. Additionally, literature exploring the cost-effectiveness of these emerging approaches for training medical professionals is almost nonexistent. Although there is an underlying assumption that there is a direct relationship between continuing professional development and the performance of recipients, only a few studies have attempted to validate this assumption. According to Bloom et al, [<xref ref-type="bibr" rid="ref19">19</xref>] and Umble and Cervero [<xref ref-type="bibr" rid="ref20">20</xref>], continuing professional development can be effective, but its effectiveness varies.</p>
        <p>In addition, a plethora of studies [<xref ref-type="bibr" rid="ref21">21</xref>-<xref ref-type="bibr" rid="ref24">24</xref>] have explored how emerging technologies can change the learning landscape across sectors. For instance, Dubey and Gunasekaran [<xref ref-type="bibr" rid="ref25">25</xref>] investigated how AI can impact the transportation sector, and Gavish et al [<xref ref-type="bibr" rid="ref26">26</xref>] explored how augmented reality (AR) and VR can transform industrial training. However, few studies have explored how emerging technologies can impact professional training and education in the life sciences field.</p>
      </sec>
      <sec>
        <title>Objectives</title>
        <p>There is a limited amount of research into the educational tools and approaches currently employed within the life sciences. Furthermore, there is a lack of studies investigating how life sciences training might evolve under the influence of emerging technologies and the increasing emphasis on cost-effectiveness. This study aims to understand the current state of teaching and learning in the life sciences, and teaching professionals’ perceptions of the impact of new technologies and practices on the field. Specifically, this study will investigate the following:</p>
        <list list-type="bullet">
          <list-item>
            <p>What technologies and pedagogies are educational professionals in Life Sciences Trainers &#38; Educators Network (LTEN) member organizations using now?</p>
          </list-item>
          <list-item>
            <p>Which currently used approaches are most cost-effective and judged as most satisfactory by training and education professionals in the life sciences?</p>
          </list-item>
          <list-item>
            <p>How do life science training and education professionals think emerging technologies might change current practice in the near future?</p>
          </list-item>
        </list>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Overview</title>
        <p>This study used a mixed methods approach [<xref ref-type="bibr" rid="ref27">27</xref>] to better understand the current state of teaching and learning in life sciences and training professionals’ perceptions of the impacts of emerging technologies. Mixed methods research requires data triangulation from quantitative and qualitative approaches, which strengthens the construct validity of the study [<xref ref-type="bibr" rid="ref28">28</xref>]. In addition, 57 members from a life sciences education not-for-profit organization, LTEN, participated in the survey, and 9 participants who responded to the survey were interviewed. In compliance with the Pennsylvania State University Institutional Review Board protocols, all participants signed the informed consent release form.</p>
      </sec>
      <sec>
        <title>Quantitative Method</title>
        <p>The survey consists of 18 broad questions with 15 subquestions in five specific sectors of the life sciences field (sales, clinical, product-related, customer-related, and other). Questions included demographic information along with detailed questions on the use and perceptions of pedagogies and tools.</p>
      </sec>
      <sec>
        <title>Qualitative Method</title>
        <p>The primary data were collected through semistructured interviews. Interviews were conducted by phone and lasted approximately 40 minutes, covering 18 questions designed to follow-up findings from the survey questions. The interview protocol was designed based on the theoretical framework proposed by Seidmen [<xref ref-type="bibr" rid="ref29">29</xref>], which consists of three general genres: personal experiences of emerging technology, attitudes toward specific technologies, and future expectations of emerging technologies. Researcher memos also served as a secondary data source [<xref ref-type="bibr" rid="ref30">30</xref>].</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Demographics</title>
        <p>Survey respondents represented the diversity of the LTEN membership. Founded in 1971, LTEN has grown to more than 1900 individual members who work in pharmaceuticals, biotech, medical device, and diagnostic companies, and industry partners who support the life sciences training departments. Additionally, LTEN has members across noncommercial disciplines including clinical, manufacturing, compliance, regulatory, quality, and general practice training roles [<xref ref-type="bibr" rid="ref31">31</xref>]. This study invited 326 active members who are directly involved in the training department of member organizations, to participate in the survey through email. A total of 57 participants completed the survey.</p>
        <p>As shown in <xref ref-type="table" rid="table1">Table 1</xref>, of the 57 participants, 24 (42%) of the respondents were education/training directors, 15 (27%) were corporate executives, 11 (19%) were education/training managers, and 7 (13%) were training developers. The respondents also had diverse responsibilities within their organizations (<xref ref-type="table" rid="table2">Table 2</xref>), with 23 (40%) working at US commercial-only organizations and 15 (26%) working at an entirely global organization.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Respondents’ role.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="750"/>
            <col width="250"/>
            <thead>
              <tr valign="bottom">
                <td>Role</td>
                <td>Respondents, n (%)<sup>a</sup></td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Education/training developer</td>
                <td>7 (13)</td>
              </tr>
              <tr valign="top">
                <td>Education/training manager</td>
                <td>11 (19)</td>
              </tr>
              <tr valign="top">
                <td>Education/training director</td>
                <td>24 (42)</td>
              </tr>
              <tr valign="top">
                <td>Corporate executive responsible for education and training</td>
                <td>15 (27)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>Percentages may not add up to 100 due to rounding.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Respondents’ responsibilities.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="750"/>
            <col width="250"/>
            <thead>
              <tr valign="top">
                <td>Responsibility</td>
                <td>Respondents, n (%)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>US commercial operations only</td>
                <td>23 (40)</td>
              </tr>
              <tr valign="top">
                <td>An entire US organization</td>
                <td>10 (18)</td>
              </tr>
              <tr valign="top">
                <td>Global organization, but commercial operations only</td>
                <td>9 (16)</td>
              </tr>
              <tr valign="top">
                <td>An entire global organization</td>
                <td>15 (26)</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Quantitative Results</title>
        <sec>
          <title>Current Pedagogies Identified</title>
          <p>A set of survey questions asked what pedagogies respondents currently use as their teaching strategies. Once they indicated their pedagogies, the respondents were asked to rank their selected pedagogies in order of importance. Fourteen pedagogies were presented as choices in the survey, and a weighted vote methodology was used to compare the pedagogies most commonly used and those perceived to be most important. A weighted ranking was produced by assigning a rank of 14 points to the item identified as most important, a score of 13 to the second most important, and so forth. The process was repeated for each category of trainee and for each training topic category, and a sum was calculated within each category of trainee and topic. The number of all the ranked items in each topic was used to produce a weighted percentage of the approaches used, which was divided by the total of all scores.</p>
          <p>As shown in <xref ref-type="table" rid="table3">Table 3</xref>, there were 4 pedagogies (instructor-led training, virtual instructor-led training, online readings, and role play activities) that captured about 55% of the weighted importance rankings, with the other 10 pedagogies comprising the other half.</p>
          <table-wrap position="float" id="table3">
            <label>Table 3</label>
            <caption>
              <p>Respondent-ranked importance of pedagogies.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="750"/>
              <col width="250"/>
              <thead>
                <tr valign="bottom">
                  <td>Pedagogies</td>
                  <td>Weighted percentages<sup>a</sup></td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Instructor-led training</td>
                  <td>17</td>
                </tr>
                <tr valign="top">
                  <td>Virtual instructor-led training</td>
                  <td>10</td>
                </tr>
                <tr valign="top">
                  <td>Role play activities</td>
                  <td>10</td>
                </tr>
                <tr valign="top">
                  <td>Competency-based learning</td>
                  <td>9</td>
                </tr>
                <tr valign="top">
                  <td>Case studies</td>
                  <td>9</td>
                </tr>
                <tr valign="top">
                  <td>Simulations</td>
                  <td>8</td>
                </tr>
                <tr valign="top">
                  <td>Field-based activities</td>
                  <td>8</td>
                </tr>
                <tr valign="top">
                  <td>Online readings</td>
                  <td>8</td>
                </tr>
                <tr valign="top">
                  <td>Collaborative learning</td>
                  <td>7</td>
                </tr>
                <tr valign="top">
                  <td>Problem-based learning</td>
                  <td>6</td>
                </tr>
                <tr valign="top">
                  <td>Games</td>
                  <td>4</td>
                </tr>
                <tr valign="top">
                  <td>Online discussions</td>
                  <td>2</td>
                </tr>
                <tr valign="top">
                  <td>Project-based learning</td>
                  <td>2</td>
                </tr>
                <tr valign="top">
                  <td>Other (please indicate)</td>
                  <td>1</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table3fn1">
                <p><sup>a</sup>Percentages may not add up to 100 due to rounding.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
          <p>Additionally, we identified some relationships in the differences between the pedagogies that respondents generally use and those they judged as most important. <xref rid="figure1" ref-type="fig">Figure 1</xref> shows the top 5 ranked pedagogies in terms of use and importance (shown with numbers representing their ranks in each category) in sales across 3 training topics (clinical, product, and skills). Across all 3 topics, instructor-led training (ILT) is perceived to be the most used and most important pedagogy for salespeople, and virtual instructor-led training (VILT) methods are among the top 3 in terms of use but are much lower in terms of importance in product and skills topics.</p>
          <fig id="figure1" position="float">
            <label>Figure 1</label>
            <caption>
              <p>Weighted comparison between pedagogy use and importance in sales. ILT: instructor-led training; VILT: virtual instructor-led training.</p>
            </caption>
            <graphic xlink:href="mededu_v6i1e15877_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
        <sec>
          <title>Cost-effectiveness of Current Pedagogies Used</title>
          <p>To determine respondents’ perceptions of the pedagogies and tools they use, we asked them to indicate a number between 0 and 100 that best indicated their satisfaction with each approach as well as its cost-effectiveness. To minimize the work and time required for respondents, we only asked them to do this for the top 3 pedagogies they had selected. We then averaged these satisfaction ratings and ranked approaches based on these averages. As shown in <xref ref-type="table" rid="table4">Table 4</xref>, the order of the satisfaction list indicates which approaches practitioners favor. There is a notable discrepancy between their satisfaction with a given approach and its cost-effectiveness.</p>
          <p>In terms of satisfaction with the approaches, the most used approaches (ILT and VILT) are not the ones with which respondents are most satisfied. Project-based learning, the option with which most respondents were satisfied, was not among the most used.</p>
          <p>In terms of cost-effectiveness, role-playing activities are most highly ranked, but are among the least frequently used. This may be due to the fact that role playing generally involves several trainees and trainers simultaneously, which may present logistical difficulties especially when conducted face-to-face. Interestingly, project-based learning, ranked as the most satisfying approach, is ranked as among the least cost-effective.</p>
          <table-wrap position="float" id="table4">
            <label>Table 4</label>
            <caption>
              <p>Comparison between the most satisfying and most cost-effective approaches.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="300"/>
              <col width="350"/>
              <col width="350"/>
              <thead>
                <tr valign="top">
                  <td>Rank</td>
                  <td>Most satisfying approach</td>
                  <td>Most cost-effective approach</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>1</td>
                  <td>Project-based learning</td>
                  <td>Role-play activities</td>
                </tr>
                <tr valign="top">
                  <td>2</td>
                  <td>Case study</td>
                  <td>Competency-based learning</td>
                </tr>
                <tr valign="top">
                  <td>3</td>
                  <td>Online discussion</td>
                  <td>Instructor-led training</td>
                </tr>
                <tr valign="top">
                  <td>4</td>
                  <td>Instructor-led training</td>
                  <td>Field-based learning</td>
                </tr>
                <tr valign="top">
                  <td>5</td>
                  <td>Problem-based learning</td>
                  <td>Problem-based learning</td>
                </tr>
                <tr valign="top">
                  <td>6</td>
                  <td>Other</td>
                  <td>Simulations</td>
                </tr>
                <tr valign="top">
                  <td>7</td>
                  <td>Competency-based learning</td>
                  <td>Other</td>
                </tr>
                <tr valign="top">
                  <td>8</td>
                  <td>Simulation</td>
                  <td>Online-reading</td>
                </tr>
                <tr valign="top">
                  <td>9</td>
                  <td>Virtual instructor</td>
                  <td>Project-based learning</td>
                </tr>
                <tr valign="top">
                  <td>10</td>
                  <td>Field-based learning</td>
                  <td>Case study</td>
                </tr>
                <tr valign="top">
                  <td>11</td>
                  <td>Collaborative learning</td>
                  <td>Virtual instructor</td>
                </tr>
                <tr valign="top">
                  <td>12</td>
                  <td>Role-play activities</td>
                  <td>Collaborative learning</td>
                </tr>
                <tr valign="top">
                  <td>13</td>
                  <td>Online reading</td>
                  <td>Games</td>
                </tr>
                <tr valign="top">
                  <td>14</td>
                  <td>Games</td>
                  <td>Online discussion</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
        </sec>
        <sec>
          <title>Now Versus the Future</title>
          <p>We also asked respondents to consider a list of 8 technologies and asked how important they felt these technologies are now and how important respondents felt these technologies will be in 5 years, on a scale of 1-10. Respondents predicted a decrease in the importance of course or learning management systems (LMS) and a large increase in the importance of AI (<xref ref-type="table" rid="table5">Table 5</xref>). They also predicted that webinars and course development systems will decrease slightly in importance, while predicting that simulation creation tools will become the most important approach. A rather significant decline in the use of online games was also projected.</p>
          <table-wrap position="float" id="table5">
            <label>Table 5</label>
            <caption>
              <p>Comparison between tools perceived to be most important now and in 5 years.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="300"/>
              <col width="350"/>
              <col width="350"/>
              <thead>
                <tr valign="top">
                  <td>Rank</td>
                  <td>Now</td>
                  <td>Future</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>1</td>
                  <td>LMS<sup>a</sup></td>
                  <td>Simulation creation tools</td>
                </tr>
                <tr valign="top">
                  <td>2</td>
                  <td>Webinars (live)</td>
                  <td>Artificial intelligence</td>
                </tr>
                <tr valign="top">
                  <td>3</td>
                  <td>Course development systems</td>
                  <td>Webinars (live)</td>
                </tr>
                <tr valign="top">
                  <td>4</td>
                  <td>Simulation creation tools</td>
                  <td>Course development systems</td>
                </tr>
                <tr valign="top">
                  <td>5</td>
                  <td>Online games</td>
                  <td>Virtual reality</td>
                </tr>
                <tr valign="top">
                  <td>6</td>
                  <td>Virtual reality</td>
                  <td>LMS</td>
                </tr>
                <tr valign="top">
                  <td>7</td>
                  <td>Augmented reality</td>
                  <td>Augmented reality</td>
                </tr>
                <tr valign="top">
                  <td>8</td>
                  <td>Artificial intelligence</td>
                  <td>Online games</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table5fn1">
                <p><sup>a</sup>LMS: learning management systems.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
        </sec>
      </sec>
      <sec>
        <title>Qualitative Results</title>
        <p>Qualitative phone interviews consisted of 18 questions related to contextual background and detailed information on both current learning strategies and experiences with technology. Participants’ perceptions of the future of training were also queried. After using an open coding approach [<xref ref-type="bibr" rid="ref30">30</xref>] and thematic analysis [<xref ref-type="bibr" rid="ref32">32</xref>] of 9 interview transcripts and researcher’s memos, the results revealed the emergence of three themes.</p>
        <sec>
          <title>The Status Quo</title>
          <p>All 9 respondents recognized that current training and education strategies in this field are at a clear risk of being abandoned in favor of rapidly evolving technologies. Most of the current techniques and tools have been used for some time, and the respondents noted that they are not very satisfied with them and that they do not perceive them to be very cost-effective. In particular, 8 of 9 respondents mentioned that the dominant learning tool, the LMS, is not conducive to learning, and the purpose of it is largely for administrative record tracking. One respondent said:</p>
          <disp-quote>
            <p>Most LMS have not developed with the user experiences in mind. In a global organizational level, LMS is just, “read the pdfs and do the quiz.” It’s absolutely boring.</p>
          </disp-quote>
          <p>Another respondent mentioned that LMS are generally not “mobile friendly,” adding that his sales team cannot have access to the learning material on-demand:</p>
          <disp-quote>
            <p>They are on the move all the time. It is unrealistic to expect them to have time to sit in the office and go through all the learning information.</p>
          </disp-quote>
          <p>Additionally, respondents raised concerns about measuring learning. Two respondents confessed that they have no valid understanding of whether learners are engaged with the tools and solutions currently employed and whether they are devoting the effort required to benefit in a meaningful way. One respondent said:</p>
          <disp-quote>
            <p>We only have a self-assessed checklist for our folks to fill out after the training session, but we do not know their progress at all.</p>
          </disp-quote>
          <p>Another respondent raised concern about how to devise valid evaluation metrics to assess his staff members. He provided an example that there are some staff members who got perfect scores on sales training assessments, yet have among the worst sales performance, while other staff members might have poor scores on sales training quizzes but have the highest sales performance in their district.</p>
          <p>Respondents acknowledged the reality that making any change involving control and administration of the learning enterprise is challenging. However, respondents also pointed out that if properly leveraged, technologies offer vast opportunities for learning and could demonstrably enhance business productivity, even in the heavily regulated life science market where disruptive change is difficult to enact.</p>
        </sec>
        <sec>
          <title>Call for Change</title>
          <p>All the respondents identified that there is a clear call to action for change within both the life sciences learning professionals and the companies in which they work. More specifically, respondents indicated the need to provide new learning solutions and leverage technologies for both employees and customers. They reported the belief that this would help drive a successful business and that organizations not taking this approach would be at risk of being left behind. One respondent even joked that she did not want to be “asleep at the wheel.”</p>
          <p>Throughout the interviews, all the respondents were reflective about their roles and performance as learning leaders and practitioners, coupled with a strong desire to be strategic in their decision-making concerning technology and to be open to change. One participant mentioned:</p>
          <disp-quote>
            <p>We, as practitioners and leaders alike, face a compelling need to improve in terms of competency, speed, quality, cost and overall return on investment.</p>
          </disp-quote>
          <p>Incorporating new technologies is perceived by these professionals as offering great promise, despite their awareness of the challenges that they realize will inevitably arise as they attempt to change the perceptions of those at the highest levels of their organizations toward embracing new solutions that leverage technology.</p>
          <p>Additionally, respondents addressed the need for change to meet the training requirements for different generations and different ways of learning. One respondent said:</p>
          <disp-quote>
            <p>The old ways of teaching and training will not work on the young generations. Millennials are on their smartphones all the time; they are addicted to the technologies.</p>
          </disp-quote>
          <p>Other respondents indicated that people come from different backgrounds in learning and have different learning styles; therefore, the traditional ways of learning will become obsolete.</p>
        </sec>
        <sec>
          <title>The Future of Teaching and Learning</title>
          <p>When asked about the future and the roles technology might play in learning, all of the respondents described a future in which training programs and processes are accelerated, impactful, and engaging. One respondent said:</p>
          <disp-quote>
            <p>Our goal is to create learning experiences, so that when people walk away, they are like, wow, that never happened to me before, I am going to remember that. We are now integrating technologies to involve all the senses to create a new unique learning events to improve impacts and effectiveness.</p>
          </disp-quote>
          <p>They also mentioned seeing solutions evolve in which learners are given more control over the learning, and where learning practitioners shift from content developers to content curators. These predictions were based on observable trends, encompassing where technologies seem to be heading as well as the changing behaviors and product lines from the providers of learning materials and training development tools.</p>
          <p>Additionally, respondents identified the potential to increase their impact through enhanced ability to develop social connectivity during learning, and to make learning personally relevant, interesting, engaging, easily accessible, self-driven, and even fun. Expanding on the concept of social connectivity, many commented on the growing nature of learner-centered environments in which learners engage interactive resources to meet their needs, working “on <italic>their</italic> terms” through learning experiences increasingly embedded in the workflow rather than as a separate formal learning event. One respondent said:</p>
          <disp-quote>
            <p>Learning is socio-cultural, if you limit the level of interactivity, you limit learning. We always need to look for opportunities for the learners to take control of the ability to connect and learn from each other. We need to look for ways to democratize data and have learning occurs [sic] down to the peer to peer level.</p>
          </disp-quote>
          <p>All respondents perceived AI, AR, and VR as having transformative near-term potential. One respondent commented that these emerging technologies are the ones “to take people’s knowledge and skills to the next level.” At the same time, two respondents indicated concern regarding how to adopt AI in this heavily regulated field.</p>
        </sec>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>This study explored the current state of training in the life science professional field through a mixed methods approach. Survey results indicated that instructor-led training is perceived to be the most used and most important pedagogy, and virtual instructor-led methods are among the top 3 in terms of use but are much lower in importance. In terms of satisfaction with the approaches, it is interesting to note that the most used approaches (instructor-led training and virtual instructor-led training) are not the approaches with which respondents were most satisfied. Although they were most satisfied with project-based learning, this approach was among the least cost-effective ones.</p>
        <p>From the cost-effectiveness perspective, role-playing activities are most highly ranked, but are among the least frequently used. This may be due to the fact that role-playing generally involves several trainees and trainers simultaneously, which may present logistical difficulties, especially when conducted face-to-face, which may cause busy professionals to dislike the activity.</p>
        <p>Looking into the future, both survey and interview results indicated that respondents are not satisfied with the status quo and that teaching and learning in this field need to change. Most of the techniques and tools currently used have been used for a long time, and the professionals are not very satisfied with them and do not find them very cost-effective. Unfortunately, the market and organizations in which these practitioners work are very complex, making change difficult. The interview results indicate that there is an evolution underway, but also highlight the need to get better at incorporating new technologies. This will require changing the perception that training and education are large expenses incurred by the organization without much evidence-based justification regarding effectiveness, and will require design thinking [<xref ref-type="bibr" rid="ref33">33</xref>] to consider both new approaches and new ways to demonstrate the effectiveness of training efforts based on the contributions training and education make to the providing organizations. Interviewees described a future in which training programs and processes are accelerated, impactful, and engaging, and in which learners are given more control. Emerging technologies such as AI and VR were seen to have increasingly important roles to play, allowing learning to become more interesting, engaging, and perhaps even fun. The vision is of learner-centered environments in which learners engage interactive resources to meet their needs, perhaps in learning experiences that are increasingly embedded in the workflow. These professionals see a bright, exciting future and a challenging path to realize this vision.</p>
      </sec>
      <sec>
        <title>Limitations</title>
        <p>Two limitations should be considered when interpreting these results. In the quantitative component of the study, 57 participants responded to the survey, which represents 17% of the sample pool of 326 people. However, this result is consistent with a typical noncompensated survey response rate (10%-15%) [<xref ref-type="bibr" rid="ref34">34</xref>]. Future studies should expand the survey to a larger membership body to increase the number of respondents. In addition, the participants were recruited through a single organization (LTEN); future studies should expand recruitment to encompass a broader spectrum of education and training professionals in the life sciences.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>This is the first study in a series designed to better understand education and training in the life sciences on a macro level, in order to build a foundation for progress and evolution of the future landscape. All respondents in this study seemed very aware that rapid and potentially beneficial change is underway, fueled by emerging technologies. They acknowledge that while the pace of its emergence is increasing in less complex contexts, aspects of this particular industry seem likely to inhibit the pace of change. In addition, respondents also acknowledge that the adoption of emerging technologies is impeded by the absence of data demonstrating a compelling return on investment, and insufficient time and resources. As important as this perspective is, understanding is a necessary but not sufficient first step. Next steps involve developing strategies for how to extend this vision throughout the individual organizations. Beyond this, we will need to determine how to expose the existing dissatisfaction with traditional, ineffective ways of operating, and create realistic “first steps” in the desired direction. Then, leaders in the field must gather data, make modifications, adjust, and document the effects. The process is not unlike the development of the organization’s core products, from research and design to operational practices. The process will be challenging, particularly in the highly regulated, relatively conservative life sciences market. This set of challenges might become less daunting through projects like this initial study, and the extended conversations it may generate among those ready to act as pioneers.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AI</term>
          <def>
            <p>artificial intelligence</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">AR</term>
          <def>
            <p>augmented reality</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">ILT</term>
          <def>
            <p>instructor-led training</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">LMS</term>
          <def>
            <p>learning management systems</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">LTEN</term>
          <def>
            <p>Life Sciences Trainers &#38; Educators Network</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">VILT</term>
          <def>
            <p>virtual instructor-led training</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">VR</term>
          <def>
            <p>virtual reality</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This research is supported by LTEN. LTEN is the only global 501(c)(3) nonprofit organization specializing in meeting the needs of life sciences learning professionals.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kaufmann</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Computers in surgical education and the operating room</article-title>
          <source>Annales Chirurgiae et Gynaecologiae</source>
          <year>2001</year>
          <fpage>141</fpage>
          <lpage>146</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gorman</surname>
              <given-names>PJ</given-names>
            </name>
            <name name-style="western">
              <surname>Meier</surname>
              <given-names>AH</given-names>
            </name>
            <name name-style="western">
              <surname>Rawn</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Krummel</surname>
              <given-names>TM</given-names>
            </name>
          </person-group>
          <article-title>The future of medical education is no longer blood and guts, it is bits and bytes</article-title>
          <source>The American Journal of Surgery</source>
          <year>2000</year>
          <month>11</month>
          <volume>180</volume>
          <issue>5</issue>
          <fpage>353</fpage>
          <lpage>356</lpage>
          <pub-id pub-id-type="doi">10.1016/s0002-9610(00)00514-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Becker</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Frishman</surname>
              <given-names>WH</given-names>
            </name>
            <name name-style="western">
              <surname>Scurlock</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Telemedicine and Tele-ICU: The Evolution and Differentiation of a New Medical Field</article-title>
          <source>The American Journal of Medicine</source>
          <year>2016</year>
          <month>12</month>
          <volume>129</volume>
          <issue>12</issue>
          <fpage>e333</fpage>
          <lpage>e334</lpage>
          <pub-id pub-id-type="doi">10.1016/j.amjmed.2016.05.045</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Khosla</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>White</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Medina</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ouwens</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Emmas</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Koder</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Male</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Leonard</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Real world evidence (RWE) – a disruptive innovation or the quiet evolution of medical evidence generation?</article-title>
          <source>F1000Res</source>
          <year>2018</year>
          <month>8</month>
          <day>29</day>
          <volume>7</volume>
          <fpage>111</fpage>
          <pub-id pub-id-type="doi">10.12688/f1000research.13585.2</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hosny</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Parmar</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Quackenbush</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>LH</given-names>
            </name>
            <name name-style="western">
              <surname>Aerts</surname>
              <given-names>HJWL</given-names>
            </name>
          </person-group>
          <article-title>Artificial intelligence in radiology</article-title>
          <source>Nat Rev Cancer</source>
          <year>2018</year>
          <month>5</month>
          <day>17</day>
          <volume>18</volume>
          <issue>8</issue>
          <fpage>500</fpage>
          <lpage>510</lpage>
          <pub-id pub-id-type="doi">10.1038/s41568-018-0016-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Langlotz</surname>
              <given-names>CP</given-names>
            </name>
            <name name-style="western">
              <surname>Allen</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Erickson</surname>
              <given-names>BJ</given-names>
            </name>
            <name name-style="western">
              <surname>Kalpathy-Cramer</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bigelow</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Cook</surname>
              <given-names>TS</given-names>
            </name>
            <name name-style="western">
              <surname>Flanders</surname>
              <given-names>AE</given-names>
            </name>
            <name name-style="western">
              <surname>Lungren</surname>
              <given-names>MP</given-names>
            </name>
            <name name-style="western">
              <surname>Mendelson</surname>
              <given-names>DS</given-names>
            </name>
            <name name-style="western">
              <surname>Rudie</surname>
              <given-names>JD</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Kandarpa</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>A Roadmap for Foundational Research on Artificial Intelligence in                     Medical Imaging: From the 2018 NIH/RSNA/ACR/The Academy Workshop</article-title>
          <source>Radiology</source>
          <year>2019</year>
          <month>06</month>
          <volume>291</volume>
          <issue>3</issue>
          <fpage>781</fpage>
          <lpage>791</lpage>
          <pub-id pub-id-type="doi">10.1148/radiol.2019190613</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Park</surname>
              <given-names>SH</given-names>
            </name>
            <name name-style="western">
              <surname>Han</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Methodologic Guide for Evaluating Clinical Performance and Effect of Artificial Intelligence Technology for Medical Diagnosis and Prediction</article-title>
          <source>Radiology</source>
          <year>2018</year>
          <month>03</month>
          <volume>286</volume>
          <issue>3</issue>
          <fpage>800</fpage>
          <lpage>809</lpage>
          <pub-id pub-id-type="doi">10.1148/radiol.2017171920</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>CJP</given-names>
            </name>
            <name name-style="western">
              <surname>Jiang</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Song</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>SY</given-names>
            </name>
            <name name-style="western">
              <surname>Fang</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Ming</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Artificial Intelligence Versus Clinicians in Disease Diagnosis: Systematic Review</article-title>
          <source>JMIR Med Inform</source>
          <year>2019</year>
          <month>08</month>
          <day>16</day>
          <volume>7</volume>
          <issue>3</issue>
          <fpage>e10010</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2019/3/e10010/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/10010</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bonk</surname>
              <given-names>CJ</given-names>
            </name>
            <name name-style="western">
              <surname>Hara</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Dennen</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Malikowski</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Supplee</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>We're in TITLE to Dream: Envisioning a Community of Practice, The Intraplanetary Teacher Learning Exchange</article-title>
          <source>CyberPsychology &#38; Behavior</source>
          <year>2000</year>
          <month>02</month>
          <volume>3</volume>
          <issue>1</issue>
          <fpage>25</fpage>
          <lpage>39</lpage>
          <pub-id pub-id-type="doi">10.1089/109493100316201</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>García-Pallarés</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sánchez-Medina</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Carrasco</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Díaz</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Izquierdo</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Endurance and neuromuscular changes in world-class level kayakers during a periodized training cycle</article-title>
          <source>Eur J Appl Physiol</source>
          <year>2009</year>
          <month>4</month>
          <day>26</day>
          <volume>106</volume>
          <issue>4</issue>
          <fpage>629</fpage>
          <lpage>638</lpage>
          <pub-id pub-id-type="doi">10.1007/s00421-009-1061-2</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tymczynska</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Integrating in-class and online learning activities in a healthcare interpreting course using Moodle</article-title>
          <source>The Journal of Specialised Translation, 12</source>
          <year>2009</year>
          <fpage>148</fpage>
          <lpage>164</lpage>
          <pub-id pub-id-type="doi">10.1057/9781137478412.0005</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mantovani</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Riva</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Galimberti</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>VR learning: potential challenges for the use of 3D environments in education training</article-title>
          <source>Towards Cyberpsychology</source>
          <year>2001</year>
          <publisher-loc>Amsterdam</publisher-loc>
          <publisher-name>Ios Press</publisher-name>
          <fpage>207</fpage>
          <lpage>225</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Stansfield</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Shawver</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Sobel</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Prasad</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Tapia</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Design and Implementation of a Virtual Reality System and Its Application to Training Medical First Responders</article-title>
          <source>Presence: Teleoperators and Virtual Environments</source>
          <year>2000</year>
          <month>12</month>
          <volume>9</volume>
          <issue>6</issue>
          <fpage>524</fpage>
          <lpage>556</lpage>
          <pub-id pub-id-type="doi">10.1162/105474600300040376</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Barsom</surname>
              <given-names>EZ</given-names>
            </name>
            <name name-style="western">
              <surname>Graafland</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schijven</surname>
              <given-names>MP</given-names>
            </name>
          </person-group>
          <article-title>Systematic review on the effectiveness of augmented reality applications in medical training</article-title>
          <source>Surg Endosc</source>
          <year>2016</year>
          <month>2</month>
          <day>23</day>
          <volume>30</volume>
          <issue>10</issue>
          <fpage>4174</fpage>
          <lpage>4183</lpage>
          <pub-id pub-id-type="doi">10.1007/s00464-016-4800-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jiang</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Jiang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Zhi</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Dong</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Ma</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Dong</surname>
              <given-names>Q</given-names>
            </name>
            <name name-style="western">
              <surname>Shen</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Artificial intelligence in healthcare: past, present and future</article-title>
          <source>Stroke Vasc Neurol</source>
          <year>2017</year>
          <month>06</month>
          <day>21</day>
          <volume>2</volume>
          <issue>4</issue>
          <fpage>230</fpage>
          <lpage>243</lpage>
          <pub-id pub-id-type="doi">10.1136/svn-2017-000101</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>A</article-title>
          <source>, Belfield, C. R., &#38; Field, S. J. Cost effectiveness of continuing professional development in health care: a critical review of the evidence.Bmj</source>
          <year>2002</year>
          <volume>324</volume>
          <issue>7338</issue>
          <fpage>652</fpage>
          <lpage>655</lpage>
          <pub-id pub-id-type="doi">10.1136/bmj.324.7338.652</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vaughn</surname>
              <given-names>HT</given-names>
            </name>
            <name name-style="western">
              <surname>Rogers</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Freeman</surname>
              <given-names>JK</given-names>
            </name>
          </person-group>
          <article-title>Does Requiring Continuing Education Units for Professional Licensing Renewal Assure Quality Patient Care?</article-title>
          <source>The Health Care Manager</source>
          <year>2006</year>
          <volume>25</volume>
          <issue>1</issue>
          <fpage>78</fpage>
          <lpage>84</lpage>
          <pub-id pub-id-type="doi">10.1097/00126450-200601000-00011</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Peck</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>McCall</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>McLaren</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Rotem</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Continuing medical education and continuing professional development: international comparisons</article-title>
          <source>BMJ</source>
          <year>2000</year>
          <month>03</month>
          <day>12</day>
          <volume>320</volume>
          <issue>7232</issue>
          <fpage>432</fpage>
          <lpage>5</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/10669451"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bmj.320.7232.432</pub-id>
          <pub-id pub-id-type="medline">10669451</pub-id>
          <pub-id pub-id-type="pmcid">PMC1117549</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bloom</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <source>Blended coaching: Skills and strategies to support principal development</source>
          <year>2005</year>
          <publisher-loc>Thousand Oaks, CA</publisher-loc>
          <publisher-name>Corwin Press</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Umble</surname>
              <given-names>KE</given-names>
            </name>
            <name name-style="western">
              <surname>Cervero</surname>
              <given-names>RM</given-names>
            </name>
          </person-group>
          <article-title>Impact Studies in Continuing Education For Health Professionals</article-title>
          <source>Eval Health Prof</source>
          <year>2016</year>
          <month>06</month>
          <day>29</day>
          <volume>19</volume>
          <issue>2</issue>
          <fpage>148</fpage>
          <lpage>174</lpage>
          <pub-id pub-id-type="doi">10.1177/016327879601900202</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Hsu</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Zheng</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>The Application of AR Technology to Spatial Skills Learning in Carpentry Training</article-title>
          <source>IJIET</source>
          <year>2019</year>
          <volume>9</volume>
          <issue>1</issue>
          <fpage>56</fpage>
          <lpage>60</lpage>
          <pub-id pub-id-type="doi">10.18178/ijiet.2019.9.1.1173</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Tang</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Yuan</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Nie</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Ma</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Wei</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>A VR-based the emergency rescue training system of railway accident</article-title>
          <source>Entertainment Computing</source>
          <year>2018</year>
          <month>08</month>
          <volume>27</volume>
          <fpage>23</fpage>
          <lpage>31</lpage>
          <pub-id pub-id-type="doi">10.1016/j.entcom.2018.03.002</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kang</surname>
              <given-names>SG</given-names>
            </name>
            <name name-style="western">
              <surname>Ryu</surname>
              <given-names>BJ</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>KS</given-names>
            </name>
            <name name-style="western">
              <surname>Ko</surname>
              <given-names>YH</given-names>
            </name>
            <name name-style="western">
              <surname>Cho</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Kang</surname>
              <given-names>SH</given-names>
            </name>
            <name name-style="western">
              <surname>Patel</surname>
              <given-names>VR</given-names>
            </name>
            <name name-style="western">
              <surname>Cheon</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>An Effective Repetitive Training Schedule to Achieve Skill Proficiency Using a Novel Robotic Virtual Reality Simulator</article-title>
          <source>Journal of Surgical Education</source>
          <year>2015</year>
          <month>05</month>
          <volume>72</volume>
          <issue>3</issue>
          <fpage>369</fpage>
          <lpage>376</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jsurg.2014.06.023</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Clifford</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Evaluating the effects of realistic communication disruptions in VR training for aerial firefighting</article-title>
          <source>10th International Conference on Virtual Worlds and Games for Serious Applications (VS-Games)</source>
          <year>2018</year>
          <month>09</month>
          <day>05</day>
          <conf-name>10th International Conference on Virtual Worlds and Games for Serious Applications (VS-Games)</conf-name>
          <conf-date>2018</conf-date>
          <conf-loc>Würzburg, Germany</conf-loc>
          <fpage>1</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1109/vs-games.2018.8493423</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Dubey</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Gunasekaran</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>The role of truck driver on sustainable transportation and logistics</article-title>
          <source>Ind and Commercial Training</source>
          <year>2015</year>
          <month>04</month>
          <day>07</day>
          <volume>47</volume>
          <issue>3</issue>
          <fpage>127</fpage>
          <lpage>134</lpage>
          <pub-id pub-id-type="doi">10.1108/ict-08-2014-0053</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gavish</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Gutiérrez</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Webel</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Rodríguez</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Peveri</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Bockholt</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Tecchia</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>Evaluating virtual reality and augmented reality training for industrial maintenance and assembly tasks</article-title>
          <source>Interactive Learning Environments</source>
          <year>2013</year>
          <month>07</month>
          <day>18</day>
          <volume>23</volume>
          <issue>6</issue>
          <fpage>778</fpage>
          <lpage>798</lpage>
          <pub-id pub-id-type="doi">10.1080/10494820.2013.815221</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Creswell</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <source>A Concise Introduction To Mixed Methods Research</source>
          <year>2014</year>
          <publisher-loc>Thousand Oaks, California</publisher-loc>
          <publisher-name>Sage Publications, Inc</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Creswell</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Editorial: Mapping the Field of Mixed Methods Research</article-title>
          <source>Journal of Mixed Methods Research</source>
          <year>2008</year>
          <month>11</month>
          <day>15</day>
          <volume>3</volume>
          <issue>2</issue>
          <fpage>95</fpage>
          <lpage>108</lpage>
          <pub-id pub-id-type="doi">10.1177/1558689808330883</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Seidman</surname>
              <given-names>I</given-names>
            </name>
          </person-group>
          <source>Interviewing As Qualitative Research: A Guide For Researchers In Education And The Social Sciences</source>
          <year>2013</year>
          <publisher-loc>New York</publisher-loc>
          <publisher-name>Teachers College Press</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Saldaña</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <source>Ethnotheatre: Research from page to stage</source>
          <year>2016</year>
          <publisher-loc>New York</publisher-loc>
          <publisher-name>Routledge</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="web">
          <source>LTEN website</source>
          <year>2018</year>
          <access-date>2018-11-01</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.l-ten.org/page/LTEN_History">https://www.l-ten.org/page/LTEN_History</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Braun</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Clarke</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Using thematic analysis in psychology</article-title>
          <source>Qualitative Research in Psychology</source>
          <year>2006</year>
          <month>01</month>
          <volume>3</volume>
          <issue>2</issue>
          <fpage>77</fpage>
          <lpage>101</lpage>
          <pub-id pub-id-type="doi">10.1191/1478088706qp063oa</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Wyatt</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Design Thinking for Social Innovation</article-title>
          <source>Development Outreach</source>
          <year>2010</year>
          <month>07</month>
          <volume>12</volume>
          <issue>1</issue>
          <fpage>29</fpage>
          <lpage>43</lpage>
          <pub-id pub-id-type="doi">10.1596/1020-797x_12_1_29</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Conn</surname>
              <given-names>KM</given-names>
            </name>
            <name name-style="western">
              <surname>Mo</surname>
              <given-names>CH</given-names>
            </name>
            <name name-style="western">
              <surname>Sellers</surname>
              <given-names>LM</given-names>
            </name>
          </person-group>
          <article-title>When Less Is More in Boosting Survey Response Rates</article-title>
          <source>Social Science Quarterly</source>
          <year>2019</year>
          <month>04</month>
          <day>12</day>
          <volume>100</volume>
          <issue>4</issue>
          <fpage>1445</fpage>
          <lpage>1458</lpage>
          <pub-id pub-id-type="doi">10.1111/ssqu.12625</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
