<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
    <front>
        <journal-meta>
            <journal-id journal-id-type="publisher-id">JME</journal-id>
            <journal-id journal-id-type="nlm-ta">JMIR Medical Education</journal-id>
            <journal-title>JMIR Medical Education</journal-title>
            <issn pub-type="epub">2369-3762</issn>
            <publisher>
                <publisher-name>JMIR Publications Inc.</publisher-name>
                <publisher-loc>Toronto, Canada</publisher-loc>
            </publisher>
        </journal-meta>
        <article-meta>
            <article-id pub-id-type="publisher-id">v1i2e15</article-id>
            <article-id pub-id-type="pmid">27731849</article-id>
            <article-id pub-id-type="doi">10.2196/mededu.4958</article-id>
            <article-categories>
                <subj-group subj-group-type="heading">
                    <subject>Original Paper</subject>
                </subj-group>
                <subj-group subj-group-type="article-type">
                    <subject>Original Paper</subject>
                </subj-group>
            </article-categories>
            <title-group>
                <article-title>Mobile Virtual Learning Object for the Assessment of Acute Pain as a Learning Tool to Assess Acute Pain in Nursing: An Analysis of the Mental Workload</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="editor">
                    <name>
                        <surname>Eysenbach</surname>
                        <given-names>Gunther</given-names>
                    </name>
                </contrib>
            </contrib-group>
            <contrib-group>
                <contrib contrib-type="reviewer">
                    <name>
                        <surname>Barra</surname>
                        <given-names>Daniela</given-names>
                    </name>
                </contrib>
            </contrib-group>
            <contrib-group>
                <contrib contrib-type="author" id="contrib1" corresp="yes" equal-contrib="yes">
                    <name name-style="western">
                        <surname>Alvarez</surname>
                        <given-names>Ana Graziela</given-names>
                    </name>
                    <degrees>PhD</degrees>
                    <xref rid="aff1" ref-type="aff">1</xref>
                    <address>
                        <institution>Science Health Center</institution>
                        <institution>Department of Nursing</institution>
                        <institution>Federal University of Santa Catarina</institution>
                        <addr-line>Campus Reitor João David Ferreira Lima</addr-line>
                        <addr-line>Bairro Trindade</addr-line>
                        <addr-line>Florianópolis, 88040-900</addr-line>
                        <country>Brazil</country>
                        <phone>55 48 3721 4910</phone>
                        <fax>55 48 3721 9921</fax>
                        <email>grazielaalvarez@gmail.com</email>
                    </address>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-3943-9884</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib2" equal-contrib="yes">
                    <name name-style="western">
                        <surname>Sasso</surname>
                        <given-names>Grace</given-names>
                    </name>
                    <degrees>PhD</degrees>
                    <xref rid="aff1" ref-type="aff">1</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0001-7702-1190</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib3" equal-contrib="yes">
                    <name name-style="western">
                        <surname>Iyengar</surname>
                        <given-names>Sriram</given-names>
                    </name>
                    <degrees>PhD</degrees>
                    <xref rid="aff2" ref-type="aff">2</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-7242-430X</ext-link>
                </contrib>
            </contrib-group>
            <aff id="aff1">
                <sup>1</sup>
                <institution>Science Health Center</institution>
                <institution>Department of Nursing</institution>
                <institution>Federal University of Santa Catarina</institution>
                <addr-line>Florianópolis</addr-line>
                <country>Brazil</country>
            </aff>
            <aff id="aff2">
                <sup>2</sup>
                <institution>Texas A&#38;M University</institution>
                <institution>Biomedical Informatics Core, Clinical Science and Translational Research</institution>
                <institution>University of Texas</institution>
                <addr-line>Houston, TX</addr-line>
                <country>United States</country>
            </aff>
            <author-notes>
                <corresp>Corresponding Author: Ana Graziela Alvarez <email>grazielaalvarez@gmail.com</email>
                </corresp>
            </author-notes>
            <pub-date pub-type="collection">
                <season>Jul-Dec</season>
                <year>2015</year>
            </pub-date>
            <pub-date pub-type="epub">
                <day>06</day>
                <month>11</month>
                <year>2015</year>
            </pub-date>
            <volume>1</volume>
            <issue>2</issue>
            <elocation-id>e15</elocation-id>
            <!--history from ojs - api-xml-->
            <history>
                <date date-type="received">
                    <day>26</day>
                    <month>7</month>
                    <year>2015</year>
                </date>
                <date date-type="rev-request">
                    <day>11</day>
                    <month>8</month>
                    <year>2015</year>
                </date>
                <date date-type="rev-recd">
                    <day>7</day>
                    <month>9</month>
                    <year>2015</year>
                </date>
                <date date-type="accepted">
                    <day>22</day>
                    <month>9</month>
                    <year>2015</year>
                </date>
            </history>
            <!--(c) the authors - correct author names and publication date here if necessary. Date in form ', dd.mm.yyyy' after jmir.org-->
            <copyright-statement>©Ana Graziela Alvarez, Grace Sasso, Sriram Iyengar. Originally published in JMIR Medical Education (http://mededu.jmir.org), 06.11.2015. </copyright-statement>
            <copyright-year>2015</copyright-year>
            <license license-type="open-access" xlink:href="http://creativecommons.org/licenses/by/2.0/">
                <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (http://creativecommons.org/licenses/by/2.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Medical Education, is properly cited. The complete bibliographic information, a link to the original publication on http://mededu.jmir.org/, as well as this copyright and license information must be included.</p>
            </license>
            <self-uri xlink:href="http://mededu.jmir.org/2015/2/e15/" xlink:type="simple"/>
            <abstract>
                <sec sec-type="background">
                    <title>Background</title>
                    <p>The inclusion of new technologies in education has motivated the development of studies on mental workload. These technologies are now being used in the teaching and learning process. The analysis enables identification of factors intervening in this workload as well as planning of overload prevention for educational activities using these technologies.</p>
                </sec>
                <sec sec-type="objective">
                    <title>Objective</title>
                    <p>To analyze the mental workload of an educational intervention with the Mobile Virtual Learning Object for the Assessment of Acute Pain in adults and newborns, according to the NASA Task Load Index criteria.</p>
                </sec>
                <sec sec-type="methods">
                    <title>Methods</title>
                    <p>A methodological study with data collected from 5 nurses and 75 students, from November of 2013 to February of 2014.</p>
                </sec>
                <sec sec-type="results">
                    <title>Results</title>
                    <p>The highest students’ and specialists’ means were in the dimensions of “Mental demand” (57.20 ± 22.27; 51 ± 29.45) and “Performance” (58.47 ± 24.19; 73 ± 28.85). The specialists’ mental workload index was higher (50.20 ± 7.28) when compared with students’ (47.87 ± 16.85) on a scale from 0 to 100 (<italic>P</italic>=.557).</p>
                </sec>
                <sec sec-type="conclusions">
                    <title>Conclusions</title>
                    <p>The instrument allowed for the assessment of mental workload after an online educational intervention with a mobile learning virtual object. An excessive overload was not identified among participants. Assessing mental workload from the use of educational technologies at the end of a task is a key to their applicability, with the aim of providing a more effective, stimulating, and long-lasting experience of the learning process.</p>
                </sec>
            </abstract>
            <kwd-group>
                <kwd>nursing</kwd>
                <kwd>nursing informatics</kwd>
                <kwd>distance learning</kwd>
                <kwd>computer-assisted instruction</kwd>
                <kwd>educational technology</kwd>
                <kwd>nursing education</kwd>
                <kwd>acute pain</kwd>
                <kwd>persuasive technology</kwd>
                <kwd>mental workload</kwd>
            </kwd-group>
        </article-meta>
    </front>
    <body>
        <sec sec-type="introduction">
            <title>Introduction</title>
            <p>Following the evolution of information and communications technologies (ICTs), the concept of the virtual learning object (VLO) has been emphasized in the educational scenario. It is defined as a unit that creates an educational context, is reusable in different learning contexts, and consists of an active and constructive teaching-learning strategy [<xref ref-type="bibr" rid="ref1">1</xref>-<xref ref-type="bibr" rid="ref3">3</xref>].</p>
            <p>The interest in research on the organization of mental workload has gradually increased given the popularization of information and communications technologies (ICTs) in the workplace. The use of technology has also increased in teaching and learning, resulting in a growing interest for understanding the flow of information, as well as the mental workload generated by the use of these technologies [<xref ref-type="bibr" rid="ref4">4</xref>].</p>
            <p>Considering the current technological advances, some issues related to the teaching and learning process need to be reviewed; additionally, there is a need to establish metrics for the effective assessment of important parameters that can affect students’ performance, such as mental workload.</p>
            <p>Mental workload can be defined as the interaction between internal and external factors and the individual, resulting in a subjectively described experience [<xref ref-type="bibr" rid="ref5">5</xref>]. This is the effect that a particular demand has on the individual in terms of mental and physical effort, which can be related to the amount of processed information, as well as the effort for the performance of that particular task [<xref ref-type="bibr" rid="ref6">6</xref>].</p>
            <p>It should be noted that the assessment of mental workload is subjective, namely, it is assessed from the perspective of the person performing the task, not the task itself. The assessment depends on individual characteristics that can increase or decrease this individual perception. These characteristics include the individual’s knowledge for performing the task, experience with the activity, age, education, humor, etc. [<xref ref-type="bibr" rid="ref7">7</xref>-<xref ref-type="bibr" rid="ref9">9</xref>].</p>
            <p>The assessment of mental workload assumes that an individual’s burnout level is directly associated with his/her ability to perform a certain activity [<xref ref-type="bibr" rid="ref10">10</xref>-<xref ref-type="bibr" rid="ref12">12</xref>]. In this sense, the NASA Task Load Index (NASA TLX) has been more widely used, which is freely available to researchers (see <xref ref-type="app" rid="app1">Multimedia Appendix 1</xref>).</p>
            <p>Although the interest in measuring mental workload as a means of understanding human-machine interfaces has been around for some years, only for the last 20 years has there been interest in the study of this effect on Web-based learning in humans [<xref ref-type="bibr" rid="ref13">13</xref>-<xref ref-type="bibr" rid="ref19">19</xref>].</p>
            <p>By identifying the rates of the dimensions that construct the mental workload, it can be classified as either “overloaded” or “underloaded.” Overload corresponds to a saturation of resource consumption, whereas underload results from the lack of stimulus to perform the task [<xref ref-type="bibr" rid="ref20">20</xref>].</p>
            <p>The calculation of mental workload through the NASA TLX combines the assessment rates of different dimensions and their weights, weighted according to their subjective importance for a given task [<xref ref-type="bibr" rid="ref21">21</xref>]. In addition, studying the factors intervening in mental workload from a given task enables the planning of overload prevention during activities performed by these individuals [<xref ref-type="bibr" rid="ref20">20</xref>], and can assist with the planning and developing of educational technologies.</p>
            <p>The aim of this study was to analyze the mental workload of an educational intervention using the Mobile Virtual Learning Object for the Assessment of Acute Pain (m-OVADor) in adults and newborns, according to the NASA TLX criteria.</p>
        </sec>
        <sec sec-type="methods">
            <title>Methods</title>
            <p>A methodological study [<xref ref-type="bibr" rid="ref22">22</xref>] approved by the Research Ethics Committee at the Universidade Federal de Santa Catarina (UFSC) in 2012 (certificate 2456), with online data collection from November 1/2013 to February 15/2014.</p>
            <p>A total of 170 students in the second to the eighth semesters of the undergraduate nursing course were eligible to participate in this study, of which 120 agreed to do so. The nonprobabilistic and intentional final sample, after applying the exclusion criteria, included 75 students and 5 specialist nurses, who completed all planned steps. All participants were identified by alphanumeric codes, students S1 to S75 and nurse specialists NS1 to NS5.</p>
            <p>The inclusion criteria for students were (1) being regularly enrolled in the second to the eighth semester of the undergraduate nursing course at UFSC during the study period and (2) being available for online participation in the study during an extracurricular period. The inclusion criteria for nurse specialists were (1) minimum of a master’s degree; (2) a nurse with minimum experience of 2 years in the critical care area, neonatology/pediatrics, and/or medical-surgical clinic; (3) experience with the use or development of ICTs in the teaching-learning process in nursing; (4) having a mobile device with Internet access (cell phone, smartphone, personal digital assistant, or tablet). The exclusion criterion was missing any of the study steps.</p>
            <p>All participants received the instructions to participate in the study via email and Facebook. The m-OVADor was accessed from their own mobile devices (see <xref ref-type="app" rid="app2">Multimedia Appendix 2</xref>).</p>
            <p>The technology developed for the study had 3 different simulated clinical scenarios (surgical clinic, adult intensive care, and neonatology; see <xref ref-type="app" rid="app3">Multimedia Appendix 3</xref>), which enabled the study of different variables involved in the assessment of acute pain (<xref ref-type="fig" rid="figure1">Figure 1</xref>).</p>
            <p>Identification of the measures required for the analysis of mental workload of students and specialists (rate, weight, magnitude, and mental workload index) was performed by application of the NASA TLX instrument [<xref ref-type="bibr" rid="ref11">11</xref>], according to the following protocol (<xref ref-type="fig" rid="figure2">Figure 2</xref>).</p>
            <p>The NASA TLX used in the study is a version that was translated and adapted by the authors, available for free use on the NASA website (see <xref ref-type="app" rid="app1">Multimedia Appendix 1</xref>). Access to the electronic instrument for data collection was made available to participants through the Google Drive tool.</p>
            <p>The application of the NASA TLX provides a subjective assessment of mental workload, namely, an assessment from the perspective of the person performing the task (self-assessment). This tool was chosen because of its ease of application, objectivity, and low cost of implementation [<xref ref-type="bibr" rid="ref7">7</xref>].</p>
            <p>The assessment of the participants’ mental workload was performed in 3 stages: assessment of weights for each dimension, assessment of rates for each dimension, and calculation of mental workload index (overload).</p>
            <p>Initially, for the assessment of the weights for each dimension, 15 pairs of combinations between 6 assessment dimensions were presented to the participants (effort and physical demand, mental demand and effort, temporal demand and frustration, frustration and effort, performance and frustration, temporal demand and mental demand, temporal demand and effort, physical demand and temporal demand, effort and performance, frustration and mental demand, physical demand and frustration, performance and mental demand, mental demand and physical demand, performance and temporal demand, and performance and physical demand). At this stage, only 1 option should be selected in each pair, which should be the most prominent one during the proposed activity [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
            <p>The frequencies assigned to each dimension were summed up and the results were expressed on a scale of 0-5. Then the rates assigned to each dimension were identified from the participants’ indication on a scale of 0-100 with 20 five-point equal intervals [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
            <p>The weight of each dimension was multiplied by its respective rate, and the magnitude of mental workload was obtained. The magnitude values of the 6 dimensions were summed up, their results divided by 15, and the mental workload (overload) index was obtained [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
            <p>The data were exported to electronic spreadsheets using Excel for Mac 2011 for data classification and analysis of the mental workload (overload) index. The dimensions comprising this mental workload were analyzed by descriptive (mean, minimum, maximum, standard deviation, and variance) and inferential (Student t test, analysis of variance [ANOVA] and Bonferroni) statistics. A significance level of <italic>P</italic>&#60;.05 for a 95% confidence interval was used in this study.</p>
            <fig id="figure1" position="float">
                <label>Figure 1</label>
                <caption>
                    <p>m-OVADor structure for access and assessment of acute pain in clinical scenarios.</p>
                </caption>
                <graphic xlink:href="mededu_v1i2e15_fig1.jpg" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
            </fig>
            <fig id="figure2" position="float">
                <label>Figure 2</label>
                <caption>
                    <p>Protocol for the analysis of mental workload from the NASA TLX instrument.</p>
                </caption>
                <graphic xlink:href="mededu_v1i2e15_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
            </fig>
        </sec>
        <sec sec-type="results">
            <title>Results</title>
            <sec>
                <title>Students’ and Specialist Nurses’ Characteristics</title>
                <p>The sample (n=75) comprised young students between 19 and 32 years of age (22.65 ± 3.12 years). Most were women (92%, 69/75) and exclusively dedicated to studying (79%, 59/75); only 8% (n=6) were working as nursing technicians.</p>
                <p>Regarding the semester of the students (n=75) at the time of data collection, the following distribution was observed: 1% (n=1) in the second semester; 27% (n=20) in the third semester; 17% (n=13) in the fourth semester; 9% (n=7) in the fifth semester; 9% (n=7) in the sixth semester; 12% (n=9) in seventh semester, and 24% (n=18) in the eighth semester.</p>
                <p>As for the use of the Internet in their personal and academic environments (multiple-choice questions), 131 answers were obtained, distributed as follows: 46% (60/131) accessed the Internet on their laptops; 34% (45/131) on mobile phones; 13% (17/131) on desktops, and 7% (9/131) on tablets.</p>
                <p>The results draw attention to the issue of connectivity from small-sized mobile devices, thereby confirming the global trend of popularization of these technologies among young people [<xref ref-type="bibr" rid="ref24">24</xref>].</p>
                <p>Regarding the sociodemographic characteristics of specialist nurses (n=5), 100% (n=5) were female, 80% (n=4) had a master’s degree, and 20% (n=1) had a doctorate. As for the area of experience, the participants were distributed in the following areas: education and/or providing care for pain (40%, 2/5), nursing informatics (40%, 2/5), and medical surgical (20%, 1/5). All used mobile technologies and the Internet for personal and academic reasons every day.</p>
            </sec>
            <sec>
                <title>Mental Workload Assessment</title>
                <p>The application of NASA TLX enabled identification of the rates, weights, and magnitudes assigned to 6 assessment dimensions, which were used to calculate the overload. <xref ref-type="table" rid="table1">Tables 1</xref> and <xref ref-type="table" rid="table2">2</xref> show the results obtained from the assessment performed by students and specialists, as well as the mean overload for each group, respectively.</p>
                <table-wrap position="float" id="table1">
                    <label>Table 1</label>
                    <caption>
                        <p>Rate, weight, magnitude and mean overload<sup>a</sup> of students (n=75).</p>
                    </caption>
                    <table width="527" border="1" cellpadding="5" cellspacing="0" rules="groups" frame="hsides">
                        <col width="102"/>
                        <col width="63"/>
                        <col width="66"/>
                        <col width="56"/>
                        <col width="37"/>
                        <col width="75"/>
                        <col width="56"/>
                        <thead>
                            <tr valign="top">
                                <td>Dimensions</td>
                                <td colspan="2">Rate</td>
                                <td colspan="2">Weight</td>
                                <td colspan="2">Magnitude</td>
                            </tr>
                            <tr valign="top">
                                <td>
                                    <break/>
                                </td>
                                <td>Mean</td>
                                <td>SD</td>
                                <td>Mean</td>
                                <td>SD</td>
                                <td>Mean</td>
                                <td>SD</td>
                            </tr>
                        </thead>
                        <tbody>
                            <tr valign="top">
                                <td>Mental demand</td>
                                <td>57.20</td>
                                <td>22.27</td>
                                <td>3.80</td>
                                <td>.92</td>
                                <td>217.27</td>
                                <td>110.41</td>
                            </tr>
                            <tr valign="top">
                                <td>Physical demand</td>
                                <td>29.27</td>
                                <td>27.91</td>
                                <td>1.08</td>
                                <td>.93</td>
                                <td>30.47</td>
                                <td>42.83</td>
                            </tr>
                            <tr valign="top">
                                <td>Temporal demand</td>
                                <td>37.67</td>
                                <td>24.14</td>
                                <td>3.08</td>
                                <td>1.47</td>
                                <td>122.27</td>
                                <td>102.26</td>
                            </tr>
                            <tr valign="top">
                                <td>Performance</td>
                                <td>58.47</td>
                                <td>24.19</td>
                                <td>3.55</td>
                                <td>1.21</td>
                                <td>216.73</td>
                                <td>121.65</td>
                            </tr>
                            <tr valign="top">
                                <td>Effort</td>
                                <td>40.73</td>
                                <td>23.24</td>
                                <td>2.69</td>
                                <td>.84</td>
                                <td>112.13</td>
                                <td>72.75</td>
                            </tr>
                            <tr valign="top">
                                <td>Frustration</td>
                                <td>30.60</td>
                                <td>27.80</td>
                                <td>0.80</td>
                                <td>1.15</td>
                                <td>36.93</td>
                                <td>71.89</td>
                            </tr>
                        </tbody>
                    </table>
                    <table-wrap-foot>
                        <fn id="table1fn1">
                            <p>
                                <sup>a</sup>Mean overload of the students is 47.87.</p>
                        </fn>
                    </table-wrap-foot>
                </table-wrap>
                <table-wrap position="float" id="table2">
                    <label>Table 2</label>
                    <caption>
                        <p>Rate, weight, magnitude, and mean overload of specialists (n=5).<sup>a</sup>
                        </p>
                    </caption>
                    <table width="557" border="1" cellpadding="5" cellspacing="0" rules="groups" frame="hsides">
                        <col width="119"/>
                        <col width="56"/>
                        <col width="66"/>
                        <col width="66"/>
                        <col width="56"/>
                        <col width="40"/>
                        <col width="82"/>
                        <thead>
                            <tr valign="top">
                                <td>Dimensions</td>
                                <td colspan="2">Rate</td>
                                <td colspan="2">Weight</td>
                                <td colspan="2">Magnitude</td>
                            </tr>
                            <tr valign="top">
                                <td>
                                    <break/>
                                </td>
                                <td>Mean</td>
                                <td>SD</td>
                                <td>Mean</td>
                                <td>SD</td>
                                <td>Mean</td>
                                <td>SD</td>
                            </tr>
                        </thead>
                        <tbody>
                            <tr valign="top">
                                <td>Mental demand</td>
                                <td>51</td>
                                <td>29.45</td>
                                <td>3.40</td>
                                <td>.89</td>
                                <td>175</td>
                                <td>110.79</td>
                            </tr>
                            <tr valign="top">
                                <td>Physical demand</td>
                                <td>10</td>
                                <td>8.66</td>
                                <td>.40</td>
                                <td>.55</td>
                                <td>2</td>
                                <td>2.74</td>
                            </tr>
                            <tr valign="top">
                                <td>Temporal demand</td>
                                <td>30</td>
                                <td>34.10</td>
                                <td>2.60</td>
                                <td>1.82</td>
                                <td>89</td>
                                <td>112.05</td>
                            </tr>
                            <tr valign="top">
                                <td>Performance</td>
                                <td>73</td>
                                <td>28.85</td>
                                <td>4</td>
                                <td>1</td>
                                <td>294</td>
                                <td>134.69</td>
                            </tr>
                            <tr valign="top">
                                <td>Effort</td>
                                <td>49</td>
                                <td>34.53</td>
                                <td>2.20</td>
                                <td>1.30</td>
                                <td>121</td>
                                <td>129.05</td>
                            </tr>
                            <tr valign="top">
                                <td>Frustration</td>
                                <td>21</td>
                                <td>19.49</td>
                                <td>2.40</td>
                                <td>1.82</td>
                                <td>72</td>
                                <td>80.44</td>
                            </tr>
                        </tbody>
                    </table>
                    <table-wrap-foot>
                        <fn id="table2fn1">
                            <p>
                                <sup>a</sup>Mean overload of the specialists is 50.20.</p>
                        </fn>
                    </table-wrap-foot>
                </table-wrap>
                <p>Students and specialists obtained the highest means in these respective dimensions: “Performance” (58.47 ± 24.19, 73 ± 28.85) and “Mental Demand” (57.10 ± 22.27, 51 ± 29.45). It should be noted that the dimension “Physical Demand” (29.27 ± 27.91, 10 ± 8.66) was identified as the one that least contributed to mental workload during the educational intervention. In general, the distribution of rates assigned by the participants for each dimension had the same behavior (<xref ref-type="fig" rid="figure3">Figure 3</xref>).</p>
                <p>The repeated measures one-way ANOVA was used to identify statistically significant differences in the mean values of the 6 dimensions of mental workload for the group of students (<xref ref-type="table" rid="table3">Table 3</xref>).</p>
                <table-wrap position="float" id="table3">
                    <label>Table 3</label>
                    <caption>
                        <p>Analysis of variance between means of assessments of the 6 dimensions of NASA TLX by students.</p>
                    </caption>
                    <table width="560" border="1" cellpadding="5" cellspacing="0" rules="groups" frame="hsides">
                        <col width="148"/>
                        <col width="66"/>
                        <col width="85"/>
                        <col width="84"/>
                        <col width="56"/>
                        <col width="59"/>
                        <thead>
                            <tr valign="top">
                                <td>Assessment sources</td>
                                <td>Sum of squares</td>
                                <td>Degrees of freedom</td>
                                <td>Mean squares</td>
                                <td>Critical factor</td>
                                <td>
                                    <italic>P</italic> value</td>
                            </tr>
                        </thead>
                        <tbody>
                            <tr valign="top">
                                <td>Factor 1 (6 dimensions)</td>
                                <td>61.053</td>
                                <td>5</td>
                                <td>13.570</td>
                                <td>29.45</td>
                                <td>&#60;.001</td>
                            </tr>
                            <tr valign="top">
                                <td>Error</td>
                                <td>153.417</td>
                                <td>333</td>
                                <td>461</td>
                                <td>—</td>
                                <td>—</td>
                            </tr>
                        </tbody>
                    </table>
                </table-wrap>
                <p>To identify the significant variables according to the ANOVA test, the Bonferroni test was applied, with no statistically significant difference in the mean values of the following pairs: “Mental Demand” and “Performance,” “Physical Demand” and “Temporal Demand,” “Physical demand” and “Frustration,” “Temporal Demand” and “Effort,” and also “Temporal Demand” and “Frustration.” For the other possible comparisons, significant differences between means were identified (<xref ref-type="table" rid="table4">Table 4</xref>).</p>
                <table-wrap position="float" id="table4">
                    <label>Table 4</label>
                    <caption>
                        <p>Comparison between the means of mental workload in 6 dimensions analyzed for students (n=75).<sup>a</sup>
                        </p>
                    </caption>
                    <table width="453" border="1" cellpadding="5" cellspacing="0" rules="groups" frame="hsides">
                        <col width="148"/>
                        <col width="85"/>
                        <col width="189"/>
                        <thead>
                            <tr valign="top">
                                <td>Dimensions</td>
                                <td>Mean</td>
                                <td>95% Confidence interval</td>
                            </tr>
                        </thead>
                        <tbody>
                            <tr valign="bottom">
                                <td>Mental demand</td>
                                <td>57.20<sup>b</sup>
                                </td>
                                <td>52.07-62.35</td>
                            </tr>
                            <tr valign="bottom">
                                <td>Physical demand</td>
                                <td>29.27<sup>c</sup>
                                </td>
                                <td>22.85-35.69</td>
                            </tr>
                            <tr valign="bottom">
                                <td>Temporal demand</td>
                                <td>37.67<sup>cd</sup>
                                </td>
                                <td>32.11-43.22</td>
                            </tr>
                            <tr valign="bottom">
                                <td>Performance</td>
                                <td>58.47<sup>b</sup>
                                </td>
                                <td>52.90-64.03</td>
                            </tr>
                            <tr valign="bottom">
                                <td>Effort</td>
                                <td>40.73<sup>d</sup>
                                </td>
                                <td>35.38-46.09</td>
                            </tr>
                            <tr valign="bottom">
                                <td>Frustration</td>
                                <td>30.60<sup>c</sup>
                                </td>
                                <td>24.20-37.00</td>
                            </tr>
                        </tbody>
                    </table>
                    <table-wrap-foot>
                        <fn id="table4fn1">
                            <p>
                                <sup>a</sup>Equal lowercase letters do not statistically differ according to the Bonferroni test.</p>
                        </fn>
                    </table-wrap-foot>
                </table-wrap>
                <p>The ANOVA test could not be performed for comparison of means of assessment of the instrument dimensions due to the small number of specialist nurses. With regard to the mental workload index (overload), the specialists’ mean was higher (50.20 ± 7.28) when compared with the students’ mean (47.87 ± 16.85). Among the students, 2 outliers were identified (S47 and S67), which had high overload means (90.33 and 100), moving away from the means of most students (<xref ref-type="fig" rid="figure4">Figure 4</xref>).</p>
                <fig id="figure3" position="float">
                    <label>Figure 3</label>
                    <caption>
                        <p>Mean rates of students (n=75) and specialists (n=5) for the NASA TLX dimensions.</p>
                    </caption>
                    <graphic xlink:href="mededu_v1i2e15_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
                </fig>
                <fig id="figure4" position="float">
                    <label>Figure 4</label>
                    <caption>
                        <p>Box plot of the students’ (n=75) and specialists’ (n=5) overload means.</p>
                    </caption>
                    <graphic xlink:href="mededu_v1i2e15_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
                </fig>
            </sec>
        </sec>
        <sec sec-type="discussion">
            <title>Discussion</title>
            <sec>
                <title>Preliminary Findings</title>
                <p>The increasing inclusion of VLOs and mobile devices for teaching and learning in nursing has also been raising concerns about the assessment of mental workload that these resources can cause in their users. The assessment of these factors becomes essential to develop new directions for technological productions using these resources.</p>
                <p>The analysis of the perceived mental workload from a given task is complex and personal, because it includes both specific characteristics of the task and the effort required for its development, which also has a direct relationship with the individual’s personal factors such as motivation, skills [<xref ref-type="bibr" rid="ref3">3</xref>].</p>
                <p>From the application of the NASA TLX instrument after an educational intervention using a VLO, we present the discussion of our results, starting with the sample characterization, followed by an analysis of individual assessment rates of the 6 dimensions of the instrument and the overload of students and specialists.</p>
                <p>During the analysis of the students’ sociodemographic characteristics, it was found that although UFSC provides free Wi-Fi networks for students, students report that they most frequently access the Internet at their own homes 63% (70/111). This finding can be explained by the fact that the students are more focused on actual classes at the university, using other spaces to complement their online learning.</p>
                <p>The fact that 7% (8/111) of the students access the Internet regularly during commuting also drew attention, confirming the global trend of mobility of individuals made possible by the use of mobile devices. Currently, technologies are very popular, especially for providing easy access to information, regardless of time or space [<xref ref-type="bibr" rid="ref25">25</xref>].</p>
                <p>Regarding rates, the 6 dimensions were similarly assessed by both students and specialists, with the dimensions “Performance” and “Mental Demand” standing out, as these had the highest assessment means.</p>
                <p>The dimension “Performance” aimed at measuring how satisfied participants felt with their own level of performance for the assessment of pain using the m-OVADor. It had the highest rate of assessment, both among students (58.47 ± 24.19) and specialists (73 ± 28.85).</p>
                <p>The assessment indicates a positive contribution of technology to learning acute pain assessment using mobile devices, suggesting it as a viable process to be used in the teaching-learning process in nursing, comparable to results of previous studies [<xref ref-type="bibr" rid="ref7">7</xref>-<xref ref-type="bibr" rid="ref9">9</xref>].</p>
                <p>By contrast, “Mental Demand” had the second highest rate, both among students (57.20 ± 22.27) and specialists (51 ± 29.45), indicating that the educational intervention through m-OVADor demanded a greater mental workload, especially regarding the amount of mental activity for execution (think, decide, remember, look, and search).</p>
                <p>This may be associated with the fact that the simulation of pain assessment in clinical scenarios requires constant involvement and judgment by the individuals, and may also be due to accessing content from mobile devices.</p>
                <p>By contrast, “Physical Demand,” defined as the amount of physical activity that the online educational intervention demands from the individual, was indicated by students (29.27 ± 27.91) and specialists (10 ± 8.66) as the dimension that contributed least to the mental workload, just as reported in a previous nursing study [<xref ref-type="bibr" rid="ref18">18</xref>].</p>
                <p>For some authors [<xref ref-type="bibr" rid="ref26">26</xref>], this result is justified by the low physical demand required for the handling of small mobile devices, which are very popular nowadays. The result is justified by both the familiarity of these individuals with access to the Internet, and the use of their own mobile devices in their academic and personal daily life, which would facilitate interaction with them. In addition, there is the project presentation, which sought to simplify the presentation layout of m-OVADor, thereby facilitating navigation through clinical scenarios.</p>
                <p>The presence of 2 outliers (S47 and S67), which were distanced from the mean of most participants, with high levels of overload from the online educational intervention, can be explained by the individual perception of these individuals at the time of data collection, occurring at the end of the school year, when all students had examinations and term papers.</p>
                <p>It must be considered that the assessment of mental workload is considered a complex and personal function, which also involves the specific characteristics of the task, the effort for its performance, and the direct relationship between motivation, emotional state, skills, among others [<xref ref-type="bibr" rid="ref7">7</xref>].</p>
                <p>Still, workload can be influenced by the nature of the task, especially when considering that the educational intervention with m-OVADor included access to 3 different clinical scenarios, in which there were a diversity of resources (animation, short texts with definitions and/or descriptions, assessment exercises, and also the clinical judgment to determine nursing diagnoses and interventions), required for the assessment of each presented context.</p>
                <p>One should also consider that mental workload generated from VLOs can relate to factors such as the complexity of the individual, resources used for the development of the activity, and factors unrelated to content, such as the multimedia resources used [<xref ref-type="bibr" rid="ref27">27</xref>-<xref ref-type="bibr" rid="ref29">29</xref>].</p>
                <p>In this sense, one should also reflect on increasing access to mobile devices with Internet access, and the potential of these technologies when applied in the teaching-learning process in nursing.</p>
                <p>Arguably these technologies have become popular, especially for the flexibility that allows users to quickly access information at any place or time, to expand their potential to transform individuals’ attitudes and behaviors [<xref ref-type="bibr" rid="ref30">30</xref>,<xref ref-type="bibr" rid="ref31">31</xref>].</p>
                <p>Specifically in the context of this study, an assessment of mental workload has indicated that the use of these technologies is feasible in higher education in nursing, and can influence the way students build their knowledge, establishing an innovative process for teaching/learning.</p>
            </sec>
            <sec>
                <title>Conclusions</title>
                <p>The NASA TLX instrument demonstrated applicability for the assessment of mental workload from the use of m-OVADor. The results enabled analysis of the sources related to students’ and specialists’ mental workload from the use of a VLO accessed from small mobile devices. The analysis revealed that the dimension “Mental demand” obtained a greater weight both from the students’ and specialists’ perceptions, which is expected from technologies for online learning, due to the level of attention that they require. Although the study achieved the proposed aim, in no way did it intend to exhaust the possible approaches to the theme. The development of other studies to explore the theme from other perspectives is recommended, to deepen our understanding of the effects of mental workload on learning outcomes and even when planning new strategies for mobile learning in nursing education as well as for validation of the instrument in Brazil.</p>
            </sec>
        </sec>
    </body>
    <back>
        <app-group>
            <app id="app1">
                <title>Multimedia Appendix 1</title>
                <p>Nasa Task Load Index Form.</p>
                <media xlink:href="mededu_v1i2e15_app1.png" xlink:title="PNG File, 70KB"/>
            </app>
            <app id="app2">
                <title>Multimedia Appendix 2</title>
                <p>m-OVADor cover page.</p>
                <media xlink:href="mededu_v1i2e15_app2.png" xlink:title="PNG File, 279KB"/>
            </app>
            <app id="app3">
                <title>Multimedia Appendix 3</title>
                <p>Clinical scenarios selection by m-OVADor.</p>
                <media xlink:href="mededu_v1i2e15_app3.png" xlink:title="PNG File, 285KB"/>
            </app>
        </app-group>
        <glossary>
            <title>Abbreviations</title>
            <def-list>
                <def-item>
                    <term id="abb1">ANOVA</term>
                    <def>
                        <p>analysis of variance</p>
                    </def>
                </def-item>
                <def-item>
                    <term id="abb2">ICTs</term>
                    <def>
                        <p>information and communications technologies</p>
                    </def>
                </def-item>
                <def-item>
                    <term id="abb3">m-OVADor</term>
                    <def>
                        <p>Mobile Virtual Learning Object for the Assessment of Acute Pain</p>
                    </def>
                </def-item>
                <def-item>
                    <term id="abb4">NASA TLX</term>
                    <def>
                        <p>NASA Task Load Index</p>
                    </def>
                </def-item>
                <def-item>
                    <term id="abb5">VLO</term>
                    <def>
                        <p>virtual learning object</p>
                    </def>
                </def-item>
                <def-item>
                    <term id="abb6">UFSC</term>
                    <def>
                        <p>Universidade Federal de Santa Catarina</p>
                    </def>
                </def-item>
            </def-list>
        </glossary>
        <ack>
            <p>We thank the International Association for the Study of Pain for the grant awarded for technological production; CNPQ for providing the internship at the School of Biomedical Informatics at University of Texas Health Science Center through the SWE grant, Science Without Borders Program, and CAPES for the doctoral grant awarded during the research development.</p>
        </ack>
        <fn-group>
            <fn fn-type="conflict">
                <p>None declared.</p>
            </fn>
        </fn-group>
        <ref-list>
            <ref id="ref1">
                <label>1</label>
                <nlm-citation citation-type="web">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Garcia</surname>
                            <given-names>PS</given-names>
                        </name>
                    </person-group>
                    <source>A Internet Como Nova Mídia na Educação</source>
                    <year>2010</year>
                    <month>8</month>
                    <day>19</day>
                    <access-date>2015-08-20</access-date>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.educadores.diaadia.pr.gov.br/arquivos/File/2010/artigos_teses/EAD/NOVAMIDIA.PDF">http://www.educadores.diaadia.pr.gov.br/arquivos/File/2010/artigos_teses/EAD/NOVAMIDIA.PDF</ext-link>
                        <ext-link ext-link-type="webcite" xlink:href="6aueJEJDm"/>
                    </comment>
                    <!-- <pub-id pub-id-type="other">6aueJEJDm</pub-id> -->
                </nlm-citation>
            </ref>
            <ref id="ref2">
                <label>2</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Schibeci</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lake</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Phillips</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lowe</surname>
                            <given-names>K</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Cummings</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Miller</surname>
                            <given-names>E</given-names>
                        </name>
                    </person-group>
                    <article-title>Evaluating the use of learning objects in Australian and New Zealand schools</article-title>
                    <source>Comput Educ</source>
                    <year>2008</year>
                    <month>1</month>
                    <volume>50</volume>
                    <issue>1</issue>
                    <fpage>271</fpage>
                    <lpage>283</lpage>
                    <pub-id pub-id-type="doi">10.1016/j.compedu.2006.05.006</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref3">
                <label>3</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>García-Barriocanal</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Sicilia</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lytras</surname>
                            <given-names>M</given-names>
                        </name>
                    </person-group>
                    <article-title>Evaluating pedagogical classification frameworks for learning objects: A case study</article-title>
                    <source>Comput Hum Behav</source>
                    <year>2007</year>
                    <month>11</month>
                    <volume>23</volume>
                    <issue>6</issue>
                    <fpage>2641</fpage>
                    <lpage>2655</lpage>
                    <pub-id pub-id-type="doi">10.1016/j.chb.2006.08.001</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref4">
                <label>4</label>
                <nlm-citation citation-type="web">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Borgman</surname>
                            <given-names>C</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Abelson</surname>
                            <given-names>H</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Dirks</surname>
                            <given-names>L</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Johson</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Koedinger</surname>
                            <given-names>KR</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Linn</surname>
                            <given-names>MC</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lynch</surname>
                            <given-names>CA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Oblinger</surname>
                            <given-names>DG</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Pea</surname>
                            <given-names>RD</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Salen</surname>
                            <given-names>K</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Smith</surname>
                            <given-names>MS</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Szalay</surname>
                            <given-names>A</given-names>
                        </name>
                    </person-group>
                    <source>Fostering Learning in the Networked World: The Cyberlearning Opportunity and Challenge</source>
                    <year>2006</year>
                    <month>6</month>
                    <day>24</day>
                    <access-date>2015-08-20</access-date>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.nsf.gov/pubs/2008/nsf08204/nsf08204.pdf">http://www.nsf.gov/pubs/2008/nsf08204/nsf08204.pdf</ext-link>
                        <ext-link ext-link-type="webcite" xlink:href="6aufF3WAF"/>
                    </comment>
                    <!-- <pub-id pub-id-type="other">6aufF3WAF</pub-id> -->
                </nlm-citation>
            </ref>
            <ref id="ref5">
                <label>5</label>
                <nlm-citation citation-type="web">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Hart</surname>
                            <given-names>SG</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Battiste</surname>
                            <given-names>V</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lester</surname>
                            <given-names>PT</given-names>
                        </name>
                    </person-group>
                    <source>POPCORN: A Supervisory Control Simulation for Workload and Performance Research</source>
                    <year>1984</year>
                    <month>8</month>
                    <day>19</day>
                    <access-date>2015-08-19</access-date>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://ntrs.nasa.gov/search.jsp?R=19850006206">http://ntrs.nasa.gov/search.jsp?R=19850006206</ext-link>
                        <ext-link ext-link-type="webcite" xlink:href="6aufWo97M"/>
                    </comment>
                    <!-- <pub-id pub-id-type="other">6aufWo97M</pub-id> -->
                </nlm-citation>
            </ref>
            <ref id="ref6">
                <label>6</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Diniz</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Guimarães</surname>
                            <given-names>LBM</given-names>
                        </name>
                    </person-group>
                    <person-group person-group-type="editor">
                        <name name-style="western">
                            <surname>Guimarães</surname>
                            <given-names>LBM</given-names>
                        </name>
                    </person-group>
                    <article-title>Avaliação da carga mental de trabalho mental</article-title>
                    <source>Ergonomia Cognitiva</source>
                    <year>2004</year>
                    <publisher-loc>Porto Alegre, Rio Grande do Sul, Brazil</publisher-loc>
                    <publisher-name>FEENG</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref7">
                <label>7</label>
                <nlm-citation citation-type="web">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Corrêa</surname>
                            <given-names>FP</given-names>
                        </name>
                    </person-group>
                    <source>Carga Mental e Ergonomia</source>
                    <year>2003</year>
                    <access-date>2015-08-20</access-date>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="https://repositorio.ufsc.br/bitstream/handle/123456789/86036/191674.pdf?sequence=1">https://repositorio.ufsc.br/bitstream/handle/123456789/86036/191674.pdf?sequence=1</ext-link>
                        <ext-link ext-link-type="webcite" xlink:href="6aufx2TJB"/>
                    </comment>
                    <!-- <pub-id pub-id-type="other">6aufx2TJB</pub-id> -->
                </nlm-citation>
            </ref>
            <ref id="ref8">
                <label>8</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Stanton</surname>
                            <given-names>N</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hedge</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Brookhuis</surname>
                            <given-names>K</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Salas</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hendrick</surname>
                            <given-names>Hw</given-names>
                        </name>
                    </person-group>
                    <article-title>Mental workload</article-title>
                    <source>Handbook of Human Factors and Ergonomics Methods</source>
                    <year>2005</year>
                    <publisher-loc>Boca Raton, FL</publisher-loc>
                    <publisher-name>CRC Press</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref9">
                <label>9</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Perry</surname>
                            <given-names>C</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Sheik-Nainar</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Segall</surname>
                            <given-names>N</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ma</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kaber</surname>
                            <given-names>D</given-names>
                        </name>
                    </person-group>
                    <article-title>Effects of physical workload on cognitive task performance and situation awareness</article-title>
                    <source>Theor Issues Ergon Sci</source>
                    <year>2008</year>
                    <month>3</month>
                    <volume>9</volume>
                    <issue>2</issue>
                    <fpage>95</fpage>
                    <lpage>113</lpage>
                    <pub-id pub-id-type="doi">10.1080/14639220600959237</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref10">
                <label>10</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cardoso</surname>
                            <given-names>MS</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Gontijo</surname>
                            <given-names>LA</given-names>
                        </name>
                    </person-group>
                    <article-title>Avaliação da carga mental de trabalho e do desempenho de medidas de mensuração: NASA TLX e SWAT</article-title>
                    <source>Gest Prod</source>
                    <year>2012</year>
                    <month>12</month>
                    <volume>19</volume>
                    <issue>4</issue>
                    <fpage>873</fpage>
                    <lpage>884</lpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.scielo.br/pdf/gp/v19n4/a15v19n4.pdf"/>
                    </comment>
                    <pub-id pub-id-type="doi">10.1590/S0104-530X2012000400015</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref11">
                <label>11</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Hart</surname>
                            <given-names>SG</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Staveland</surname>
                            <given-names>LE</given-names>
                        </name>
                    </person-group>
                    <article-title>Development of NASA-TLX (Task Load Index): Results of empirical and theoretical research</article-title>
                    <source>Adv Psychol</source>
                    <year>1988</year>
                    <volume>52</volume>
                    <fpage>139</fpage>
                    <lpage>183</lpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://humansystems.arc.nasa.gov/groups/tlx/downloads/NASA-TLXChapter.pdf"/>
                    </comment>
                    <pub-id pub-id-type="doi">10.1016/S0166-4115(08)62386-9</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref12">
                <label>12</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Hart</surname>
                            <given-names>SG</given-names>
                        </name>
                    </person-group>
                    <article-title>Nasa-Task Load Index (NASA-TLX); 20 Years Later</article-title>
                    <source>Proc Hum Fact Ergon Soc Annu Meet</source>
                    <year>2006</year>
                    <month>10</month>
                    <day>01</day>
                    <volume>50</volume>
                    <issue>9</issue>
                    <fpage>904</fpage>
                    <lpage>908</lpage>
                    <pub-id pub-id-type="doi">10.1177/154193120605000909</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref13">
                <label>13</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Ayres</surname>
                            <given-names>P</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Gog</surname>
                            <given-names>TV</given-names>
                        </name>
                    </person-group>
                    <article-title>State of the art research into cognitive load theory</article-title>
                    <source>Comput Hum Behav</source>
                    <year>2009</year>
                    <month>3</month>
                    <volume>25</volume>
                    <issue>2</issue>
                    <fpage>253</fpage>
                    <lpage>257</lpage>
                    <pub-id pub-id-type="doi">10.1016/j.chb.2008.12.007</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref14">
                <label>14</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mayer</surname>
                            <given-names>RE</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Moreno</surname>
                            <given-names>R</given-names>
                        </name>
                    </person-group>
                    <article-title>A split-attention effect in multimedia learning: Evidence for dual processing systems in working memory</article-title>
                    <source>J Educ Psychol</source>
                    <year>1998</year>
                    <volume>90</volume>
                    <issue>2</issue>
                    <fpage>312</fpage>
                    <lpage>320</lpage>
                    <pub-id pub-id-type="doi">10.1037/0022-0663.90.2.312</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref15">
                <label>15</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Sweller</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Van</surname>
                            <given-names>MJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Paas</surname>
                            <given-names>F</given-names>
                        </name>
                    </person-group>
                    <article-title>Cognitive architecture and instructional design</article-title>
                    <source>Educ Psychol Rev</source>
                    <year>1998</year>
                    <month>9</month>
                    <volume>10</volume>
                    <issue>3</issue>
                    <fpage>251</fpage>
                    <lpage>296</lpage>
                    <pub-id pub-id-type="doi">10.1023/A:1022193728205</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref16">
                <label>16</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Wiebe</surname>
                            <given-names>EN</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Roberts</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Behrend</surname>
                            <given-names>TS</given-names>
                        </name>
                    </person-group>
                    <article-title>An examination of two mental workload measurement approaches to understanding multimedia learning</article-title>
                    <source>Comput Hum Behav</source>
                    <year>1998</year>
                    <volume>10</volume>
                    <issue>3</issue>
                    <fpage>474</fpage>
                    <lpage>481</lpage>
                    <pub-id pub-id-type="doi">10.1023/A:1022193728205</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref17">
                <label>17</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mayer</surname>
                            <given-names>RE</given-names>
                        </name>
                    </person-group>
                    <article-title>Elements of a science of e-learning</article-title>
                    <source>J Educ Comput Res</source>
                    <year>2003</year>
                    <volume>29</volume>
                    <issue>3</issue>
                    <fpage>297</fpage>
                    <lpage>313</lpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://cecs5580.pbworks.com/f/elements+of+a+science+of+e-learning.pdf"/>
                    </comment>
                </nlm-citation>
            </ref>
            <ref id="ref18">
                <label>18</label>
                <nlm-citation citation-type="confproc">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Sasso</surname>
                            <given-names>GTM</given-names>
                        </name>
                    </person-group>
                    <article-title>Mobile learning object in CPR—Advanced cardiac life support: An application of the persuasive technology in nursing</article-title>
                    <source>Congresso Nacional de Ambientes Hipermídia para Aprendizagem</source>
                    <year>2009</year>
                    <conf-name>4º Congresso Nacional de Ambientes Hipermídia para Aprendizagem (4º Conahpa)</conf-name>
                    <conf-date>Nov 05, 2009</conf-date>
                    <conf-loc>Florianópolis, Brazil</conf-loc>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://wright.ava.ufsc.br/~alice/conahpa/anais/2009/"/>
                    </comment>
                </nlm-citation>
            </ref>
            <ref id="ref19">
                <label>19</label>
                <nlm-citation citation-type="web">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Barra</surname>
                            <given-names>DCC</given-names>
                        </name>
                    </person-group>
                    <source>Computadorized Nursing Process and Patient Safety in Intensive Care from the ICNP Version 1</source>
                    <year>2012</year>
                    <access-date>2015-09-07</access-date>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="https://repositorio.ufsc.br/handle/123456789/100938">https://repositorio.ufsc.br/handle/123456789/100938</ext-link>
                        <ext-link ext-link-type="webcite" xlink:href="6bNaave9Q"/>
                    </comment>
                    <!-- <pub-id pub-id-type="other">6bNaave9Q</pub-id> -->
                </nlm-citation>
            </ref>
            <ref id="ref20">
                <label>20</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Amalberti</surname>
                            <given-names>R</given-names>
                        </name>
                    </person-group>
                    <article-title>La Conduite de systèmes à risques</article-title>
                    <source>Le Travail Humain</source>
                    <year>2001</year>
                    <publisher-loc>Paris, France</publisher-loc>
                    <publisher-name>Presses Universitaires de France</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref21">
                <label>21</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cao</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Chintamani</surname>
                            <given-names>KK</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Pandya</surname>
                            <given-names>AK</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ellis</surname>
                            <given-names>RD</given-names>
                        </name>
                    </person-group>
                    <article-title>NASA TLX: Software for assessing subjective mental workload</article-title>
                    <source>Behav Res Methods</source>
                    <year>2009</year>
                    <volume>41</volume>
                    <issue>1</issue>
                    <fpage>113</fpage>
                    <lpage>117</lpage>
                    <pub-id pub-id-type="doi">10.3758/BRM.41.1.113</pub-id>
                    <pub-id pub-id-type="medline">19182130</pub-id>
                    <pub-id pub-id-type="pii">41/1/113</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref22">
                <label>22</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Abdellah</surname>
                            <given-names>FG</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Levine</surname>
                            <given-names>E</given-names>
                        </name>
                    </person-group>
                    <article-title>Better patient care through nursing research</article-title>
                    <source>Int J Nurs Stud</source>
                    <year>1965</year>
                    <month>4</month>
                    <volume>2</volume>
                    <issue>1</issue>
                    <fpage>1</fpage>
                    <lpage>12</lpage>
                    <pub-id pub-id-type="doi">10.1016/0020-7489(65)90013-1</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref23">
                <label>23</label>
                <nlm-citation citation-type="web">
                    <person-group person-group-type="author">
                        <collab>Human Performance Research Group (NASA Ames Research Center)</collab>
                    </person-group>
                    <source>NASA Task Load Index (NASA TLX): Paper-and-Pencil Package (Version 1)</source>
                    <year>1986</year>
                    <access-date>2013-02-10</access-date>
                    <publisher-loc>Washington, DC</publisher-loc>
                    <publisher-name>NASA</publisher-name>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://humansystems.arc.nasa.gov/groups/tlx/downloads/TLX_pappen_manual.pdf">http://humansystems.arc.nasa.gov/groups/tlx/downloads/TLX_pappen_manual.pdf</ext-link>
                        <ext-link ext-link-type="webcite" xlink:href="6bNb0ck9O"/>
                    </comment>
                    <!-- <pub-id pub-id-type="other">6bNb0ck9O</pub-id> -->
                </nlm-citation>
            </ref>
            <ref id="ref24">
                <label>24</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Saccol</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Schlemmer</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Barbosa</surname>
                            <given-names>J</given-names>
                        </name>
                    </person-group>
                    <source>M-Learning e u-Learning: Novas Perspectivas da Aprendizagem Móvel e Ubíqua</source>
                    <year>2011</year>
                    <publisher-loc>São Paulo, Brazil</publisher-loc>
                    <publisher-name>Pearson Prentice Hall</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref25">
                <label>25</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Clay</surname>
                            <given-names>CA</given-names>
                        </name>
                    </person-group>
                    <article-title>Exploring the use of mobile technologies for the acquisition of clinical skills</article-title>
                    <source>Nurse Educ Today</source>
                    <year>2011</year>
                    <month>08</month>
                    <volume>31</volume>
                    <issue>6</issue>
                    <fpage>582</fpage>
                    <lpage>586</lpage>
                    <pub-id pub-id-type="doi">10.1016/j.nedt.2010.10.011</pub-id>
                    <pub-id pub-id-type="medline">21112132</pub-id>
                    <pub-id pub-id-type="pii">S0260-6917(10)00194-2</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref26">
                <label>26</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Iyengar</surname>
                            <given-names>MS</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Florez-Arango</surname>
                            <given-names>JF</given-names>
                        </name>
                    </person-group>
                    <article-title>Decreasing workload among community health workers using interactive, structured, rich-media guidelines on smartphones</article-title>
                    <source>Technol Health Care</source>
                    <year>2013</year>
                    <volume>21</volume>
                    <issue>2</issue>
                    <fpage>113</fpage>
                    <lpage>123</lpage>
                    <pub-id pub-id-type="doi">10.3233/THC-130713</pub-id>
                    <pub-id pub-id-type="medline">23510972</pub-id>
                    <pub-id pub-id-type="pii">9T83393414644065</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref27">
                <label>27</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Van Merriënboer</surname>
                            <given-names>JJG</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Sweller</surname>
                            <given-names>J</given-names>
                        </name>
                    </person-group>
                    <article-title>Cognitive load theory and complex learning: Recent developments and future directions</article-title>
                    <source>Educ Psychol Rev</source>
                    <year>2005</year>
                    <month>6</month>
                    <volume>17</volume>
                    <issue>2</issue>
                    <fpage>147</fpage>
                    <lpage>177</lpage>
                    <pub-id pub-id-type="doi">10.1007/s10648-005-3951-0</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref28">
                <label>28</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Rowe</surname>
                            <given-names>G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hirsh</surname>
                            <given-names>JB</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Anderson</surname>
                            <given-names>AK</given-names>
                        </name>
                    </person-group>
                    <article-title>Positive affect increases the breadth of attentional selection</article-title>
                    <source>Proc Natl Acad Sci USA</source>
                    <year>2007</year>
                    <volume>104</volume>
                    <issue>1</issue>
                    <fpage>383</fpage>
                    <lpage>388</lpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.pnas.org/cgi/pmidlookup?view=long&#38;pmid=17182749"/>
                    </comment>
                    <pub-id pub-id-type="doi">10.1073/pnas.0605198104</pub-id>
                    <pub-id pub-id-type="medline">17182749</pub-id>
                    <pub-id pub-id-type="pii">0605198104</pub-id>
                    <pub-id pub-id-type="pmcid">PMC1765470</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref29">
                <label>29</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Patel</surname>
                            <given-names>VL</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Arocha</surname>
                            <given-names>JF</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Diermeier</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Greenes</surname>
                            <given-names>RA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Shortliffe</surname>
                            <given-names>EH</given-names>
                        </name>
                    </person-group>
                    <article-title>Methods of cognitive analysis to support the design and evaluation of biomedical systems: The case of clinical practice guidelines</article-title>
                    <source>J Biomed Inform</source>
                    <year>2001</year>
                    <volume>34</volume>
                    <issue>1</issue>
                    <fpage>52</fpage>
                    <lpage>66</lpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://linkinghub.elsevier.com/retrieve/pii/S1532-0464(01)91002-0"/>
                    </comment>
                    <pub-id pub-id-type="doi">10.1006/jbin.2001.1002</pub-id>
                    <pub-id pub-id-type="medline">11376543</pub-id>
                    <pub-id pub-id-type="pii">S1532-0464(01)91002-0</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref30">
                <label>30</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Fogg</surname>
                            <given-names>BJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Eckles</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Bogost</surname>
                            <given-names>I</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Consolvo</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Holmen</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Spasojevic</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ulm</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Tanguay</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Walker</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>White</surname>
                            <given-names>S</given-names>
                        </name>
                    </person-group>
                    <source>Mobile Persuasion: 20 Perspectives on the Future of Behavior Change</source>
                    <year>2007</year>
                    <publisher-loc>Standford, CA</publisher-loc>
                    <publisher-name>Stanford Captology Media</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref31">
                <label>31</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Fogg</surname>
                            <given-names>BJ</given-names>
                        </name>
                    </person-group>
                    <source>Persuasive Technology: Using Computers to Change What We Think and Do</source>
                    <year>2002</year>
                    <publisher-loc>Stanford, CA</publisher-loc>
                    <publisher-name>Morgan Kaufmann</publisher-name>
                    <fpage>1</fpage>
                    <lpage>312</lpage>
                </nlm-citation>
            </ref>
        </ref-list>
    </back>
</article>
