@Article{info:doi/10.2196/67552,
author="Chen, Wei-Pin
and Teng, Wei-Guang
and Kuo, Benson C.
and Yen, Yu-Jui
and Lian, Jian-Yu
and Sing, Matthew
and Chen, Peng-Ting",
title="Regulatory Insights From 27 Years of Artificial Intelligence/Machine Learning--Enabled Medical Device Recalls in the United States: Implications for Future Governance",
journal="JMIR Med Inform",
year="2025",
month="Jul",
day="11",
volume="13",
pages="e67552",
keywords="AI/ML-enabled medical device",
keywords="recall",
keywords="SaMD",
keywords="510(k)",
keywords="FDA",
keywords="Food and Drug Administration",
keywords="artificial intelligence/machine learning",
abstract="Background: Artificial intelligence/machine learning (AI/ML) has revolutionized the health care industry, particularly in the development and use of medical devices. The US Food and Drug Administration (FDA) has authorized over 878 AI/ML--enabled medical devices, reflecting a growing trend in both quantity and application scope. Understanding the distinct challenges these devices present in terms of FDA regulation violations is crucial for effectively avoiding recalls. This is particularly pertinent for proactive measures regarding medical devices. Objective: This study explores the impact of AI/ML on medical device recalls, focusing on the distinct causes associated with AI/ML--enabled devices compared with other device types. Recall information associated with 510(k)-cleared devices was obtained from openFDA. Three recall cohorts were established: ``All 510(k) devices recall,'' ``software-related devices recall,'' and ``AI/ML devices recall.'' Methods: Recall information for 510(k)-cleared devices was obtained from openFDA. AI/ML-enabled medical devices were identified based on FDA listings. Three cohorts were established: ``All 510(k) devices recall,'' ``software-related devices recall,'' and ``AI/ML devices recall.'' Root cause analysis was conducted for each recall event. Results: The results indicate that while the top 5 recall root causes are relatively similar across the 3 control groups, the proportions vary, with AI/ML devices showing a higher impact for 87\% of all recalls. Design and development--related factors play a significant role in recalls of AI/ML devices with root causes related to device design and software design accounting for 50\% of recalls, emphasizing the importance of thorough planning, user feedback incorporation, and validation during the development process to reduce the probability of recalls. In addition, changes in software, including design changes and control changes, also contribute substantially to recalls in AI/ML devices. Conclusions: In conclusion, this study provides valuable insights into the unique challenges and considerations associated with AI/ML--enabled medical device recalls, offering guidance for manufacturers to enhance verification plans and mitigate risks in this rapidly evolving technological landscape. ",
doi="10.2196/67552",
url="https://medinform.jmir.org/2025/1/e67552"
}


@Article{info:doi/10.2196/60399,
author="Cho, Yunah
and Talboys, L. Sharon",
title="Trends in South Korean Medical Device Development for Attention-Deficit/Hyperactivity Disorder and Autism Spectrum Disorder: Narrative Review",
journal="JMIR Biomed Eng",
year="2024",
month="Oct",
day="15",
volume="9",
pages="e60399",
keywords="ADHD",
keywords="attention-deficit/hyperactivity disorder",
keywords="ASD",
keywords="autism spectrum disorder",
keywords="medical device",
keywords="digital therapeutics",
abstract="Background: Attention-deficit/hyperactivity disorder (ADHD) and autism spectrum disorder (ASD) are among the most prevalent mental disorders among school-aged youth in South Korea and may play a role in the increasing pressures on teachers and school-based special education programming. A lack of support for special education; tensions between teachers, students, and parents; and limited backup for teacher absences are common complaints among Korean educators. New innovations in technology to screen and treat ADHD and ASD may offer relief to students, parents, and teachers through earlier and efficient diagnosis; access to treatment options; and ultimately, better-managed care and expectations. Objective: This narrative literature review provides an account of medical device use and development in South Korea for the diagnosis and management of ADHD and ASD and highlights research gaps. Methods: A narrative review was conducted across 4 databases (PubMed, Korean National Assembly Library, Scopus, and PsycINFO). Journal articles, dissertations, and government research and development reports were included if they discussed medical devices for ADHD and ASD. Only Korean or English papers were included. Resources were excluded if they did not correspond to the research objective or did not discuss at least 1 topic about medical devices for ADHD and ASD. Journal articles were excluded if they were not peer reviewed. Resources were limited to publications between 2013 and July 22, 2024. Results: A total of 1794 records about trends in Korean medical device development were categorized into 2 major groups: digital therapeutics and traditional therapy. Digital therapeutics resulted in 5 subgroups: virtual reality and artificial intelligence, machine learning and robot, gaming and visual contents, eye-feedback and movement intervention, and electroencephalography and neurofeedback. Traditional therapy resulted in 3 subgroups: cognitive behavioral therapy and working memory; diagnosis and rating scale; and musical, literary therapy, and mindfulness-based stress reduction. Digital therapeutics using artificial intelligence, machine learning, and electroencephalography technologies account for the biggest portions of development in South Korea, rather than traditional therapies. Most resources, 94.15\% (1689/1794), were from the Korean National Assembly Library. Conclusions: Limitations include small sizes of populations to conclude findings in many articles, a lower number of articles discussing medical devices for ASD, and a majority of articles being dissertations. Emerging digital medical devices and those integrated with traditional therapies are important solutions to reducing the prevalence rates of ADHD and ASD in South Korea by promoting early diagnosis and intervention. Furthermore, their application will relieve pressures on teachers and school-based special education programming by providing direct supporting resources to students with ADHD or ASD. Future development of medical devices for ADHD and ASD is predicted to heavily rely on digital technologies, such as those that sense people's behaviors, eye movement, and brainwaves. ",
doi="10.2196/60399",
url="https://biomedeng.jmir.org/2024/1/e60399"
}


@Article{info:doi/10.2196/62769,
author="Singh, Sanidhya
and Bennett, Romney Miles
and Chen, Chen
and Shin, Sooyoon
and Ghanbari, Hamid
and Nelson, W. Benjamin",
title="Impact of Skin Pigmentation on Pulse Oximetry Blood Oxygenation and Wearable Pulse Rate Accuracy: Systematic Review and Meta-Analysis",
journal="J Med Internet Res",
year="2024",
month="Oct",
day="10",
volume="26",
pages="e62769",
keywords="photoplethysmography",
keywords="pulse oximetry",
keywords="arterial blood gas",
keywords="skin tone",
keywords="skin pigmentation",
keywords="bias",
keywords="digital technology",
abstract="Background: Photoplethysmography (PPG) is a technology routinely used in clinical practice to assess blood oxygenation (SpO2) and pulse rate (PR). Skin pigmentation may influence accuracy, leading to health outcomes disparities. Objective: This systematic review and meta-analysis primarily aimed to evaluate the accuracy of PPG-derived SpO2 and PR by skin pigmentation. Secondarily, we aimed to evaluate statistical biases and the clinical relevance of PPG-derived SpO2 and PR according to skin pigmentation. Methods: We identified 23 pulse oximetry studies (n=59,684; 197,353 paired SpO2-arterial blood observations) and 4 wearable PR studies (n=176; 140,771 paired PPG-electrocardiography observations). We evaluated accuracy according to skin pigmentation group by comparing SpO2 accuracy root-mean-square values to the regulatory threshold of 3\% and PR 95\% limits of agreement values to +5 or --5 beats per minute (bpm), according to the standards of the American National Standards Institute, Association for the Advancement of Medical Instrumentation, and the International Electrotechnical Commission. We evaluated biases and clinical relevance using mean bias and 95\% CI. Results: For SpO2, accuracy root-mean-square values were 3.96\%, 4.71\%, and 4.15\%, and pooled mean biases were 0.70\% (95\% CI 0.17\%-1.22\%), 0.27\% (95\% CI --0.64\% to 1.19\%), and 1.27\% (95\% CI 0.58\%-1.95\%) for light, medium, and dark pigmentation, respectively. For PR, 95\% limits of agreement values were from --16.02 to 13.54, from --18.62 to 16.84, and from --33.69 to 32.54, and pooled mean biases were --1.24 (95\% CI --5.31 to 2.83) bpm, --0.89 (95\% CI --3.70 to 1.93) bpm, and --0.57 (95\% CI --9.44 to 8.29) bpm for light, medium, and dark pigmentation, respectively. Conclusions: SpO2 and PR measurements may be inaccurate across all skin pigmentation groups, breaching U.S. Food and Drug Administration guidance and industry standard thresholds. Pulse oximeters significantly overestimate SpO2 for both light and dark skin pigmentation, but this overestimation may not be clinically relevant. PRs obtained from wearables exhibit no statistically or clinically significant bias based on skin pigmentation. ",
doi="10.2196/62769",
url="https://www.jmir.org/2024/1/e62769"
}


@Article{info:doi/10.2196/56034,
author="Ahmed, Wasim
and Hardey, Mariann
and Winters, David Bradford
and Sarwal, Aarti",
title="Racial Biases Associated With Pulse Oximetry: Longitudinal Social Network Analysis of Social Media Advocacy Impact",
journal="J Med Internet Res",
year="2024",
month="Oct",
day="8",
volume="26",
pages="e56034",
keywords="social media",
keywords="X",
keywords="racial biases",
keywords="pulse oximetry",
keywords="advocacy",
keywords="impact",
keywords="awareness",
keywords="racial",
keywords="bias",
keywords="biases",
keywords="longitudinal study",
keywords="information",
keywords="dissemination",
keywords="disparity",
keywords="disparities",
keywords="accuracy",
keywords="social network analysis",
keywords="Academic Track application programming interface",
keywords="API",
abstract="Background: Pulse oximetry is a noninvasive method widely used in critical care and various clinical settings to monitor blood oxygen saturation. During the COVID-19 pandemic, its application for at-home oxygen saturation monitoring became prevalent. Further investigations found that pulse oximetry devices show decreased accuracy when used on individuals with darker skin tones. This study aimed to investigate the influence of X (previously known as Twitter) on the dissemination of information and the extent to which it raised health care sector awareness regarding racial disparities in pulse oximetry. Objective: This study aimed to explore the impact of social media, specifically X, on increasing awareness of racial disparities in the accuracy of pulse oximetry and to map this analysis against the evolution of published literature on this topic. Methods: We used social network analysis drawing upon Network Overview Discovery and Exploration for Excel Pro (NodeXL Pro; Social Media Research Foundation) to examine the impact of X conversations concerning pulse oximetry devices. Searches were conducted using the Twitter Academic Track application programming interface (as it was known then). These searches were performed each year (January to December) from 2012 to 2022 to cover 11 years with up to 52,052 users, generating 188,051 posts. We identified the nature of influencers in this field and monitored the temporal dissemination of information about social events and regulatory changes. Furthermore, our social media analysis was mapped against the evolution of published literature on this topic, which we located using PubMed. Results: Conversations on X increased health care awareness of racial bias in pulse oximetry. They also facilitated the rapid dissemination of information, attaining a substantial audience within a compressed time frame, which may have impacted regulatory action announced concerning the investigation of racial biases in pulse oximetry. This increased awareness led to a surge in scientific research on the subject, highlighting a growing recognition of the necessity to understand and address these disparities in medical technology and its usage. Conclusions: Social media platforms such as X enabled researchers, health experts, patients, and the public to rapidly share information, increasing awareness of potential racial bias. These platforms also helped connect individuals interested in these topics and facilitated discussions that spurred further research. Our research provides a basis for understanding the role of X and other social media platforms in spreading health-related information about potential biases in medical devices such as pulse oximeters. ",
doi="10.2196/56034",
url="https://www.jmir.org/2024/1/e56034"
}


@Article{info:doi/10.2196/55466,
author="Herrera, Nierva Claire
and Gimenes, Escobar Fernanda Raphael
and Herrera, Paulo Jo{\~a}o
and Cavalli, Ricardo",
title="Development of Automated Triggers in Ambulatory Settings in Brazil: Protocol for a Machine Learning--Based Design Thinking Study",
journal="JMIR Res Protoc",
year="2024",
month="Aug",
day="12",
volume="13",
pages="e55466",
keywords="machine learning",
keywords="ambulatory care",
keywords="patient safety",
keywords="medical records systems",
keywords="computerized",
keywords="technology",
keywords="quality of care",
keywords="automated triggers",
keywords="limitation",
keywords="predict",
keywords="potential risk",
keywords="outpatient",
keywords="ambulatory patient",
keywords="walk-in",
keywords="adverse events",
keywords="evidence-based",
keywords="preventive",
keywords="low-income countries",
keywords="middle-income countries",
keywords="data",
keywords="scarcity",
keywords="standardization",
keywords="quality intervention",
abstract="Background: The use of technologies has had a significant impact on patient safety and the quality of care and has increased globally. In the literature, it has been reported that people die annually due to adverse events (AEs), and various methods exist for investigating and measuring AEs. However, some methods have a limited scope, data extraction, and the need for data standardization. In Brazil, there are few studies on the application of trigger tools, and this study is the first to create automated triggers in ambulatory care. Objective: This study aims to develop a machine learning (ML)--based automated trigger for outpatient health care settings in Brazil. Methods: A mixed methods research will be conducted within a design thinking framework and the principles will be applied in creating the automated triggers, following the stages of (1) empathize and define the problem, involving observations and inquiries to comprehend both the user and the challenge at hand; (2) ideation, where various solutions to the problem are generated; (3) prototyping, involving the construction of a minimal representation of the best solutions; (4) testing, where user feedback is obtained to refine the solution; and (5) implementation, where the refined solution is tested, changes are assessed, and scaling is considered. Furthermore, ML methods will be adopted to develop automated triggers, tailored to the local context in collaboration with an expert in the field. Results: This protocol describes a research study in its preliminary stages, prior to any data gathering and analysis. The study was approved by the members of the organizations within the institution in January 2024 and by the ethics board of the University of S{\~a}o Paulo and the institution where the study will take place. in May 2024. As of June 2024, stage 1 commenced with data gathering for qualitative research. A separate paper focused on explaining the method of ML will be considered after the outcomes of stages 1 and 2 in this study. Conclusions: After the development of automated triggers in the outpatient setting, it will be possible to prevent and identify potential risks of AEs more promptly, providing valuable information. This technological innovation not only promotes advances in clinical practice but also contributes to the dissemination of techniques and knowledge related to patient safety. Additionally, health care professionals can adopt evidence-based preventive measures, reducing costs associated with AEs and hospital readmissions, enhancing productivity in outpatient care, and contributing to the safety, quality, and effectiveness of care provided. Additionally, in the future, if the outcome is successful, there is the potential to apply it in all units, as planned by the institutional organization. International Registered Report Identifier (IRRID): PRR1-10.2196/55466 ",
doi="10.2196/55466",
url="https://www.researchprotocols.org/2024/1/e55466"
}


@Article{info:doi/10.2196/59459,
author="Caserman, Polona
and Yum, Sungsoo
and G{\"o}bel, Stefan
and Reif, Andreas
and Matura, Silke",
title="Assessing the Accuracy of Smartwatch-Based Estimation of Maximum Oxygen Uptake Using the Apple Watch Series 7: Validation Study",
journal="JMIR Biomed Eng",
year="2024",
month="Jul",
day="31",
volume="9",
pages="e59459",
keywords="maximal oxygen uptake",
keywords="oxygen consumption",
keywords="cardiorespiratory fitness",
keywords="physical fitness",
keywords="physical activity",
keywords="fitness tracker",
keywords="wearables",
keywords="wearable",
keywords="exercise",
keywords="fitness",
keywords="tracker",
keywords="trackers",
keywords="cardiorespiratory",
keywords="wrist worn device",
keywords="devices",
keywords="validation study",
keywords="VO2max",
keywords="sport watch",
keywords="fitness level",
keywords="mobile phone",
abstract="Background: Determining maximum oxygen uptake (VO2max) is essential for evaluating cardiorespiratory fitness. While laboratory-based testing is considered the gold standard, sports watches or fitness trackers offer a convenient alternative. However, despite the high number of wrist-worn devices, there is a lack of scientific validation for VO2max estimation outside the laboratory setting. Objective: This study aims to compare the Apple Watch Series 7's performance against the gold standard in VO2max estimation and Apple's validation findings. Methods: A total of 19 participants (7 female and 12 male), aged 18 to 63 (mean 28.42, SD 11.43) years were included in the validation study. VO2max for all participants was determined in a controlled laboratory environment using a metabolic gas analyzer. Thereby, they completed a graded exercise test on a cycle ergometer until reaching subjective exhaustion. This value was then compared with the estimated VO2max value from the Apple Watch, which was calculated after wearing the watch for at least 2 consecutive days and measured directly after an outdoor running test. Results: The measured VO2max (mean 45.88, SD 9.42 mL/kg/minute) in the laboratory setting was significantly higher than the predicted VO2max (mean 41.37, SD 6.5 mL/kg/minute) from the Apple Watch (t18=2.51; P=.01) with a medium effect size (Hedges g=0.53). The Bland-Altman analysis revealed a good overall agreement between both measurements. However, the intraclass correlation coefficient ICC(2,1)=0.47 (95\% CI 0.06-0.75) indicated poor reliability. The mean absolute percentage error between the predicted and the actual VO2max was 15.79\%, while the root mean square error was 8.85 mL/kg/minute. The analysis further revealed higher accuracy when focusing on participants with good fitness levels (mean absolute percentage error=14.59\%; root-mean-square error=7.22 ml/kg/minute; ICC(2,1)=0.60 95\% CI 0.09-0.87). Conclusions: Similar to other smartwatches, the Apple Watch also overestimates or underestimates the VO2max in individuals with poor or excellent fitness levels, respectively. Assessing the accuracy and reliability of the Apple Watch's VO2max estimation is crucial for determining its suitability as an alternative to laboratory testing. The findings of this study will apprise researchers, physical training professionals, and end users of wearable technology, thereby enhancing the knowledge base and practical application of such devices in assessing cardiorespiratory fitness parameters. ",
doi="10.2196/59459",
url="https://biomedeng.jmir.org/2024/1/e59459"
}


@Article{info:doi/10.2196/50505,
author="Straw, Isabel
and Brass, Irina
and Mkwashi, Andrew
and Charles, Inika
and Soares, Amelie
and Steer, Caroline",
title="Insights From a Clinically Orientated Workshop on Health Care Cybersecurity and Medical Technology: Observational Study and Thematic Analysis",
journal="J Med Internet Res",
year="2024",
month="Jul",
day="11",
volume="26",
pages="e50505",
keywords="digital health",
keywords="clinical medicine",
keywords="biotechnology",
keywords="medical device",
keywords="device regulation",
keywords="medical education",
keywords="eHealth",
keywords="digital medicine",
keywords="health care",
keywords="health care cybersecurity",
keywords="internet of medical things",
abstract="Background: Health care professionals receive little training on the digital technologies that their patients rely on. Consequently, practitioners may face significant barriers when providing care to patients experiencing digitally mediated harms (eg, medical device failures and cybersecurity exploits). Here, we explore the impact of technological failures in clinical terms. Objective: Our study explored the key challenges faced by frontline health care workers during digital events, identified gaps in clinical training and guidance, and proposes a set of recommendations for improving digital clinical practice. Methods: A qualitative study involving a 1-day workshop of 52 participants, internationally attended, with multistakeholder participation. Participants engaged in table-top exercises and group discussions focused on medical scenarios complicated by technology (eg, malfunctioning ventilators and malicious hacks on health care apps). Extensive notes from 5 scribes were retrospectively analyzed and a thematic analysis was performed to extract and synthesize data. Results: Clinicians reported novel forms of harm related to technology (eg, geofencing in domestic violence and errors related to interconnected fetal monitoring systems) and barriers impeding adverse event reporting (eg, time constraints and postmortem device disposal). Challenges to providing effective patient care included a lack of clinical suspicion of device failures, unfamiliarity with equipment, and an absence of digitally tailored clinical protocols. Participants agreed that cyberattacks should be classified as major incidents, with the repurposing of existing crisis resources. Treatment of patients was determined by the role technology played in clinical management, such that those reliant on potentially compromised laboratory or radiological facilities were prioritized. Conclusions: Here, we have framed digital events through a clinical lens, described in terms of their end-point impact on the patient. In doing so, we have developed a series of recommendations for ensuring responses to digital events are tailored to clinical needs and center patient care. ",
doi="10.2196/50505",
url="https://www.jmir.org/2024/1/e50505",
url="http://www.ncbi.nlm.nih.gov/pubmed/38990611"
}


@Article{info:doi/10.2196/48156,
author="Kale, U. Aditya
and Dattani, Riya
and Tabansi, Ashley
and Hogg, Jeffry Henry David
and Pearson, Russell
and Glocker, Ben
and Golder, Su
and Waring, Justin
and Liu, Xiaoxuan
and Moore, J. David
and Denniston, K. Alastair",
title="AI as a Medical Device Adverse Event Reporting in Regulatory Databases: Protocol for a Systematic Review",
journal="JMIR Res Protoc",
year="2024",
month="Jul",
day="11",
volume="13",
pages="e48156",
keywords="adverse event",
keywords="artificial intelligence",
keywords="regulatory science",
keywords="regulatory database",
keywords="safety issue",
keywords="feedback",
keywords="health care product",
keywords="artificial intelligence health technology",
keywords="reporting system",
keywords="safety",
keywords="medical devices",
keywords="safety monitoring",
keywords="risks",
keywords="descriptive analysis",
abstract="Background: The reporting of adverse events (AEs) relating to medical devices is a long-standing area of concern, with suboptimal reporting due to a range of factors including a failure to recognize the association of AEs with medical devices, lack of knowledge of how to report AEs, and a general culture of nonreporting. The introduction of artificial intelligence as a medical device (AIaMD) requires a robust safety monitoring environment that recognizes both generic risks of a medical device and some of the increasingly recognized risks of AIaMD (such as algorithmic bias). There is an urgent need to understand the limitations of current AE reporting systems and explore potential mechanisms for how AEs could be detected, attributed, and reported with a view to improving the early detection of safety signals. Objective: The systematic review outlined in this protocol aims to yield insights into the frequency and severity of AEs while characterizing the events using existing regulatory guidance. Methods: Publicly accessible AE databases will be searched to identify AE reports for AIaMD. Scoping searches have identified 3 regulatory territories for which public access to AE reports is provided: the United States, the United Kingdom, and Australia. AEs will be included for analysis if an artificial intelligence (AI) medical device is involved. Software as a medical device without AI is not within the scope of this review. Data extraction will be conducted using a data extraction tool designed for this review and will be done independently by AUK and a second reviewer. Descriptive analysis will be conducted to identify the types of AEs being reported, and their frequency, for different types of AIaMD. AEs will be analyzed and characterized according to existing regulatory guidance. Results: Scoping searches are being conducted with screening to begin in April 2024. Data extraction and synthesis will commence in May 2024, with planned completion by August 2024. The review will highlight the types of AEs being reported for different types of AI medical devices and where the gaps are. It is anticipated that there will be particularly low rates of reporting for indirect harms associated with AIaMD. Conclusions: To our knowledge, this will be the first systematic review of 3 different regulatory sources reporting AEs associated with AIaMD. The review will focus on real-world evidence, which brings certain limitations, compounded by the opacity of regulatory databases generally. The review will outline the characteristics and frequency of AEs reported for AIaMD and help regulators and policy makers to continue developing robust safety monitoring processes. International Registered Report Identifier (IRRID): PRR1-10.2196/48156 ",
doi="10.2196/48156",
url="https://www.researchprotocols.org/2024/1/e48156",
url="http://www.ncbi.nlm.nih.gov/pubmed/38990628"
}


@Article{info:doi/10.2196/51614,
author="Kale, U. Aditya
and Hogg, Jeffry Henry David
and Pearson, Russell
and Glocker, Ben
and Golder, Su
and Coombe, April
and Waring, Justin
and Liu, Xiaoxuan
and Moore, J. David
and Denniston, K. Alastair",
title="Detecting Algorithmic Errors and Patient Harms for AI-Enabled Medical Devices in Randomized Controlled Trials: Protocol for a Systematic Review",
journal="JMIR Res Protoc",
year="2024",
month="Jun",
day="28",
volume="13",
pages="e51614",
keywords="patient safety",
keywords="adverse events",
keywords="randomized controlled trials",
keywords="medical device",
keywords="systematic review",
keywords="algorithmic",
keywords="artificial intelligence",
keywords="AI",
keywords="AI health technology",
keywords="safety",
keywords="algorithm error",
abstract="Background: Artificial intelligence (AI) medical devices have the potential to transform existing clinical workflows and ultimately improve patient outcomes. AI medical devices have shown potential for a range of clinical tasks such as diagnostics, prognostics, and therapeutic decision-making such as drug dosing. There is, however, an urgent need to ensure that these technologies remain safe for all populations. Recent literature demonstrates the need for rigorous performance error analysis to identify issues such as algorithmic encoding of spurious correlations (eg, protected characteristics) or specific failure modes that may lead to patient harm. Guidelines for reporting on studies that evaluate AI medical devices require the mention of performance error analysis; however, there is still a lack of understanding around how performance errors should be analyzed in clinical studies, and what harms authors should aim to detect and report. Objective: This systematic review will assess the frequency and severity of AI errors and adverse events (AEs) in randomized controlled trials (RCTs) investigating AI medical devices as interventions in clinical settings. The review will also explore how performance errors are analyzed including whether the analysis includes the investigation of subgroup-level outcomes. Methods: This systematic review will identify and select RCTs assessing AI medical devices. Search strategies will be deployed in MEDLINE (Ovid), Embase (Ovid), Cochrane CENTRAL, and clinical trial registries to identify relevant papers. RCTs identified in bibliographic databases will be cross-referenced with clinical trial registries. The primary outcomes of interest are the frequency and severity of AI errors, patient harms, and reported AEs. Quality assessment of RCTs will be based on version 2 of the Cochrane risk-of-bias tool (RoB2). Data analysis will include a comparison of error rates and patient harms between study arms, and a meta-analysis of the rates of patient harm in control versus intervention arms will be conducted if appropriate. Results: The project was registered on PROSPERO in February 2023. Preliminary searches have been completed and the search strategy has been designed in consultation with an information specialist and methodologist. Title and abstract screening started in September 2023. Full-text screening is ongoing and data collection and analysis began in April 2024. Conclusions: Evaluations of AI medical devices have shown promising results; however, reporting of studies has been variable. Detection, analysis, and reporting of performance errors and patient harms is vital to robustly assess the safety of AI medical devices in RCTs. Scoping searches have illustrated that the reporting of harms is variable, often with no mention of AEs. The findings of this systematic review will identify the frequency and severity of AI performance errors and patient harms and generate insights into how errors should be analyzed to account for both overall and subgroup performance. Trial Registration: PROSPERO CRD42023387747; https://www.crd.york.ac.uk/prospero/display\_record.php?RecordID=387747 International Registered Report Identifier (IRRID): PRR1-10.2196/51614 ",
doi="10.2196/51614",
url="https://www.researchprotocols.org/2024/1/e51614",
url="http://www.ncbi.nlm.nih.gov/pubmed/38941147"
}


@Article{info:doi/10.2196/52495,
author="Lau, Y. Erica
and Cragg, Amber
and Small, S. Serena
and Butcher, Katherine
and Hohl, M. Corinne",
title="Characterizing and Comparing Adverse Drug Events Documented in 2 Spontaneous Reporting Systems in the Lower Mainland of British Columbia, Canada: Retrospective Observational Study",
journal="JMIR Hum Factors",
year="2024",
month="Jan",
day="18",
volume="11",
pages="e52495",
keywords="adverse drug event reporting systems",
keywords="side effect",
keywords="side effects",
keywords="drug",
keywords="drugs",
keywords="pharmacy",
keywords="pharmacology",
keywords="pharmacotherapy",
keywords="pharmaceutic",
keywords="pharmaceutics",
keywords="pharmaceuticals",
keywords="pharmaceutical",
keywords="medication",
keywords="medications",
keywords="patient safety",
keywords="health information technology",
keywords="pharmacovigilance",
keywords="adverse",
keywords="safety",
keywords="HIT",
keywords="information system",
keywords="information systems",
keywords="reporting",
keywords="descriptive statistics",
keywords="monitoring",
abstract="Background: Robust adverse drug event (ADE) reporting systems are crucial to monitor and identify drug safety signals, but the quantity and type of ADEs captured may vary by system characteristics. Objective: We compared ADEs reported in 2 different reporting systems in the same jurisdictions, the Patient Safety and Learning System--Adverse Drug Reaction (PSLS-ADR) and ActionADE, to understand report variation. Methods: This retrospective observational study analyzed reports entered into PSLS-ADR and ActionADE systems between December 1, 2019, and December 31, 2022. We conducted a comprehensive analysis including all events from both reporting systems to examine coverage and usage and understand the types of events captured in both systems. We calculated descriptive statistics for reporting facility type, patient demographics, serious events, and most reported drugs. We conducted a subanalysis focused on adverse drug reactions to enable direct comparisons between systems in terms of the volume and events reported. We stratified results by reporting system. Results: We performed the comprehensive analysis on 3248 ADE reports, of which 12.4\% (375/3035) were reported in PSLS-ADR and 87.6\% (2660/3035) were reported in ActionADE. Distribution of all events and serious events varied slightly between the 2 systems. Iohexol, gadobutrol, and empagliflozin were the most common culprit drugs (173/375, 46.2\%) in PSLS-ADR, while hydrochlorothiazide, apixaban, and ramipril (308/2660, 11.6\%) were common in ActionADE. We included 2728 reports in the subanalysis of adverse drug reactions, of which 12.9\% (353/2728) were reported in PSLS-ADR and 86.4\% (2357/2728) were reported in ActionADE. ActionADE captured 4- to 6-fold more comparable events than PSLS-ADR over this study's period. Conclusions: User-friendly and robust reporting systems are vital for pharmacovigilance and patient safety. This study highlights substantial differences in ADE data that were generated by different reporting systems. Understanding system factors that lead to varying reporting patterns can enhance ADE monitoring and should be taken into account when evaluating drug safety signals. ",
doi="10.2196/52495",
url="https://humanfactors.jmir.org/2024/1/e52495",
url="http://www.ncbi.nlm.nih.gov/pubmed/38236629"
}


@Article{info:doi/10.2196/35706,
author="Schneider, Vest Uffe
and Knudsen, Dahl Jenny
and Koch, Anders
and Kirkby, S{\o}ren Nikolai
and Lisby, Gorm Jan",
title="An Agreement of Antigen Tests on Oral Pharyngeal Swabs or Less Invasive Testing With Reverse Transcription Polymerase Chain Reaction for Detecting SARS-CoV-2 in Adults: Protocol for a Prospective Nationwide Observational Study",
journal="JMIR Res Protoc",
year="2022",
month="May",
day="4",
volume="11",
number="5",
pages="e35706",
keywords="SARS-CoV-2",
keywords="COVID-19",
keywords="point of care",
keywords="PoC",
keywords="antigen test",
keywords="anatomic sampling location",
keywords="Reverse Transcription Polymerase Chain Reaction",
keywords="RT-PCR",
keywords="rapid antigen test",
keywords="RAT",
keywords="testing",
keywords="antigen",
keywords="sampling",
keywords="PCR",
keywords="rapid",
keywords="protocol",
keywords="prospective",
keywords="observational",
keywords="agreement",
keywords="oral",
keywords="adult",
keywords="sensitivity",
keywords="specificity",
keywords="test location",
keywords="anatomy",
keywords="saliva",
keywords="swab",
keywords="nasopharyngeal",
keywords="nasal",
abstract="Background: The SARS-CoV-2 pandemic has resulted in an unprecedented level of worldwide testing for epidemiologic and diagnostic purposes, and due to the extreme need for tests, the gold-standard Reverse Transcription Polymerase Chain Reaction (RT-PCR) testing capacity has been unable to meet the overall worldwide testing demand. Consequently, although the current literature has shown the sensitivity of rapid antigen tests (RATs) to be inferior to RT-PCR, RATs have been implemented on a large scale without solid data on performance. Objective: This study will compare analytical and clinical sensitivities and specificities of 50 lateral flow-- or laboratory-based RATs and 3 strand invasion--based amplification (SIBA)-RT-PCR tests from 30 manufacturers to RT-PCR testing of samples obtained from the deep oropharynx. In addition, the study will compare sensitivities and specificities of the included RATs as well as RT-PCR on clinical samples obtained from the deep oropharynx, the anterior nasal cavity, saliva, the deep nasopharynx, and expired air to RT-PCR on deep oropharyngeal samples. Methods: In the prospective part of the study, 200 individuals found SARS-CoV-2 positive and 200 individuals found SARS-CoV-2 negative by routine RT-PCR testing will be retested with each RAT, applying RT-PCR as the reference method. In the retrospective part of the study, 304 deep oropharyngeal cavity swabs divided into 4 groups based on RT-PCR quantification cycle (Cq) levels will be tested with each RAT. Results: The results will be reported in several papers with different aims. The first paper will report retrospective (analytical sensitivity, overall and stratified into different Cq range groups) and prospective (clinical sensitivity) data for RATs, with RT-PCR as the reference method. The second paper will report results for RAT based on anatomical sampling location. The third paper will compare different anatomical sampling locations by RT-PCR testing. The fourth paper will focus on RATs that rely on central laboratory testing. Tests from 4 different manufacturers will be compared for analytical performance data on retrospective deep oropharyngeal swab samples. The fifth paper will report the results of 4 RATs applied both as professional use and as self-tests. The last paper will report the results from 2 breath tests in the study. A comparison of sensitivity and specificity between RATs will be conducted using the McNemar test for paired samples and the chi-squared test for unpaired samples. Comparison of the positive predictive value (PPV) and negative predictive value (NPV) between RATs will be performed by the bootstrap test, and 95\% CIs for sensitivity, specificity, PPV, and NPV will be calculated as bootstrap CIs. Conclusions: The study will compare the sensitivities of a large number of RATs for SARS-CoV-2 to with those of RT-PCR and will address whether lateral flow--based RATs differ significantly from laboratory-based RATs. The anatomical test locations for both RATs and RT-PCR will also be compared. Trial Registration: ClinicalTrials.gov NCT04913116; https://clinicaltrials.gov/ct2/show/NCT04913116 International Registered Report Identifier (IRRID): DERR1-10.2196/35706 ",
doi="10.2196/35706",
url="https://www.researchprotocols.org/2022/5/e35706",
url="http://www.ncbi.nlm.nih.gov/pubmed/35394449"
}


@Article{info:doi/10.2196/25453,
author="Jain, Deeptee
and Norman, Kevin
and Werner, Zachary
and Makovoz, Bar
and Baker, Turner
and Huber, Stephan",
title="Using Postmarket Surveillance to Assess Safety-Related Events in a Digital Rehabilitation App (Kaia App): Observational Study",
journal="JMIR Hum Factors",
year="2021",
month="Nov",
day="9",
volume="8",
number="4",
pages="e25453",
keywords="lower back pain",
keywords="digital therapeutics",
keywords="adverse event",
keywords="pain",
keywords="safety",
keywords="digital health",
keywords="multidisciplinary pain treatment",
abstract="Background: Low back pain (LBP) affects nearly 4 out of 5 individuals during their lifetime and is the leading cause of disability globally. Digital therapeutics are emerging as effective treatment options for individuals experiencing LBP. Despite the growth of evidence demonstrating the benefits of these therapeutics in reducing LBP and improving functional outcomes, little data has been systematically collected on their safety profiles. Objective: This study aims to evaluate the safety profile of a multidisciplinary digital therapeutic for LBP, the Kaia App, by performing a comprehensive assessment of reported adverse events (AEs) by users as captured by a standardized process for postmarket surveillance. Methods: All users of a multidisciplinary digital app that includes physiotherapy, mindfulness techniques, and education for LBP (Kaia App) from 2018 to 2019 were included. Relevant messages sent by users via the app were collected according to a standard operating procedure regulating postmarket surveillance of the device. These messages were then analyzed to determine if they described an adverse event (AE). Messages describing an AE were then categorized based on the type of AE, its seriousness, and its relatedness to the app, and they were described by numerical counts. User demographics, including age and gender, and data on app use were collected and evaluated to determine if they were risk factors for increased AE reporting. Results: Of the 138,337 active users of the Kaia App, 125 (0.09\%) reported at least one AE. Users reported 0.00014 AEs per active day on the app. The most common nonserious AE reported was increased pain. Other nonserious AEs reported included muscle issues, unpleasant sensations, headache, dizziness, and sleep disturbances. One serious AE, a surgery, was reported. Details of the event and its connection to the intervention were not obtainable, as the user did not provide more information when asked to do so; therefore, it was considered to be possibly related to the intervention. There was no relationship between gender and AE reporting (P>.99). Users aged 25 to 34 years had reduced odds (odds ratio [OR] 0.31, 95\% CI 0.08-0.95; P=.03) of reporting AEs, while users aged 55 to 65 years (OR 2.53, 95\% CI 1.36-4.84, P=.002) and ?75 years (OR 4.36, 95\% CI 1.07-13.26; P=.02) had increased odds. AEs were most frequently reported by users who had 0 to 99 active days on the app, and less frequently reported by users with more active days on the app. Conclusions: This study on the Kaia App provides the first comprehensive assessment of reported AEs associated with real-world use of digital therapeutics for lower back pain. The overall rate of reported AEs was very low, but significant reporting bias is likely to be present. The AEs reported were generally consistent with those described for in-person therapies for LBP. ",
doi="10.2196/25453",
url="https://humanfactors.jmir.org/2021/4/e25453",
url="http://www.ncbi.nlm.nih.gov/pubmed/34751664"
}


@Article{info:doi/10.2196/20652,
author="Ceross, Aaron
and Bergmann, Jeroen",
title="Tracking the Presence of Software as a Medical Device in US Food and Drug Administration Databases: Retrospective Data Analysis",
journal="JMIR Biomed Eng",
year="2021",
month="Nov",
day="3",
volume="6",
number="4",
pages="e20652",
keywords="regulation",
keywords="software",
keywords="medical device",
abstract="Background: Software as a medical device (SaMD) has gained the attention of medical device regulatory bodies as the prospects of standalone software for use in diagnositic and therapeutic settings have increased. However, to date, figures related to SaMD have not been made available by regulators, which limits the understanding of how prevalent these devices are and what actions should be taken to regulate them. Objective: The aim of this study is to empirically evaluate the market approvals and clearances related to SaMD and identify adverse incidents related to these devices. Methods: Using databases managed by the US medical device regulator, the US Food and Drug Administration (FDA), we identified the counts of SaMD registered with the FDA since 2016 through the use of product codes, mapped the path SaMD takes toward classification, and recorded adverse events. Results: SaMD does not seem to be registered at a rate dissimilar to that of other medical devices; thus, adverse events for SaMD only comprise a small portion of the total reported number. Conclusions: Although SaMD has been identified in the literature as an area of development, our analysis suggests that this growth has been modest. These devices are overwhelmingly classified as moderate to high risk, and they take a very particular path to that classification. The digital revolution in health care is less pronounced when evidence related to SaMD is considered. In general, the addition of SaMD to the medical device market seems to mimic that of other medical devices. ",
doi="10.2196/20652",
url="https://biomedeng.jmir.org/2021/4/e20652"
}


@Article{info:doi/10.2196/26556,
author="Alwashmi, F. Meshari
and Mugford, Gerald
and Vokey, Brett
and Abu-Ashour, Waseem
and Hawboldt, John",
title="Effectiveness of the BreatheSuite Device in Assessing the Technique of Metered-Dose Inhalers: Validation Study",
journal="JMIR Biomed Eng",
year="2021",
month="Nov",
day="3",
volume="6",
number="4",
pages="e26556",
keywords="digital health",
keywords="asthma",
keywords="smartphone",
keywords="mHealth",
keywords="mobile health",
keywords="effective",
keywords="observational",
keywords="treatment",
keywords="chronic obstructive pulmonary disease",
keywords="inhaler",
keywords="app",
abstract="Background: The majority of medications used in treating asthma and chronic obstructive pulmonary disease (COPD) are taken through metered-dose inhalers (MDIs). Studies have reported that most patients demonstrate poor inhaler technique, which has resulted in poor disease control. Digital Health applications have the potential to improve the technique and adherence of inhaled medications. Objective: This study aimed to validate the effectiveness of the BreatheSuite MDI device in assessing the technique of taking a dose via an MDI. Methods: The study was a validation study. Thirty participants who self-reported a diagnosis of asthma or COPD were recruited from community pharmacies in Newfoundland and Labrador, Canada. Participants used a BreatheSuite MDI device attached to a placebo MDI and resembled taking 3 doses. Pharmacists used a scoring sheet to evaluate the technique of using the MDI. An independent researcher compared the results of the pharmacist's scoring sheet with the results of the BreatheSuite device. Results: This study found that the BreatheSuite MDI can objectively detect several errors in the MDI technique. The data recorded by the BreatheSuite MDI device showed that all participants performed at least one error in using the MDI. The BreatheSuite device captured approximately 40\% (143/360) more errors compared to observation alone. The distribution of participants who performed errors in MDI steps as recorded by BreatheSuite compared to errors reported by observation alone were as follows: shaking before actuation, 33.3\% (30/90) versus 25.5\% (23/90); upright orientation of the inhaler during actuation, 66.7\% (60/90) versus 18.87\% (17/90); coordination (actuating after the start of inhalation), 76.6\% (69/90) versus 35.5\% (32/90); and duration of inspiration, 96.7\% (87/90) versus 34.4\% (31/90). Conclusions: The BreatheSuite MDI can objectively detect several errors in the MDI technique, which were missed by observation alone. It has the potential to enhance treatment outcomes among patients with chronic lung diseases. ",
doi="10.2196/26556",
url="https://biomedeng.jmir.org/2021/4/e26556"
}


@Article{info:doi/10.2196/22911,
author="Groenendaal, Willemijn
and Lee, Seulki
and van Hoof, Chris",
title="Wearable Bioimpedance Monitoring: Viewpoint for Application in Chronic Conditions",
journal="JMIR Biomed Eng",
year="2021",
month="May",
day="11",
volume="6",
number="2",
pages="e22911",
keywords="wearable monitoring",
keywords="bioimpedance",
keywords="impedance pneumography",
keywords="impedance cardiography",
keywords="body composition",
keywords="imaging",
doi="10.2196/22911",
url="https://biomedeng.jmir.org/2021/2/e22911"
}


@Article{info:doi/10.2196/18649,
author="Georgiou, Theodoros
and Holland, Simon
and van der Linden, Janet",
title="Rhythmic Haptic Cueing for Gait Rehabilitation of People With Hemiparesis: Quantitative Gait Study",
journal="JMIR Biomed Eng",
year="2020",
month="Nov",
day="24",
volume="5",
number="1",
pages="e18649",
keywords="hemiparetic gait",
keywords="stroke",
keywords="technology assisted rehabilitation",
keywords="quantitative study",
keywords="gait analysis",
keywords="gait asymmetry",
keywords="gait",
keywords="neurology",
keywords="hemiparesis",
keywords="rehabilitation",
keywords="brain injury",
abstract="Background: Rhythm, brain, and body are closely linked. Humans can synchronize their movement to auditory rhythms in ways that can improve the regularity of movement while reducing perceived effort. However, the ability to perform rhythmic movement may be disrupted by various neurological conditions. Many such conditions impair mechanisms that control movement, such as gait, but typically without rhythmic perception being affected. This paper focuses on hemiparetic stroke, a neurological condition that affects one side of the body. Hemiparetic stroke can cause severe asymmetries in gait, leading to numerous physical problems ranging from muscle degeneration to bone fractures. Movement synchronization via entrainment to auditory metronomes is known to improve asymmetry and related gait problems; this paper presents the first systematic study of entrainment for gait rehabilitation via the haptic modality. Objective: This paper explores the gait rehabilitation of people with hemiparesis following a stroke or brain injury, by a process of haptic entrainment to rhythmic cues. Various objective measures, such as stride length and stride time, are considered. Methods: This study is a quantitative gait study combining temporal and spatial data on haptically cued participants with hemiparetic stroke and brain injury. We designed wearable devices to deliver the haptic rhythm, called Haptic Bracelets, which were placed on the leg near the knee.   Spatial data were recorded using a Qualisys optical motion capturing system, consisting of 8 optoelectronic cameras, and 20 markers placed on anatomical lower limb landmarks and 4 additional tracking clusters placed on the right and left shank and thigh. Gait characteristics were measured before, during, and after cueing. Results: All 11 successfully screened participants were able to synchronize their steps to a haptically presented rhythm. Specifically, 6 participants demonstrated immediate improvements regarding their temporal gait characteristics, and 3 of the 6 improved their gait in terms of spatial characteristics. Conclusions: Considering the great variability between survivors of stroke and brain injury and the limited number of available participants in our study, there is no claim of statistical evidence that supports a formal experimental result of improved gait. However, viewing this empirical gait investigation as a set of 11 case studies, more modest empirical claims can be made. All participants were able to synchronize their steps to a haptically presented rhythm. For a substantial proportion of participants, an immediate (though not necessarily lasting) improvement of temporal gait characteristics was found during cueing. Some improvements over baseline occurred immediately after, rather than during, haptic cueing. Design issues and trade-offs are identified, and interactions between perception, sensory deficit, attention, memory, cognitive load, and haptic entrainment are noted. ",
doi="10.2196/18649",
url="http://biomedeng.jmir.org/2020/1/e18649/"
}


@Article{info:doi/10.2196/14443,
author="Beach, Bradley
and Scansen, A. Brian",
title="Modular Catheter Systems in Minimally Invasive Interventional Medical Procedures: Case Study",
journal="JMIR Biomed Eng",
year="2019",
month="Jul",
day="29",
volume="4",
number="1",
pages="e14443",
keywords="catheters",
keywords="angioplasty",
keywords="balloon valvuloplasty",
keywords="medical device design",
keywords="minimally invasive surgical procedures",
keywords="endovascular procedures",
abstract="Background: Medical device catheters that are used in minimally invasive interventional medical procedures all follow the same integrated design and use paradigm. The features and elements of any catheter device are combined in a single unitary construction. A modular approach to the design, construction, and use of these types of interventional catheters may provide significant advantages and benefits not available with an integrated design paradigm. Objective: This paper aimed to present the design of a modular catheter system and the findings from an initial veterinary use as a case study for the potential of modular catheter systems in general. Methods: A modular catheter system was designed using commercially available angioplasty balloon dilatation catheters as one module in the system and a custom designed scoring adapter as the other module. The scoring adapter incorporates wires to add scoring features to the angioplasty balloon catheter to improve the dilatation performance during a pulmonary valvuloplasty procedure. The scoring adapter also includes a novel attachment mechanism to couple the scoring adapter to any 0.035-inch guidewire--compatible angioplasty balloon catheter. Results: The modular catheter system was successfully designed, manufactured, and used in an initial minimally invasive veterinary cardiovascular intervention to treat a case of canine subvalvular pulmonary stenosis. The scoring adapter and angioplasty balloon catheter were successfully combined tableside in the operating room at the time of the procedure and used to successfully dilate the subvalvular obstruction. Conclusions: The successful design and use of the presented modular catheter system demonstrates the feasibility and potential advantages of this type of paradigm to enable physicians to create interventional catheter devices at the time of a procedure guided by the procedural needs. ",
doi="10.2196/14443",
url="http://biomedeng.jmir.org/2019/1/e14443/"
}


@Article{info:doi/10.2196/biomedeng.9062,
author="Stubberud, Anker
and Omland, Moe Petter
and Tronvik, Erling
and Olsen, Alexander
and Sand, Trond
and Linde, Mattias",
title="Wireless Surface Electromyography and Skin Temperature Sensors for Biofeedback Treatment of Headache: Validation Study with Stationary Control Equipment",
journal="JMIR Biomed Eng",
year="2018",
month="Feb",
day="23",
volume="3",
number="1",
pages="e1",
keywords="biofeedback",
keywords="mobile phone",
keywords="app",
keywords="migraine",
keywords="pediatric",
abstract="Background: The use of wearables and mobile phone apps in medicine is gaining attention. Biofeedback has the potential to exploit the recent advances in mobile health (mHealth) for the treatment of headaches. Objectives: The aim of this study was to assess the validity of selected wireless wearable health monitoring sensors (WHMS) for measuring surface electromyography (SEMG) and peripheral skin temperature in combination with a mobile phone app. This proof of concept will form the basis for developing innovative mHealth delivery of biofeedback treatment among young persons with primary headache. Methods: Sensors fulfilling the following predefined criteria were identified: wireless, small size, low weight, low cost, and simple to use. These sensors were connected to an app and used by 20 healthy volunteers. Validity was assessed through the agreement with simultaneous control measurements made with stationary neurophysiological equipment. The main variables were (1) trapezius muscle tension during different degrees of voluntary contraction and (2) voluntary increase in finger temperature. Data were statistically analyzed using Bland-Altman plots, intraclass correlation coefficient (ICC), and concordance correlation coefficient (CCC). Results: The app was programmed to receive data from the wireless sensors, process them, and feed them back to the user through a simple interface. Excellent agreement was found for the temperature sensor regarding increase in temperature (CCC .90; 95\% CI 0.83-0.97). Excellent to fair agreement was found for the SEMG sensor. The ICC for the average of 3 repetitions during 4 different target levels ranged from .58 to .81. The wireless sensor showed consistency in muscle tension change during moderate muscle activity. Electrocardiography artifacts were avoided through right-sided use of the SEMG sensors. Participants evaluated the setup as usable and tolerable. Conclusions: This study confirmed the validity of wireless WHMS connected to a mobile phone for monitoring neurophysiological parameters of relevance for biofeedback therapy. ",
doi="10.2196/biomedeng.9062",
url="http://biomedeng.jmir.org/2018/1/e1/"
}