@article {2603, title = {Using Automatic Item Generation to Create Solutions and Rationales for Computerized Formative Testing}, journal = {Applied Psychological Measurement}, volume = {42}, number = {1}, year = {2018}, pages = {42-57}, abstract = {Computerized testing provides many benefits to support formative assessment. However, the advent of computerized formative testing has also raised formidable new challenges, particularly in the area of item development. Large numbers of diverse, high-quality test items are required because items are continuously administered to students. Hence, hundreds of items are needed to develop the banks necessary for computerized formative testing. One promising approach that may be used to address this test development challenge is automatic item generation. Automatic item generation is a relatively new but rapidly evolving research area where cognitive and psychometric modeling practices are used to produce items with the aid of computer technology. The purpose of this study is to describe a new method for generating both the items and the rationales required to solve the items to produce the required feedback for computerized formative testing. The method for rationale generation is demonstrated and evaluated in the medical education domain.}, doi = {10.1177/0146621617726788}, url = {https://doi.org/10.1177/0146621617726788}, author = {Mark J. Gierl and Hollis Lai} } @conference {2628, title = {Using Automated Item Generation in a Large-scale Medical Licensure Exam Program: Lessons Learned.}, booktitle = {2017 IACAT Conference}, year = {2017}, month = {08.2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

On-demand testing has become commonplace with most large-scale testing programs. Continuous testing is appealing for candidates in that it affords greater flexibility in scheduling a session at the desired location. Furthermore, the push for more comprehensive systems of assessment (e.g. CBAL) is predicated on the availability of more frequently administered tasks given the purposeful link between instruction and assessment in these frameworks. However, continuous testing models impose several challenges to programs, including overexposure of items. Robust item banks are therefore needed to support routine retirement and replenishment of items. In a traditional approach to developing items, content experts select a topic and then develop an item consisting of a stem, lead-in question, a correct answer and list of distractors. The item then undergoes review by a panel of experts to validate the content and identify any potential flaws. The process involved in developing quality MCQ items can be time-consuming as well as costly, with estimates as high as $1500-$2500 USD per item (Rudner, 2010). The Medical Council of Canada (MCC) has been exploring a novel item development process to supplement traditional approaches. Specifically, the use of automated item generation (AIG), which uses technology to generate test items from cognitive models, has been studied for over five years. Cognitive models are representations of the knowledge and skills that are required to solve any given problem. While developing a cognitive model for a medical scenario, for example, content experts are asked to deconstruct the (clinical) reasoning process involved via clearly stated variables and related elements. The latter information is then entered into a computer program that uses algorithms to generate MCQs. The MCC has been piloting AIG \–based items for over five years with the MCC Qualifying Examination Part I (MCCQE I), a pre-requisite for licensure in Canada. The aim of this presentation is to provide an overview of the practical lessons learned in the use and operational rollout of AIG with the MCCQE I. Psychometrically, the quality of the items is at least equal, and in many instances superior, to that of traditionally written MCQs, based on difficulty, discrimination, and information. In fact, 96\% of the AIG based items piloted in a recent administration were retained for future operational scoring based on pre-defined inclusion criteria. AIG also offers a framework for the systematic creation of plausible distractors, in that the content experts not only need to provide the clinical reasoning underlying a correct response but also the cognitive errors associated with each of the distractors (Lai et al. 2016). Consequently, AIG holds great promise in regard to improving and tailoring diagnostic feedback for remedial purposes (Pugh, De Champlain, Gierl, Lai, Touchie, 2016). Furthermore, our test development process has been greatly enhanced by the addition of AIG as it requires that item writers use metacognitive skills to describe how they solve problems. We are hopeful that sharing our experiences with attendees might not only help other testing organizations interested in adopting AIG, but also foster discussion which might benefit all participants.

References

Lai, H., Gierl, M.J., Touchie, C., Pugh, D., Boulais, A.P., \& De Champlain, A.F. (2016). Using automatic item generation to improve the quality of MCQ distractors. Teaching and Learning in Medicine, 28, 166-173.

Pugh, D., De Champlain, A.F., Lai, H., Gierl, M., \& Touchie, C. (2016). Using cognitive models to develop quality multiple choice questions. Medical Teacher, 38, 838-843.

Rudner, L. (2010). Implementing the Graduate Management Admission Test Computerized Adaptive Test. In W. van der Linden \& C. Glass (Eds.), Elements of adaptive testing (pp. 151-165). New York, NY: Springer.\ 

Presentation Video

}, keywords = {Automated item generation, large scale, medical licensure}, url = {https://drive.google.com/open?id=14N8hUc8qexAy5W_94TykEDABGVIJHG1h}, author = {Andr{\'e} F. De Champlain} } @conference {2633, title = {Using Bayesian Decision Theory in Cognitive Diagnosis Computerized Adaptive Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata Japan}, abstract = {

Cognitive diagnosis computerized adaptive testing (CD-CAT) purports to provide each individual a profile about the strengths and weaknesses of attributes or skills with computerized adaptive testing. In the CD-CAT literature, researchers dedicated to evolving item selection algorithms to improve measurement efficiency, and most algorithms were developed based on information theory. By the discontinuous nature of the latent variables in CD-CAT, this study introduced an alternative for item selection, called the minimum expected cost (MEC) method, which was derived based on Bayesian decision theory. Using simulations, the MEC method was evaluated against the posterior weighted Kullback-Leibler (PWKL) information, the modified PWKL (MPWKL), and the mutual information (MI) methods by manipulating item bank quality, item selection algorithm, and termination rule. Results indicated that, regardless of item quality and termination criterion, the MEC, MPWKL, and MI methods performed very similarly and they all outperformed the PWKL method in classification accuracy and test efficiency, especially in short tests; the MEC method had more efficient item bank usage than the MPWKL and MI methods. Moreover, the MEC method could consider the costs of incorrect decisions and improve classification accuracy and test efficiency when a particular profile was of concern. All the results suggest the practicability of the MEC method in CD-CAT.

Session Video

}, keywords = {Bayesian Decision Theory, CD-CAT}, author = {Chia-Ling Hsu and Wen-Chung Wang and ShuYing Chen} } @conference {2661, title = {Using Computerized Adaptive Testing to Detect Students{\textquoteright} Misconceptions: Exploration of Item Selection}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Owning misconceptions impedes learning, thus detecting misconceptions through assessments is crucial to facilitate teaching. However, most computerized adaptive testing (CAT) applications to diagnose examinees\’ attribute profiles focus on whether examinees mastering correct concepts or not. In educational scenario, teachers and students have to figure out the misconceptions underlying incorrect answers after obtaining the scores from assessments and then correct the corresponding misconceptions. The Scaling Individuals and Classifying Misconceptions (SICM) models proposed by Bradshaw and Templin (2014) fill this gap. SICMs can identify a student\’s misconceptions directly from the distractors of multiple-choice questions and report whether s/he own the misconceptions or not. Simultaneously, SICM models are able to estimate a continuous ability within the item response theory (IRT) framework to fulfill the needs of policy-driven assessment systems relying on scaling examinees\’ ability. However, the advantage of providing estimations for two types of latent variables also causes complexity of model estimation. More items are required to achieve the same accuracies for both classification and estimation compared to dichotomous DCMs and to IRT, respectively. Thus, we aim to develop a CAT using the SICM models (SICM-CAT) to estimate students\’ misconceptions and continuous abilities simultaneously using fewer items than a linear test.

To achieve this goal, in this study, our research questions mainly focus on establishing several item selection rules that target on providing both accurate classification results and continuous ability estimations using SICM-CAT. The first research question is which information criterion to be used. The Kullback\–Leibler (KL) divergence is the first choice, as it can naturally combine the continuous and discrete latent variables. Based on this criterion, we propose an item selection index that can nicely integrate the two types of information. Based on this index, the items selected in real time could discriminate the examinee\’s current misconception profile and ability estimates from other possible estimates to the most extent. The second research question is about how to adaptively balance the estimations of the misconception profile and the continuous latent ability. Mimic the idea of the Hybrid Design proposed by Wang et al. (2016), we propose a design framework which makes the item selection transition from the group-level to the item-level. We aim to explore several design questions, such as how to select the transiting point and which latent variable estimation should be targeted first.

Preliminary results indicated that the SICM-CAT based on the proposed item selection index could classify examinees into different latent classes and measure their latent abilities compared with the random selection method more accurately and reliably under all the simulation conditions. We plan to compare different CAT designs based on our proposed item selection rules with the best linear test as the next step. We expect that the SICM-CAT is able to use shorter test length while retaining the same accuracies and reliabilities.

References

Bradshaw, L., \& Templin, J. (2014). Combining item response theory and diagnostic classification models: A psychometric model for scaling ability and diagnosing misconceptions. Psychometrika, 79(3), 403-425.

Wang, S., Lin, H., Chang, H. H., \& Douglas, J. (2016). Hybrid computerized adaptive testing: from group sequential design to fully sequential design. Journal of Educational Measurement, 53(1), 45-62.

Session Video

}, keywords = {CAT, incorrect answering, Student Misconception}, author = {Yawei Shen and Yu Bao and Shiyu Wang and Laine Bradshaw} } @conference {2659, title = {Using Determinantal Point Processes for Multistage Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Multistage tests are a generalization of computerized adaptive tests (CATs), that allow to ask batches of questions before starting to adapt the process, instead of asking questions one by one. In order to be provided in real-world scenarios, they should be assembled on the fly, and recent models have been designed accordingly (Zheng \& Chang, 2015). We will present a new algorithm for assembling multistage tests, based on a recent technique in machine learning called determinantal point processes. We will illustrate this technique on various student data that come from fraction subtraction items, or massive online open courses.

In multidimensional CATs, feature vectors are estimated for students and questions, and the probability that a student gets a question correct depends on how much their feature vector is correlated with the question feature vector. In other words, questions that are close in space lead to similar response patterns from the students. Therefore, in order to maximize the information of a batch of questions, the volume spanned by their feature vectors should be as large as possible. Determinantal point processes allow to sample efficiently batches of items from a bank that are diverse, i.e., that span a large volume: it is actually possible to draw k items among n with a O(nk3 ) complexity, which is convenient for large databases of 10,000s of items.

References

Zheng, Y., \& Chang, H. H. (2015). On-the-fly assembled multistage adaptive testing. Applied Psychological Measurement, 39(2), 104-118.

Session Video

}, keywords = {Multidimentional CAT, multistage testing}, url = {https://drive.google.com/open?id=1GkJkKTEFWK3srDX8TL4ra_Xbsliemu1R}, author = {Jill-J{\^e}nn Vie} } @article {2482, title = {Using Response Time to Detect Item Preknowledge in Computer?Based Licensure Examinations}, journal = {Educational Measurement: Issues and Practice.}, volume = {35}, number = {38{\textendash}47}, year = {2016}, abstract = {This article addresses the issue of how to detect item preknowledge using item response time data in two computer-based large-scale licensure examinations. Item preknowledge is indicated by an unexpected short response time and a correct response. Two samples were used for detecting item preknowledge for each examination. The first sample was from the early stage of the operational test and was used for item calibration. The second sample was from the late stage of the operational test, which may feature item preknowledge. The purpose of this research was to explore whether there was evidence of item preknowledge and compromised items in the second sample using the parameters estimated from the first sample. The results showed that for one nonadaptive operational examination, two items (of 111) were potentially exposed, and two candidates (of 1,172) showed some indications of preknowledge on multiple items. For another licensure examination that featured computerized adaptive testing, there was no indication of item preknowledge or compromised items. Implications for detected aberrant examinees and compromised items are discussed in the article.}, doi = {http://dx.doi.org/10.1111/emip.12102}, author = {Qian H. and Staniewska, D. and Reckase, M. and Woo, A.} } @article {2483, title = {Using Out-of-Level Items in Computerized Adaptive Testing}, journal = {International Journal of Testing}, volume = {15}, number = {50-70}, year = {2015}, abstract = {Out-of-level testing refers to the practice of assessing a student with a test that is intended for students at a higher or lower grade level. Although the appropriateness of out-of-level testing for accountability purposes has been questioned by educators and policymakers, incorporating out-of-level items in formative assessments for accurate feedback is recommended. This study made use of a commercial item bank with vertically scaled items across grades and simulated student responses in a computerized adaptive testing (CAT) environment. Results of the study suggested that administration of out-of-level items improved measurement accuracy and test efficiency for students who perform significantly above or below their grade-level peers. This study has direct implications with regards to the relevance, applicability, and benefits of using out-of-level items in CAT.}, doi = {http://dx.doi.org/10.1080/15305058.2014.979492}, author = {Wei,H. and Lin,J.} } @article {2403, title = {Utilizing Response Times in Computerized Classification Testing}, journal = {Applied Psychological Measurement}, volume = {39}, number = {5}, year = {2015}, pages = {389-405}, abstract = {A well-known approach in computerized mastery testing is to combine the Sequential Probability Ratio Test (SPRT) stopping rule with item selection to maximize Fisher information at the mastery threshold. This article proposes a new approach in which a time limit is defined for the test and examinees{\textquoteright} response times are considered in both item selection and test termination. Item selection is performed by maximizing Fisher information per time unit, rather than Fisher information itself. The test is terminated once the SPRT makes a classification decision, the time limit is exceeded, or there is no remaining item that has a high enough probability of being answered before the time limit. In a simulation study, the new procedure showed a substantial reduction in average testing time while slightly improving classification accuracy compared with the original method. In addition, the new procedure reduced the percentage of examinees who exceeded the time limit.}, doi = {10.1177/0146621615569504}, url = {http://apm.sagepub.com/content/39/5/389.abstract}, author = {Sie, Haskell and Finkelman, Matthew D. and Riley, Barth and Smits, Niels} } @article {2344, title = {Using Multidimensional CAT to Administer a Short, Yet Precise, Screening Test}, journal = {Applied Psychological Measurement}, volume = {38}, number = {8}, year = {2014}, pages = {614-631}, abstract = {

Multidimensional computerized adaptive testing (MCAT) provides a mechanism by which the simultaneous goals of accurate prediction and minimal testing time for a screening test could both be met. This article demonstrates the use of MCAT to administer a screening test for the Computerized Adaptive Testing\–Armed Services Vocational Aptitude Battery (CAT-ASVAB) under a variety of manipulated conditions. CAT-ASVAB is a test battery administered via unidimensional CAT (UCAT) that is used to qualify applicants for entry into the U.S. military and assign them to jobs. The primary research question being evaluated is whether the use of MCAT to administer a screening test can lead to significant reductions in testing time from the full-length selection test, without significant losses in score precision. Different stopping rules, item selection methods, content constraints, time constraints, and population distributions for the MCAT administration are evaluated through simulation, and compared with results from a regular full-length UCAT administration.

}, doi = {10.1177/0146621614541514}, url = {http://apm.sagepub.com/content/38/8/614.abstract}, author = {Yao, Lihua and Pommerich, Mary and Segall, Daniel O.} } @article {2317, title = {The Utility of Adaptive Testing in Addressing the Problem of Unmotivated Examinees}, journal = {Journal of Computerized Adaptive Testing}, volume = {2}, year = {2014}, pages = {1-17}, doi = {10.7333/1401-02010001}, author = {Steven L. Wise} } @article {2282, title = {Uncertainties in the Item Parameter Estimates and Robust Automated Test Assembly}, journal = {Applied Psychological Measurement}, volume = {37}, number = {2}, year = {2013}, pages = {123-139}, abstract = {

Item response theory parameters have to be estimated, and because of the estimation process, they do have uncertainty in them. In most large-scale testing programs, the parameters are stored in item banks, and automated test assembly algorithms are applied to assemble operational test forms. These algorithms treat item parameters as fixed values, and uncertainty is not taken into account. As a consequence, resulting tests might be off target or less informative than expected. In this article, the process of parameter estimation is described to provide insight into the causes of uncertainty in the item parameters. The consequences of uncertainty are studied. Besides, an alternative automated test assembly algorithm is presented that is robust against uncertainties in the data. Several numerical examples demonstrate the performance of the robust test assembly algorithm, and illustrate the consequences of not taking this uncertainty into account. Finally, some recommendations about the use of robust test assembly and some directions for further research are given.

}, doi = {10.1177/0146621612469825}, url = {http://apm.sagepub.com/content/37/2/123.abstract}, author = {Veldkamp, Bernard P. and Matteucci, Mariagiulia and de Jong, Martijn G.} } @article {2045, title = {Unproctored Internet test verification: Using adaptive confirmation testing}, journal = {Organizational Research Methods}, volume = {14}, year = {2011}, pages = {608-630}, author = {Makransky, G. and Glas, C. A. W.} } @conference {2107, title = {The Use of Decision Trees for Adaptive Item Selection and Score Estimation}, booktitle = {Annual Conference of the International Association for Computerized Adaptive Testing}, year = {2011}, month = {10/2011}, abstract = {

Conducted post-hoc simulations comparing the relative efficiency, and precision of decision trees (using CHAID and CART) vs. IRT-based CAT.

Conclusions

Decision tree methods were more efficient than CAT

But,...

Conclusions

CAT selects items based on two criteria: Item location relative to current estimate of theta, Item discrimination

Decision Trees select items that best discriminate between groups defined by the total score.

CAT is optimal only when trait level is well estimated.
Findings suggest that combining decision tree followed by CAT item selection may be advantageous.

}, keywords = {adaptive item selection, CAT, decision tree}, author = {Barth B. Riley and Rodney Funk and Michael L. Dennis and Richard D. Lennox and Matthew Finkelman} } @article {2243, title = {Using Item Response Theory and Adaptive Testing in Online Career Assessment}, journal = {Journal of Career Assessment}, volume = {19}, number = {3}, year = {2011}, pages = {274-286}, abstract = {

The present article describes the potential utility of item response theory (IRT) and adaptive testing for scale evaluation and for web-based career assessment. The article describes the principles of both IRT and adaptive testing and then illustrates these with reference to data analyses and simulation studies of the Career Confidence Inventory (CCI). The kinds of information provided by IRT are shown to give a more precise look at scale quality across the trait continuum and also to permit the use of adaptive testing, where the items administered are tailored to the individual being tested. Such tailoring can significantly reduce testing time while maintaining high quality of measurement. This efficiency is especially useful when multiscale inventories and/or a large number of scales are to be administered. Readers are encouraged to consider using these advances in career assessment.

}, doi = {10.1177/1069072710395534}, url = {http://jca.sagepub.com/cgi/content/abstract/19/3/274}, author = {Betz, Nancy E. and Turner, Brandon M.} } @article {151, title = {The use of PROMIS and assessment center to deliver patient-reported outcome measures in clinical research}, journal = {Journal of Applied Measurement}, volume = {11}, number = {3}, year = {2010}, pages = {304-314}, abstract = {The Patient-Reported Outcomes Measurement Information System (PROMIS) was developed as one of the first projects funded by the NIH Roadmap for Medical Research Initiative to re-engineer the clinical research enterprise. The primary goal of PROMIS is to build item banks and short forms that measure key health outcome domains that are manifested in a variety of chronic diseases which could be used as a "common currency" across research projects. To date, item banks, short forms and computerized adaptive tests (CAT) have been developed for 13 domains with relevance to pediatric and adult subjects. To enable easy delivery of these new instruments, PROMIS built a web-based resource (Assessment Center) for administering CATs and other self-report data, tracking item and instrument development, monitoring accrual, managing data, and storing statistical analysis results. Assessment Center can also be used to deliver custom researcher developed content, and has numerous features that support both simple and complicated accrual designs (branching, multiple arms, multiple time points, etc.). This paper provides an overview of the development of the PROMIS item banks and details Assessment Center functionality.}, isbn = {1529-7713}, author = {Gershon, R. C. and Rothrock, N. and Hanrahan, R. and Bass, M. and Cella, D.} } @inbook {1817, title = {Using automatic item generation to address item demands for CAT}, year = {2009}, note = {{PDF File, 320 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing.}, author = {Lai, H. and Alves, C. and Gierl, M. J.} } @inbook {1912, title = {Utilizing the generalized likelihood ratio as a termination criterion}, year = {2009}, note = {{PDF File, 194 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing.}, abstract = {Computer-based testing can be used to classify examinees into mutually exclusive groups. Currently, the predominant psychometric algorithm for designing computerized classification tests (CCTs) is the sequential probability ratio test (SPRT; Reckase, 1983) based on item response theory (IRT). The SPRT has been shown to be more efficient than confidence intervals around θ estimates as a method for CCT delivery (Spray \& Reckase, 1996; Rudner, 2002). More recently, it was demonstrated that the SPRT, which only uses fixed values, is less efficient than a generalized form which tests whether a given examinee{\textquoteright}s θ is below θ1or above θ2 (Thompson, 2007). This formulation allows the indifference region to vary based on observed data. Moreover, this composite hypothesis formulation better represents the conceptual purpose of the test, which is to test whether θ is above or below the cutscore. The purpose of this study was to explore the specifications of the new generalized likelihood ratio (GLR; Huang, 2004). As with the SPRT, the efficiency of the procedure depends on the nominal error rates and the distance between θ1 and θ2 (Eggen, 1999). This study utilized a monte-carlo approach, with 10,000 examinees simulated under each condition, to evaluate differences in efficiency and accuracy due to hypothesis structure, nominal error rate, and indifference region size. The GLR was always at least as efficient as the fixed-point SPRT while maintaining equivalent levels of accuracy. }, author = {Thompson, N. A.} } @article {152, title = {Using computerized adaptive testing to reduce the burden of mental health assessment}, journal = {Psychiatric Services}, volume = {59}, number = {4}, year = {2008}, note = {Gibbons, Robert DWeiss, David JKupfer, David JFrank, EllenFagiolini, AndreaGrochocinski, Victoria JBhaumik, Dulal KStover, AngelaBock, R DarrellImmekus, Jason CR01-MH-30915/MH/United States NIMHR01-MH-66302/MH/United States NIMHResearch Support, N.I.H., ExtramuralUnited StatesPsychiatric services (Washington, D.C.)Psychiatr Serv. 2008 Apr;59(4):361-8.}, month = {Apr}, pages = {361-8}, edition = {2008/04/02}, abstract = {OBJECTIVE: This study investigated the combination of item response theory and computerized adaptive testing (CAT) for psychiatric measurement as a means of reducing the burden of research and clinical assessments. METHODS: Data were from 800 participants in outpatient treatment for a mood or anxiety disorder; they completed 616 items of the 626-item Mood and Anxiety Spectrum Scales (MASS) at two times. The first administration was used to design and evaluate a CAT version of the MASS by using post hoc simulation. The second confirmed the functioning of CAT in live testing. RESULTS: Tests of competing models based on item response theory supported the scale{\textquoteright}s bifactor structure, consisting of a primary dimension and four group factors (mood, panic-agoraphobia, obsessive-compulsive, and social phobia). Both simulated and live CAT showed a 95\% average reduction (585 items) in items administered (24 and 30 items, respectively) compared with administration of the full MASS. The correlation between scores on the full MASS and the CAT version was .93. For the mood disorder subscale, differences in scores between two groups of depressed patients--one with bipolar disorder and one without--on the full scale and on the CAT showed effect sizes of .63 (p<.003) and 1.19 (p<.001) standard deviation units, respectively, indicating better discriminant validity for CAT. CONCLUSIONS: Instead of using small fixed-length tests, clinicians can create item banks with a large item pool, and a small set of the items most relevant for a given individual can be administered with no loss of information, yielding a dramatic reduction in administration time and patient and clinician burden.}, keywords = {*Diagnosis, Computer-Assisted, *Questionnaires, Adolescent, Adult, Aged, Agoraphobia/diagnosis, Anxiety Disorders/diagnosis, Bipolar Disorder/diagnosis, Female, Humans, Male, Mental Disorders/*diagnosis, Middle Aged, Mood Disorders/diagnosis, Obsessive-Compulsive Disorder/diagnosis, Panic Disorder/diagnosis, Phobic Disorders/diagnosis, Reproducibility of Results, Time Factors}, isbn = {1075-2730 (Print)}, author = {Gibbons, R. D. and Weiss, D. J. and Kupfer, D. J. and Frank, E. and Fagiolini, A. and Grochocinski, V. J. and Bhaumik, D. K. and Stover, A. and Bock, R. D. and Immekus, J. C.} } @article {132, title = {Using item banks to construct measures of patient reported outcomes in clinical trials: investigator perceptions}, journal = {Clinical Trials}, volume = {5}, number = {6}, year = {2008}, note = {Flynn, Kathryn EDombeck, Carrie BDeWitt, Esi MorganSchulman, Kevin AWeinfurt, Kevin P5U01AR052186/AR/NIAMS NIH HHS/United StatesResearch Support, N.I.H., ExtramuralEnglandClinical trials (London, England)Clin Trials. 2008;5(6):575-86.}, pages = {575-86}, edition = {2008/11/26}, abstract = {BACKGROUND: Item response theory (IRT) promises more sensitive and efficient measurement of patient-reported outcomes (PROs) than traditional approaches; however, the selection and use of PRO measures from IRT-based item banks differ from current methods of using PRO measures. PURPOSE: To anticipate barriers to the adoption of IRT item banks into clinical trials. METHODS: We conducted semistructured telephone or in-person interviews with 42 clinical researchers who published results from clinical trials in the Journal of the American Medical Association, the New England Journal of Medicine, or other leading clinical journals from July 2005 through May 2006. Interviews included a brief tutorial on IRT item banks. RESULTS: After the tutorial, 39 of 42 participants understood the novel products available from an IRT item bank, namely customized short forms and computerized adaptive testing. Most participants (38/42) thought that item banks could be useful in their clinical trials, but they mentioned several potential barriers to adoption, including economic and logistical constraints, concerns about whether item banks are better than current PRO measures, concerns about how to convince study personnel or statisticians to use item banks, concerns about FDA or sponsor acceptance, and the lack of availability of item banks validated in specific disease populations. LIMITATIONS: Selection bias might have led to more positive responses to the concept of item banks in clinical trials. CONCLUSIONS: Clinical investigators are open to a new method of PRO measurement offered in IRT item banks, but bank developers must address investigator and stakeholder concerns before widespread adoption can be expected.}, isbn = {1740-7745 (Print)}, author = {Flynn, K. E. and Dombeck, C. B. and DeWitt, E. M. and Schulman, K. A. and Weinfurt, K. P.} } @article {743, title = {Using response times for item selection in adaptive testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {33}, year = {2008}, pages = {5{\textendash}20}, author = {van der Linden, W. J.} } @article {129, title = {On using stochastic curtailment to shorten the SPRT in sequential mastery testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {33}, number = {4}, year = {2008}, pages = {442}, author = {Finkelman, M. D.} } @article {420, title = {Utilizing Rasch measurement models to develop a computer adaptive self-report of walking, climbing, and running}, journal = {Disability and Rehabilitation}, volume = {30}, number = {6}, year = {2008}, pages = {458-467}, abstract = {Purpose.The purpose of this paper is to show how the Rasch model can be used to develop a computer adaptive self-report of walking, climbing, and running.Method.Our instrument development work on the walking/climbing/running construct of the ICF Activity Measure was used to show how to develop a computer adaptive test (CAT). Fit of the items to the Rasch model and validation of the item difficulty hierarchy was accomplished using Winsteps software. Standard error was used as a stopping rule for the CAT. Finally, person abilities were connected to items difficulties using Rasch analysis {\textquoteleft}maps{\textquoteright}.Results.All but the walking one mile item fit the Rasch measurement model. A CAT was developed which selectively presented items based on the last calibrated person ability measure and was designed to stop when standard error decreased to a pre-set criterion. Finally, person ability measures were connected to the ability to perform specific walking/climbing/running activities using Rasch maps.Conclusions.Rasch measurement models can be useful in developing CAT measures for rehabilitation and disability. In addition to CATs reducing respondent burden, the connection of person measures to item difficulties may be important for the clinical interpretation of measures.Read More: http://informahealthcare.com/doi/abs/10.1080/09638280701617317}, author = {Velozo, C. A. and Wang, Y. and Lehman, L. A. and Wang, J. H.} } @inbook {1894, title = {Up-and-down procedures for approximating optimal designs using person-response functions}, year = {2007}, note = {{PDF file, 1,042 KB}}, address = {D. J. Weiss (Ed.). Proceedings of the 2007 GMAC Conference on Computerized Adaptive Testing.}, author = {Sheng, Y. and Flournoy, N. and Osterlind, S. J.} } @inbook {1753, title = {Use of CAT in dynamic testing}, year = {2007}, note = {{PDF file, 133 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2007 GMAC Conference on Computerized Adaptive Testing.}, author = {De Beer, M.} } @proceedings {83, title = {The use of computerized adaptive testing to assess psychopathology using the Global Appraisal of Individual Needs}, journal = {American Evaluation Association}, year = {2007}, month = {November}, publisher = {American Evaluation Association }, address = {Portland, OR USA}, author = {Conrad, K. J. and Riley, B. B. and Dennis, M. L.} } @article {281, title = {The use of person-fit statistics in computerized adaptive testing}, year = {2005}, month = {September, 2005}, institution = {Law School Administration Council}, address = {Newton, PA. USA}, isbn = {Computerized Testing Report 97-14}, author = {Meijer, R. R. and van Krimpen-Stoop, E. M. L. A.} } @inbook {1742, title = {Understanding computerized adaptive testing: From Robbins-Munro to Lord and beyond}, year = {2004}, address = {D. Kaplan (Ed.), The Sage handbook of quantitative methodology for the social sciences (pp. 117-133). New York: Sage.}, author = {Chang, Hua-Hua} } @article {278, title = {Using patterns of summed scores in paper-and-pencil tests and computer-adaptive tests to detect misfitting item score patterns}, journal = {Journal of Educational Measurement}, volume = {41}, number = {2}, year = {2004}, pages = {119-136}, abstract = {Two new methods have been proposed to determine unexpected sum scores on subtests (testlets) both for paper-and-pencil tests and computer adaptive tests. A method based on a conservative bound using the hypergeometric distribution, denoted ρ, was compared with a method where the probability for each score combination was calculated using a highest density region (HDR). Furthermore, these methods were compared with the standardized log-likelihood statistic with and without a correction for the estimated latent trait value (denoted as l-super(*)-sub(z) and l-sub(z), respectively). Data were simulated on the basis of the one-parameter logistic model, and both parametric and nonparametric logistic regression was used to obtain estimates of the latent trait. Results showed that it is important to take the trait level into account when comparing subtest scores. In a nonparametric item response theory (IRT) context, on adapted version of the HDR method was a powerful alterative to ρ. In a parametric IRT context, results showed that l-super(*)-sub(z) had the highest power when the data were simulated conditionally on the estimated latent trait level. (PsycINFO Database Record (c) 2005 APA ) (journal abstract)}, keywords = {Computer Assisted Testing, Item Response Theory, person Fit, Test Scores}, author = {Meijer, R. R.} } @article {2138, title = {Using Set Covering with Item Sampling to Analyze the Infeasibility of Linear Programming Test Assembly Models}, journal = {Applied Psychological Measurement}, volume = {28}, number = {5}, year = {2004}, pages = {355-375}, abstract = {

This article shows how set covering with item sampling (SCIS) methods can be used in the analysis and preanalysis of linear programming models for test assembly (LPTA). LPTA models can construct tests, fulfilling a set of constraints set by the test assembler. Sometimes, no solution to the LPTA model exists. The model is then said to be infeasible. Causes of infeasibility can be difficult to find. A method is proposed that constitutes a helpful tool for test assemblers to detect infeasibility before hand and, in the case of infeasibility, give insight into its causes. This method is based on SCIS. Although SCIS can help to detect feasibility or infeasibility, its power lies in pinpointing causes of infeasibility such as irreducible infeasible sets of constraints. Methods to resolve infeasibility are also given, minimizing the model deviations. A simulation study is presented, offering a guide to test assemblers to analyze and solve infeasibility.

}, doi = {10.1177/0146621604266152}, url = {http://apm.sagepub.com/content/28/5/355.abstract}, author = {Huitzing, Hiddo A.} } @booklet {1394, title = {Using moving averages to assess test and item security in computer-based testing (Center for Educational Assessment Research Report No 468)}, year = {2003}, address = {Amherst, MA: University of Massachusetts, School of Education.}, author = {Han, N.} } @article {407, title = {Using response times to detect aberrant responses in computerized adaptive testing}, journal = {Psychometrika}, volume = {68}, number = {2}, year = {2003}, pages = {251-265}, abstract = {A lognormal model for response times is used to check response times for aberrances in examinee behavior on computerized adaptive tests. Both classical procedures and Bayesian posterior predictive checks are presented. For a fixed examinee, responses and response times are independent; checks based on response times offer thus information independent of the results of checks on response patterns. Empirical examples of the use of classical and Bayesian checks for detecting two different types of aberrances in response times are presented. The detection rates for the Bayesian checks outperformed those for the classical checks, but at the cost of higher false-alarm rates. A guideline for the choice between the two types of checks is offered.}, keywords = {Adaptive Testing, Behavior, Computer Assisted Testing, computerized adaptive testing, Models, person Fit, Prediction, Reaction Time}, author = {van der Linden, W. J. and van Krimpen-Stoop, E. M. L. A.} } @conference {1171, title = {Updated item parameter estimates using sparse CAT data}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2002}, note = {{PDF file, 986 KB}}, address = {New Orleans LA}, author = {Smith, R. L. and Rizavi, S. and Paez, R. and Rotou, O.} } @conference {1224, title = {Using judgments of item difficulty to change answers on computerized adaptive vocabulary tests}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {$\#$VI02-01}, address = {New Orleans LA}, author = {Vispoel, W. P. and Clough, S. J. and Bleiler, T.} } @conference {1261, title = {Using testlet response theory to evaluate the equivalence of automatically generated multiple-choice items}, booktitle = {Symposium conducted at the annual meeting of the National Council on Measurement in Eucation}, year = {2002}, address = {New Orleans LA}, author = {Williamson, D. M. and Bejar, I. I.} } @conference {900, title = {Utility of Learning Potential Computerised Adaptive Test (LPCAT) scores in predicting academic performance of bridging students: A comparison with other predictors}, booktitle = {Paper presented at the 5th Annual Society for Industrial and Organisational Psychology Congress}, year = {2002}, address = {Pretoria, South Africa}, author = {De Beer, M.} } @booklet {1560, title = {User{\textquoteright}s guide for SCORIGHT (version 1): A computer program for scoring tests built of testlets (Research Report 01-06)}, year = {2001}, note = {$\#$WA01-06 {PDF file, 2.349 MB}}, address = {Princeton NJ: Educational Testing Service.}, author = {Wang, X and Bradlow, E. T. and Wainer, H.,} } @booklet {1558, title = {Users guide for SCORIGHT (version 2) : A computer program for scoring tests built of testlets (Research Report 01-06)}, year = {2001}, address = {Princeton NJ: Educational Testing Service.}, author = {Wang, X and Bradlow, E. T. and Wainer, H.,} } @conference {1212, title = {Using response times to detect aberrant behavior in computerized adaptive testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2001}, address = {Seattle WA}, author = {van der Linden, W. J. and van Krimpen-Stoop, E. M. L. A.} } @inbook {1856, title = {Using Bayesian Networks in Computerized Adaptive Tests}, year = {2000}, address = {M. Ortega and J. Bravo (Eds.),Computers and Education in the 21st Century. Kluwer, pp. 217228.}, author = {Millan, E. and Trella, M and Perez-de-la-Cruz, J.-L. and Conejo, R} } @conference {827, title = {Using constraints to develop and deliver adaptive tests}, booktitle = {Paper presented at the Computer-Assisted Testing Conference.}, year = {2000}, note = {{PDF file, 46 KB}}, author = {Abdullah, S. C and Cooley, R. E.} } @booklet {1538, title = {Using response times to detect aberrant behavior in computerized adaptive testing (Research Report 00-09)}, year = {2000}, address = {Enschede, The Netherlands: University of Twente, Faculty of Educational Science and Technology, Department of Measurement and Data Analysis}, author = {van der Linden, W. J. and van Krimpen-Stoop, E. M. L. A.} } @conference {828, title = {Use of conditional item exposure methodology for an operational CAT}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, address = {Montreal, Canada}, author = {Anderson, D.} } @conference {866, title = {The use of linear-on-the-fly testing for TOEFL Reading}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, address = {Montreal, Canada}, author = {Carey, P. A.} } @article {419, title = {The use of Rasch analysis to produce scale-free measurement of functional ability}, journal = {American Journal of Occupational Therapy}, volume = {53}, number = {1}, year = {1999}, note = {991250470272-9490Journal Article}, pages = {83-90}, abstract = {Innovative applications of Rasch analysis can lead to solutions for traditional measurement problems and can produce new assessment applications in occupational therapy and health care practice. First, Rasch analysis is a mechanism that translates scores across similar functional ability assessments, thus enabling the comparison of functional ability outcomes measured by different instruments. This will allow for the meaningful tracking of functional ability outcomes across the continuum of care. Second, once the item-difficulty order of an instrument or item bank is established by Rasch analysis, computerized adaptive testing can be used to target items to the patient{\textquoteright}s ability level, reducing assessment length by as much as one half. More importantly, Rasch analysis can provide the foundation for "equiprecise" measurement or the potential to have precise measurement across all levels of functional ability. The use of Rasch analysis to create scale-free measurement of functional ability demonstrates how this methodlogy can be used in practical applications of clinical and outcome assessment.}, keywords = {*Activities of Daily Living, Disabled Persons/*classification, Human, Occupational Therapy/*methods, Predictive Value of Tests, Questionnaires/standards, Sensitivity and Specificity}, author = {Velozo, C. A. and Kielhofner, G. and Lai, J-S.} } @article {772, title = {Using Bayesian decision theory to design a computerized mastery test}, journal = {Journal of Educational and Behavioral Statistics}, volume = {24(3)}, year = {1999}, pages = {271{\textendash}292}, author = {Vos, H. J.} } @article {406, title = {Using response-time constraints to control for differential speededness in computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {23}, number = {3}, year = {1999}, note = {Sage Publications, US}, pages = {195-210}, abstract = {An item-selection algorithm is proposed for neutralizing the differential effects of time limits on computerized adaptive test scores. The method is based on a statistical model for distributions of examinees{\textquoteright} response times on items in a bank that is updated each time an item is administered. Predictions from the model are used as constraints in a 0-1 linear programming model for constrained adaptive testing that maximizes the accuracy of the trait estimator. The method is demonstrated empirically using an item bank from the Armed Services Vocational Aptitude Battery. }, keywords = {computerized adaptive testing}, author = {van der Linden, W. J. and Scrams, D. J. and Schnipke, D. L.} } @booklet {1537, title = {Using response-time constraints to control for differential speededness in adaptive testing (Research Report 98-06)}, year = {1998}, address = {Enschede, The Netherlands: University of Twente, Faculty of Educational Science and Technology, Department of Measurement and Data Analysis}, author = {van der Linden, W. J. and Scrams, D. J. and Schnipke, D. L.} } @article {678, title = {Una soluci{\'o}n a la estimati{\'o}n inicial en los tests adaptivos informatizados [A solution to initial estimation in CATs.] }, journal = {Revista Electr{\'o}nica de Metodolog{\'\i}a Aplicada}, volume = {2}, year = {1997}, pages = {1-6}, author = {Revuelta, J. and Ponsoda, V.} } @booklet {1606, title = {Unidimensional approximations for a computerized adaptive test when the item pool and latent space are multidimensional (Research Report 97-5)}, year = {1997}, address = {Iowa City IA: ACT Inc}, author = {Spray, J. A. and Abdel-Fattah, A. A. and Huang, C.-Y. and Lau, CA} } @book {1644, title = {Users manual for the MicroCAT testing system, Version 3.5}, year = {1996}, note = {.}, address = {St Paul MN: Assessment Systems Corporation}, author = {Assessment-Systems-Corporation.} } @conference {991, title = {Using unidimensional IRT models for dichotomous classification via CAT with multidimensional data}, booktitle = {Poster session presented at the annual meeting of the American Educational Research Association}, year = {1996}, address = {Boston MA}, author = {Lau, CA and Abdel-Fattah, A. A. and Spray, J. A.} } @conference {918, title = {Utility of Fisher information, global information and different starting abilities in mini CAT}, booktitle = {Paper presented at the Annual Meeting of the National Council on Measurement in Education}, year = {1996}, note = {$\#$FA96-01}, address = {New York NY}, author = {Fan, M. and Hsu, Y.} } @booklet {1408, title = {Using simulation to select an adaptive testing strategy: An item bank evaluation program}, year = {1995}, address = {Unpublished manuscript, University of Pittsburgh}, author = {Hsu, T. C. and Tseng, F. L.} } @article {807, title = {Understanding self-adapted testing: The perceived control hypothesis}, journal = {Applied Measurement in Education}, volume = {7}, year = {1994}, pages = {15-24}, author = {Wise, S. L.} } @inbook {1870, title = {Utilisation de la simulation en tant que m{\'e}thodologie de recherche [Simulation methodology in research]}, year = {1994}, address = {Association pour la recherche au coll{\'e}gial (Ed.) : L{\textquoteright}en-qu{\^e}te de la cr{\'e}ativit{\'e} [In quest of creativity]. Proceeding of the 6th Congress of the ARC. Montr{\'e}al: Association pour la recherche au coll{\'e}gial (ARC).}, author = {Ra{\^\i}che, G.} } @inbook {1819, title = {Un test adaptatif en langue seconde : la perception des apprenants}, year = {1993}, address = {R.Hivon ({\'E}d.),L{\textquoteright}{\'e}valuation des apprentissages. Sherbrooke : {\'E}ditions du CRP.}, author = {Laurier, M.} } @conference {134, title = {The use of the graded response model in computerized adaptive testing of the attitudes to science scale}, booktitle = {annual meeting of the American Education Research Association}, year = {1991}, month = {April 3-7, 1991}, address = {Chicago, IL USA}, abstract = {The graded response model for two-stage testing was applied to an attitudes toward science scale using real-data simulation. The 48-item scale was administered to 920 students at a grade-8 equivalent in Singapore. A two-stage 16-item computerized adaptive test was developed. In two-stage testing an initial, or routing, test is followed by a second-stage testlet of greater or lesser difficulty based on performance. A conventional test of the same length as the adaptive two-stage test was selected from the 48-item pool. Responses to the conventional test, the routing test, and a testlet were simulated. The algorithm of E. Balas (1965) and the multidimensional knapsack problem of optimization theory were used in test development. The simulation showed the efficiency and accuracy of the two-stage test with the graded response model in estimating attitude trait levels, as evidenced by better results from the two-stage test than its conventional counterpart and the reduction to one-third of the length of the original measure. Six tables and three graphs are included. (SLD)}, author = {Foong, Y-Y. and Lam, T-L.} } @article {434, title = {The use of unidimensional parameter estimates of multidimensional items in adaptive testing}, journal = {Applied Psychological Measurement}, volume = {15}, year = {1991}, pages = {13-24}, author = {Ackerman, T. A.} } @article {2023, title = {The Use of Unidimensional Parameter Estimates of Multidimensional Items in Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {15}, year = {1991}, pages = {13-24}, author = {Ackerman, T. A.} } @article {2022, title = {Using Bayesian Decision Theory to Design a Computerized Mastery Test}, journal = {Applied Psychological Measurement}, volume = {14}, year = {1990}, pages = {367-386}, author = {Lewis, C. and Sheehan, K.} } @article {614, title = {Using Bayesian decision theory to design a computerized mastery test}, journal = {Applied Psychological Measurement}, volume = {14}, year = {1990}, pages = {367-386}, author = {Lewis, C., and Sheehan, K.} } @booklet {1390, title = {Utility of predicting starting abilities in sequential computer-based adaptive tests (Research Report 90-1)}, year = {1990}, address = {Baltimore MD: Johns Hopkins University, Department of Psychology}, author = {Green, B. F. and Thomas, T. J.} } @book {1643, title = {Users manual for the MicroCAT Testing System, Version 3}, year = {1988}, address = {St. Paul MN: Author.}, author = {Assessment-Systems-Corporation} } @article {3, title = {The use of unidimensional item parameter estimates of multidimensional items in adaptive testing}, number = {87-13}, year = {1987}, month = {September, 1987}, pages = {33}, institution = {ACT}, address = {Iowa City, IA}, abstract = {Investigated the effect of using multidimensional (MDN) items in a computer adaptive test setting that assumes a unidimensional item response theory model in 2 experiments, using generated and real data in which difficulty was known to be confounded with dimensionality. Results from simulations suggest that univariate calibration of MDN data filtered out multidimensionality. The closer an item{\textquoteright}s MDN composite aligned itself with the calibrated univariate ability scale{\textquoteright}s orientation, the larger was the estimated discrimination parameter. (PsycINFO Database Record (c) 2003 APA, all rights reserved).}, isbn = {87-13}, author = {Ackerman, T. A.} } @article {725, title = {Using microcomputer-based assessment in career counseling}, journal = {Journal of Employment Counseling}, volume = {23}, year = {1986}, pages = {50-56}, author = {Thompson, D. L.} } @proceedings {271, title = {Unidimensional and multidimensional models for item response theory}, journal = {Proceedings of the 1982 Computerized Adaptive Testing Conference}, year = {1985}, month = {06/1982}, pages = {127-148}, publisher = {University of Minnesota, Department of Psychology, Psychometrics Methods Program}, address = {Minneapolis, MN. USA}, author = {McDonald, R. P.} } @book {1642, title = {Users manual for the MicroCAT Testing System}, year = {1984}, address = {St. Paul MN: Author}, author = {Assessment-Systems-Corporation} } @article {788, title = {Using microcomputers to administer tests}, journal = {Educational Measurement: Issues and Practice}, volume = {3(2)}, year = {1984}, pages = {16-20}, author = {W. C. Ward} } @article {643, title = {Using microcomputers to administer tests: An alternate point of view}, journal = {Educational Measurement: Issues and Practice}, volume = {3(2)}, year = {1984}, pages = {20-21}, author = {Millman, J.} } @inbook {1953, title = {Use of Sequential Testing to Prescreen Prospective Entrants to Military Service.}, year = {1982}, note = {{PDF file, 483 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 1982 Item Response Theory and Computerized Adaptive Testing Conference Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program.}, author = {Weitzman, R. A.} } @booklet {1501, title = {The use of the sequential probability ratio test in making grade classifications in conjunction with tailored testing (Research Report 81-4)}, year = {1981}, address = {Columbia MO: University of Missouri, Department of Educational Psychology}, author = {Reckase, M. D.} } @article {2005, title = {A Use of the Information Function in Tailored Testing}, journal = {Applied Psychological Measurement}, volume = {1}, year = {1977}, pages = {233-247}, author = {Samejima, F.} } @inbook {1748, title = {Using computerized tests to add new dimensions to the measurement of abilities which are important for on-job performance: An exploratory study}, year = {1976}, note = {{PDF file, 632 KB}}, address = {C. K. Clark (Ed.), Proceedings of the First Conference on Computerized Adaptive Testing (pp. 64-74). Washington DC: U.S. Government Printing Office.}, author = {Cory, C. H.} } @article {589, title = {Use of an on-line computer for psychological testing with the up-and-down method}, journal = {American Psychologist}, volume = {24}, year = {1969}, pages = {207-211}, author = {Kappauf, W. E.} }