@conference {2631, title = {Generating Rationales to Support Formative Feedback in Adaptive Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Computer adaptive testing offers many important benefits to support and promote life-long learning. Computers permit testing on-demand thereby allowing students to take the test at any time during instruction; items on computerized tests are scored immediately thereby providing students with instant feedback; computerized tests permit continuous administration thereby allowing students to have more choice about when they write their exams. But despite these important benefits, the advent of computer adaptive testing has also raised formidable challenges, particularly in the area of item development. Educators must have access to large numbers of diverse, high-quality test items to implement computerize adaptive testing because items are continuously administered to students. Hence, hundreds or even thousands of items are needed to develop the test item banks necessary for computer adaptive testing. Unfortunately, educational test items, as they are currently created, are time consuming and expensive to develop because each individual item is written, initially, by a content specialist and, then, reviewed, edited, and revised by groups of content specialists to ensure the items yield reliable and valid information. Hence, item development is one of the most important problems that must be solved before we can migrate to computer adaptive testing to support life-long learning because large numbers of high-quality, content-specific, test items are required.

One promising item development method that may be used to address this challenge is with automatic item generation. Automatic item generation is a relatively new but rapidly evolving research area where cognitive and psychometric modelling practices are used produce hundreds of new test items with the aid of computer technology. The purpose of our presentation is to describe a new methodology for generating both the items and the rationales required to solve each generated item in order to produce the feedback needed to support life-long learning. Our item generation methodology will first be described. To ensure our description is practical, the method will also be demonstrated using generated items from the health sciences to demonstrate how item generation can promote life-long learning for medical educators and practitioners.

Session Video

}, keywords = {Adaptive Testing, formative feedback, Item generation}, url = {https://drive.google.com/open?id=1O5KDFtQlDLvhNoDr7X4JO4arpJkIHKUP}, author = {Mark Gierl and Okan Bulut} } @conference {2647, title = {Grow a Tiger out of Your CAT }, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

The main focus in the community of test developers and researchers is on improving adaptive test procedures and methodologies. Yet, the transition from research projects to larger-scale operational CATs is facing its own challenges. Usually, these operational CATs find their origin in government tenders. \“Scalability\”, \“Interoperability\” and \“Transparency\” are three keywords often found in these documents. Scalability is concerned with parallel system architectures which are based upon stateless selection algorithms. Design capacities often range from 10,000 to well over 100,000 concurrent students. Interoperability is implemented in standards like QTI, standards that were not designed with adaptive testing in mind. Transparency is being realized by open source software: the adaptive test should not be a black box. These three requirements often complicate the development of an adaptive test, or sometimes even conflict.

Session Video

}, keywords = {interoparability, Scalability, transparency}, author = {Angela Verschoor} } @article {2325, title = {General Test Overlap Control: Improved Algorithm for CAT and CCT}, journal = {Applied Psychological Measurement}, volume = {38}, number = {3}, year = {2014}, pages = {229-244}, abstract = {

This article proposed a new online test overlap control algorithm that is an improvement of Chen\’s algorithm in controlling general test overlap rate for item pooling among a group of examinees. Chen\’s algorithm is not very efficient in that not only item pooling between current examinee and prior examinees is controlled for but also item pooling between previous examinees, which would have been controlled for when they were current examinees. The proposed improvement increases efficiency by only considering item pooling between current and previous examinees, and its improved performance over Chen is demonstrated in a simulated computerized adaptive testing (CAT) environment. Moreover, the proposed algorithm is adapted for computerized classification testing (CCT) using the sequential probability ratio test procedure and is evaluated against some existing exposure control procedures. The proposed algorithm appears to work best in controlling general test overlap rate among the exposure control procedures examined without sacrificing much classification precision, though longer tests might be required for more stringent control of item pooling among larger groups. Given the capability of the proposed algorithm in controlling item pooling among a group of examinees of any size and its ease of implementation, it appears to be a good test overlap control method.

}, doi = {10.1177/0146621613513494}, url = {http://apm.sagepub.com/content/38/3/229.abstract}, author = {Chen, Shu-Ying and Lei, Pui-Wa and Chen, Jyun-Hong and Liu, Tzu-Chen} } @article {182, title = {Gradual maximum information ratio approach to item selection in computerized adaptive testing}, number = {RR-09-07}, year = {2009}, institution = {Graduate Management Admissions Council}, address = {McLean, VA. USA}, author = {Han, K. T.} } @inbook {1789, title = {A gradual maximum information ratio approach to item selection in computerized adaptive testing}, year = {2009}, note = {{PDF file, 391 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing.}, author = {Han, K. T.} } @inbook {1905, title = {Guess what? Score differences with rapid replies versus omissions on a computerized adaptive test}, year = {2009}, note = {{PDF File, 215 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing.}, author = {Talento-Miller, E. and Guo, F.} } @inbook {119, title = {Generating abstract reasoning items with cognitive theory}, booktitle = {Item generation for test development}, year = {2002}, note = {Using Smart Source ParsingItem generation for test development. (pp. 219-250). Mahwah, NJ : Lawrence Erlbaum Associates, Publishers. xxxii, 412 pp}, pages = {219-250}, publisher = {Lawrence Erlbaum Associates, Inc.}, organization = {Lawrence Erlbaum Associates, Inc.}, address = {Mahwah, N.J. USA}, abstract = {(From the chapter) Developed and evaluated a generative system for abstract reasoning items based on cognitive theory. The cognitive design system approach was applied to generate matrix completion problems. Study 1 involved developing the cognitive theory with 191 college students who were administered Set I and Set II of the Advanced Progressive Matrices. Study 2 examined item generation by cognitive theory. Study 3 explored the psychometric properties and construct representation of abstract reasoning test items with 728 young adults. Five structurally equivalent forms of Abstract Reasoning Test (ART) items were prepared from the generated item bank and administered to the Ss. In Study 4, the nomothetic span of construct validity of the generated items was examined with 728 young adults who were administered ART items, and 217 young adults who were administered ART items and the Advanced Progressive Matrices. Results indicate the matrix completion items were effectively generated by the cognitive design system approach. (PsycINFO Database Record (c) 2005 APA )}, keywords = {Cognitive Processes, Measurement, Reasoning, Test Construction, Test Items, Test Validity, Theories}, author = {Embretson, S. E.}, editor = {P. Kyllomen} } @inbook {1858, title = {The GRE computer adaptive test: Operational issues}, year = {2000}, address = {W. J. van der Linden and C. A. W. Glas (Eds.), Computerized adaptive testing: Theory and practice (pp. 75-99). Dordrecht, Netherlands: Kluwer.}, author = {Mills, C. N. and Steffen, M.} } @article {539, title = {Generating items during testing: Psychometric issues and models}, journal = {Psychometrika}, volume = {64}, year = {1999}, pages = {407-433}, author = {Embretson, S. E.} } @article {6, title = {Graphical models and computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {23}, number = {3}, year = {1999}, pages = {223-37}, abstract = {Considers computerized adaptive testing from the perspective of graphical modeling (GM). GM provides methods for making inferences about multifaceted skills and knowledge and for extracting data from complex performances. Provides examples from language-proficiency assessment. (SLD)}, keywords = {computerized adaptive testing}, author = {Almond, R. G. and Mislevy, R. J.} } @conference {984, title = {Getting more precision on computer adaptive testing}, booktitle = {Paper presented at the 62nd Annual meeting of Psychometric Society}, year = {1997}, address = {University of Tennessee, Knoxville, TN}, author = {Krass, I. A.} } @conference {1199, title = {The goal of equity within and between computerized adaptive tests and paper and pencil forms. }, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1997}, address = {Chicago IL}, author = {Thomasson, G. L.} } @article {2032, title = {A Global Information Approach to Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {20}, year = {1996}, pages = {213-229}, author = {Chang, H.-H. and Ying, Z.} } @article {58, title = {A global information approach to computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {20}, number = {3}, year = {1996}, pages = {213-229}, abstract = {based on Fisher information (or item information). At each stage, an item is selected to maximize the Fisher information at the currently estimated trait level (\&thetas;). However, this application of Fisher information could be much less efficient than assumed if the estimators are not close to the true \&thetas;, especially at early stages of an adaptive test when the test length (number of items) is too short to provide an accurate estimate for true \&thetas;. It is argued here that selection procedures based on global information should be used, at least at early stages of a test when \&thetas; estimates are not likely to be close to the true \&thetas;. For this purpose, an item selection procedure based on average global information is proposed. Results from pilot simulation studies comparing the usual maximum item information item selection with the proposed global information approach are reported, indicating that the new method leads to improvement in terms of bias and mean squared error reduction under many circumstances. Index terms: computerized adaptive testing, Fisher information, global information, information surface, item information, item response theory, Kullback-Leibler information, local information, test information.}, isbn = {0146-6216}, author = {Chang, Hua-Hua and Ying, Z.} } @conference {869, title = {A global information approach to computerized adaptive testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1995}, address = {San Francisco CA}, author = {Chang, Hua-Hua} } @book {1641, title = {Guidelines for computer-adaptive test development and use in education}, year = {1995}, address = {Washington DC: Author}, author = {American-Council-on-Education.} } @article {473, title = {A general approach to algorithmic design of fixed-form tests, adaptive tests, and testlets}, journal = {Applied Psychological Measurement}, volume = {18}, year = {1994}, pages = {141-153}, author = {Berger, M. P. F.} } @article {2029, title = {A General Approach to Algorithmic Design of Fixed-Form Tests, Adaptive Tests, and Testlets}, journal = {Applied Psychological Measurement}, volume = {18}, year = {1994}, pages = {141-153}, author = {Berger, M. P. F.} } @booklet {1559, title = {A general Bayesian model for testlets: theory and applications (Research Report 92-21; GRE Board Professional Report No 99-01P)}, year = {1992}, address = {Princeton NJ: Educational Testing Service.}, author = {Wang, X and Bradlow, E. T. and Wainer, H.,} } @booklet {1579, title = {Generative adaptive testing with digit span items}, year = {1990}, address = {San Diego, CA: Testing Systems Department, Navy Personnel Research and Development Center}, author = {Wolfe, J. H. and Larson, G. E.} } @conference {1271, title = {Golden section search strategies for computerized adaptive testing}, booktitle = {Paper presented at the Fifth International Objective Measurement Workshop}, year = {1989}, note = {$\#$XI89-01}, address = {Berkeley CA}, author = {Xiao, B.} } @conference {1108, title = {A generalization of sequential analysis to decision making with tailored testing}, booktitle = {Paper presented at the meeting of the Military Testing Association}, year = {1978}, address = {Oklahoma City OK}, author = {Reckase, M. D.} } @conference {2226, title = {Group tailored tests and some problems of their utlization}, booktitle = {Third international Symposium on Educational testing}, year = {1977}, month = {06/1977}, address = {Leyden, The Netherlands}, author = {Lewy, A and Doron, R} } @inbook {1880, title = {The graded response model of latent trait theory and tailored testing}, year = {1976}, address = {C. K. Clark (Ed.), Proceedings of the First Conference on Computerized Adaptive Testing (pp. 5-17). Washington DC: U.S. Government Printing Office.}, author = {Samejima, F.} }