@article {415, title = {Bayesian item selection in constrained adaptive testing}, journal = {Psicologica}, volume = {31}, number = {1}, year = {2010}, pages = {149-169}, abstract = {Application of Bayesian item selection criteria in computerized adaptive testing might result in improvement of bias and MSE of the ability estimates. The question remains how to apply Bayesian item selection criteria in the context of constrained adaptive testing, where large numbers of specifications have to be taken into account in the item selection process. The Shadow Test Approach is a general purpose algorithm for administering constrained CAT. In this paper it is shown how the approach can be slightly modified to handle Bayesian item selection criteria. No differences in performance were found between the shadow test approach and the modifiedapproach. In a simulation study of the LSAT, the effects of Bayesian item selection criteria are illustrated. The results are compared to item selection based on Fisher Information. General recommendations about the use of Bayesian item selection criteria are provided.}, keywords = {computerized adaptive testing}, author = {Veldkamp, B. P.} } @inbook {2060, title = {Designing Item Pools for Adaptive Testing}, booktitle = {Elements of Adaptive Testing}, year = {2010}, pages = {231-245}, chapter = {12}, doi = {10.1007/978-0-387-85461-8}, author = {Veldkamp, B. P. and van der Linden, W. J.} } @article {113, title = {Detection of aberrant item score patterns in computerized adaptive testing: An empirical example using the CUSUM}, journal = {Personality and Individual Differences}, volume = {48}, number = {8}, year = {2010}, pages = {921-925}, abstract = {The scalability of individual trait scores on a computerized adaptive test (CAT) was assessed through investigating the consistency of individual item score patterns. A sample of N = 428 persons completed a personality CAT as part of a career development procedure. To detect inconsistent item score patterns, we used a cumulative sum (CUSUM) procedure. Combined information from the CUSUM, other personality measures, and interviews showed that similar estimated trait values may have a different interpretation.Implications for computer-based assessment are discussed.}, keywords = {CAT, computerized adaptive testing, CUSUM approach, person Fit}, isbn = {01918869}, author = {Egberink, I. J. L. and Meijer, R. R. and Veldkamp, B. P. and Schakel, L. and Smid, N. G.} } @article {15, title = {Comparison of methods for controlling maximum exposure rates in computerized adaptive testing}, journal = {Psicothema}, volume = {21}, number = {2}, year = {2009}, note = {Barrada, Juan RamonAbad, Francisco JoseVeldkamp, Bernard PComparative StudySpainPsicothemaPsicothema. 2009 May;21(2):313-20.}, month = {May}, pages = {313-320}, edition = {2009/05/01}, abstract = {This paper has two objectives: (a) to provide a clear description of three methods for controlling the maximum exposure rate in computerized adaptive testing {\textemdash}the Symson-Hetter method, the restricted method, and the item-eligibility method{\textemdash} showing how all three can be interpreted as methods for constructing the variable sub-bank of items from which each examinee receives the items in his or her test; (b) to indicate the theoretical and empirical limitations of each method and to compare their performance. With the three methods, we obtained basically indistinguishable results in overlap rate and RMSE (differences in the third decimal place). The restricted method is the best method for controlling exposure rate, followed by the item-eligibility method. The worst method is the Sympson-Hetter method. The restricted method presents problems of sequential overlap rate. Our advice is to use the item-eligibility method, as it saves time and satisfies the goals of restricting maximum exposure. Comparaci{\'o}n de m{\'e}todos para el control de tasa m{\'a}xima en tests adaptativos informatizados. Este art{\'\i}culo tiene dos objetivos: (a) ofrecer una descripci{\'o}n clara de tres m{\'e}todos para el control de la tasa m{\'a}xima en tests adaptativos informatizados, el m{\'e}todo Symson-Hetter, el m{\'e}todo restringido y el m{\'e}todode elegibilidad del {\'\i}tem, mostrando c{\'o}mo todos ellos pueden interpretarse como m{\'e}todos para la construcci{\'o}n del subbanco de {\'\i}tems variable, del cual cada examinado recibe los {\'\i}tems de su test; (b) se{\~n}alar las limitaciones te{\'o}ricas y emp{\'\i}ricas de cada m{\'e}todo y comparar sus resultados. Se obtienen resultados b{\'a}sicamente indistinguibles en tasa de solapamiento y RMSE con los tres m{\'e}todos (diferencias en la tercera posici{\'o}n decimal). El m{\'e}todo restringido es el mejor en el control de la tasa de exposici{\'o}n,seguido por el m{\'e}todo de elegibilidad del {\'\i}tem. El peor es el m{\'e}todo Sympson-Hetter. El m{\'e}todo restringido presenta un problema de solapamiento secuencial. Nuestra recomendaci{\'o}n ser{\'\i}a utilizar el m{\'e}todo de elegibilidad del {\'\i}tem, puesto que ahorra tiempo y satisface los objetivos de limitar la tasa m{\'a}xima de exposici{\'o}n.}, keywords = {*Numerical Analysis, Computer-Assisted, Psychological Tests/*standards/*statistics \& numerical data}, isbn = {0214-9915 (Print)0214-9915 (Linking)}, author = {Barrada, J and Abad, F. J. and Veldkamp, B. P.} } @inbook {1763, title = {The development of a computerized adaptive test for integrity}, year = {2007}, note = {{PDf file, 290 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2007 GMAC Conference on Computerized Adaptive Testing.}, author = {Egberink, I. J. L. and Veldkamp, B. P.} } @article {2208, title = {A multiple objective test assembly approach for exposure control problems in computerized adaptive testing}, number = {2007-1}, year = {2007}, institution = {Cito}, address = {Arnhem, The Netherlands}, author = {Veldkamp, B. P. and Verschoor, Angela J. and Theo Eggen} } @article {401, title = {Assembling a computerized adaptive testing item pool as a set of linear tests}, journal = {Journal of Educational and Behavioral Statistics}, volume = {31}, number = {1}, year = {2006}, pages = {81-99}, publisher = {Sage Publications: US}, abstract = {Test-item writing efforts typically results in item pools with an undesirable correlational structure between the content attributes of the items and their statistical information. If such pools are used in computerized adaptive testing (CAT), the algorithm may be forced to select items with less than optimal information, that violate the content constraints, and/or have unfavorable exposure rates. Although at first sight somewhat counterintuitive, it is shown that if the CAT pool is assembled as a set of linear test forms, undesirable correlations can be broken down effectively. It is proposed to assemble such pools using a mixed integer programming model with constraints that guarantee that each test meets all content specifications and an objective function that requires them to have maximal information at a well-chosen set of ability values. An empirical example with a previous master pool from the Law School Admission Test (LSAT) yielded a CAT with nearly uniform bias and mean-squared error functions for the ability estimator and item-exposure rates that satisfied the target for all items in the pool. }, keywords = {Algorithms, computerized adaptive testing, item pool, linear tests, mathematical models, statistics, Test Construction, Test Items}, isbn = {1076-9986 (Print)}, author = {van der Linden, W. J. and Ariel, A. and Veldkamp, B. P.} } @conference {838, title = {Multiple maximum exposure rates in computerized adaptive testing}, booktitle = {Paper presented at the SMABS-EAM Conference}, year = {2006}, address = {Budapest, Hungary}, author = {Barrada, J and Veldkamp, B. P. and Olea, J.} } @booklet {1540, title = {Constraining item exposure in computerized adaptive testing with shadow tests}, year = {2005}, address = {Law School Admission Council Computerized Testing Report 02-03}, author = {van der Linden, W. J. and Veldkamp, B. P.} } @article {208, title = {Infeasibility in automated test assembly models: A comparison study of different methods}, journal = {Journal of Educational Measurement}, volume = {42}, number = {3}, year = {2005}, pages = {223-243}, abstract = {Several techniques exist to automatically put together a test meeting a number of specifications. In an item bank, the items are stored with their characteristics. A test is constructed by selecting a set of items that fulfills the specifications set by the test assembler. Test assembly problems are often formulated in terms of a model consisting of restrictions and an objective to be maximized or minimized. A problem arises when it is impossible to construct a test from the item pool that meets all specifications, that is, when the model is not feasible. Several methods exist to handle these infeasibility problems. In this article, test assembly models resulting from two practical testing programs were reconstructed to be infeasible. These models were analyzed using methods that forced a solution (Goal Programming, Multiple-Goal Programming, Greedy Heuristic), that analyzed the causes (Relaxed and Ordered Deletion Algorithm (RODA), Integer Randomized Deletion Algorithm (IRDA), Set Covering (SC), and Item Sampling), or that analyzed the causes and used this information to force a solution (Irreducible Infeasible Set-Solver). Specialized methods such as the IRDA and the Irreducible Infeasible Set-Solver performed best. Recommendations about the use of different methods are given. (PsycINFO Database Record (c) 2005 APA ) (journal abstract)}, keywords = {Algorithms, Item Content (Test), Models, Test Construction}, author = {Huitzing, H. A. and Veldkamp, B. P. and Verschoor, A. J.} } @article {408, title = {Constraining item exposure in computerized adaptive testing with shadow tests}, journal = {Journal of Educational and Behavioral Statistics}, volume = {29}, number = {3}, year = {2004}, pages = {273-291}, publisher = {American Educational Research Assn: US}, abstract = {Item-exposure control in computerized adaptive testing is implemented by imposing item-ineligibility constraints on the assembly process of the shadow tests. The method resembles Sympson and Hetter{\textquoteright}s (1985) method of item-exposure control in that the decisions to impose the constraints are probabilistic. The method does not, however, require time-consuming simulation studies to set values for control parameters before the operational use of the test. Instead, it can set the probabilities of item ineligibility adaptively during the test using the actual item-exposure rates. An empirical study using an item pool from the Law School Admission Test showed that application of the method yielded perfect control of the item-exposure rates and had negligible impact on the bias and mean-squared error functions of the ability estimator. }, keywords = {computerized adaptive testing, item exposure control, item ineligibility constraints, Probability, shadow tests}, isbn = {1076-9986 (Print)}, author = {van der Linden, W. J. and Veldkamp, B. P.} } @article {10, title = {Constructing rotating item pools for constrained adaptive testing}, journal = {Journal of Educational Measurement}, volume = {41}, number = {4}, year = {2004}, pages = {345-359}, publisher = {Blackwell Publishing: United Kingdom}, abstract = {Preventing items in adaptive testing from being over- or underexposed is one of the main problems in computerized adaptive testing. Though the problem of overexposed items can be solved using a probabilistic item-exposure control method, such methods are unable to deal with the problem of underexposed items. Using a system of rotating item pools, on the other hand, is a method that potentially solves both problems. In this method, a master pool is divided into (possibly overlapping) smaller item pools, which are required to have similar distributions of content and statistical attributes. These pools are rotated among the testing sites to realize desirable exposure rates for the items. A test assembly model, motivated by Gulliksen{\textquoteright}s matched random subtests method, was explored to help solve the problem of dividing a master pool into a set of smaller pools. Different methods to solve the model are proposed. An item pool from the Law School Admission Test was used to evaluate the performances of computerized adaptive tests from systems of rotating item pools constructed using these methods. (PsycINFO Database Record (c) 2007 APA, all rights reserved)}, keywords = {computerized adaptive tests, constrained adaptive testing, item exposure, rotating item pools}, isbn = {0022-0655 (Print)}, author = {Ariel, A. and Veldkamp, B. P. and van der Linden, W. J.} } @conference {1213, title = {Constraining item exposure in computerized adaptive testing with shadow tests}, booktitle = {Paper presented at the Annual meeting of the National Council on Measurement in Education}, year = {2003}, note = {$\#$vdLI03-02}, address = {Chicago IL}, author = {van der Linden, W. J. and Veldkamp, B. P.} } @conference {1215, title = {Controlling item exposure and item eligibility in computerized adaptive testing}, year = {2003}, author = {van der Linden, W. J. and Veldkamp, B. P.} } @conference {1216, title = {Implementing an alternative to Sympson-Hetter item-exposure control in constrained adaptive testing}, booktitle = {Paper presented at the Annual meeting of the National Council on Measurement in Education}, year = {2003}, address = {Chicago IL}, author = {Veldkamp, B. P. and van der Linden, W. J.} } @inbook {414, title = {Item selection in polytomous CAT}, booktitle = {New developments in psychometrics}, year = {2003}, pages = {207{\textendash}214}, publisher = {Psychometric Society, Springer}, organization = {Psychometric Society, Springer}, address = {Tokyo, Japan}, keywords = {computerized adaptive testing}, author = {Veldkamp, B. P.}, editor = {A. Okada and K. Shigenasu and Y. Kano and J. Meulman} } @inbook {1927, title = {Item selection in polytomous CAT}, year = {2003}, note = {$\#$VE03207 {PDF file, 79 KB}}, address = {H. Yanai, A. Okada, K. Shigemasu, Y Kano, and J. J. Meulman (eds.), New developments in psychometrics (pp. 207-214). Tokyo, Japan: Springer-Verlag.}, author = {Veldkamp, B. P.} } @booklet {1539, title = {Constraining item exposure in computerized adaptive testing with shadow tests (Research Report No. 02-06)}, year = {2002}, address = {University of Twente, The Netherlands}, author = {van der Linden, W. J. and Veldkamp, B. P.} } @article {418, title = {Mathematical-programming approaches to test item pool design}, number = {RR 02-09}, year = {2002}, note = {Using Smart Source ParsingAdvances in psychology research, Vol. ( Hauppauge, NY : Nova Science Publishers, Inc, [URL:http://www.Novapublishers.com]. vi, 228 pp}, pages = {93-108}, institution = {University of Twente, Faculty of Educational Science and Technology}, address = {Twente, The Netherlands}, abstract = {(From the chapter) This paper presents an approach to item pool design that has the potential to improve on the quality of current item pools in educational and psychological testing and hence to increase both measurement precision and validity. The approach consists of the application of mathematical programming techniques to calculate optimal blueprints for item pools. These blueprints can be used to guide the item-writing process. Three different types of design problems are discussed, namely for item pools for linear tests, item pools computerized adaptive testing (CAT), and systems of rotating item pools for CAT. The paper concludes with an empirical example of the problem of designing a system of rotating item pools for CAT.}, keywords = {Adaptive Testing, Computer Assisted, Computer Programming, Educational Measurement, Item Response Theory, Mathematics, Psychometrics, Statistical Rotation computerized adaptive testing, Test Items, Testing}, isbn = {02-09}, author = {Veldkamp, B. P. and van der Linden, W. J. and Ariel, A.} } @article {417, title = {Multidimensional adaptive testing with constraints on test content}, journal = {Psychometrika}, volume = {67}, number = {4}, year = {2002}, note = {Psychometric Society, US}, pages = {575-588}, abstract = {The case of adaptive testing under a multidimensional response model with large numbers of constraints on the content of the test is addressed. The items in the test are selected using a shadow test approach. The 0{\textendash}1 linear programming model that assembles the shadow tests maximizes posterior expected Kullback-Leibler information in the test. The procedure is illustrated for five different cases of multidimensionality. These cases differ in (a) the numbers of ability dimensions that are intentional or should be considered as ldquonuisance dimensionsrdquo and (b) whether the test should or should not display a simple structure with respect to the intentional ability dimensions.}, author = {Veldkamp, B. P. and van der Linden, W. J.} } @booklet {1547, title = {Implementing constrained CAT with shadow tests for large item pools}, year = {2001}, address = {Submitted for publication}, author = {Veldkamp, B. P.} } @inbook {416, title = {Designing item pools for computerized adaptive testing}, booktitle = {Computerized adaptive testing: Theory and practice}, year = {2000}, pages = {149{\textendash}162}, publisher = {Kluwer Academic Publishers}, organization = {Kluwer Academic Publishers}, address = {Dendrecht, The Netherlands}, author = {Veldkamp, B. P. and van der Linden, W. J.} } @article {409, title = {An integer programming approach to item bank design}, journal = {Applied Psychological Measurement}, volume = {24}, number = {2}, year = {2000}, pages = {139-150}, abstract = {An integer programming approach to item bank design is presented that can be used to calculate an optimal blueprint for an item bank, in order to support an existing testing program. The results are optimal in that they minimize the effort involved in producing the items as revealed by current item writing patterns. Also presented is an adaptation of the models, which can be used as a set of monitoring tools in item bank management. The approach is demonstrated empirically for an item bank that was designed for the Law School Admission Test. }, keywords = {Aptitude Measures, Item Analysis (Test), Item Response Theory, Test Construction, Test Items}, author = {van der Linden, W. J. and Veldkamp, B. P. and Reese, L. M.} } @booklet {1546, title = {Modifications of the branch-and-bound algorithm for application in constrained adaptive testing (Research Report 00-05)}, year = {2000}, address = {Enschede, The Netherlands: University of Twente, Faculty of Educational Science and Technology, Department of Measurement and Data Analysis}, author = {Veldkamp, B. P.} } @booklet {1549, title = {Multidimensional adaptive testing with constraints on test content (Research Report 00-11)}, year = {2000}, address = {Enschede, The Netherlands: University of Twente, Faculty of Educational Science and Technology, Department of Measurement and Data Analysis}, author = {Veldkamp, B. P. and van der Linden, W. J.} } @booklet {1548, title = {Designing item pools for computerized adaptive testing (Research Report 99-03 )}, year = {1999}, address = {Enschede, The Netherlands: University of Twente, Faculty of Educational Science and Technology, Department of Measurement and Data Analysis}, author = {Veldkamp, B. P. and van der Linden, W. J.} }