@conference {2651, title = {Adaptive Item and Feedback Selection in Personalized Learning with a Network Approach}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Personalized learning is a term used to describe educational systems that adapt student-specific curriculum sequencing, pacing, and presentation based on their unique backgrounds, knowledge, preferences, interests, and learning goals. (Chen, 2008; Netcoh, 2016). The technological approach to personalized learning provides data-driven models to incorporate these adaptations automatically. Examples of applications include online learning systems, educational games, and revision-aid systems. In this study we introduce Bayesian networks as a methodology to implement an adaptive framework within a personalized learning environment. Existing ideas from Computerized Adaptive Testing (CAT) with Item Response Theory (IRT), where choices about content provision are based on maximizing information, are related to the goals of personalized learning environments. Personalized learning entails other goals besides efficient ability estimation by maximizing information, such as an adaptive configuration of preferences and feedback to the student. These considerations will be discussed and their application in networks will be illustrated.

Adaptivity in Personalized Learning.In standard CAT\’s there is a focus on selecting items that provide maximum information about the ability of an individual at a certain point in time (Van der Linden \& Glas, 2000). When learning is the main goal of testing, alternative adaptive item selection methods were explored by Eggen (2012). The adaptive choices made in personalized learning applications require additional adaptivity with respect to the following aspects; the moment of feedback, the kind of feedback, and the possibility for students to actively influence the learning process.

Bayesian Networks and Personalized Learning.Personalized learning aims at constructing a framework to incorporate all the aspects mentioned above. Therefore, the goal of this framework is not only to focus on retrieving ability estimates by choosing items on maximum information, but also to construct a framework that allows for these other factors to play a role. Plajner and Vomlel (2016) have already applied Bayesian Networks to adaptive testing, selecting items with help of entropy reduction. Almond et al. (2015) provide a reference work on Bayesian Networks in Educational Assessment. Both acknowledge the potential of the method in terms of features such as modularity options to build finer-grained models. IRT does not allow to model sub-skills very easily and to gather information on fine-grained level, due to its dependency on the assumption of generally one underlying trait. The local independence assumption in IRT implies being interested in mainly the student\’s overall ability on the subject of interest. When the goal is to improve student\’s learning, we are not just interested in efficiently coming to their test score on a global subject. One wants a model that is able to map\ educational problems and talents in detail over the whole educational program, while allowing for dependency between items. The moment in time can influence topics to be better mastered than others, and this is exactly what we can to get out of a model. The possibility to model flexible structures, estimate abilities on a very detailed level for sub-skills and to easily incorporate other variables such as feedback in Bayesian Networks makes it a very promising method for making adaptive choices in personalized learning. It is shown in this research how item and feedback selection can be performed with help of the promising Bayesian Networks. A student involvement possibility is also introduced and evaluated.

References

Almond, R. G., Mislevy, R. J., Steinberg, L. S., Yan, D., \& Williamson, D. M. (2015). Bayesian Networks in Educational Assessment. Test. New York: Springer Science+Business Media. http://doi.org/10.1007/978-0-387-98138-3

Eggen, T.J.H.M. (2012) Computerized Adaptive Testing Item Selection in Computerized Adaptive Learning Systems. In: Eggen. TJHM \& Veldkamp, BP.. (Eds). Psychometrics in Practice at RCEC. Enschede: RCEC

Netcoh, S. (2016, March). \“What Do You Mean by \‘Personalized Learning?\’. Croscutting Conversations in Education \– Research, Reflections \& Practice. Blogpost.

Plajner, M., \& Vomlel, J. (2016). Student Skill Models in Adaptive Testing. In Proceedings of the Eighth International Conference on Probabilistic Graphical Models (pp. 403-414).

Van der Linden, W. J., \& Glas, C. A. (2000). Computerized adaptive testing: Theory and practice. Dordrecht: Kluwer Academic Publishers.

Session Video

}, keywords = {feedback selection, item selection, network approach, personalized learning}, author = {Nikky van Buuren and Hendrik Straat and Theo Eggen and Jean-Paul Fox} } @conference {2652, title = {Efficiency of Targeted Multistage Calibration Designs under Practical Constraints: A Simulation Study}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Calibration of an item bank for computer adaptive testing requires substantial resources. In this study, we focused on two related research questions. First, we investigated whether the efficiency of item calibration under the Rasch model could be enhanced by calibration designs that optimize the match between item difficulty and student ability (Berger, 1991). Therefore, we introduced targeted multistage calibration designs, a design type that refers to a combination of traditional targeted calibration designs and multistage designs. As such, targeted multistage calibration designs consider ability-related background variables (e.g., grade in school), as well as performance (i.e., outcome of a preceding test stage) for assigning students to suitable items.

Second, we explored how limited a priori knowledge about item difficulty affects the efficiency of both targeted calibration designs and targeted multistage calibration designs. When arranging items within a given calibration design, test developers need to know the item difficulties to locate items optimally within the design. However, usually, no empirical information about item difficulty is available before item calibration. Owing to missing empirical data, test developers might fail to assign all items to the most suitable location within a calibration design.

Both research questions were addressed in a simulation study in which we varied the calibration design, as well as the accuracy of item distribution across the different booklets or modules within each design (i.e., number of misplaced items). The results indicated that targeted multistage calibration designs were more efficient than ordinary targeted designs under optimal conditions. Especially, targeted multistage calibration designs provided more accurate estimates for very easy and 52 IACAT 2017 ABSTRACTS BOOKLET very difficult items. Limited knowledge about item difficulty during test construction impaired the efficiency of all designs. The loss of efficiency was considerably large for one of the two investigated targeted multistage calibration designs, whereas targeted designs were more robust.

References

Berger, M. P. F. (1991). On the efficiency of IRT models when applied to different sampling designs. Applied Psychological Measurement, 15(3), 293\–306. doi:10.1177/014662169101500310

Session Video

}, keywords = {CAT, Efficiency, Multistage Calibration}, url = {https://drive.google.com/file/d/1ko2LuiARKqsjL_6aupO4Pj9zgk6p_xhd/view?usp=sharing}, author = {Stephanie Berger and Angela J. Verschoor and Theo Eggen and Urs Moser} } @conference {2642, title = {The Implementation of Nationwide High Stakes Computerized (adaptive) Testing in the Netherlands}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

In this presentation the challenges of implementation of (adaptive) digital testing in the Facet system in the Netherlands is discussed. In the Netherlands there is a long tradition of implementing adaptive testing in educational settings. Already since the late nineties of the last century adaptive testing was used mostly in low stakes testing. Several CATs were implemented in student monitoring systems for primary education and in the general subjects language and arithmetic in vocational education. The only nationwide implemented high stakes CAT is the WISCAT-pabo: an arithmetic test for students in the first year of primary school teacher colleges. The psychometric advantages of item based adaptive testing are obvious. For example efficiency and high measurement precision. But there are also some disadvantages such as the impossibility of reviewing items during and after the test. During the test the student is not in control of his own test; e.q . he can only navigate forward to the next item. This is one of the reasons other methods of testing, such as multistage-testing, with adaptivity not on the item level but on subtest level, has become more popular to use in high stakes testing.

A main challenge of computerized (adaptive) testing is the implementation of the item bank and the test workflow in a digital system. Since 2014 a nationwide new digital system (Facet) was introduced in the Netherlands, with connections to the digital systems of different parties based on international standards (LTI and QTI). The first nationwide tests in the Facet-system were flexible exams Dutch and arithmetic for vocational (and secondary) education, taken as item response theory-based equated linear multiple forms tests, which are administered during 5 periods in a year. Nowadays there are some implementations of different methods of (multistage) adaptive testing in the same Facet system (DTT en Acet).

In this conference, other presenters of Cito will elaborate on the psychometric characteristics of this other adaptive testing methods. In this contribution, the system architecture and interoperability of the Facet system will be explained. The emphasis is on the implementation and the problems to be solved by using this digital system in all phases of the (adaptive) testing process: item banking, test construction, designing, publication, test taking, analyzing and reporting to the student. An evaluation of the use of the system will be presented.

Session Video

}, keywords = {High stakes CAT, Netherlands, WISCAT}, url = {https://drive.google.com/open?id=1Kn1PvgioUYaOJ5pykq-_XWnwDU15rRsf}, author = {Mia van Boxel and Theo Eggen} } @conference {2106, title = {Item Selection Methods based on Multiple Objective Approaches for Classification of Respondents into Multiple Levels}, booktitle = {Annual Conference of the International Association for Computerized Adaptive Testing}, year = {2011}, month = {10/2011}, abstract = {

Is it possible to develop new item selection methods which take advantage of the fact that we want to classify into multiple categories? New methods: Taking multiple points on the ability scale into account; Based on multiple objective approaches.

Conclusions

}, keywords = {adaptive classification test, CAT, item selection, sequential classification test}, author = {Maaike van Groen and Theo Eggen and Bernard Veldkamp} } @conference {2105, title = {A Test Assembly Model for MST}, booktitle = {Annual Conference of the International Association for Computerized Adaptive Testing}, year = {2011}, month = {10/2011}, abstract = {

This study is just a short exploration in the matter of optimization of a MST. It is extremely hard or maybe impossible to chart influence of item pool and test specifications on optimization process. Simulations are very helpful in finding an acceptable MST.

}, keywords = {CAT, mst, multistage testing, Rasch, routing, tif}, author = {Angela Verschoor and Ingrid Radtke and Theo Eggen} } @inbook {2067, title = {Three-Category Adaptive Classification Testing}, booktitle = {Elements of Adaptive Testing}, year = {2010}, pages = {373-387}, chapter = {19}, doi = {10.1007/978-0-387-85461-8}, author = {Theo Eggen} } @inbook {1959, title = {Computerized classification testing in more than two categories by using stochastic curtailment}, year = {2009}, note = {{PDF file, 298 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing.}, author = {Wouda, J. T. and Theo Eggen} } @inbook {1960, title = {Constrained item selection using a stochastically curtailed SPRT}, year = {2009}, note = {{PDF File, 298 KB}{PDF File, 298 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing.}, author = {Wouda, J. T. and Theo Eggen} } @inbook {1764, title = {Choices in CAT models in the context of educational testing}, year = {2007}, note = {{PDF file, 123 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2007 GMAC Conference on Computerized Adaptive Testing.}, author = {Theo Eggen} } @conference {114, title = {Choices in CAT models in the context of educattional testing}, booktitle = {GMAC Conference on Computerized Adaptive Testing}, year = {2007}, month = {June 7, 2007}, publisher = {Graduate Management Admission Council}, organization = {Graduate Management Admission Council}, address = {St. Paul, MN}, author = {Theo Eggen} } @article {2208, title = {A multiple objective test assembly approach for exposure control problems in computerized adaptive testing}, number = {2007-1}, year = {2007}, institution = {Cito}, address = {Arnhem, The Netherlands}, author = {Veldkamp, B. P. and Verschoor, Angela J. and Theo Eggen} } @article {116, title = {Optimal testing with easy or difficult items in computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {30}, number = {5}, year = {2006}, pages = {379-393}, publisher = {Sage Publications: US}, abstract = {Computerized adaptive tests (CATs) are individualized tests that, from a measurement point of view, are optimal for each individual, possibly under some practical conditions. In the present study, it is shown that maximum information item selection in CATs using an item bank that is calibrated with the one- or the two-parameter logistic model results in each individual answering about 50\% of the items correctly. Two item selection procedures giving easier (or more difficult) tests for students are presented and evaluated. Item selection on probability points of items yields good results only with the one-parameter logistic model and not with the two-parameter logistic model. An alternative selection procedure, based on maximum information at a shifted ability level, gives satisfactory results with both models. (PsycINFO Database Record (c) 2007 APA, all rights reserved)}, keywords = {computer adaptive tests, individualized tests, Item Response Theory, item selection, Measurement}, isbn = {0146-6216 (Print)}, author = {Theo Eggen and Verschoor, Angela J.} } @article {2147, title = {Optimal Testing With Easy or Difficult Items in Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {30}, number = {5}, year = {2006}, pages = {379-393}, abstract = {

Computerized adaptive tests (CATs) are individualized tests that, from a measurement point of view, are optimal for each individual, possibly under some practical conditions. In the present study, it is shown that maximum information item selection in CATs using an item bank that is calibrated with the one or the two-parameter logistic model results in each individual answering about 50\% of the items correctly. Two item selection procedures giving easier (or more difficult) tests for students are presented and evaluated. Item selection on probability points of items yields good results only with the one-parameter logistic model and not with the two-parameter logistic model. An alternative selection procedure, based on maximum information at a shifted ability level, gives satisfactory results with both models. Index terms: computerized adaptive testing, item selection, item response theory

}, doi = {10.1177/0146621606288890}, url = {http://apm.sagepub.com/content/30/5/379.abstract}, author = {Theo Eggen and Verschoor, Angela J.} } @book {1667, title = {Contributions to the theory and practice of computerized adaptive testing}, year = {2004}, address = {Arnhem, The Netherlands: Citogroep}, author = {Theo Eggen} } @booklet {1370, title = {Optimal testing with easy items in computerized adaptive testing (Measurement and Research Department Report 2004-2)}, year = {2004}, address = {Arnhem, The Netherlands: Cito Group}, author = {Theo Eggen and Verschoor, A. J.} } @conference {912, title = {Optimal testing with easy items in computerized adaptive testing}, booktitle = {Paper presented at the conference of the International Association for Educational Assessment}, year = {2003}, address = {Manchester UK}, author = {Theo Eggen and Verschoor, A.} } @article {681, title = {Evaluation of selection procedures for computerized adaptive testing with polytomous items}, journal = {Applied Psychological Measurement}, volume = {26}, year = {2002}, pages = {393-411}, author = {van Rijn, P. W. and Theo Eggen and Hemker, B. T. and Sanders, P. F.} } @article {412, title = {Evaluation of selection procedures for computerized adaptive testing with polytomous items}, journal = {Applied Psychological Measurement}, volume = {26}, number = {4}, year = {2002}, note = {References .Sage Publications, US}, pages = {393-411}, abstract = {In the present study, a procedure that has been used to select dichotomous items in computerized adaptive testing was applied to polytomous items. This procedure was designed to select the item with maximum weighted information. In a simulation study, the item information function was integrated over a fixed interval of ability values and the item with the maximum area was selected. This maximum interval information item selection procedure was compared to a maximum point information item selection procedure. Substantial differences between the two item selection procedures were not found when computerized adaptive tests were evaluated on bias and the root mean square of the ability estimate. }, keywords = {computerized adaptive testing}, author = {van Rijn, P. W. and Theo Eggen and Hemker, B. T. and Sanders, P. F.} } @booklet {1368, title = {Overexposure and underexposure of items in computerized adaptive testing (Measurement and Research Department Reports 2001-1)}, year = {2001}, note = {{PDF file, 276 KB}}, address = {Arnhem, The Netherlands: CITO Groep}, author = {Theo Eggen} } @article {115, title = {Computerized adaptive testing for classifying examinees into three categories}, journal = {Educational and Psychological Measurement}, volume = {60}, number = {5}, year = {2000}, pages = {713-34}, abstract = {The objective of this study was to explore the possibilities for using computerized adaptive testing in situations in which examinees are to be classified into one of three categories.Testing algorithms with two different statistical computation procedures are described and evaluated. The first computation procedure is based on statistical testing and the other on statistical estimation. Item selection methods based on maximum information (MI) considering content and exposure control are considered. The measurement quality of the proposed testing algorithms is reported. The results of the study are that a reduction of at least 22\% in the mean number of items can be expected in a computerized adaptive test (CAT) compared to an existing paper-and-pencil placement test. Furthermore, statistical testing is a promising alternative to statistical estimation. Finally, it is concluded that imposing constraints on the MI selection strategy does not negatively affect the quality of the testing algorithms}, keywords = {computerized adaptive testing, Computerized classification testing}, author = {Theo Eggen and Straetmans, G. J. J. M.} } @booklet {1509, title = {A selection procedure for polytomous items in computerized adaptive testing (Measurement and Research Department Reports 2000-5)}, year = {2000}, address = {Arnhem, The Netherlands: Cito}, author = {Rijn, P. W. van, and Theo Eggen and Hemker, B. T. and Sanders, P. F.} } @article {536, title = {Item selection in adaptive testing with the sequential probability ratio test}, journal = {Applied Psychological Measurement}, volume = {23}, year = {1999}, note = {[Reprinted as Chapter 6 in $\#$EG04-01]}, pages = {249-261}, author = {Theo Eggen} } @article {377, title = {Computerized adaptive testing: What it is and how it works}, journal = {Educational Technology}, volume = {38}, number = {1}, year = {1998}, pages = {45-52}, abstract = {Describes the workings of computerized adaptive testing (CAT). Focuses on the key concept of information and then discusses two important components of a CAT system: the calibrated item bank and the testing algorithm. Describes a CAT that was designed for making placement decisions on the basis of two typical test administrations and notes the most significant differences between traditional paper-based testing and CAT. (AEF)}, author = {Straetmans, G. J. J. M. and Theo Eggen} } @booklet {1367, title = {Item selection in adaptive testing with the sequential probability ratio test (Measurement and Research Department Report, 98-1)}, year = {1998}, note = {[see APM paper, 1999; also reprinted as Chapter 6 in $\#$EG04-01.]}, address = {Arnhem, The Netherlands: Cito.}, author = {Theo Eggen} } @booklet {1369, title = {Computerized adaptive testing for classifying examinees into three categories (Measurement and Research Department Rep 96-3)}, year = {1996}, note = {$\#$EG96-3 . [Reprinted in Chapter 5 in $\#$EG04-01]}, address = {Arnhem, The Netherlands: Cito}, author = {Theo Eggen and Straetmans, G. J. J. M.} }