@inbook {1884, title = {Criterion-related validity of an innovative CAT-based personality measure}, year = {2009}, note = {{PDF File, 163 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing.}, abstract = {This paper describes development and initial criterion-related validation of the PreVisor Computer Adaptive Personality Scales (PCAPS), a computerized adaptive testing-based personality measure that uses an ideal point IRT model based on forced-choice, paired-comparison responses. Based on results from a large consortium study, a composite of six PCAPS scales identified as relevant to the population of interest (first-line supervisors) had an estimated operational validity against an overall job performance criterion of ρ = .25. Uncorrected and corrected criterion-related validity results for each of the six PCAPS scales making up the composite are also reported. Because the PCAPS algorithm computes intermediate scale scores until a stopping rule is triggered, we were able to graph number of statement-pairs presented against criterion-related validities. Results showed generally monotonically increasing functions. However, asymptotic validity levels, or at least a reduction in the rate of increase in slope, were often reached after 5-7 statement-pairs were presented. In the case of the composite measure, there was some evidence that validities decreased after about six statement-pairs. A possible explanation for this is provided.}, author = {Schneider, R. J. and McLellan, R. A. and Kantrowitz, T. M. and Houston, J. S. and Borman, W. C.} } @article {349, title = {Computerized adaptive rating scales for measuring managerial performance}, journal = {International Journal of Selection and Assessment}, volume = {11}, number = {2-3}, year = {2003}, pages = {237-246}, abstract = {Computerized adaptive rating scales (CARS) had been developed to measure contextual or citizenship performance. This rating format used a paired-comparison protocol, presenting pairs of behavioral statements scaled according to effectiveness levels, and an iterative item response theory algorithm to obtain estimates of ratees{\textquoteright} citizenship performance (W. C. Borman et al, 2001). In the present research, we developed CARS to measure the entire managerial performance domain, including task and citizenship performance, thus addressing a major limitation of the earlier CARS. The paper describes this development effort, including an adjustment to the algorithm that reduces substantially the number of item pairs required to obtain almost as much precision in the performance estimates. (PsycINFO Database Record (c) 2005 APA )}, keywords = {Adaptive Testing, Algorithms, Associations, Citizenship, Computer Assisted Testing, Construction, Contextual, Item Response Theory, Job Performance, Management, Management Personnel, Rating Scales, Test}, author = {Schneider, R. J. and Goff, M. and Anderson, S. and Borman, W. C.} } @article {36, title = {An examination of the comparative reliability, validity, and accuracy of performance ratings made using computerized adaptive rating scales}, journal = {Journal of Applied Psychology}, volume = {86}, number = {5}, year = {2001}, note = {214803450021-9010Journal ArticleValidation Studies}, pages = {965-973}, abstract = {This laboratory research compared the reliability, validity, and accuracy of a computerized adaptive rating scale (CARS) format and 2 relatively common and representative rating formats. The CARS is a paired-comparison rating task that uses adaptive testing principles to present pairs of scaled behavioral statements to the rater to iteratively estimate a ratee{\textquoteright}s effectiveness on 3 dimensions of contextual performance. Videotaped vignettes of 6 office workers were prepared, depicting prescripted levels of contextual performance, and 112 subjects rated these vignettes using the CARS format and one or the other competing format. Results showed 23\%-37\% lower standard errors of measurement for the CARS format. In addition, validity was significantly higher for the CARS format (d = .18), and Cronbach{\textquoteright}s accuracy coefficients showed significantly higher accuracy, with a median effect size of .08. The discussion focuses on possible reasons for the results.}, keywords = {*Computer Simulation, *Employee Performance Appraisal, *Personnel Selection, Adult, Automatic Data Processing, Female, Human, Male, Reproducibility of Results, Sensitivity and Specificity, Support, U.S. Gov{\textquoteright}t, Non-P.H.S., Task Performance and Analysis, Video Recording}, author = {Borman, W. C. and Buck, D. E. and Hanson, M. A. and Motowidlo, S. J. and Stark, S. and F Drasgow} } @booklet {1336, title = {Computerized adaptive rating scales (CARS): Development and evaluation of the concept}, year = {2000}, address = {(Institute Rep No. 350). Tampa FL: Personnel Decisions Research Institute.}, author = {Borman, W. C. and Hanson, M. A. and Kubisiak, U. C. and Buck, D. E.} } @conference {864, title = {Computerized adaptive rating scales that measure contextual performance}, booktitle = {Paper presented at the 3th annual conference of the Society for Industrial and Organizational Psychology}, year = {1998}, address = {Dallas TX}, author = {Borman, W. C. and Hanson, M. A. and Montowidlo, S. J and F Drasgow and Foster, L and Kubisiak, U. C.} }