@EdWorkingPaper{ai26-1476,
title = "The Challenge of Capturing Higher-Order Thinking Skills at Scale",
author = "John Wang, Beth E. Schueler",
institution = "Annenberg Institute at Brown University",
number = "1476",
year = "2026",
month = "May",
URL = "http://www.edworkingpapers.com/ai26-1476",
abstract = {Higher-order thinking skills are important for K-12 students’ long-term success. However, the lack of widely administered assessments designed to capture this construct has made it difficult to measure higher-order skills at scale. This paper examines the measurement properties of an approach to capturing higher-order skills using extant statewide standardized testing data. We use item-level data from Massachusetts’s English Language Arts exams for over two decades (2001 to 2023) and pair this with a novel coding which flags whether an item captures higher-order skills based on the Webb’s Depth of Knowledge framework. Overall, we find that this is a challenging approach for separating out higher-order from lower-order skills because state assessments were designed to measure unidimensional constructs. However, there are some cases where the data do appear to distinguish higher-order from lower-order skills in testing years prior to the introduction of Common Core-aligned assessments. Data from these grade-years could be used to isolate and study higher-order skills. However, we encourage researchers to proceed with caution when relying on this approach and to also pursue alternative avenues to measuring these important skills.},
}