diff --git a/src/glum/_glm.py b/src/glum/_glm.py
index 43f6516e..33afb37f 100644
--- a/src/glum/_glm.py
+++ b/src/glum/_glm.py
@@ -2711,7 +2711,8 @@ def _set_up_and_check_fit_args(
                 self.feature_dtypes_ = X.dtypes.to_dict()
                 self.has_missing_category_ = {
                     col: (self.cat_missing_method == "convert") and X[col].isna().any()
-                    for col in self.feature_dtypes_.keys()
+                    for col, dtype in self.feature_dtypes_.items()
+                    if isinstance(dtype, pd.CategoricalDtype)
                 }
 
                 if any(X.dtypes == "category"):