diff --git a/pandas/_testing/asserters.py b/pandas/_testing/asserters.py
index daa5187cdb636..8ecc114293f5f 100644
--- a/pandas/_testing/asserters.py
+++ b/pandas/_testing/asserters.py
@@ -35,6 +35,7 @@
 import pandas as pd
 from pandas import (
     Categorical,
+    CategoricalIndex,
     DataFrame,
     DatetimeIndex,
     Index,
@@ -322,7 +323,15 @@ def _check_types(left, right, obj: str = "Index") -> None:
     # skip exact index checking when `check_categorical` is False
     elif check_exact and check_categorical:
         if not left.equals(right):
-            mismatch = left._values != right._values
+            try:
+                mismatch = left._values != right._values
+            except TypeError:
+                if isinstance(left, CategoricalIndex) and isinstance(
+                    right, CategoricalIndex
+                ):
+                    mismatch = left.codes != right.codes
+                else:
+                    mismatch = left.values != right.values
 
             if not isinstance(mismatch, np.ndarray):
                 mismatch = cast("ExtensionArray", mismatch).fillna(True)
diff --git a/pandas/tests/util/test_assert_index_equal.py b/pandas/tests/util/test_assert_index_equal.py
index ab52d6c8e9f39..caf33f16638f1 100644
--- a/pandas/tests/util/test_assert_index_equal.py
+++ b/pandas/tests/util/test_assert_index_equal.py
@@ -317,3 +317,15 @@ def test_assert_multi_index_dtype_check_categorical(check_categorical):
             tm.assert_index_equal(idx1, idx2, check_categorical=check_categorical)
     else:
         tm.assert_index_equal(idx1, idx2, check_categorical=check_categorical)
+
+
+def test_assert_index_equal_categorical_mismatch_categories():
+    # GH#61941
+    ci = CategoricalIndex(["a", "b", "c"], categories=["a", "b", "c"], ordered=False)
+    idx = Index(["a", "b", "c"])
+
+    with pytest.raises(AssertionError, match="Index are different"):
+        tm.assert_index_equal(
+            ci,
+            idx,
+        )