diff --git a/src/cmap/_colormap.py b/src/cmap/_colormap.py
index 97a0272e4..ed75c4b3f 100644
--- a/src/cmap/_colormap.py
+++ b/src/cmap/_colormap.py
@@ -415,8 +415,7 @@ def __call__(
         xa = np.array(x, copy=True)
         if not xa.dtype.isnative:
             # Native byteorder is faster.
-            native: Literal[">", "<"] = ">" if xa.dtype.byteorder in ("<", "=") else "<"
-            xa = xa.view(xa.dtype.newbyteorder(native))
+            xa = xa.byteswap().view(xa.dtype.newbyteorder())
         if xa.dtype.kind == "f":
             xa *= N
             # xa == 1 (== N after multiplication) is not out of range.
diff --git a/tests/test_colormap.py b/tests/test_colormap.py
index c17ad6f7a..fa75b4885 100644
--- a/tests/test_colormap.py
+++ b/tests/test_colormap.py
@@ -144,6 +144,14 @@ def test_colormap_apply() -> None:
     assert cmap1(swapped).shape == (10, 10, 4)
 
 
+def test_non_native_byte_order_maps_the_same_colors() -> None:
+    cmap = Colormap(["red", "blue"], under="green", over="yellow", bad="black")
+    native = np.array([-0.5, 0.0, 0.5, 1.0, 1.5, np.nan])
+    non_native = native.astype(native.dtype.newbyteorder())
+
+    npt.assert_array_equal(cmap(non_native), cmap(native))
+
+
 def test_colormap_masked_array_with_unmasked_nan() -> None:
     cmap = Colormap("viridis", bad="red")
     mask = [True, False, False, False]