diff --git a/src/lib.rs b/src/lib.rs
index b53eb96..eaa9f18 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -246,6 +246,12 @@ impl<T, const N: usize> SmallVec<T, N> {
         } else {
             let mut vec = ManuallyDrop::new(vec);
             let length = vec.len();
+
+            // A heap-allocated `SmallVec` must always observe the invariant
+            // that `cap > N`.
+            if vec.capacity() <= N {
+                vec.reserve(N + 1 - length);
+            }
             let cap = vec.capacity();
             // SAFETY: vec.capacity is not `0` (checked above), so the pointer
             // can not dangle and thus specifically cannot be null.
@@ -642,14 +648,15 @@ impl<T, const N: usize, A: Allocator> SmallVec<T, N, A> {
             .unwrap_or_else(SmallVecError::handle);
     }
 
-    #[cold]
     pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), SmallVecError> {
         if Self::IS_ZST {
             return Ok(());
         }
 
         let (length, on_heap) = self.length.parts();
-        assert!(new_capacity >= length);
+        if new_capacity <= length {
+            return Ok(());
+        }
 
         if new_capacity > Self::inline_size() {
             // SAFETY: we checked all the preconditions
@@ -697,11 +704,18 @@ impl<T, const N: usize, A: Allocator> SmallVec<T, N, A> {
     #[inline]
     pub fn try_reserve(&mut self, additional: usize) -> Result<(), SmallVecError> {
         if additional > self.capacity() - self.len() {
-            let new_capacity = self
+            let required = self
                 .len()
                 .checked_add(additional)
-                .and_then(usize::checked_next_power_of_two)
                 .ok_or(SmallVecError::CapacityOverflow)?;
+
+            let double_cap = self.capacity().saturating_mul(2);
+
+            let new_capacity = required
+                .max(double_cap)
+                .checked_next_power_of_two()
+                .ok_or(SmallVecError::CapacityOverflow)?;
+
             self.try_grow(new_capacity)
         } else {
             Ok(())
diff --git a/tests/main.rs b/tests/main.rs
index 9f565ba..0c230d8 100644
--- a/tests/main.rs
+++ b/tests/main.rs
@@ -470,11 +470,11 @@ fn append() {
 }
 
 #[test]
-#[should_panic(expected = "new_capacity >= length")]
 fn invalid_grow() {
     let mut v: SmallVec<u8, 8> = SmallVec::new();
     v.extend(0..8);
     v.grow(5);
+    assert_eq!(v.capacity(), 8);
 }
 
 #[test]