diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index cd8caa0c8a..ba1734bc30 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -23,6 +23,9 @@
   ([#4532](https://github.com/google/ExoPlayer/issues/4532)).
 * Suppress spurious "references unknown class member" shrinking warning
   ([#4890](https://github.com/google/ExoPlayer/issues/4890)).
+* Fix issue where a `NullPointerException` is thrown when removing an unprepared
+  media source from a `ConcatenatingMediaSource` with the `useLazyPreparation`
+  option enabled ([#4986](https://github.com/google/ExoPlayer/issues/4986)).
 
 ### 2.9.0 ###
 
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java b/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java
index 7418e84449..1b614e18a9 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java
@@ -502,9 +502,7 @@ public class ConcatenatingMediaSource extends CompositeMediaSource<MediaSourceHo
         Assertions.checkNotNull(mediaSourceByMediaPeriod.remove(mediaPeriod));
     ((DeferredMediaPeriod) mediaPeriod).releasePeriod();
     holder.activeMediaPeriods.remove(mediaPeriod);
-    if (holder.activeMediaPeriods.isEmpty() && holder.isRemoved) {
-      releaseChildSource(holder);
-    }
+    maybeReleaseChildSource(holder);
   }
 
   @Override
@@ -747,9 +745,7 @@ public class ConcatenatingMediaSource extends CompositeMediaSource<MediaSourceHo
         -oldTimeline.getWindowCount(),
         -oldTimeline.getPeriodCount());
     holder.isRemoved = true;
-    if (holder.activeMediaPeriods.isEmpty()) {
-      releaseChildSource(holder);
-    }
+    maybeReleaseChildSource(holder);
   }
 
   private void moveMediaSourceInternal(int currentIndex, int newIndex) {
@@ -778,6 +774,16 @@ public class ConcatenatingMediaSource extends CompositeMediaSource<MediaSourceHo
     }
   }
 
+  private void maybeReleaseChildSource(MediaSourceHolder mediaSourceHolder) {
+    // Release if the source has been removed from the playlist, but only if it has been previously
+    // prepared and only if we are not waiting for an existing media period to be released.
+    if (mediaSourceHolder.isRemoved
+        && mediaSourceHolder.hasStartedPreparing
+        && mediaSourceHolder.activeMediaPeriods.isEmpty()) {
+      releaseChildSource(mediaSourceHolder);
+    }
+  }
+
   /** Return uid of media source holder from period uid of concatenated source. */
   private static Object getMediaSourceHolderUid(Object periodUid) {
     return ConcatenatedTimeline.getChildTimelineUidFromConcatenatedUid(periodUid);
diff --git a/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java b/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java
index d3d3b39ea4..dd1221f160 100644
--- a/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java
+++ b/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java
@@ -925,6 +925,22 @@ public final class ConcatenatingMediaSourceTest {
     testRunner.createPeriod(mediaPeriodId);
   }
 
+  @Test
+  public void testRemoveUnpreparedChildSourceWithLazyPreparation() throws IOException {
+    FakeMediaSource[] childSources = createMediaSources(/* count= */ 2);
+    mediaSource =
+        new ConcatenatingMediaSource(
+            /* isAtomic= */ false,
+            /* useLazyPreparation= */ true,
+            new DefaultShuffleOrder(0),
+            childSources);
+    testRunner = new MediaSourceTestRunner(mediaSource, /* allocator= */ null);
+    testRunner.prepareSource();
+
+    // Check that removal doesn't throw even though the child sources are unprepared.
+    mediaSource.removeMediaSource(0);
+  }
+
   @Test
   public void testSetShuffleOrderBeforePreparation() throws Exception {
     mediaSource.setShuffleOrder(new ShuffleOrder.UnshuffledShuffleOrder(/* length= */ 0));