diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index 060b5f4daf..17e445dd98 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -30,6 +30,9 @@
 * IMA extension:
   * For preroll to live stream transitions, project forward the loading position
     to avoid being behind the live window.
+* Fix issue where a `NullPointerException` is thrown when removing an unprepared
+  media source from a `ConcatenatingMediaSource` with the `useLazyPreparation`
+  option enabled ([#4986](https://github.com/google/ExoPlayer/issues/4986)).
 
 ### 2.9.0 ###
 
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java b/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java
index fb99eef6e7..830ab52e19 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/source/ConcatenatingMediaSource.java
@@ -502,9 +502,7 @@ public class ConcatenatingMediaSource extends CompositeMediaSource<MediaSourceHo
         Assertions.checkNotNull(mediaSourceByMediaPeriod.remove(mediaPeriod));
     ((DeferredMediaPeriod) mediaPeriod).releasePeriod();
     holder.activeMediaPeriods.remove(mediaPeriod);
-    if (holder.activeMediaPeriods.isEmpty() && holder.isRemoved) {
-      releaseChildSource(holder);
-    }
+    maybeReleaseChildSource(holder);
   }
 
   @Override
@@ -747,9 +745,7 @@ public class ConcatenatingMediaSource extends CompositeMediaSource<MediaSourceHo
         -oldTimeline.getWindowCount(),
         -oldTimeline.getPeriodCount());
     holder.isRemoved = true;
-    if (holder.activeMediaPeriods.isEmpty()) {
-      releaseChildSource(holder);
-    }
+    maybeReleaseChildSource(holder);
   }
 
   private void moveMediaSourceInternal(int currentIndex, int newIndex) {
@@ -778,6 +774,16 @@ public class ConcatenatingMediaSource extends CompositeMediaSource<MediaSourceHo
     }
   }
 
+  private void maybeReleaseChildSource(MediaSourceHolder mediaSourceHolder) {
+    // Release if the source has been removed from the playlist, but only if it has been previously
+    // prepared and only if we are not waiting for an existing media period to be released.
+    if (mediaSourceHolder.isRemoved
+        && mediaSourceHolder.hasStartedPreparing
+        && mediaSourceHolder.activeMediaPeriods.isEmpty()) {
+      releaseChildSource(mediaSourceHolder);
+    }
+  }
+
   /** Return uid of media source holder from period uid of concatenated source. */
   private static Object getMediaSourceHolderUid(Object periodUid) {
     return ConcatenatedTimeline.getChildTimelineUidFromConcatenatedUid(periodUid);
diff --git a/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java b/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java
index d3d3b39ea4..dd1221f160 100644
--- a/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java
+++ b/library/core/src/test/java/com/google/android/exoplayer2/source/ConcatenatingMediaSourceTest.java
@@ -925,6 +925,22 @@ public final class ConcatenatingMediaSourceTest {
     testRunner.createPeriod(mediaPeriodId);
   }
 
+  @Test
+  public void testRemoveUnpreparedChildSourceWithLazyPreparation() throws IOException {
+    FakeMediaSource[] childSources = createMediaSources(/* count= */ 2);
+    mediaSource =
+        new ConcatenatingMediaSource(
+            /* isAtomic= */ false,
+            /* useLazyPreparation= */ true,
+            new DefaultShuffleOrder(0),
+            childSources);
+    testRunner = new MediaSourceTestRunner(mediaSource, /* allocator= */ null);
+    testRunner.prepareSource();
+
+    // Check that removal doesn't throw even though the child sources are unprepared.
+    mediaSource.removeMediaSource(0);
+  }
+
   @Test
   public void testSetShuffleOrderBeforePreparation() throws Exception {
     mediaSource.setShuffleOrder(new ShuffleOrder.UnshuffledShuffleOrder(/* length= */ 0));