diff --git a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
index 00e4e4f513..1432212b86 100644
--- a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
@@ -316,7 +316,8 @@ public class HlsSampleSource implements SampleSource, Loader.Callback {
   }
 
   private void maybeThrowLoadableException() throws IOException {
-    if (currentLoadableException != null && currentLoadableExceptionCount > minLoadableRetryCount) {
+    if (currentLoadableException != null && (currentLoadableExceptionFatal
+        || currentLoadableExceptionCount > minLoadableRetryCount)) {
       throw currentLoadableException;
     }
   }