diff --git a/player/loadfile.c b/player/loadfile.c
index e73f6f9f83..1071ec2488 100644
--- a/player/loadfile.c
+++ b/player/loadfile.c
@@ -473,7 +473,7 @@ void add_demuxer_tracks(struct MPContext *mpctx, struct demuxer *demuxer)
  */
 // Return whether t1 is preferred over t2
 static bool compare_track(struct track *t1, struct track *t2, char **langs, bool os_langs,
-                          struct MPOpts *opts, int preferred_program)
+                          struct MPOpts *opts)
 {
     bool sub = t2->type == STREAM_SUB;
     bool ext1 = t1->is_external && !t1->no_default;
@@ -486,11 +486,6 @@ static bool compare_track(struct track *t1, struct track *t2, char **langs, bool
     }
     if (t1->auto_loaded != t2->auto_loaded)
         return !t1->auto_loaded;
-    if (preferred_program != -1 && t1->program_id != -1 && t2->program_id != -1) {
-        if ((t1->program_id == preferred_program) !=
-            (t2->program_id == preferred_program))
-            return t1->program_id == preferred_program;
-    }
     int l1 = mp_match_lang(langs, t1->lang), l2 = mp_match_lang(langs, t2->lang);
     if (!os_langs && l1 != l2)
         return l1 > l2;
@@ -605,6 +600,9 @@ struct track *select_default_track(struct MPContext *mpctx, int order,
             continue;
         if (!opts->autoload_files && track->is_external)
             continue;
+        if (preferred_program != -1 && !track->is_external &&
+            track->program_id != -1 && track->program_id != preferred_program)
+            continue;
         if (duplicate_track(mpctx, order, type, track))
             continue;
         if (sub) {
@@ -623,7 +621,7 @@ struct track *select_default_track(struct MPContext *mpctx, int order,
                 continue;
             track->forced_select = forced;
         }
-        if (!pick || compare_track(track, pick, langs, os_langs, mpctx->opts, preferred_program)) {
+        if (!pick || compare_track(track, pick, langs, os_langs, mpctx->opts)) {
             pick = track;
         }
     }
diff --git a/test/libmpv_test_track_selection.c b/test/libmpv_test_track_selection.c
index 85f5cc553e..9a517aafb7 100644
--- a/test/libmpv_test_track_selection.c
+++ b/test/libmpv_test_track_selection.c
@@ -309,12 +309,13 @@ static void test_track_selection(char *file, char *path)
         check_string("track-list/1/selected", "no");
         check_string("track-list/3/selected", "no");
 
-        // because subs-fallback=default, the jpn track cannot be chosen
-        // so we choose the eng track despite it having the wrong program ID
+        // the eng track cannot be selected due to having the wrong program ID
+        // because subs-fallback=default, the jpn track cannot be chosen either
+        // so we get no subs
         set_property_string("slang", "eng");
         reload_file(path);
-        check_string("current-tracks/sub/lang", "eng");
-        check_string("track-list/1/selected", "yes");
+        check_string("track-list/1/selected", "no");
+        check_string("track-list/3/selected", "no");
 
         // the jpn track is selected to match video program ID
         // even though the user requested eng subtitles