diff --git a/src/utils/thumbnails.ts b/src/utils/thumbnails.ts
index f6f82a60..5b98cc44 100644
--- a/src/utils/thumbnails.ts
+++ b/src/utils/thumbnails.ts
@@ -1,6 +1,6 @@
 import { isOnInvidious, parseYouTubeVideoIDFromURL } from "../../maze-utils/src/video";
 import Config from "../config";
-import { getVideoLabel } from "./videoLabels";
+import { getHasStartSegment, getVideoLabel } from "./videoLabels";
 import { getThumbnailSelector, setThumbnailListener } from "../../maze-utils/src/thumbnailManagement";
 import { VideoID } from "../types";
 import { getSegmentsForVideo } from "./segmentData";
@@ -59,14 +59,14 @@ function thumbnailHoverListener(e: MouseEvent) {
 
     // Pre-fetch data for this video
     let fetched = false;
-    const preFetch = () => {
+    const preFetch = async () => {
         fetched = true;
         const videoID = extractVideoID(thumbnail);
-        if (videoID) {
+        if (videoID && await getHasStartSegment(videoID)) {
             void getSegmentsForVideo(videoID, false);
         }
     };
-    const timeout = setTimeout(preFetch, 200);
+    const timeout = setTimeout(preFetch, 100);
     const onMouseDown = () => {
         clearTimeout(timeout);
         if (!fetched) {
diff --git a/src/utils/videoLabels.ts b/src/utils/videoLabels.ts
index 82af788c..4b9c71c4 100644
--- a/src/utils/videoLabels.ts
+++ b/src/utils/videoLabels.ts
@@ -6,9 +6,14 @@ import { asyncRequestToServer } from "./requests";
 
 const utils = new Utils();
 
+interface VideoLabelsCacheData {
+    category: Category;
+    hasStartSegment: boolean;
+}
+
 export interface LabelCacheEntry {
     timestamp: number;
-    videos: Record<VideoID, Category>;
+    videos: Record<VideoID, VideoLabelsCacheData>;
 }
 
 const labelCache: Record<string, LabelCacheEntry> = {};
@@ -21,7 +26,7 @@ async function getLabelHashBlock(hashPrefix: string): Promise<LabelCacheEntry |
         return cachedEntry;
     }
 
-    const response = await asyncRequestToServer("GET", `/api/videoLabels/${hashPrefix}`);
+    const response = await asyncRequestToServer("GET", `/api/videoLabels/${hashPrefix}?hasStartSegment=true`);
     if (response.status !== 200) {
         // No video labels or server down
         labelCache[hashPrefix] = {
@@ -36,7 +41,10 @@ async function getLabelHashBlock(hashPrefix: string): Promise<LabelCacheEntry |
 
         const newEntry: LabelCacheEntry = {
             timestamp: Date.now(),
-            videos: Object.fromEntries(data.map(video => [video.videoID, video.segments[0].category])),
+            videos: Object.fromEntries(data.map(video => [video.videoID, {
+                category: video.segments[0]?.category,
+                hasStartSegment: video.hasStartSegment
+            }])),
         };
         labelCache[hashPrefix] = newEntry;
 
@@ -55,11 +63,11 @@ async function getLabelHashBlock(hashPrefix: string): Promise<LabelCacheEntry |
 }
 
 export async function getVideoLabel(videoID: VideoID): Promise<Category | null> {
-    const prefix = (await getHash(videoID, 1)).slice(0, 3);
+    const prefix = (await getHash(videoID, 1)).slice(0, 4);
     const result = await getLabelHashBlock(prefix);
 
     if (result) {
-        const category = result.videos[videoID];
+        const category = result.videos[videoID]?.category;
         if (category && utils.getCategorySelection(category).option !== CategorySkipOption.Disabled) {
             return category;
         } else {
@@ -67,5 +75,16 @@ export async function getVideoLabel(videoID: VideoID): Promise<Category | null>
         }
     }
 
+    return null;
+}
+
+export async function getHasStartSegment(videoID: VideoID): Promise<boolean | null> {
+    const prefix = (await getHash(videoID, 1)).slice(0, 4);
+    const result = await getLabelHashBlock(prefix);
+
+    if (result) {
+        return result?.videos[videoID]?.hasStartSegment ?? false;
+    }
+
     return null;
 }
\ No newline at end of file