diff --git a/public/_locales/en/messages.json b/public/_locales/en/messages.json
index d334e3c3..11807407 100644
--- a/public/_locales/en/messages.json
+++ b/public/_locales/en/messages.json
@@ -534,5 +534,11 @@
     },
     "bracketEnd": {
         "message": "(End)"
+    },
+    "hiddenDueToDownvote": {
+        "message": "hidden: downvote"
+    },
+    "hiddenDueToDuration": {
+        "message": "hidden: too short"
     }
 }
diff --git a/src/components/SkipNoticeComponent.tsx b/src/components/SkipNoticeComponent.tsx
index 08d817e6..8d53efbb 100644
--- a/src/components/SkipNoticeComponent.tsx
+++ b/src/components/SkipNoticeComponent.tsx
@@ -1,6 +1,6 @@
 import * as React from "react";
 import Config from "../config"
-import { ContentContainer } from "../types";
+import { ContentContainer, SponsorHideType } from "../types";
 
 import Utils from "../utils";
 var utils = new Utils();
@@ -269,7 +269,7 @@ class SkipNoticeComponent extends React.Component<SkipNoticeProps, SkipNoticeSta
                 //this one is the one to hide
                 
                 //add this as a hidden sponsorTime
-                this.contentContainer().hiddenSponsorTimes.push(i);
+                this.contentContainer().sponsorTimes[i].hidden = SponsorHideType.Downvoted;
             
                 this.contentContainer().updatePreviewBar();
                 break;
diff --git a/src/content.ts b/src/content.ts
index d9d018ab..901ce761 100644
--- a/src/content.ts
+++ b/src/content.ts
@@ -1,6 +1,6 @@
 import Config from "./config";
 
-import { SponsorTime, CategorySkipOption, CategorySelection, VideoID } from "./types";
+import { SponsorTime, CategorySkipOption, CategorySelection, VideoID, SponsorHideType } from "./types";
 
 import { ContentContainer } from "./types";
 import Utils from "./utils";
@@ -30,9 +30,6 @@ var sponsorVideoID: VideoID = null;
 var currentSkipSchedule: NodeJS.Timeout = null;
 var seekListenerSetUp = false
 
-//these are sponsors that have been downvoted
-var hiddenSponsorTimes: number[] = [];
-
 /** @type {Array[boolean]} Has the sponsor been skipped */
 var sponsorSkipped: boolean[] = [];
 
@@ -110,7 +107,6 @@ var skipNoticeContentContainer: ContentContainer = () => ({
     unskipSponsorTime,
     sponsorTimes,
     sponsorTimesSubmitting,
-    hiddenSponsorTimes,
     v: video,
     sponsorVideoID,
     reskipSponsorTime,
@@ -143,8 +139,7 @@ function messageListener(request: any, sender: any, sendResponse: (response: any
             //send the sponsor times along with if it's found
             sendResponse({
                 found: sponsorDataFound,
-                sponsorTimes: sponsorTimes,
-                hiddenSponsorTimes: hiddenSponsorTimes
+                sponsorTimes: sponsorTimes
             });
 
             if (popupInitialised && document.getElementById("sponsorBlockPopupContainer") != null) {
@@ -287,9 +282,6 @@ async function videoIDChange(id) {
     //if the id has not changed return
     if (sponsorVideoID === id) return;
 
-    // Reset hidden times (only do this when the ID has changed)
-    hiddenSponsorTimes = [];
-
     //set the global videoID
     sponsorVideoID = id;
 
@@ -640,17 +632,13 @@ function sponsorsLookup(id: string, channelIDPromise?) {
 
             sponsorTimes = recievedSegments;
 
-            // Remove all submissions smaller than the minimum duration
+            // Hide all submissions smaller than the minimum duration
             if (Config.config.minDuration !== 0) {
-                let smallSegments: SponsorTime[] = [];
-
                 for (let i = 0; i < sponsorTimes.length; i++) {
-                    if (sponsorTimes[i].segment[1] - sponsorTimes[i].segment[0] >= Config.config.minDuration) {
-                        smallSegments.push(sponsorTimes[i]);
+                    if (sponsorTimes[i].segment[1] - sponsorTimes[i].segment[0] < Config.config.minDuration) {
+                        sponsorTimes[i].hidden = SponsorHideType.MinimumDuration;
                     }
                 }
-
-                sponsorTimes = smallSegments;
             }
 
             if (!switchingVideos) {
@@ -838,7 +826,7 @@ function updatePreviewBar() {
     //create an array of the sponsor types
     let types = [];
     for (let i = 0; i < localSponsorTimes.length; i++) {
-        if (!hiddenSponsorTimes.includes(i)) {
+        if (localSponsorTimes[i].hidden === SponsorHideType.Visible) {
             types.push(localSponsorTimes[i].category);
         } else {
             // Don't show this sponsor
@@ -927,7 +915,7 @@ function getLatestEndTimeIndex(sponsorTimes: SponsorTime[], index: number, hideH
         let latestEndTime = sponsorTimes[latestEndTimeIndex].segment[1];
 
         if (currentSegment[0] <= latestEndTime && currentSegment[1] > latestEndTime 
-            && (!hideHiddenSponsors || !hiddenSponsorTimes.includes(i))
+            && (!hideHiddenSponsors || sponsorTimes[i].hidden === SponsorHideType.Visible)
             && utils.getCategorySelection(sponsorTimes[i].category).option === CategorySkipOption.AutoSkip) {
                 // Overlapping segment
                 latestEndTimeIndex = i;
@@ -961,7 +949,7 @@ function getStartTimes(sponsorTimes: SponsorTime[], includeIntersectingSegments:
     for (let i = 0; i < sponsorTimes.length; i++) {
         if ((minimum === undefined || (sponsorTimes[i].segment[0] >= minimum || (includeIntersectingSegments && sponsorTimes[i].segment[1] > minimum))) 
                 && (!onlySkippableSponsors || utils.getCategorySelection(sponsorTimes[i].category).option !== CategorySkipOption.ShowOverlay)
-                && (!hideHiddenSponsors || !hiddenSponsorTimes.includes(i))) {
+                && (!hideHiddenSponsors || sponsorTimes[i].hidden === SponsorHideType.Visible)) {
 
             startTimes.push(sponsorTimes[i].segment[0]);
         } 
diff --git a/src/popup.ts b/src/popup.ts
index 14a75a94..14041132 100644
--- a/src/popup.ts
+++ b/src/popup.ts
@@ -1,7 +1,7 @@
 import Config from "./config";
 
 import Utils from "./utils";
-import { SponsorTime } from "./types";
+import { SponsorTime, SponsorHideType } from "./types";
 var utils = new Utils();
 
 interface MessageListener {
@@ -273,7 +273,7 @@ async function runThePopup(messageListener?: MessageListener) {
         );
     }
   
-    function infoFound(request: {found: boolean, sponsorTimes: SponsorTime[], hiddenSponsorTimes: number[]}) {
+    function infoFound(request: {found: boolean, sponsorTimes: SponsorTime[]}) {
         if(chrome.runtime.lastError) {
             //This page doesn't have the injected content script, or at least not yet
             displayNoVideo();
@@ -364,7 +364,7 @@ async function runThePopup(messageListener?: MessageListener) {
     }
   
     //display the video times from the array at the top, in a different section
-    function displayDownloadedSponsorTimes(request: {found: boolean, sponsorTimes: SponsorTime[], hiddenSponsorTimes: number[]}) {
+    function displayDownloadedSponsorTimes(request: {found: boolean, sponsorTimes: SponsorTime[]}) {
         if (request.sponsorTimes != undefined) {
             //set it to the message
             if (PageElements.downloadedSponsorMessageTimes.innerText != chrome.i18n.getMessage("channelWhitelisted")) {
@@ -378,9 +378,12 @@ async function runThePopup(messageListener?: MessageListener) {
                 sponsorTimeButton.className = "warningButton popupElement";
 
                 let extraInfo = "";
-                if (request.hiddenSponsorTimes.includes(i)) {
-                    //this one is hidden
-                    extraInfo = " (hidden)";
+                if (request.sponsorTimes[i].hidden === SponsorHideType.Downvoted) {
+                    //this one is downvoted
+                    extraInfo = " (" + chrome.i18n.getMessage("hiddenDueToDownvote") + ")";
+                } else if (request.sponsorTimes[i].hidden === SponsorHideType.MinimumDuration) {
+                    //this one is too short
+                    extraInfo = " (" + chrome.i18n.getMessage("hiddenDueToDuration") + ")";
                 }
 
                 sponsorTimeButton.innerText = getFormattedTime(request.sponsorTimes[i].segment[0]) + " to " + getFormattedTime(request.sponsorTimes[i].segment[1]) + extraInfo;
@@ -444,6 +447,14 @@ async function runThePopup(messageListener?: MessageListener) {
                     //add commas if necessary
                     timeMessage = ", " + timeMessage;
                 }
+
+                if (sponsorTimes[i].hidden === SponsorHideType.Downvoted) {
+                    //this one is downvoted
+                    timeMessage += " (" + chrome.i18n.getMessage("hiddenDueToDownvote") + ")";
+                } else if (sponsorTimes[i].hidden === SponsorHideType.MinimumDuration) {
+                    //this one is too short
+                    timeMessage += " (" + chrome.i18n.getMessage("hiddenDueToDuration") + ")";
+                }
   
                 sponsorTimesMessage += timeMessage;
             }
diff --git a/src/types.ts b/src/types.ts
index f61e8d69..befa1fc0 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -8,7 +8,6 @@ interface ContentContainer {
         unskipSponsorTime: (UUID: any) => void,
         sponsorTimes: SponsorTime[],
         sponsorTimesSubmitting: SponsorTime[],
-        hiddenSponsorTimes: number[],
         v: HTMLVideoElement,
         sponsorVideoID,
         reskipSponsorTime: (UUID: any) => void,
@@ -36,11 +35,19 @@ interface CategorySelection {
     option: CategorySkipOption
 }
 
+enum SponsorHideType {
+    Visible = undefined,
+    Downvoted = 1,
+    MinimumDuration
+}
+
 interface SponsorTime {
     segment: number[];
     UUID: string;
 
     category: string;
+
+    hidden?: SponsorHideType;
 }
 
 type VideoID = string;
@@ -51,5 +58,6 @@ export {
     CategorySelection,
     CategorySkipOption,
     SponsorTime,
-    VideoID
+    VideoID,
+    SponsorHideType
 };
\ No newline at end of file