diff --git a/src/components/SkipNoticeComponent.tsx b/src/components/SkipNoticeComponent.tsx
index f7fa6022..74dcee86 100644
--- a/src/components/SkipNoticeComponent.tsx
+++ b/src/components/SkipNoticeComponent.tsx
@@ -517,9 +517,10 @@ class SkipNoticeComponent extends React.Component<SkipNoticeProps, SkipNoticeSta
             source: SponsorSourceType.Local
         };
 
-        const segmentTimes = Config.config.segmentTimes.get(sponsorVideoID) || [];
+        const segmentTimes = Config.config.unsubmittedSegments[sponsorVideoID] || [];
         segmentTimes.push(sponsorTimesSubmitting);
-        Config.config.segmentTimes.set(sponsorVideoID, segmentTimes);
+        Config.config.unsubmittedSegments[sponsorVideoID] = segmentTimes;
+        Config.forceUpdate("unsubmittedSegments");
 
         this.props.contentContainer().sponsorTimesSubmitting.push(sponsorTimesSubmitting);
         this.props.contentContainer().updatePreviewBar();
diff --git a/src/components/SponsorTimeEditComponent.tsx b/src/components/SponsorTimeEditComponent.tsx
index cdc21617..b0ea504d 100644
--- a/src/components/SponsorTimeEditComponent.tsx
+++ b/src/components/SponsorTimeEditComponent.tsx
@@ -509,8 +509,9 @@ class SponsorTimeEditComponent extends React.Component<SponsorTimeEditProps, Spo
         const inputActionType = this.actionTypeOptionRef?.current?.value as ActionType;
         sponsorTimesSubmitting[this.props.index].actionType = this.getNextActionType(category, inputActionType);
 
-        Config.config.segmentTimes.set(this.props.contentContainer().sponsorVideoID, sponsorTimesSubmitting);
-
+        Config.config.unsubmittedSegments[this.props.contentContainer().sponsorVideoID] = sponsorTimesSubmitting;
+        Config.forceUpdate("unsubmittedSegments");
+        
         this.props.contentContainer().updatePreviewBar();
 
         if (sponsorTimesSubmitting[this.props.index].actionType === ActionType.Full 
@@ -555,7 +556,8 @@ class SponsorTimeEditComponent extends React.Component<SponsorTimeEditProps, Spo
         sponsorTimes.splice(index, 1);
   
         //save this
-        Config.config.segmentTimes.set(this.props.contentContainer().sponsorVideoID, sponsorTimes);
+        Config.config.unsubmittedSegments[this.props.contentContainer().sponsorVideoID] = sponsorTimes;
+        Config.forceUpdate("unsubmittedSegments");
 
         this.props.contentContainer().updatePreviewBar();
         
diff --git a/src/config.ts b/src/config.ts
index 7414b762..c496e828 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -8,7 +8,7 @@ interface SBConfig {
     isVip: boolean,
     lastIsVipUpdate: number,
     /* Contains unsubmitted segments that the user has created. */
-    segmentTimes: SBMap<string, SponsorTime[]>,
+    unsubmittedSegments: Record<string, SponsorTime[]>,
     defaultCategory: Category,
     whitelistedChannels: string[],
     forceChannelCheck: boolean,
@@ -99,73 +99,7 @@ export interface SBObject {
     defaults: SBConfig;
     localConfig: SBConfig;
     config: SBConfig;
-
-    // Functions
-    encodeStoredItem<T>(data: T): T | UnencodedSegmentTimes;
-    convertJSON(): void;
-}
-
-// Allows a SBMap to be conveted into json form
-// Currently used for local storage
-class SBMap<T, U> extends Map {
-    id: string;
-
-    constructor(id: string, entries?: [T, U][]) {
-        super();
-
-        this.id = id;
-
-        // Import all entries if they were given
-        if (entries !== undefined) {
-            for (const item of entries) {
-                super.set(item[0], item[1])
-            }
-        }
-    }
-
-    get(key): U {
-        return super.get(key);
-    }
-
-    rawSet(key, value) {
-        return super.set(key, value);
-    }
-
-    update() {
-        // Store updated SBMap locally
-        chrome.storage.sync.set({
-            [this.id]: encodeStoredItem(this)
-        });
-    }
-
-    set(key: T, value: U) {
-        const result = super.set(key, value);
-
-        this.update();
-        return result;
-    }
-	
-    delete(key) {
-        const result = super.delete(key);
-
-        // Make sure there are no empty elements
-        for (const entry of this.entries()) {
-            if (entry[1].length === 0) {
-                super.delete(entry[0]);
-            }
-        }
-
-        this.update();
-
-        return result;
-    }
-
-    clear() {
-        const result = super.clear();
-
-        this.update();
-        return result;
-    }
+    forceUpdate(prop: string): void;
 }
 
 const Config: SBObject = {
@@ -177,7 +111,7 @@ const Config: SBObject = {
         userID: null,
         isVip: false,
         lastIsVipUpdate: 0,
-        segmentTimes: new SBMap("segmentTimes"),
+        unsubmittedSegments: {},
         defaultCategory: "chooseACategory" as Category,
         whitelistedChannels: [],
         forceChannelCheck: false,
@@ -334,52 +268,15 @@ const Config: SBObject = {
     },
     localConfig: null,
     config: null,
-    
-    // Functions
-    encodeStoredItem,
-    convertJSON
+    forceUpdate
 };
 
 // Function setup
 
-/**
- * A SBMap cannot be stored in the chrome storage. 
- * This data will be encoded into an array instead
- * 
- * @param data 
- */
-function encodeStoredItem<T>(data: T): T | UnencodedSegmentTimes  {
-    // if data is SBMap convert to json for storing
-    if(!(data instanceof SBMap)) return data;
-    return Array.from(data.entries()).filter((element) => element[1].length > 0); // Remove empty entries
-}
-
-/**
- * An SBMap cannot be stored in the chrome storage. 
- * This data will be decoded from the array it is stored in
- * 
- * @param {*} data 
- */
-function decodeStoredItem<T>(id: string, data: T): T | SBMap<string, SponsorTime[]> {
-    if (!Config.defaults[id]) return data;
-
-    if (Config.defaults[id] instanceof SBMap) {
-        try {
-            if (!Array.isArray(data)) return data;
-            return new SBMap(id, data as UnencodedSegmentTimes);
-        } catch(e) {
-            console.error("Failed to parse SBMap: " + id);
-        }
-    }
-
-    // If all else fails, return the data
-    return data;
-}
-
 function configProxy(): SBConfig {
     chrome.storage.onChanged.addListener((changes: {[key: string]: chrome.storage.StorageChange}) => {
         for (const key in changes) {
-            Config.localConfig[key] = decodeStoredItem(key, changes[key].newValue);
+            Config.localConfig[key] = changes[key].newValue;
         }
 
         for (const callback of Config.configListeners) {
@@ -392,7 +289,7 @@ function configProxy(): SBConfig {
             Config.localConfig[prop] = value;
 
             chrome.storage.sync.set({
-                [prop]: encodeStoredItem(value)
+                [prop]: value
             });
 
             return true;
@@ -415,6 +312,12 @@ function configProxy(): SBConfig {
     return new Proxy<SBConfig>({handler} as unknown as SBConfig, handler);
 }
 
+function forceUpdate(prop: string): void {
+    chrome.storage.sync.set({
+        [prop]: Config.localConfig[prop]
+    });
+}
+
 function fetchConfig(): Promise<void> { 
     return new Promise((resolve) => {
         chrome.storage.sync.get(null, function(items) {
@@ -425,6 +328,14 @@ function fetchConfig(): Promise<void> {
 }
 
 function migrateOldFormats(config: SBConfig) {
+    if (config["segmentTimes"]) {
+        for (const item of config["segmentTimes"]) {
+            config.unsubmittedSegments[item[0]] = item[1];
+        }
+
+        chrome.storage.sync.remove("segmentTimes");
+    }
+
     if (!config["exclusive_accessCategoryAdded"] && !config.categorySelections.some((s) => s.name === "exclusive_access")) {
         config["exclusive_accessCategoryAdded"] = true;
 
@@ -512,19 +423,12 @@ function migrateOldFormats(config: SBConfig) {
 async function setupConfig() {
     await fetchConfig();
     addDefaults();
-    convertJSON();
     const config = configProxy();
     migrateOldFormats(config);
 
     Config.config = config;
 }
 
-function convertJSON(): void {
-    Object.keys(Config.localConfig).forEach(key => {
-        Config.localConfig[key] = decodeStoredItem(key, Config.localConfig[key]);
-    });
-}
-
 // Add defaults
 function addDefaults() {
     for (const key in Config.defaults) {
diff --git a/src/content.ts b/src/content.ts
index c3ddd635..8cbb412e 100644
--- a/src/content.ts
+++ b/src/content.ts
@@ -1510,7 +1510,8 @@ function startOrEndTimingNewSegment() {
     }
 
     // Save the newly created segment
-    Config.config.segmentTimes.set(sponsorVideoID, sponsorTimesSubmitting);
+    Config.config.unsubmittedSegments[sponsorVideoID] = sponsorTimesSubmitting;
+    Config.forceUpdate("unsubmittedSegments");
 
     // Make sure they know if someone has already submitted something it while they were watching
     sponsorsLookup(sponsorVideoID);
@@ -1532,7 +1533,8 @@ function isSegmentCreationInProgress(): boolean {
 function cancelCreatingSegment() {
     if (isSegmentCreationInProgress()) {
         sponsorTimesSubmitting.splice(sponsorTimesSubmitting.length - 1, 1);
-        Config.config.segmentTimes.set(sponsorVideoID, sponsorTimesSubmitting);
+        Config.config.unsubmittedSegments[sponsorVideoID] = sponsorTimesSubmitting;
+        Config.forceUpdate("unsubmittedSegments");
 
         if (sponsorTimesSubmitting.length <= 0) resetSponsorSubmissionNotice();
     }
@@ -1542,7 +1544,7 @@ function cancelCreatingSegment() {
 }
 
 function updateSponsorTimesSubmitting(getFromConfig = true) {
-    const segmentTimes = Config.config.segmentTimes.get(sponsorVideoID);
+    const segmentTimes = Config.config.unsubmittedSegments[sponsorVideoID];
 
     //see if this data should be saved in the sponsorTimesSubmitting variable
     if (getFromConfig && segmentTimes != undefined) {
@@ -1671,7 +1673,7 @@ function closeInfoMenuAnd<T>(func: () => T): T {
 function clearSponsorTimes() {
     const currentVideoID = sponsorVideoID;
 
-    const sponsorTimes = Config.config.segmentTimes.get(currentVideoID);
+    const sponsorTimes = Config.config.unsubmittedSegments[currentVideoID];
 
     if (sponsorTimes != undefined && sponsorTimes.length > 0) {
         const confirmMessage = chrome.i18n.getMessage("clearThis") + getSegmentsMessage(sponsorTimes)
@@ -1681,7 +1683,8 @@ function clearSponsorTimes() {
         resetSponsorSubmissionNotice();
 
         //clear the sponsor times
-        Config.config.segmentTimes.delete(currentVideoID);
+        delete Config.config.unsubmittedSegments[currentVideoID];
+        Config.forceUpdate("unsubmittedSegments");
 
         //clear sponsor times submitting
         sponsorTimesSubmitting = [];
@@ -1828,7 +1831,8 @@ async function sendSubmitMessage() {
     }
 
     //update sponsorTimes
-    Config.config.segmentTimes.set(sponsorVideoID, sponsorTimesSubmitting);
+    Config.config.unsubmittedSegments[sponsorVideoID] = sponsorTimesSubmitting;
+    Config.forceUpdate("unsubmittedSegments");
 
     // Check to see if any of the submissions are below the minimum duration set
     if (Config.config.minDuration > 0) {
@@ -1855,7 +1859,8 @@ async function sendSubmitMessage() {
         stopAnimation();
 
         // Remove segments from storage since they've already been submitted
-        Config.config.segmentTimes.delete(sponsorVideoID);
+        delete Config.config.unsubmittedSegments[sponsorVideoID];
+        Config.forceUpdate("unsubmittedSegments");
 
         const newSegments = sponsorTimesSubmitting;
         try {
@@ -2079,6 +2084,7 @@ function checkForPreloadedSegment() {
     }
 
     if (pushed) {
-        Config.config.segmentTimes.set(sponsorVideoID, sponsorTimesSubmitting);
+        Config.config.unsubmittedSegments[sponsorVideoID] = sponsorTimesSubmitting;
+        Config.forceUpdate("unsubmittedSegments");
     }
 }
\ No newline at end of file
diff --git a/src/options.ts b/src/options.ts
index 21de0859..b3f423c5 100644
--- a/src/options.ts
+++ b/src/options.ts
@@ -494,20 +494,14 @@ function activatePrivateTextChange(element: HTMLElement) {
     }
     
     let result = Config.config[option];
-
     // See if anything extra must be done
     switch (option) {
         case "*": {
-            const jsonData = JSON.parse(JSON.stringify(Config.localConfig));
-
-            // Fix segmentTimes data as it is destroyed from the JSON stringify
-            jsonData.segmentTimes = Config.encodeStoredItem(Config.localConfig.segmentTimes);
-
-            result = JSON.stringify(jsonData);
+            result = JSON.stringify(Config.localConfig);
             break;
         }
     }
-
+    
     textBox.value = result;
     
     const setButton = element.querySelector(".text-change-set");
@@ -557,7 +551,6 @@ async function setTextOption(option: string, element: HTMLElement, value: string
                     for (const key in newConfig) {
                         Config.config[key] = newConfig[key];
                     }
-                    Config.convertJSON();
 
                     if (newConfig.supportInvidious) {
                         const checkbox = <HTMLInputElement> document.querySelector("#support-invidious > div > label > input");
@@ -585,7 +578,6 @@ async function setTextOption(option: string, element: HTMLElement, value: string
 function downloadConfig() {
     const file = document.createElement("a");
     const jsonData = JSON.parse(JSON.stringify(Config.localConfig));
-    jsonData.segmentTimes = Config.encodeStoredItem(Config.localConfig.segmentTimes);
     file.setAttribute("href", "data:text/json;charset=utf-8," + encodeURIComponent(JSON.stringify(jsonData)));
     file.setAttribute("download", "SponsorBlockConfig.json");
     document.body.append(file);
@@ -642,9 +634,6 @@ function copyDebugOutputToClipboard() {
         config: JSON.parse(JSON.stringify(Config.localConfig)) // Deep clone config object
     };
 
-    // Fix segmentTimes data as it is destroyed from the JSON stringify
-    output.config.segmentTimes = Config.encodeStoredItem(Config.localConfig.segmentTimes);
-    
     // Sanitise sensitive user config values
     delete output.config.userID;
     output.config.serverAddress = (output.config.serverAddress === CompileConfig.serverAddress) 
diff --git a/src/popup.ts b/src/popup.ts
index d6885759..bcdb63aa 100644
--- a/src/popup.ts
+++ b/src/popup.ts
@@ -262,7 +262,7 @@ async function runThePopup(messageListener?: MessageListener): Promise<void> {
             return;
         }
 
-        sponsorTimes = Config.config.segmentTimes.get(currentVideoID) ?? [];
+        sponsorTimes = Config.config.unsubmittedSegments[currentVideoID] ?? [];
         updateSegmentEditingUI();
 
         messageHandler.sendMessage(
@@ -360,7 +360,7 @@ async function runThePopup(messageListener?: MessageListener): Promise<void> {
 
         // Only update the segments after a segment was created
         if (!creatingSegment) {
-            sponsorTimes = Config.config.segmentTimes.get(currentVideoID) || [];
+            sponsorTimes = Config.config.unsubmittedSegments[currentVideoID] || [];
         }
 
         // Update the UI