diff --git a/test/selenium.test.ts b/test/selenium.test.ts
index e6db6cdb..bf10d8fd 100644
--- a/test/selenium.test.ts
+++ b/test/selenium.test.ts
@@ -12,6 +12,11 @@ test("Selenium Chrome test", async () => {
         console.warn("A browser is probably not installed, skipping selenium tests");
         console.warn(e);
 
+        if (String(e).includes("This version of ChromeDriver only supports")) {
+            // Count as failure
+            throw e;
+        }
+
         return;
     }