Added support for JSON and SRS rulesets

.shellcheckrc

@@ -0,0 +1 @@
+disable=SC3036,SC3010,SC3014,SC3015,SC3020,SC3003

podkop/files/usr/bin/podkop

@@ -1,4 +1,5 @@
 #!/bin/ash
+# shellcheck shell=dash
 
 [ -r /lib/functions.sh ] && . /lib/functions.sh
 [ -r /lib/config/uci.sh ] && . /lib/config/uci.sh
@@ -107,7 +108,9 @@ start_main() {
     config_foreach sing_box_rule_preset
     config_foreach process_domains_list_local
     config_foreach process_subnet_for_section
-    config_foreach process_remote_ruleset_srs
+    config_foreach configure_community_lists
+    config_foreach configure_remote_domain_lists
+    config_foreach configure_remote_subnet_lists
     config_foreach process_all_traffic_for_section
     config_foreach add_cron_job
 
@@ -572,6 +575,7 @@ prepare_custom_ruleset() {
         sing_box_rules $tag $section
         sing_box_dns_rule_fakeip_section $tag $tag
 
+        # TODO(ampetelin): Who is 'test' rule_set? (need fix)
         log "Added 'test' rule_set to sing-box config"
     fi
 }
@@ -619,10 +623,10 @@ list_update() {
     fi
 
     echolog "📥 Downloading and processing lists..."
-    
+
-    config_foreach process_remote_ruleset_subnet
+    config_foreach import_community_subnet_lists
-    config_foreach process_domains_list_url
+    config_foreach import_domains_from_remote_domain_lists
-    config_foreach process_subnet_for_section_remote
+    config_foreach import_subnets_from_remote_subnet_lists
 
     if [ $? -eq 0 ]; then
         echolog "✅ Lists update completed successfully"
@@ -964,7 +968,7 @@ sing_box_dns_rule_fakeip() {
 
 sing_box_dns_rule_fakeip_section() {
     local rule_set=$1
-    echo $rule_set
+
     log "Adding section to fakeip route rules in sing-box"
 
     jq \
@@ -1465,13 +1469,109 @@ sing_box_ruleset_subnets_json() {
     log "$subnet added to $section-custom-domains-subnets.json"
 }
 
+#######################################
+# Adds a new remote ruleset to the sing-box configuration.
+# https://sing-box.sagernet.org/configuration/rule-set/#__tabbed_1_3
+# 
+# Arguments:
+#   tag: unique identifier for the ruleset.
+#   format: format of the ruleset (e.g., "source" or "binary").
+#   url: URL from which the ruleset can be fetched.
+#   update_interval: update interval for the ruleset (e.g., "1d").
+#   detour: flag indicating whether to use a download detour ("1" or "0").
+#
+# Outputs:
+#   Modifies the sing-box configuration file by appending a new ruleset entry.
+#
+# Returns:
+#   None. Always returns 0. If a ruleset with the same tag exists, it is skipped.
+#######################################
+sing_box_config_add_remote_ruleset() {
+    local tag=$1
+    local format=$2
+    local url=$3
+    local update_interval=$4
+    local detour=$5
+
+    local tag_exists
+    tag_exists=$(jq -r --arg tag "$tag" '
+        .route.rule_set[]? | select(.tag == $tag) | .tag
+    ' "$SING_BOX_CONFIG")
+
+    if [[ -n "$tag_exists" ]]; then
+        log "Ruleset with tag $tag already exists. Skipping addition."
+    else
+        jq \
+        --arg tag "$tag" \
+        --arg format "$format" \
+        --arg url "$url" \
+        --arg update_interval "$update_interval" \
+        --arg detour "$detour" \
+        '
+        .route.rule_set += [
+            (
+                {
+                    "tag": $tag,
+                    "type": "remote",
+                    "format": $format,
+                    "url": $url,
+                    "update_interval": $update_interval
+                } +
+                    (if $detour == "1" then {"download_detour": "main"} else {} end)
+            )
+        ]' "$SING_BOX_CONFIG" | build_sing_box_config
+
+        log "Added new remote ruleset with tag $tag"
+    fi
+}
+
+#######################################
+# Adds a remote ruleset to the sing-box configuration and applies route and dns rules.
+#
+# Arguments:
+#   url: remote ruleset URL.
+#   section: configuration section where rules will be applied.
+#   ruleset_content_type: Type of ruleset content (e.g., "domains" or "subnets").
+#
+# Returns:
+#   0 on success, non-zero if the file extension is unsupported.
+#######################################
+sing_box_add_remote_ruleset_and_rules() {
+    local url="$1"
+    local section="$2"
+    local ruleset_content_type="$3"
+
+    local tag
+    local format
+    local update_interval='1d'
+    local detour
+
+    case "$(get_url_file_extension "$url")" in
+        json) format="source" ;;
+        srs)  format="binary" ;;
+        *)
+            log "Unsupported file extension: .$file_extension"
+            return 1
+            ;;
+    esac
+
+    tag=$(get_ruleset_tag_from_url "$url" "$section-remote-$ruleset_content_type")
+    config_get_bool detour "main" "detour" "0"
+
+    sing_box_config_add_remote_ruleset "$tag" "$format" "$url" "$update_interval" "$detour"
+    sing_box_rules "$tag" "$section"
+    if [[ "$ruleset_content_type" = "domains" ]]; then
+        sing_box_dns_rule_fakeip_section "$tag"
+    fi
+}
+
 process_domains_for_section() {
     local section="$1"
 
     config_get custom_domains_list_type "$section" "custom_domains_list_type" "disabled"
 
     if [ "$custom_domains_list_type" != "disabled" ]; then
-        log "Adding a custom domains list for section $section"
+        log "Adding a custom domains list for $section section"
         if [ "$custom_domains_list_type" = "dynamic" ]; then
             # Handle list domains from custom_domains
             config_list_foreach "$section" custom_domains "sing_box_ruleset_domains" "$section"
@@ -1483,96 +1583,6 @@ process_domains_for_section() {
     fi
 }
 
-sing_box_ruleset_remote() {
-    local tag=$1
-    local type=$2
-    local update_interval=$3
-    local detour=$4
-
-    url="$SRS_MAIN_URL/$tag.srs"
-
-    local tag_exists=$(jq -r --arg tag "$tag" '
-        .route.rule_set[]? | select(.tag == $tag) | .tag
-    ' "$SING_BOX_CONFIG")
-
-    if [[ -n "$tag_exists" ]]; then
-        log "Ruleset with tag $tag already exists. Skipping addition."
-    else
-        jq \
-        --arg tag "$tag" \
-        --arg type "$type" \
-        --arg url "$url" \
-        --arg update_interval "$update_interval" \
-        --arg detour "$detour" \
-        '
-        .route.rule_set += [
-            (
-                {
-                    "tag": $tag,
-                    "type": $type,
-                    "format": "binary",
-                    "url": $url,
-                    "update_interval": $update_interval
-                } +
-                    (if $detour == "1" then {"download_detour": "main"} else {} end)
-            )
-        ]' "$SING_BOX_CONFIG" | build_sing_box_config
-
-        log "Added new ruleset with tag $tag"
-    fi
-}
-
-list_subnets_download() {
-    local service="$1"
-    local table="PodkopTable"
-
-    case "$service" in
-        "twitter")
-            URL=$SUBNETS_TWITTER
-            ;;
-        "meta")
-            URL=$SUBNETS_META
-            ;;
-        "telegram")
-            URL=$SUBNETS_TELERAM
-            ;;
-        "cloudflare")
-            URL=$SUBNETS_CLOUDFLARE
-            ;;
-        "hetzner")
-            URL=$SUBNETS_HETZNER
-            ;;
-        "ovh")
-            URL=$SUBNETS_OVH
-            ;;
-        "discord")
-            URL=$SUBNETS_DISCORD
-            nft add set inet $table podkop_discord_subnets { type ipv4_addr\; flags interval\; auto-merge\; }
-            nft add rule inet $table mangle iifname "$SRC_INTERFACE" ip daddr @podkop_discord_subnets udp dport { 50000-65535 } meta mark set 0x105 counter
-            ;;
-        *)
-            return
-            ;;
-    esac
-
-    local filename=$(basename "$URL")
-
-    config_get_bool detour "main" "detour" "0"
-    if [ "$detour" -eq 1 ]; then
-        http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "/tmp/podkop/$filename" "$URL"
-    else
-        wget -O "/tmp/podkop/$filename" "$URL"
-    fi
-
-    while IFS= read -r subnet; do
-        if [ "$service" = "discord" ]; then
-            nft add element inet $table podkop_discord_subnets { $subnet }
-        else
-            nft add element inet $table podkop_subnets { $subnet }
-        fi
-    done <"/tmp/podkop/$filename"
-}
-
 sing_box_rules() {
     log "Configure rule in sing-box"
     local rule_set="$1"
@@ -1648,29 +1658,11 @@ sing_box_quic_reject() {
     fi
 }
 
-process_remote_ruleset_srs() {
+# TODO(ampetelin): function needs refactoring
-    config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
-    if [ "$domain_list_enabled" -eq 1 ]; then
-        config_get_bool detour "main" "detour" "0"
-        log "Adding a srs list for $section"
-        config_list_foreach "$section" domain_list "sing_box_ruleset_remote" "remote" "1d" "$detour"
-    fi
-}
-
-process_remote_ruleset_subnet() {
-    config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
-    if [ "$domain_list_enabled" -eq 1 ]; then
-        log "Adding a srs list for $section"
-        config_list_foreach "$section" domain_list "list_subnets_download" "$section" "$domain_list"
-    fi
-}
-
 sing_box_rule_preset() {
     config_get custom_domains_list_type "$section" "custom_domains_list_type"
     config_get custom_subnets_list_enabled "$section" "custom_subnets_list_enabled"
     config_get custom_local_domains_list_enabled "$section" "custom_local_domains_list_enabled"
-    # config_get custom_download_domains_list_enabled "$section" "custom_download_domains_list_enabled"
-    # config_get custom_download_subnets_list_enabled "$section" "custom_download_subnets_list_enabled"
 
     if [ "$custom_domains_list_type" != "disabled" ] || [ "$custom_subnets_list_enabled" != "disabled" ] ||
         [ "$custom_local_domains_list_enabled" = "1" ]; then
@@ -1715,46 +1707,12 @@ process_domains_list_local() {
     fi
 }
 
-list_custom_url_domains_create() {
-    local section="$2"
-    local URL="$1"
-    local filename=$(basename "$URL")
-    local filepath="/tmp/podkop/${filename}"
-
-    config_get_bool detour "main" "detour" "0"
-    if [ "$detour" -eq 1 ]; then
-        http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "$filepath" "$URL"
-    else
-        wget -O "$filepath" "$URL"
-    fi
-
-    if grep -q $'\r' "$filepath"; then
-        log "$filename has Windows line endings (CRLF). Converting to Unix (LF)"
-        sed -i 's/\r$//' "$filepath"
-    fi
-
-    while IFS= read -r domain; do
-        log "From downloaded file: $domain"
-        sing_box_ruleset_domains_json $domain $section
-    done <"$filepath"
-}
-
-process_domains_list_url() {
-    local section="$1"
-
-    config_get custom_download_domains_list_enabled "$section" "custom_download_domains_list_enabled"
-    if [ "$custom_download_domains_list_enabled" -eq 1 ]; then
-        log "Adding a custom domains list from URL in $section"
-        config_list_foreach "$section" "custom_download_domains" list_custom_url_domains_create "$section"
-    fi
-}
-
 process_subnet_for_section() {
     local section="$1"
 
     config_get custom_subnets_list_enabled "$section" "custom_subnets_list_enabled" "disabled"
     if [ "$custom_subnets_list_enabled" != "disabled" ]; then
-        log "Adding a custom subnet list for section $section"
+        log "Adding a custom subnet list for $section section"
         if [ "$custom_subnets_list_enabled" = "dynamic" ]; then
             # Handle list domains from custom_domains
             config_list_foreach "$section" custom_subnets "sing_box_ruleset_subnets" "$section"
@@ -1766,52 +1724,315 @@ process_subnet_for_section() {
     fi
 }
 
-list_custom_url_subnets_create() {
+configure_community_lists() {
-    local section="$2"
+    config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
-    local URL="$1"
+    if [ "$domain_list_enabled" -eq 1 ]; then
-    local filename=$(basename "$URL")
+        log "Configuring community lists for $section section"
-    local filepath="/tmp/podkop/${filename}"
+        config_list_foreach "$section" domain_list configure_community_list_handler
-
-    config_get_bool detour "main" "detour" "0"
-    if [ "$detour" -eq 1 ]; then
-        http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "$filepath" "$URL"
-    else
-        wget -O "$filepath" "$URL"
     fi
-
-    if grep -q $'\r' "$filepath"; then
-        log "$filename has Windows line endings (CRLF). Converting to Unix (LF)"
-        sed -i 's/\r$//' "$filepath"
-    fi
-
-    while IFS= read -r subnet; do
-        log "From local file: $subnet"
-        sing_box_ruleset_subnets_json $subnet $section
-        nft add element inet PodkopTable podkop_subnets { $subnet }
-    done <"$filepath"
 }
 
-process_subnet_for_section_remote() {
+configure_community_list_handler() {
+    local tag=$1
+
+    local format="binary"
+    local update_interval="1d"
+    config_get_bool detour "main" "detour" "0"
+    local url="$SRS_MAIN_URL/$tag.srs"
+    
+    sing_box_config_add_remote_ruleset "$tag" "$format" "$url" "$update_interval" "$detour"
+}
+
+configure_remote_domain_lists() {
+    local section="$1"
+    
+    config_get custom_download_domains_list_enabled "$section" custom_download_domains_list_enabled
+    if [ "$custom_download_domains_list_enabled" -eq 1 ]; then
+        log "Configuring remote domain lists for $section section"
+        config_list_foreach "$section" custom_download_domains configure_remote_domain_list_handler "$section"
+    fi
+}
+
+configure_remote_domain_list_handler() {
+    local url="$1"
+    local section="$2"
+    
+    log "Configuring remote domain list from URL: $url"
+
+    local file_extension
+    file_extension=$(get_url_file_extension "$url")
+    case "$file_extension" in
+        lst)
+            log "Detected file extension: .$file_extension → no processing needed, managed on list_update"
+            ;;
+        json|srs)
+            log "Detected file extension: .$file_extension → proceeding with processing"
+            sing_box_add_remote_ruleset_and_rules "$url" "$section" "domains"
+            ;;
+        *)
+            log "Detected file extension: .$file_extension → unsupported, skipping"
+            return 1
+            ;;
+    esac
+}
+
+configure_remote_subnet_lists() {
     local section="$1"
 
-    config_get custom_download_subnets_list_enabled "$section" "custom_download_subnets_list_enabled" "disabled"
+    config_get custom_download_subnets_list_enabled "$section" custom_download_subnets_list_enabled disabled
     if [ "$custom_download_subnets_list_enabled" -eq "1" ]; then
-            log "Adding a custom SUBNET list from URL in $section"
+        log "Configuring remote subnet lists for $section section"
-            config_list_foreach "$section" "custom_download_subnets" list_custom_url_subnets_create "$section"
+        config_list_foreach "$section" custom_download_subnets configure_remote_subnet_list_handler "$section"
     fi
 }
 
+configure_remote_subnet_list_handler() {
+    local url="$1"
+    local section="$2"
+
+    log "Configuring remote subnet list from URL: $url"
+
+    local file_extension
+    file_extension=$(get_url_file_extension "$url")
+    case "$file_extension" in
+        lst)
+            log "Detected file extension: .$file_extension → no processing needed, managed on list_update"
+            ;;
+        json|srs)
+            log "Detected file extension: .$file_extension → proceeding with processing"
+            sing_box_add_remote_ruleset_and_rules "$url" "$section" "subnets"
+            ;;
+        *)
+            log "Detected file extension: .$file_extension → unsupported, skipping"
+            return 1
+            ;;
+    esac
+}
+
 process_all_traffic_for_section() {
     local section="$1"
 
     config_get all_traffic_from_ip_enabled "$section" "all_traffic_from_ip_enabled"
     if [ "$all_traffic_from_ip_enabled" -eq "1" ]; then
         log "Adding an IP to redirect all traffic"
-        config_list_foreach $section all_traffic_ip list_all_traffic_from_ip
+        config_list_foreach $section all_traffic_ip nft_list_all_traffic_from_ip
         config_list_foreach $section all_traffic_ip sing_box_rules_source_ip_cidr $all_traffic_ip $section
     fi
 }
 
+import_community_subnet_lists() {
+    config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
+    if [ "$domain_list_enabled" -eq 1 ]; then
+        log "Importing community subnet lists for $section section"
+        config_list_foreach "$section" domain_list import_community_service_subnet_list_handler
+    fi
+}
+
+import_community_service_subnet_list_handler() {
+    local service="$1"
+    local table="PodkopTable"
+
+    case "$service" in
+        "twitter")
+            URL=$SUBNETS_TWITTER
+            ;;
+        "meta")
+            URL=$SUBNETS_META
+            ;;
+        "telegram")
+            URL=$SUBNETS_TELERAM
+            ;;
+        "cloudflare")
+            URL=$SUBNETS_CLOUDFLARE
+            ;;
+        "hetzner")
+            URL=$SUBNETS_HETZNER
+            ;;
+        "ovh")
+            URL=$SUBNETS_OVH
+            ;;
+        "discord")
+            URL=$SUBNETS_DISCORD
+            nft add set inet $table podkop_discord_subnets { type ipv4_addr\; flags interval\; auto-merge\; }
+            nft add rule inet $table mangle iifname "$SRC_INTERFACE" ip daddr @podkop_discord_subnets udp dport { 50000-65535 } meta mark set 0x105 counter
+            ;;
+        *)
+            return
+            ;;
+    esac
+
+    local filename=$(basename "$URL")
+
+    config_get_bool detour "main" "detour" "0"
+    if [ "$detour" -eq 1 ]; then
+        http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "/tmp/podkop/$filename" "$URL"
+    else
+        wget -O "/tmp/podkop/$filename" "$URL"
+    fi
+
+    while IFS= read -r subnet; do
+        if [ "$service" = "discord" ]; then
+            nft add element inet $table podkop_discord_subnets { $subnet }
+        else
+            nft add element inet $table podkop_subnets { $subnet }
+        fi
+    done <"/tmp/podkop/$filename"
+}
+
+import_domains_from_remote_domain_lists() {
+    local section="$1"
+
+    config_get custom_download_domains_list_enabled "$section" custom_download_domains_list_enabled
+    if [ "$custom_download_domains_list_enabled" -eq 1 ]; then
+        log "Importing domains from remote domain lists for $section section"
+        config_list_foreach "$section" custom_download_domains import_domains_from_remote_domain_list_handler "$section"
+    fi
+}
+
+import_domains_from_remote_domain_list_handler() {
+    local url="$1"
+    local section="$2"
+    
+    log "Importing domains from URL: $url"
+
+    local file_extension
+    file_extension=$(get_url_file_extension "$url")
+    case "$file_extension" in
+        lst)
+            log "Detected file extension: .$file_extension → proceeding with processing"
+            import_domains_from_remote_lst_file "$url" "$section"
+            ;;
+        json|srs)
+            log "Detected file extension: .$file_extension → no update needed, sing-box manages updates"
+            ;;
+        *)
+            log "Detected file extension: .$file_extension → unsupported, skipping"
+            return 1
+            ;;
+    esac
+}
+
+import_domains_from_remote_lst_file() {
+    local url="$1"
+    local section="$2"
+
+    local filename
+    filename=$(basename "$url")
+    local filepath="/tmp/podkop/${filename}"
+
+    download_to_tempfile "$url" "$filepath"
+
+    while IFS= read -r domain; do
+        sing_box_ruleset_domains_json $domain $section
+    done <"$filepath"
+
+    rm -f "$filepath"
+}
+
+import_subnets_from_remote_subnet_lists() {
+    local section="$1"
+
+    config_get custom_download_subnets_list_enabled "$section" custom_download_subnets_list_enabled disabled
+    if [ "$custom_download_subnets_list_enabled" -eq "1" ]; then
+            log "Importing subnets from remote subnet lists for $section section"
+            config_list_foreach "$section" custom_download_subnets import_subnets_from_remote_subnet_list_handler "$section"
+    fi
+}
+
+import_subnets_from_remote_subnet_list_handler() {
+    local url="$1"
+    local section="$2"
+
+    log "Importing subnets from URL: $url"
+
+    local file_extension
+    file_extension=$(get_url_file_extension "$url")
+    case "$file_extension" in
+        lst)
+            log "Detected file extension: .$file_extension → proceeding with processing"
+            import_subnets_from_remote_lst_file "$url" "$section"
+            ;;
+        json)
+            log "Detected file extension: .$file_extension → proceeding with processing"
+            import_subnets_from_remote_json_file "$url"
+            ;;
+        srs)
+            log "Detected file extension: .$file_extension → proceeding with processing"
+            import_subnets_from_remote_srs_file "$url"
+            ;;
+        *)
+            log "Detected file extension: .$file_extension → unsupported, skipping"
+            return 1
+            ;;
+    esac
+}
+
+import_subnets_from_remote_lst_file() {
+    local url="$1"
+    local section="$2"
+
+    local filename
+    filename=$(basename "$url")
+    local filepath="/tmp/podkop/${filename}"
+
+    download_to_tempfile "$url" "$filepath"
+
+    while IFS= read -r subnet; do
+        sing_box_ruleset_subnets_json "$subnet" "$section"
+        nft_add_podkop_subnet "$subnet"
+    done <"$filepath"
+
+    rm -f "$filepath"
+}
+
+import_subnets_from_remote_json_file() {
+    local url="$1"
+
+    download_to_stream "$url" | jq -r '.rules[].ip_cidr[]' | while read -r subnet; do
+        nft_add_podkop_subnet "$subnet"
+    done
+}
+
+import_subnets_from_remote_srs_file() {
+    local url="$1"
+    
+    local filename
+    filename=$(basename "$url")
+    local binary_filepath="/tmp/podkop/${filename}"
+    local json_filepath="/tmp/podkop/decompiled-${filename%%.*}.json"
+    
+    download_to_tempfile "$url" "$binary_filepath"
+
+    if ! decompile_srs_file "$binary_filepath" "$json_filepath"; then
+        return 1
+    fi
+
+    jq -r '.rules[].ip_cidr[]' "$json_filepath" | while read -r subnet; do
+        nft_add_podkop_subnet "$subnet"
+    done
+
+    rm -f "$binary_filepath" "$json_filepath"
+}
+
+# Decompiles a sing-box SRS binary file into a JSON ruleset file
+decompile_srs_file() {
+    local binary_filepath="$1"
+    local output_filepath="$2"
+
+    log "Decompiling $binary_filepath to $output_filepath"
+
+    if ! file_exists "$binary_filepath"; then
+        log "File $binary_filepath not found"
+        return 1
+    fi
+
+    sing-box rule-set decompile "$binary_filepath" -o "$output_filepath"
+    if [[ $? -ne 0 ]]; then
+        log "Decompilation command failed for $binary_filepath"
+        return 1
+    fi
+}
+
 sing_box_rules_source_ip_cidr() {
     log "Configure source_ip_cidr rule in sing-box"
     local source_ip_cidr="$1"
@@ -1865,7 +2086,7 @@ detour_mixed() {
 }
 
 ## nftables
-list_all_traffic_from_ip() {
+nft_list_all_traffic_from_ip() {
     local ip="$1"
     local table="PodkopTable"
 
@@ -1890,6 +2111,17 @@ list_all_traffic_from_ip() {
     fi
 }
 
+# Adds an IPv4 subnet to nftables firewall set
+nft_add_podkop_subnet() {
+    local subnet="$1"
+
+    if is_ipv4_cidr "$subnet"; then
+        nft add element inet PodkopTable podkop_subnets { "$subnet" }
+    else
+        log "Invalid subnet format. $subnet is not a valid IPv4 CIDR"
+    fi
+}
+
 # Diagnotics
 check_proxy() {
     if ! command -v sing-box >/dev/null 2>&1; then
@@ -2602,6 +2834,87 @@ global_check() {
     fi
 }
 
+# Download URL content directly
+download_to_stream() {
+    local url="$1"
+
+    config_get_bool detour "main" "detour" "0"
+    if [ "$detour" -eq 1 ]; then
+        http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -qO- "$filepath" "$url" | sed 's/\r$//'
+    else
+        wget -qO- "$url" | sed 's/\r$//'
+    fi
+}
+
+# Download URL to temporary file
+download_to_tempfile() {
+    local url="$1"
+    local filepath="$2"
+
+    config_get_bool detour "main" "detour" "0"
+    if [ "$detour" -eq 1 ]; then
+        http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "$filepath" "$url"
+    else
+        wget -O "$filepath" "$url"
+    fi
+
+    if grep -q $'\r' "$filepath"; then
+        log "$filename has Windows line endings (CRLF). Converting to Unix (LF)"
+        sed -i 's/\r$//' "$filepath"
+    fi
+}
+
+# helper function
+
+# check if file exists
+file_exists() {
+  local filepath="$1"
+
+  if [[ -f "$filepath" ]]; then
+    return 0   # success
+  else
+    return 1   # failure
+  fi
+}
+
+# extracts file extension from URL
+get_url_file_extension() {
+    local url="$1"
+
+    local file_extension="${url##*.}"
+
+    echo "$file_extension"
+}
+
+# extracts file extension from URL
+get_ruleset_tag_from_url() {
+    local url="$1"
+    local prefix="${2:-}"
+    local postfix="${3:-}"
+
+    local filename="${url##*/}"
+    local basename="${filename%%.*}"
+
+    local tag="$basename"
+
+    if [ -n "$prefix" ]; then
+        tag="${prefix}-${tag}"
+    fi
+
+    if [ -n "$postfix" ]; then
+        tag="${tag}-${postfix}"
+    fi
+
+    echo "$tag"
+}
+
+# check if string is valid IPv4 with CIDR mask
+is_ipv4_cidr() {
+  local ip="$1"
+  local regex="^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}(\/(3[0-2]|2[0-9]|1[0-9]|[0-9]))$"
+  [[ $ip =~ $regex ]]
+}
+
 show_help() {
     cat << EOF
 Usage: $0 COMMAND