Merge pull request #240 from itdoginfo/fix/long-nft-command

Import large subnet lists in chunks into nft sets

podkop/files/usr/bin/podkop

@@ -1032,7 +1032,7 @@ import_local_domain_or_subnet_list() {
     domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
     subnets)
         sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
-        nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items"
+        nft_add_set_elements_from_file_chunked "$filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
         ;;
     esac
 }
@@ -1213,7 +1213,7 @@ import_community_service_subnet_list_handler() {
     *) return 0 ;;
     esac
 
-    local tmpfile http_proxy_address subnets
+    local tmpfile http_proxy_address
     tmpfile=$(mktemp)
     http_proxy_address="$(get_service_proxy_address)"
 
@@ -1224,14 +1224,13 @@ import_community_service_subnet_list_handler() {
         return 1
     fi
 
-    subnets="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "subnets")"
-    rm -f "$tmpfile"
-
     if [ "$service" = "discord" ]; then
-        nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_DISCORD_SET_NAME" "$subnets"
+        nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_DISCORD_SET_NAME"
     else
-        nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets"
+        nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
     fi
+
+    rm -f "$tmpfile"
 }
 
 import_domains_from_remote_domain_lists() {
@@ -1313,8 +1312,8 @@ import_domains_or_subnets_from_remote_file() {
         return 1
     fi
 
+    convert_crlf_to_lf "$tmpfile"
     items="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "$type")"
-    rm -f "$tmpfile"
 
     if [ -z "$items" ]; then
         log "No valid $type found in $url" "warn"
@@ -1329,33 +1328,36 @@ import_domains_or_subnets_from_remote_file() {
     domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
     subnets)
         sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
-        nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items"
+        nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
         ;;
     esac
+
+    rm -f "$tmpfile"
 }
 
 import_subnets_from_remote_json_file() {
     local url="$1"
-    local tmpfile subnets http_proxy_address
+    local json_tmpfile subnets_tmpfile http_proxy_address
-    tmpfile="$(mktemp)"
+    json_tmpfile="$(mktemp)"
+    subnets_tmpfile="$(mktemp)"
     http_proxy_address="$(get_service_proxy_address)"
 
-    download_to_stream "$url" "$http_proxy_address" | jq -r '.rules[].ip_cidr[]?' > "$tmpfile"
+    download_to_file "$url" "$json_tmpfile" "$http_proxy_address"
 
-    if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then
+    if [ $? -ne 0 ] || [ ! -s "$json_tmpfile" ]; then
         log "Download $url list failed" "error"
         return 1
     fi
 
-    subnets="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "subnets")"
+    extract_ip_cidr_from_json_ruleset_to_file "$json_tmpfile" "$subnets_tmpfile"
-    rm -f "$tmpfile"
+    nft_add_set_elements_from_file_chunked "$subnets_tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
-    nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets"
+    rm -f "$json_tmpfile" "$subnets_tmpfile"
 }
 
 import_subnets_from_remote_srs_file() {
     local url="$1"
 
-    local binary_tmpfile json_tmpfile subnets_tmpfile subnets http_proxy_address
+    local binary_tmpfile json_tmpfile subnets_tmpfile http_proxy_address
     binary_tmpfile="$(mktemp)"
     json_tmpfile="$(mktemp)"
     subnets_tmpfile="$(mktemp)"
@@ -1373,10 +1375,9 @@ import_subnets_from_remote_srs_file() {
         return 1
     fi
 
-    jq -r '.rules[].ip_cidr[]' "$json_tmpfile" > "$subnets_tmpfile"
+    extract_ip_cidr_from_json_ruleset_to_file "$json_tmpfile" "$subnets_tmpfile"
-    subnets="$(parse_domain_or_subnet_file_to_comma_string "$subnets_tmpfile" "subnets")"
+    nft_add_set_elements_from_file_chunked "$subnets_tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
     rm -f "$binary_tmpfile" "$json_tmpfile" "$subnets_tmpfile"
-    nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets"
 }
 
 ## Support functions
@@ -1470,6 +1471,46 @@ nft_list_all_traffic_from_ip() {
     fi
 }
 
+nft_add_set_elements_from_file_chunked() {
+    local filepath="$1"
+    local nft_table_name="$2"
+    local nft_set_name="$3"
+    local chunk_size="${4:-5000}"
+
+    local array count
+    count=0
+    while IFS= read -r line; do
+        line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+
+        [ -z "$line" ] && continue
+
+        if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then
+            log "'$line' is not IPv4 or IPv4 CIDR" "debug"
+            continue
+        fi
+
+        if [ -z "$array" ]; then
+            array="$line"
+        else
+            array="$array,$line"
+        fi
+
+        count=$((count + 1))
+
+        if [ "$count" = "$chunk_size" ]; then
+            log "Adding $count elements to nft set $nft_set_name" "debug"
+            nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
+            array=""
+            count=0
+        fi
+    done < "$filepath"
+
+    if [ -n "$array" ]; then
+        log "Adding $count elements to nft set $nft_set_name" "debug"
+        nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
+    fi
+}
+
 # Diagnotics
 check_proxy() {
     local sing_box_config_path

podkop/files/usr/lib/helpers.sh

@@ -268,25 +268,6 @@ migration_rename_config_key() {
     fi
 }
 
-# Download URL content directly
-download_to_stream() {
-    local url="$1"
-    local http_proxy_address="$2"
-    local retries="${3:-3}"
-    local wait="${4:-2}"
-
-    for attempt in $(seq 1 "$retries"); do
-        if [ -n "$http_proxy_address" ]; then
-            http_proxy="http://$http_proxy_address" https_proxy="http://$http_proxy_address" wget -qO- "$url" | sed 's/\r$//' && break
-        else
-            wget -qO- "$url" | sed 's/\r$//' && break
-        fi
-
-        log "Attempt $attempt/$retries to download $url failed" "warn"
-        sleep "$wait"
-    done
-}
-
 # Download URL to file
 download_to_file() {
     local url="$1"
@@ -305,10 +286,17 @@ download_to_file() {
         log "Attempt $attempt/$retries to download $url failed" "warn"
         sleep "$wait"
     done
+}
+
+# Converts Windows-style line endings (CRLF) to Unix-style (LF)
+convert_crlf_to_lf() {
+    local filepath="$1"
 
     if grep -q $'\r' "$filepath"; then
-        log "Downloaded file has Windows line endings (CRLF). Converting to Unix (LF)"
+        log "File '$filepath' contains CRLF line endings. Converting to LF..." "debug"
-        sed -i 's/\r$//' "$filepath"
+        local tmpfile
+        tmpfile=$(mktemp)
+        tr -d '\r' < "$filepath" > "$tmpfile" && mv "$tmpfile" "$filepath" || rm -f "$tmpfile"
     fi
 }
 
@@ -400,3 +388,17 @@ parse_domain_or_subnet_file_to_comma_string() {
 
     echo "$result"
 }
+
+# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file.
+extract_ip_cidr_from_json_ruleset_to_file() {
+    local json_file="$1"
+    local output_file="$2"
+
+    if [ ! -f "$json_file" ]; then
+        log "JSON file not found: $json_file" "error"
+        return 1
+    fi
+
+    log "Extracting ip_cidr entries from $json_file to $output_file" "debug"
+    jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file"
+}