diff --git a/podkop/files/usr/bin/podkop b/podkop/files/usr/bin/podkop index 68577bf..b7f1295 100755 --- a/podkop/files/usr/bin/podkop +++ b/podkop/files/usr/bin/podkop @@ -1032,7 +1032,7 @@ import_local_domain_or_subnet_list() { domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; subnets) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" - nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items" + nft_add_set_elements_from_file_chunked "$filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" ;; esac } @@ -1213,7 +1213,7 @@ import_community_service_subnet_list_handler() { *) return 0 ;; esac - local tmpfile http_proxy_address subnets + local tmpfile http_proxy_address tmpfile=$(mktemp) http_proxy_address="$(get_service_proxy_address)" @@ -1224,14 +1224,13 @@ import_community_service_subnet_list_handler() { return 1 fi - subnets="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "subnets")" - rm -f "$tmpfile" - if [ "$service" = "discord" ]; then - nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_DISCORD_SET_NAME" "$subnets" + nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_DISCORD_SET_NAME" else - nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets" + nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" fi + + rm -f "$tmpfile" } import_domains_from_remote_domain_lists() { @@ -1307,15 +1306,14 @@ import_domains_or_subnets_from_remote_file() { http_proxy_address="$(get_service_proxy_address)" download_to_file "$url" "$tmpfile" "$http_proxy_address" - convert_crlf_to_lf "$tmpfile" if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then log "Download $url list failed" "error" return 1 fi + convert_crlf_to_lf "$tmpfile" items="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "$type")" - rm -f "$tmpfile" if [ -z "$items" ]; then log "No valid $type found in $url" "warn" @@ -1330,14 +1328,16 @@ import_domains_or_subnets_from_remote_file() { domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; subnets) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" - nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items" + nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" ;; esac + + rm -f "$tmpfile" } import_subnets_from_remote_json_file() { local url="$1" - local json_tmpfile subnets_tmpfile subnets http_proxy_address + local json_tmpfile subnets_tmpfile http_proxy_address json_tmpfile="$(mktemp)" subnets_tmpfile="$(mktemp)" http_proxy_address="$(get_service_proxy_address)" @@ -1349,16 +1349,15 @@ import_subnets_from_remote_json_file() { return 1 fi - jq -r '.rules[].ip_cidr[]' "$json_tmpfile" > "$subnets_tmpfile" - subnets="$(parse_domain_or_subnet_file_to_comma_string "$subnets_tmpfile" "subnets")" + extract_ip_cidr_from_json_ruleset_to_file "$json_tmpfile" "$subnets_tmpfile" + nft_add_set_elements_from_file_chunked "$subnets_tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" rm -f "$json_tmpfile" "$subnets_tmpfile" - nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets" } import_subnets_from_remote_srs_file() { local url="$1" - local binary_tmpfile json_tmpfile subnets_tmpfile subnets http_proxy_address + local binary_tmpfile json_tmpfile subnets_tmpfile http_proxy_address binary_tmpfile="$(mktemp)" json_tmpfile="$(mktemp)" subnets_tmpfile="$(mktemp)" @@ -1376,10 +1375,9 @@ import_subnets_from_remote_srs_file() { return 1 fi - jq -r '.rules[].ip_cidr[]' "$json_tmpfile" > "$subnets_tmpfile" - subnets="$(parse_domain_or_subnet_file_to_comma_string "$subnets_tmpfile" "subnets")" + extract_ip_cidr_from_json_ruleset_to_file "$json_tmpfile" "$subnets_tmpfile" + nft_add_set_elements_from_file_chunked "$subnets_tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" rm -f "$binary_tmpfile" "$json_tmpfile" "$subnets_tmpfile" - nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets" } ## Support functions @@ -1473,6 +1471,46 @@ nft_list_all_traffic_from_ip() { fi } +nft_add_set_elements_from_file_chunked() { + local filepath="$1" + local nft_table_name="$2" + local nft_set_name="$3" + local chunk_size="${4:-5000}" + + local array count + count=0 + while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + [ -z "$line" ] && continue + + if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then + log "'$line' is not IPv4 or IPv4 CIDR" "debug" + continue + fi + + if [ -z "$array" ]; then + array="$line" + else + array="$array,$line" + fi + + count=$((count + 1)) + + if [ "$count" = "$chunk_size" ]; then + log "Adding $count elements to nft set $nft_set_name" "debug" + nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" + array="" + count=0 + fi + done < "$filepath" + + if [ -n "$array" ]; then + log "Adding $count elements to nft set $nft_set_name" "debug" + nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" + fi +} + # Diagnotics check_proxy() { local sing_box_config_path diff --git a/podkop/files/usr/lib/helpers.sh b/podkop/files/usr/lib/helpers.sh index 4c005f6..6ab48c7 100644 --- a/podkop/files/usr/lib/helpers.sh +++ b/podkop/files/usr/lib/helpers.sh @@ -388,3 +388,17 @@ parse_domain_or_subnet_file_to_comma_string() { echo "$result" } + +# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file. +extract_ip_cidr_from_json_ruleset_to_file() { + local json_file="$1" + local output_file="$2" + + if [ ! -f "$json_file" ]; then + log "JSON file not found: $json_file" "error" + return 1 + fi + + log "Extracting ip_cidr entries from $json_file to $output_file" "debug" + jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file" +} \ No newline at end of file