Mirror/podkop
1
0
Fork 0
mirror of https://github.com/itdoginfo/podkop.git synced 2025-12-06 03:26:51 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix: import large subnet lists in chunks into nft sets

Browse Source
This commit is contained in:
Andrey Petelin
2025-10-30 14:07:12 +05:00
parent f8510cd828
commit cc947f9734
2 changed files with 70 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
podkop/files/usr/bin/podkop
View File

@@ -1032,7 +1032,7 @@ import_local_domain_or_subnet_list() {
domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
subnets)
sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items"
nft_add_set_elements_from_file_chunked "$filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
;;
esac
}
@@ -1213,7 +1213,7 @@ import_community_service_subnet_list_handler() {
*) return 0 ;;
esac
local tmpfile http_proxy_address subnets
local tmpfile http_proxy_address
tmpfile=$(mktemp)
http_proxy_address="$(get_service_proxy_address)"
@@ -1224,14 +1224,13 @@ import_community_service_subnet_list_handler() {
return 1
fi
subnets="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "subnets")"
rm -f "$tmpfile"
if [ "$service" = "discord" ]; then
nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_DISCORD_SET_NAME" "$subnets"
nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_DISCORD_SET_NAME"
else
nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets"
nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
fi
rm -f "$tmpfile"
}
import_domains_from_remote_domain_lists() {
@@ -1307,15 +1306,14 @@ import_domains_or_subnets_from_remote_file() {
http_proxy_address="$(get_service_proxy_address)"
download_to_file "$url" "$tmpfile" "$http_proxy_address"
convert_crlf_to_lf "$tmpfile"
if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then
log "Download $url list failed" "error"
return 1
fi
convert_crlf_to_lf "$tmpfile"
items="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "$type")"
rm -f "$tmpfile"
if [ -z "$items" ]; then
log "No valid $type found in $url" "warn"
@@ -1330,14 +1328,16 @@ import_domains_or_subnets_from_remote_file() {
domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
subnets)
sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items"
nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
;;
esac
rm -f "$tmpfile"
}
import_subnets_from_remote_json_file() {
local url="$1"
local json_tmpfile subnets_tmpfile subnets http_proxy_address
local json_tmpfile subnets_tmpfile http_proxy_address
json_tmpfile="$(mktemp)"
subnets_tmpfile="$(mktemp)"
http_proxy_address="$(get_service_proxy_address)"
@@ -1349,16 +1349,15 @@ import_subnets_from_remote_json_file() {
return 1
fi
jq -r '.rules[].ip_cidr[]' "$json_tmpfile" > "$subnets_tmpfile"
subnets="$(parse_domain_or_subnet_file_to_comma_string "$subnets_tmpfile" "subnets")"
extract_ip_cidr_from_json_ruleset_to_file "$json_tmpfile" "$subnets_tmpfile"
nft_add_set_elements_from_file_chunked "$subnets_tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
rm -f "$json_tmpfile" "$subnets_tmpfile"
nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets"
}
import_subnets_from_remote_srs_file() {
local url="$1"
local binary_tmpfile json_tmpfile subnets_tmpfile subnets http_proxy_address
local binary_tmpfile json_tmpfile subnets_tmpfile http_proxy_address
binary_tmpfile="$(mktemp)"
json_tmpfile="$(mktemp)"
subnets_tmpfile="$(mktemp)"
@@ -1376,10 +1375,9 @@ import_subnets_from_remote_srs_file() {
return 1
fi
jq -r '.rules[].ip_cidr[]' "$json_tmpfile" > "$subnets_tmpfile"
subnets="$(parse_domain_or_subnet_file_to_comma_string "$subnets_tmpfile" "subnets")"
extract_ip_cidr_from_json_ruleset_to_file "$json_tmpfile" "$subnets_tmpfile"
nft_add_set_elements_from_file_chunked "$subnets_tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
rm -f "$binary_tmpfile" "$json_tmpfile" "$subnets_tmpfile"
nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets"
}
## Support functions
@@ -1473,6 +1471,46 @@ nft_list_all_traffic_from_ip() {
fi
}
nft_add_set_elements_from_file_chunked() {
local filepath="$1"
local nft_table_name="$2"
local nft_set_name="$3"
local chunk_size="${4:-5000}"
local array count
count=0
while IFS= read -r line; do
line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
[ -z "$line" ] && continue
if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then
log "'$line' is not IPv4 or IPv4 CIDR" "debug"
continue
fi
if [ -z "$array" ]; then
array="$line"
else
array="$array,$line"
fi
count=$((count + 1))
if [ "$count" = "$chunk_size" ]; then
log "Adding $count elements to nft set $nft_set_name" "debug"
nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
array=""
count=0
fi
done < "$filepath"
if [ -n "$array" ]; then
log "Adding $count elements to nft set $nft_set_name" "debug"
nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
fi
}
# Diagnotics
check_proxy() {
local sing_box_config_path

14
podkop/files/usr/lib/helpers.sh
View File

@@ -388,3 +388,17 @@ parse_domain_or_subnet_file_to_comma_string() {
echo "$result"
}
# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file.
extract_ip_cidr_from_json_ruleset_to_file() {
local json_file="$1"
local output_file="$2"
if [ ! -f "$json_file" ]; then
log "JSON file not found: $json_file" "error"
return 1
fi
log "Extracting ip_cidr entries from $json_file to $output_file" "debug"
jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file"
}