From 32c385b309854a8e21a1456cfa8e1d245d48d332 Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Sun, 16 Nov 2025 09:55:44 +0500 Subject: [PATCH] fix: load large plain domain/subnet lists in chunks; move ruleset logic to rulesets.sh and nft chunker to nft.sh --- podkop/files/usr/bin/podkop | 141 +++++--------- podkop/files/usr/lib/helpers.sh | 64 ------- podkop/files/usr/lib/nft.sh | 40 ++++ podkop/files/usr/lib/rulesets.sh | 177 ++++++++++++++++++ .../files/usr/lib/sing_box_config_manager.sh | 45 ----- 5 files changed, 266 insertions(+), 201 deletions(-) create mode 100644 podkop/files/usr/lib/rulesets.sh diff --git a/podkop/files/usr/bin/podkop b/podkop/files/usr/bin/podkop index 6783d17..80f89bc 100755 --- a/podkop/files/usr/bin/podkop +++ b/podkop/files/usr/bin/podkop @@ -18,6 +18,7 @@ check_required_file "$PODKOP_LIB/helpers.sh" check_required_file "$PODKOP_LIB/sing_box_config_manager.sh" check_required_file "$PODKOP_LIB/sing_box_config_facade.sh" check_required_file "$PODKOP_LIB/logging.sh" +check_required_file "$PODKOP_LIB/rulesets.sh" . /lib/config/uci.sh . /lib/functions.sh . "$PODKOP_LIB/constants.sh" @@ -26,6 +27,7 @@ check_required_file "$PODKOP_LIB/logging.sh" . "$PODKOP_LIB/sing_box_config_manager.sh" . "$PODKOP_LIB/sing_box_config_facade.sh" . "$PODKOP_LIB/logging.sh" +. "$PODKOP_LIB/rulesets.sh" config_load "$PODKOP_CONFIG" @@ -907,22 +909,16 @@ prepare_common_ruleset() { log "Preparing a common $type ruleset for '$section' section" "debug" ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") - ruleset_filename="$ruleset_tag.json" - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" - if file_exists "$ruleset_filepath"; then - log "Ruleset $ruleset_filepath already exists. Skipping." "debug" - else - sing_box_cm_create_local_source_ruleset "$ruleset_filepath" - config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") - config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") - case "$type" in - domains) - config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") - ;; - subnets) ;; - *) log "Unsupported remote rule set type: $type" "error" ;; - esac - fi + ruleset_filepath=$(create_source_rule_set "$ruleset_tag") + config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") + config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") + case "$type" in + domains) + config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") + ;; + subnets) ;; + *) log "Unsupported remote rule set type: $type" "error" ;; + esac } configure_community_list_handler() { @@ -972,9 +968,9 @@ configure_user_domain_or_subnets_list() { items="$(parse_domain_or_subnet_string_to_commas_string "$items" "$type")" json_array="$(comma_string_to_json_array "$items")" case "$type" in - domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; + domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; subnets) - sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items" ;; esac @@ -985,56 +981,57 @@ configure_local_domain_or_subnet_lists() { local type="$2" local route_rule_tag="$3" - local ruleset_tag ruleset_filename ruleset_filepath + local ruleset_tag ruleset_filepath ruleset_tag="$(get_ruleset_tag "$section" "local" "$type")" - ruleset_filename="$ruleset_tag.json" - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" - - sing_box_cm_create_local_source_ruleset "$ruleset_filepath" + ruleset_filepath=$(create_source_rule_set "$ruleset_tag") config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") case "$type" in domains) - config_list_foreach "$section" "local_domain_lists" import_local_domain_or_subnet_list "$type" \ - "$section" "$ruleset_filepath" + config_list_foreach "$section" "local_domain_lists" import_local_domain_list "$ruleset_filepath" config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") ;; subnets) - config_list_foreach "$section" "local_subnet_lists" import_local_domain_or_subnet_list "$type" \ - "$section" "$ruleset_filepath" + config_list_foreach "$section" "local_subnet_lists" import_local_subnets_list "$ruleset_filepath" ;; *) log "Unsupported local rule set type: $type" "error" ;; esac } -import_local_domain_or_subnet_list() { - local filepath="$1" - local type="$2" - local section="$3" - local ruleset_filepath="$4" +import_local_domain_list() { + local local_domain_list_filepath="$1" + local ruleset_filepath="$2" - if ! file_exists "$filepath"; then - log "File $filepath not found" "error" + if ! file_exists "$local_domain_list_filepath"; then + log "Local domain list file $local_domain_list_filepath not found" "error" return 1 fi - local items json_array - items="$(parse_domain_or_subnet_file_to_comma_string "$filepath" "$type")" - - if [ -z "$items" ]; then - log "No valid $type found in $filepath" "warn" - return 0 + if ! file_exists "$ruleset_filepath"; then + log "Target ruleset file $ruleset_filepath not found" "error" + return 1 fi - json_array="$(comma_string_to_json_array "$items")" - case "$type" in - domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; - subnets) - sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" - nft_add_set_elements_from_file_chunked "$filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" - ;; - esac + import_plain_domain_list_to_local_source_ruleset_chunked "$local_domain_list_filepath" "$ruleset_filepath" +} + +import_local_subnets_list() { + local local_subnet_list_filepath="$1" + local ruleset_filepath="$2" + + if ! file_exists "$local_subnet_list_filepath"; then + log "Local subnet list file $local_subnet_list_filepath not found" "error" + return 1 + fi + + if ! file_exists "$ruleset_filepath"; then + log "Target ruleset file $ruleset_filepath not found" "error" + return 1 + fi + + import_plain_subnet_list_to_local_source_ruleset_chunked "$local_subnet_list_filepath" "$ruleset_filepath" + nft_add_set_elements_from_file_chunked "$local_subnet_list_filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" } configure_remote_domain_or_subnet_list_handler() { @@ -1353,9 +1350,9 @@ import_domains_or_subnets_from_remote_file() { ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" json_array="$(comma_string_to_json_array "$items")" case "$type" in - domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; + domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; subnets) - sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" ;; esac @@ -1398,8 +1395,8 @@ import_subnets_from_remote_srs_file() { return 1 fi - if ! decompile_srs_file "$binary_tmpfile" "$json_tmpfile"; then - log "Failed to decompile SRS file" "error" + if ! decompile_binary_ruleset "$binary_tmpfile" "$json_tmpfile"; then + log "Failed to decompile binary rule set file" "error" return 1 fi @@ -1516,46 +1513,6 @@ nft_list_all_traffic_from_ip() { fi } -nft_add_set_elements_from_file_chunked() { - local filepath="$1" - local nft_table_name="$2" - local nft_set_name="$3" - local chunk_size="${4:-5000}" - - local array count - count=0 - while IFS= read -r line; do - line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - [ -z "$line" ] && continue - - if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then - log "'$line' is not IPv4 or IPv4 CIDR" "debug" - continue - fi - - if [ -z "$array" ]; then - array="$line" - else - array="$array,$line" - fi - - count=$((count + 1)) - - if [ "$count" = "$chunk_size" ]; then - log "Adding $count elements to nft set $nft_set_name" "debug" - nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" - array="" - count=0 - fi - done < "$filepath" - - if [ -n "$array" ]; then - log "Adding $count elements to nft set $nft_set_name" "debug" - nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" - fi -} - # Diagnotics check_proxy() { local sing_box_config_path diff --git a/podkop/files/usr/lib/helpers.sh b/podkop/files/usr/lib/helpers.sh index 6ab48c7..93dcf6d 100644 --- a/podkop/files/usr/lib/helpers.sh +++ b/podkop/files/usr/lib/helpers.sh @@ -105,37 +105,6 @@ get_domain_resolver_tag() { echo "$section-$postfix" } -# Constructs and returns a ruleset tag using section, name, optional type, and a fixed postfix -get_ruleset_tag() { - local section="$1" - local name="$2" - local type="$3" - local postfix="ruleset" - - if [ -n "$type" ]; then - echo "$section-$name-$type-$postfix" - else - echo "$section-$name-$postfix" - fi -} - -# Determines the ruleset format based on the file extension (json → source, srs → binary) -get_ruleset_format_by_file_extension() { - local file_extension="$1" - - local format - case "$file_extension" in - json) format="source" ;; - srs) format="binary" ;; - *) - log "Unsupported file extension: .$file_extension" "error" - return 1 - ;; - esac - - echo "$format" -} - # Converts a comma-separated string into a JSON array string comma_string_to_json_array() { local input="$1" @@ -300,25 +269,6 @@ convert_crlf_to_lf() { fi } -# Decompiles a sing-box SRS binary file into a JSON ruleset file -decompile_srs_file() { - local binary_filepath="$1" - local output_filepath="$2" - - log "Decompiling $binary_filepath to $output_filepath" "debug" - - if ! file_exists "$binary_filepath"; then - log "File $binary_filepath not found" "error" - return 1 - fi - - sing-box rule-set decompile "$binary_filepath" -o "$output_filepath" - if [[ $? -ne 0 ]]; then - log "Decompilation command failed for $binary_filepath" "error" - return 1 - fi -} - ####################################### # Parses a whitespace-separated string, validates items as either domains # or IPv4 addresses/subnets, and returns a comma-separated string of valid items. @@ -387,18 +337,4 @@ parse_domain_or_subnet_file_to_comma_string() { done < "$filepath" echo "$result" -} - -# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file. -extract_ip_cidr_from_json_ruleset_to_file() { - local json_file="$1" - local output_file="$2" - - if [ ! -f "$json_file" ]; then - log "JSON file not found: $json_file" "error" - return 1 - fi - - log "Extracting ip_cidr entries from $json_file to $output_file" "debug" - jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file" } \ No newline at end of file diff --git a/podkop/files/usr/lib/nft.sh b/podkop/files/usr/lib/nft.sh index 3a2a3c2..8cbcc6a 100644 --- a/podkop/files/usr/lib/nft.sh +++ b/podkop/files/usr/lib/nft.sh @@ -27,4 +27,44 @@ nft_add_set_elements() { local elements="$3" nft add element inet "$table" "$set" "{ $elements }" +} + +nft_add_set_elements_from_file_chunked() { + local filepath="$1" + local nft_table_name="$2" + local nft_set_name="$3" + local chunk_size="${4:-5000}" + + local array count + count=0 + while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + [ -z "$line" ] && continue + + if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then + log "'$line' is not IPv4 or IPv4 CIDR" "debug" + continue + fi + + if [ -z "$array" ]; then + array="$line" + else + array="$array,$line" + fi + + count=$((count + 1)) + + if [ "$count" = "$chunk_size" ]; then + log "Adding $count elements to nft set $nft_set_name" "debug" + nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" + array="" + count=0 + fi + done < "$filepath" + + if [ -n "$array" ]; then + log "Adding $count elements to nft set $nft_set_name" "debug" + nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" + fi } \ No newline at end of file diff --git a/podkop/files/usr/lib/rulesets.sh b/podkop/files/usr/lib/rulesets.sh new file mode 100644 index 0000000..7fa7395 --- /dev/null +++ b/podkop/files/usr/lib/rulesets.sh @@ -0,0 +1,177 @@ +# Constructs and returns a ruleset tag using section, name, optional type, and a fixed postfix +get_ruleset_tag() { + local section="$1" + local name="$2" + local type="$3" + local postfix="ruleset" + + if [ -n "$type" ]; then + echo "$section-$name-$type-$postfix" + else + echo "$section-$name-$postfix" + fi +} + +# Creates a new ruleset JSON file if it doesn't already exist and outputs its path. +create_source_rule_set() { + local ruleset_name="$1" + + ruleset_filename="$ruleset_name.json" + ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" + if file_exists "$ruleset_filepath"; then + log "Ruleset $ruleset_filepath already exists. Skipping." "debug" + return 0 + fi + + jq -n '{version: 3, rules: []}' > "$ruleset_filepath" + + echo "$ruleset_filepath" +} + +####################################### +# Patch a source ruleset JSON file for sing-box by appending a new ruleset object containing the provided key +# and value. +# Arguments: +# filepath: path to the JSON file to patch +# key: the ruleset key to insert (e.g., "ip_cidr") +# value: a JSON array of values to assign to the key +# Example: +# patch_source_ruleset_rules "/tmp/sing-box/ruleset.json" "ip_cidr" '["1.1.1.1","2.2.2.2"]' +####################################### +patch_source_ruleset_rules() { + local filepath="$1" + local key="$2" + local value="$3" + + local content + content="$(cat "$filepath")" + + echo "$content" | jq \ + --arg key "$key" \ + --argjson value "$value" ' + .rules += [{ ($key): $value }] + ' > "$filepath" +} + +# Imports a plain domain list into a ruleset in chunks, validating domains and appending them as domain_suffix rules +import_plain_domain_list_to_local_source_ruleset_chunked() { + local plain_list_filepath="$1" + local ruleset_filepath="$2" + local chunk_size="${3:-5000}" + + local array count json_array + count=0 + while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + [ -z "$line" ] && continue + + if ! is_domain_suffix "$line"; then + log "'$line' is not a valid domain" "debug" + continue + fi + + if [ -z "$array" ]; then + array="$line" + else + array="$array,$line" + fi + + count=$((count + 1)) + + if [ "$count" = "$chunk_size" ]; then + log "Adding $count elements to rule set at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" + array="" + count=0 + fi + done < "$plain_list_filepath" + + if [ -n "$array" ]; then + log "Adding $count elements to rule set at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" + fi +} + +# Imports a plain IPv4/CIDR list into a ruleset in chunks, validating entries and appending them as ip_cidr rules +import_plain_subnet_list_to_local_source_ruleset_chunked() { + local plain_list_filepath="$1" + local ruleset_filepath="$2" + local chunk_size="${3:-5000}" + + local array count json_array + count=0 + while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + [ -z "$line" ] && continue + + if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then + log "'$line' is not IPv4 or IPv4 CIDR" "debug" + continue + fi + + if [ -z "$array" ]; then + array="$line" + else + array="$array,$line" + fi + + count=$((count + 1)) + + if [ "$count" = "$chunk_size" ]; then + log "Adding $count elements to ruleset at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + array="" + count=0 + fi + done < "$plain_list_filepath" + + if [ -n "$array" ]; then + log "Adding $count elements to ruleset at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + fi +} + +# Determines the ruleset format based on the file extension (json → source, srs → binary) +get_ruleset_format_by_file_extension() { + local file_extension="$1" + + local format + case "$file_extension" in + json) format="source" ;; + srs) format="binary" ;; + *) + log "Unsupported file extension: .$file_extension" "error" + return 1 + ;; + esac + + echo "$format" +} + +# Decompiles a sing-box SRS binary file into a JSON ruleset file +decompile_binary_ruleset() { + local binary_filepath="$1" + local output_filepath="$2" + + log "Decompiling $binary_filepath to $output_filepath" "debug" + sing-box rule-set decompile "$binary_filepath" -o "$output_filepath" + if [[ $? -ne 0 ]]; then + log "Decompilation command failed for $binary_filepath" "error" + return 1 + fi +} + +# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file. +extract_ip_cidr_from_json_ruleset_to_file() { + local json_file="$1" + local output_file="$2" + + log "Extracting ip_cidr entries from $json_file to $output_file" "debug" + jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file" +} diff --git a/podkop/files/usr/lib/sing_box_config_manager.sh b/podkop/files/usr/lib/sing_box_config_manager.sh index ff817aa..9a575d4 100644 --- a/podkop/files/usr/lib/sing_box_config_manager.sh +++ b/podkop/files/usr/lib/sing_box_config_manager.sh @@ -1365,51 +1365,6 @@ sing_box_cm_configure_clash_api() { + (if $secret != "" then { secret: $secret } else {} end)' } -####################################### -# Create a local source ruleset JSON file for sing-box. -# Arguments: -# filepath: path to the JSON file to create -# Example: -# sing_box_cm_create_local_source_ruleset "/tmp/sing-box/ruleset.json" -####################################### -sing_box_cm_create_local_source_ruleset() { - local filepath="$1" - - jq -n '{version: 3, rules: []}' > "$filepath" -} - -####################################### -# Patch a local source ruleset JSON file for sing-box by adding unique! values to a given key. -# Arguments: -# filepath: path to the JSON file to patch -# key: the ruleset key to update (e.g., "ip_cidr") -# value: a JSON array of values to add to the key -# Example: -# sing_box_cm_patch_local_source_ruleset_rules "/tmp/sing-box/ruleset.json" "ip_cidr" '["1.1.1.1","2.2.2.2"]' -####################################### -sing_box_cm_patch_local_source_ruleset_rules() { - local filepath="$1" - local key="$2" - local value="$3" - - value=$(_normalize_arg "$value") - - local content - content="$(cat "$filepath")" - - echo "$content" | jq \ - --arg key "$key" \ - --argjson value "$value" ' - ([.rules[]?[$key][]] | unique) as $existing - | ($value - $existing) as $value - | if ($value | length) > 0 then - .rules += [{($key): $value}] - else - . - end - ' > "$filepath" -} - ####################################### # Save a sing-box JSON configuration to a file, removing service-specific tags. # Arguments: