From 32c385b309854a8e21a1456cfa8e1d245d48d332 Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Sun, 16 Nov 2025 09:55:44 +0500 Subject: [PATCH 1/4] fix: load large plain domain/subnet lists in chunks; move ruleset logic to rulesets.sh and nft chunker to nft.sh --- podkop/files/usr/bin/podkop | 141 +++++--------- podkop/files/usr/lib/helpers.sh | 64 ------- podkop/files/usr/lib/nft.sh | 40 ++++ podkop/files/usr/lib/rulesets.sh | 177 ++++++++++++++++++ .../files/usr/lib/sing_box_config_manager.sh | 45 ----- 5 files changed, 266 insertions(+), 201 deletions(-) create mode 100644 podkop/files/usr/lib/rulesets.sh diff --git a/podkop/files/usr/bin/podkop b/podkop/files/usr/bin/podkop index 6783d17..80f89bc 100755 --- a/podkop/files/usr/bin/podkop +++ b/podkop/files/usr/bin/podkop @@ -18,6 +18,7 @@ check_required_file "$PODKOP_LIB/helpers.sh" check_required_file "$PODKOP_LIB/sing_box_config_manager.sh" check_required_file "$PODKOP_LIB/sing_box_config_facade.sh" check_required_file "$PODKOP_LIB/logging.sh" +check_required_file "$PODKOP_LIB/rulesets.sh" . /lib/config/uci.sh . /lib/functions.sh . "$PODKOP_LIB/constants.sh" @@ -26,6 +27,7 @@ check_required_file "$PODKOP_LIB/logging.sh" . "$PODKOP_LIB/sing_box_config_manager.sh" . "$PODKOP_LIB/sing_box_config_facade.sh" . "$PODKOP_LIB/logging.sh" +. "$PODKOP_LIB/rulesets.sh" config_load "$PODKOP_CONFIG" @@ -907,22 +909,16 @@ prepare_common_ruleset() { log "Preparing a common $type ruleset for '$section' section" "debug" ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") - ruleset_filename="$ruleset_tag.json" - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" - if file_exists "$ruleset_filepath"; then - log "Ruleset $ruleset_filepath already exists. Skipping." "debug" - else - sing_box_cm_create_local_source_ruleset "$ruleset_filepath" - config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") - config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") - case "$type" in - domains) - config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") - ;; - subnets) ;; - *) log "Unsupported remote rule set type: $type" "error" ;; - esac - fi + ruleset_filepath=$(create_source_rule_set "$ruleset_tag") + config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") + config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") + case "$type" in + domains) + config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") + ;; + subnets) ;; + *) log "Unsupported remote rule set type: $type" "error" ;; + esac } configure_community_list_handler() { @@ -972,9 +968,9 @@ configure_user_domain_or_subnets_list() { items="$(parse_domain_or_subnet_string_to_commas_string "$items" "$type")" json_array="$(comma_string_to_json_array "$items")" case "$type" in - domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; + domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; subnets) - sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items" ;; esac @@ -985,56 +981,57 @@ configure_local_domain_or_subnet_lists() { local type="$2" local route_rule_tag="$3" - local ruleset_tag ruleset_filename ruleset_filepath + local ruleset_tag ruleset_filepath ruleset_tag="$(get_ruleset_tag "$section" "local" "$type")" - ruleset_filename="$ruleset_tag.json" - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" - - sing_box_cm_create_local_source_ruleset "$ruleset_filepath" + ruleset_filepath=$(create_source_rule_set "$ruleset_tag") config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") case "$type" in domains) - config_list_foreach "$section" "local_domain_lists" import_local_domain_or_subnet_list "$type" \ - "$section" "$ruleset_filepath" + config_list_foreach "$section" "local_domain_lists" import_local_domain_list "$ruleset_filepath" config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") ;; subnets) - config_list_foreach "$section" "local_subnet_lists" import_local_domain_or_subnet_list "$type" \ - "$section" "$ruleset_filepath" + config_list_foreach "$section" "local_subnet_lists" import_local_subnets_list "$ruleset_filepath" ;; *) log "Unsupported local rule set type: $type" "error" ;; esac } -import_local_domain_or_subnet_list() { - local filepath="$1" - local type="$2" - local section="$3" - local ruleset_filepath="$4" +import_local_domain_list() { + local local_domain_list_filepath="$1" + local ruleset_filepath="$2" - if ! file_exists "$filepath"; then - log "File $filepath not found" "error" + if ! file_exists "$local_domain_list_filepath"; then + log "Local domain list file $local_domain_list_filepath not found" "error" return 1 fi - local items json_array - items="$(parse_domain_or_subnet_file_to_comma_string "$filepath" "$type")" - - if [ -z "$items" ]; then - log "No valid $type found in $filepath" "warn" - return 0 + if ! file_exists "$ruleset_filepath"; then + log "Target ruleset file $ruleset_filepath not found" "error" + return 1 fi - json_array="$(comma_string_to_json_array "$items")" - case "$type" in - domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; - subnets) - sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" - nft_add_set_elements_from_file_chunked "$filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" - ;; - esac + import_plain_domain_list_to_local_source_ruleset_chunked "$local_domain_list_filepath" "$ruleset_filepath" +} + +import_local_subnets_list() { + local local_subnet_list_filepath="$1" + local ruleset_filepath="$2" + + if ! file_exists "$local_subnet_list_filepath"; then + log "Local subnet list file $local_subnet_list_filepath not found" "error" + return 1 + fi + + if ! file_exists "$ruleset_filepath"; then + log "Target ruleset file $ruleset_filepath not found" "error" + return 1 + fi + + import_plain_subnet_list_to_local_source_ruleset_chunked "$local_subnet_list_filepath" "$ruleset_filepath" + nft_add_set_elements_from_file_chunked "$local_subnet_list_filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" } configure_remote_domain_or_subnet_list_handler() { @@ -1353,9 +1350,9 @@ import_domains_or_subnets_from_remote_file() { ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" json_array="$(comma_string_to_json_array "$items")" case "$type" in - domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; + domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; subnets) - sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" ;; esac @@ -1398,8 +1395,8 @@ import_subnets_from_remote_srs_file() { return 1 fi - if ! decompile_srs_file "$binary_tmpfile" "$json_tmpfile"; then - log "Failed to decompile SRS file" "error" + if ! decompile_binary_ruleset "$binary_tmpfile" "$json_tmpfile"; then + log "Failed to decompile binary rule set file" "error" return 1 fi @@ -1516,46 +1513,6 @@ nft_list_all_traffic_from_ip() { fi } -nft_add_set_elements_from_file_chunked() { - local filepath="$1" - local nft_table_name="$2" - local nft_set_name="$3" - local chunk_size="${4:-5000}" - - local array count - count=0 - while IFS= read -r line; do - line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - [ -z "$line" ] && continue - - if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then - log "'$line' is not IPv4 or IPv4 CIDR" "debug" - continue - fi - - if [ -z "$array" ]; then - array="$line" - else - array="$array,$line" - fi - - count=$((count + 1)) - - if [ "$count" = "$chunk_size" ]; then - log "Adding $count elements to nft set $nft_set_name" "debug" - nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" - array="" - count=0 - fi - done < "$filepath" - - if [ -n "$array" ]; then - log "Adding $count elements to nft set $nft_set_name" "debug" - nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" - fi -} - # Diagnotics check_proxy() { local sing_box_config_path diff --git a/podkop/files/usr/lib/helpers.sh b/podkop/files/usr/lib/helpers.sh index 6ab48c7..93dcf6d 100644 --- a/podkop/files/usr/lib/helpers.sh +++ b/podkop/files/usr/lib/helpers.sh @@ -105,37 +105,6 @@ get_domain_resolver_tag() { echo "$section-$postfix" } -# Constructs and returns a ruleset tag using section, name, optional type, and a fixed postfix -get_ruleset_tag() { - local section="$1" - local name="$2" - local type="$3" - local postfix="ruleset" - - if [ -n "$type" ]; then - echo "$section-$name-$type-$postfix" - else - echo "$section-$name-$postfix" - fi -} - -# Determines the ruleset format based on the file extension (json → source, srs → binary) -get_ruleset_format_by_file_extension() { - local file_extension="$1" - - local format - case "$file_extension" in - json) format="source" ;; - srs) format="binary" ;; - *) - log "Unsupported file extension: .$file_extension" "error" - return 1 - ;; - esac - - echo "$format" -} - # Converts a comma-separated string into a JSON array string comma_string_to_json_array() { local input="$1" @@ -300,25 +269,6 @@ convert_crlf_to_lf() { fi } -# Decompiles a sing-box SRS binary file into a JSON ruleset file -decompile_srs_file() { - local binary_filepath="$1" - local output_filepath="$2" - - log "Decompiling $binary_filepath to $output_filepath" "debug" - - if ! file_exists "$binary_filepath"; then - log "File $binary_filepath not found" "error" - return 1 - fi - - sing-box rule-set decompile "$binary_filepath" -o "$output_filepath" - if [[ $? -ne 0 ]]; then - log "Decompilation command failed for $binary_filepath" "error" - return 1 - fi -} - ####################################### # Parses a whitespace-separated string, validates items as either domains # or IPv4 addresses/subnets, and returns a comma-separated string of valid items. @@ -387,18 +337,4 @@ parse_domain_or_subnet_file_to_comma_string() { done < "$filepath" echo "$result" -} - -# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file. -extract_ip_cidr_from_json_ruleset_to_file() { - local json_file="$1" - local output_file="$2" - - if [ ! -f "$json_file" ]; then - log "JSON file not found: $json_file" "error" - return 1 - fi - - log "Extracting ip_cidr entries from $json_file to $output_file" "debug" - jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file" } \ No newline at end of file diff --git a/podkop/files/usr/lib/nft.sh b/podkop/files/usr/lib/nft.sh index 3a2a3c2..8cbcc6a 100644 --- a/podkop/files/usr/lib/nft.sh +++ b/podkop/files/usr/lib/nft.sh @@ -27,4 +27,44 @@ nft_add_set_elements() { local elements="$3" nft add element inet "$table" "$set" "{ $elements }" +} + +nft_add_set_elements_from_file_chunked() { + local filepath="$1" + local nft_table_name="$2" + local nft_set_name="$3" + local chunk_size="${4:-5000}" + + local array count + count=0 + while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + [ -z "$line" ] && continue + + if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then + log "'$line' is not IPv4 or IPv4 CIDR" "debug" + continue + fi + + if [ -z "$array" ]; then + array="$line" + else + array="$array,$line" + fi + + count=$((count + 1)) + + if [ "$count" = "$chunk_size" ]; then + log "Adding $count elements to nft set $nft_set_name" "debug" + nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" + array="" + count=0 + fi + done < "$filepath" + + if [ -n "$array" ]; then + log "Adding $count elements to nft set $nft_set_name" "debug" + nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array" + fi } \ No newline at end of file diff --git a/podkop/files/usr/lib/rulesets.sh b/podkop/files/usr/lib/rulesets.sh new file mode 100644 index 0000000..7fa7395 --- /dev/null +++ b/podkop/files/usr/lib/rulesets.sh @@ -0,0 +1,177 @@ +# Constructs and returns a ruleset tag using section, name, optional type, and a fixed postfix +get_ruleset_tag() { + local section="$1" + local name="$2" + local type="$3" + local postfix="ruleset" + + if [ -n "$type" ]; then + echo "$section-$name-$type-$postfix" + else + echo "$section-$name-$postfix" + fi +} + +# Creates a new ruleset JSON file if it doesn't already exist and outputs its path. +create_source_rule_set() { + local ruleset_name="$1" + + ruleset_filename="$ruleset_name.json" + ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" + if file_exists "$ruleset_filepath"; then + log "Ruleset $ruleset_filepath already exists. Skipping." "debug" + return 0 + fi + + jq -n '{version: 3, rules: []}' > "$ruleset_filepath" + + echo "$ruleset_filepath" +} + +####################################### +# Patch a source ruleset JSON file for sing-box by appending a new ruleset object containing the provided key +# and value. +# Arguments: +# filepath: path to the JSON file to patch +# key: the ruleset key to insert (e.g., "ip_cidr") +# value: a JSON array of values to assign to the key +# Example: +# patch_source_ruleset_rules "/tmp/sing-box/ruleset.json" "ip_cidr" '["1.1.1.1","2.2.2.2"]' +####################################### +patch_source_ruleset_rules() { + local filepath="$1" + local key="$2" + local value="$3" + + local content + content="$(cat "$filepath")" + + echo "$content" | jq \ + --arg key "$key" \ + --argjson value "$value" ' + .rules += [{ ($key): $value }] + ' > "$filepath" +} + +# Imports a plain domain list into a ruleset in chunks, validating domains and appending them as domain_suffix rules +import_plain_domain_list_to_local_source_ruleset_chunked() { + local plain_list_filepath="$1" + local ruleset_filepath="$2" + local chunk_size="${3:-5000}" + + local array count json_array + count=0 + while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + [ -z "$line" ] && continue + + if ! is_domain_suffix "$line"; then + log "'$line' is not a valid domain" "debug" + continue + fi + + if [ -z "$array" ]; then + array="$line" + else + array="$array,$line" + fi + + count=$((count + 1)) + + if [ "$count" = "$chunk_size" ]; then + log "Adding $count elements to rule set at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" + array="" + count=0 + fi + done < "$plain_list_filepath" + + if [ -n "$array" ]; then + log "Adding $count elements to rule set at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" + fi +} + +# Imports a plain IPv4/CIDR list into a ruleset in chunks, validating entries and appending them as ip_cidr rules +import_plain_subnet_list_to_local_source_ruleset_chunked() { + local plain_list_filepath="$1" + local ruleset_filepath="$2" + local chunk_size="${3:-5000}" + + local array count json_array + count=0 + while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + [ -z "$line" ] && continue + + if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then + log "'$line' is not IPv4 or IPv4 CIDR" "debug" + continue + fi + + if [ -z "$array" ]; then + array="$line" + else + array="$array,$line" + fi + + count=$((count + 1)) + + if [ "$count" = "$chunk_size" ]; then + log "Adding $count elements to ruleset at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + array="" + count=0 + fi + done < "$plain_list_filepath" + + if [ -n "$array" ]; then + log "Adding $count elements to ruleset at $ruleset_filepath" "debug" + json_array="$(comma_string_to_json_array "$array")" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + fi +} + +# Determines the ruleset format based on the file extension (json → source, srs → binary) +get_ruleset_format_by_file_extension() { + local file_extension="$1" + + local format + case "$file_extension" in + json) format="source" ;; + srs) format="binary" ;; + *) + log "Unsupported file extension: .$file_extension" "error" + return 1 + ;; + esac + + echo "$format" +} + +# Decompiles a sing-box SRS binary file into a JSON ruleset file +decompile_binary_ruleset() { + local binary_filepath="$1" + local output_filepath="$2" + + log "Decompiling $binary_filepath to $output_filepath" "debug" + sing-box rule-set decompile "$binary_filepath" -o "$output_filepath" + if [[ $? -ne 0 ]]; then + log "Decompilation command failed for $binary_filepath" "error" + return 1 + fi +} + +# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file. +extract_ip_cidr_from_json_ruleset_to_file() { + local json_file="$1" + local output_file="$2" + + log "Extracting ip_cidr entries from $json_file to $output_file" "debug" + jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file" +} diff --git a/podkop/files/usr/lib/sing_box_config_manager.sh b/podkop/files/usr/lib/sing_box_config_manager.sh index ff817aa..9a575d4 100644 --- a/podkop/files/usr/lib/sing_box_config_manager.sh +++ b/podkop/files/usr/lib/sing_box_config_manager.sh @@ -1365,51 +1365,6 @@ sing_box_cm_configure_clash_api() { + (if $secret != "" then { secret: $secret } else {} end)' } -####################################### -# Create a local source ruleset JSON file for sing-box. -# Arguments: -# filepath: path to the JSON file to create -# Example: -# sing_box_cm_create_local_source_ruleset "/tmp/sing-box/ruleset.json" -####################################### -sing_box_cm_create_local_source_ruleset() { - local filepath="$1" - - jq -n '{version: 3, rules: []}' > "$filepath" -} - -####################################### -# Patch a local source ruleset JSON file for sing-box by adding unique! values to a given key. -# Arguments: -# filepath: path to the JSON file to patch -# key: the ruleset key to update (e.g., "ip_cidr") -# value: a JSON array of values to add to the key -# Example: -# sing_box_cm_patch_local_source_ruleset_rules "/tmp/sing-box/ruleset.json" "ip_cidr" '["1.1.1.1","2.2.2.2"]' -####################################### -sing_box_cm_patch_local_source_ruleset_rules() { - local filepath="$1" - local key="$2" - local value="$3" - - value=$(_normalize_arg "$value") - - local content - content="$(cat "$filepath")" - - echo "$content" | jq \ - --arg key "$key" \ - --argjson value "$value" ' - ([.rules[]?[$key][]] | unique) as $existing - | ($value - $existing) as $value - | if ($value | length) > 0 then - .rules += [{($key): $value}] - else - . - end - ' > "$filepath" -} - ####################################### # Save a sing-box JSON configuration to a file, removing service-specific tags. # Arguments: From e256e4bee548a11ca287bcb85852374db55ee05b Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Sun, 16 Nov 2025 09:56:12 +0500 Subject: [PATCH 2/4] chore: shorten Text List option label by removing the detailed format hint --- .../htdocs/luci-static/resources/view/podkop/section.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luci-app-podkop/htdocs/luci-static/resources/view/podkop/section.js b/luci-app-podkop/htdocs/luci-static/resources/view/podkop/section.js index 51f188c..5d522fe 100644 --- a/luci-app-podkop/htdocs/luci-static/resources/view/podkop/section.js +++ b/luci-app-podkop/htdocs/luci-static/resources/view/podkop/section.js @@ -445,7 +445,7 @@ function createSectionContent(section) { ); o.value("disabled", _("Disabled")); o.value("dynamic", _("Dynamic List")); - o.value("text", _("Text List (comma/space/newline separated)")); + o.value("text", _("Text List")); o.default = "disabled"; o.rmempty = false; From 2bf208ecaccb7246a972f7671d5f6782478fc0f1 Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Sun, 16 Nov 2025 13:21:51 +0500 Subject: [PATCH 3/4] fix: import remote plain domain and subnet lists using chunked processing --- podkop/files/usr/bin/podkop | 119 +++++++++++++++++-------------- podkop/files/usr/lib/rulesets.sh | 10 +-- 2 files changed, 70 insertions(+), 59 deletions(-) diff --git a/podkop/files/usr/bin/podkop b/podkop/files/usr/bin/podkop index 80f89bc..82afc47 100755 --- a/podkop/files/usr/bin/podkop +++ b/podkop/files/usr/bin/podkop @@ -909,16 +909,19 @@ prepare_common_ruleset() { log "Preparing a common $type ruleset for '$section' section" "debug" ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") - ruleset_filepath=$(create_source_rule_set "$ruleset_tag") - config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") - config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") - case "$type" in - domains) - config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") - ;; - subnets) ;; - *) log "Unsupported remote rule set type: $type" "error" ;; - esac + ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" + create_source_rule_set "$ruleset_filepath" + if [ $? -eq 0 ]; then + config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") + config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") + case "$type" in + domains) + config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") + ;; + subnets) ;; + *) log "Unsupported remote rule set type: $type" "error" ;; + esac + fi } configure_community_list_handler() { @@ -983,7 +986,8 @@ configure_local_domain_or_subnet_lists() { local ruleset_tag ruleset_filepath ruleset_tag="$(get_ruleset_tag "$section" "local" "$type")" - ruleset_filepath=$(create_source_rule_set "$ruleset_tag") + ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" + create_source_rule_set "$ruleset_filepath" config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") @@ -1282,11 +1286,35 @@ import_domains_from_remote_domain_list_handler() { ;; *) log "Detected file extension: '$file_extension' → proceeding with processing" "debug" - import_domains_or_subnets_from_remote_file "$url" "$section" "domains" + import_domains_from_remote_plain_file "$url" "$section" "domains" ;; esac } +import_domains_from_remote_plain_file() { + local url="$1" + local section="$2" + local type="$3" + + local tmpfile http_proxy_address items json_array + tmpfile=$(mktemp) + http_proxy_address="$(get_service_proxy_address)" + + download_to_file "$url" "$tmpfile" "$http_proxy_address" + + if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then + log "Download $url list failed" "error" + return 1 + fi + + convert_crlf_to_lf "$tmpfile" + ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") + ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" + import_plain_domain_list_to_local_source_ruleset_chunked "$tmpfile" "$ruleset_filepath" + + rm -f "$tmpfile" +} + import_subnets_from_remote_subnet_lists() { local section="$1" local remote_subnet_lists @@ -1316,50 +1344,11 @@ import_subnets_from_remote_subnet_list_handler() { ;; *) log "Detected file extension: '$file_extension' → proceeding with processing" "debug" - import_domains_or_subnets_from_remote_file "$url" "$section" "subnets" + import_subnets_from_remote_plain_file "$url" "$section" "subnets" ;; esac } -import_domains_or_subnets_from_remote_file() { - local url="$1" - local section="$2" - local type="$3" - - local tmpfile http_proxy_address items json_array - tmpfile=$(mktemp) - http_proxy_address="$(get_service_proxy_address)" - - download_to_file "$url" "$tmpfile" "$http_proxy_address" - - if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then - log "Download $url list failed" "error" - return 1 - fi - - convert_crlf_to_lf "$tmpfile" - items="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "$type")" - - if [ -z "$items" ]; then - log "No valid $type found in $url" "warn" - return 0 - fi - - ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") - ruleset_filename="$ruleset_tag.json" - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" - json_array="$(comma_string_to_json_array "$items")" - case "$type" in - domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; - subnets) - patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" - nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" - ;; - esac - - rm -f "$tmpfile" -} - import_subnets_from_remote_json_file() { local url="$1" local json_tmpfile subnets_tmpfile http_proxy_address @@ -1405,6 +1394,32 @@ import_subnets_from_remote_srs_file() { rm -f "$binary_tmpfile" "$json_tmpfile" "$subnets_tmpfile" } +import_subnets_from_remote_plain_file() { + local url="$1" + local section="$2" + local type="$3" + + local tmpfile http_proxy_address items json_array + tmpfile=$(mktemp) + http_proxy_address="$(get_service_proxy_address)" + + download_to_file "$url" "$tmpfile" "$http_proxy_address" + + if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then + log "Download $url list failed" "error" + return 1 + fi + + convert_crlf_to_lf "$tmpfile" + + ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") + ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" + import_plain_subnet_list_to_local_source_ruleset_chunked "$tmpfile" "$ruleset_filepath" + nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" + + rm -f "$tmpfile" +} + ## Support functions get_service_proxy_address() { local download_lists_via_proxy diff --git a/podkop/files/usr/lib/rulesets.sh b/podkop/files/usr/lib/rulesets.sh index 7fa7395..29ad2ab 100644 --- a/podkop/files/usr/lib/rulesets.sh +++ b/podkop/files/usr/lib/rulesets.sh @@ -14,18 +14,14 @@ get_ruleset_tag() { # Creates a new ruleset JSON file if it doesn't already exist and outputs its path. create_source_rule_set() { - local ruleset_name="$1" + local ruleset_filepath="$1" - ruleset_filename="$ruleset_name.json" - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" if file_exists "$ruleset_filepath"; then - log "Ruleset $ruleset_filepath already exists. Skipping." "debug" - return 0 + log "Source ruleset $ruleset_filepath already exists" "debug" + return 1 fi jq -n '{version: 3, rules: []}' > "$ruleset_filepath" - - echo "$ruleset_filepath" } ####################################### From 1b7ab606bad2c3289dd6166054c5e4bb9882e06c Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Fri, 21 Nov 2025 20:37:19 +0500 Subject: [PATCH 4/4] refactor: unify source ruleset preparation and list handlers; make ruleset creation idempotent and atomic updates --- podkop/files/usr/bin/podkop | 205 +++++++++++++++---------------- podkop/files/usr/lib/rulesets.sh | 27 ++-- 2 files changed, 113 insertions(+), 119 deletions(-) diff --git a/podkop/files/usr/bin/podkop b/podkop/files/usr/bin/podkop index 82afc47..9397053 100755 --- a/podkop/files/usr/bin/podkop +++ b/podkop/files/usr/bin/podkop @@ -867,63 +867,37 @@ configure_routing_for_section_lists() { if [ "$user_domain_list_type" != "disabled" ]; then log "Processing user domains routing rules for '$section' section" - prepare_common_ruleset "$section" "domains" "$route_rule_tag" - configure_user_domain_or_subnets_list "$section" "domains" "$route_rule_tag" + configure_user_domain_list "$section" "$route_rule_tag" fi if [ "$user_subnet_list_type" != "disabled" ]; then log "Processing user subnets routing rules for '$section' section" - prepare_common_ruleset "$section" "subnets" "$route_rule_tag" - configure_user_domain_or_subnets_list "$section" "subnets" "$route_rule_tag" + configure_user_subnet_list "$section" "$route_rule_tag" fi if [ -n "$local_domain_lists" ]; then log "Processing local domains routing rules for '$section' section" - configure_local_domain_or_subnet_lists "$section" "domains" "$route_rule_tag" + configure_local_domain_lists "$section" "$route_rule_tag" fi if [ -n "$local_subnet_lists" ]; then log "Processing local subnets routing rules for '$section' section" - configure_local_domain_or_subnet_lists "$section" "subnets" "$route_rule_tag" + configure_local_subnet_lists "$section" "$route_rule_tag" fi if [ -n "$remote_domain_lists" ]; then log "Processing remote domains routing rules for '$section' section" - prepare_common_ruleset "$section" "domains" "$route_rule_tag" config_list_foreach "$section" "remote_domain_lists" configure_remote_domain_or_subnet_list_handler \ "domains" "$section" "$route_rule_tag" fi if [ -n "$remote_subnet_lists" ]; then log "Processing remote subnets routing rules for '$section' section" - prepare_common_ruleset "$section" "subnets" "$route_rule_tag" config_list_foreach "$section" "remote_subnet_lists" configure_remote_domain_or_subnet_list_handler \ "subnets" "$section" "$route_rule_tag" fi } -prepare_common_ruleset() { - local section="$1" - local type="$2" - local route_rule_tag="$3" - - log "Preparing a common $type ruleset for '$section' section" "debug" - ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" - create_source_rule_set "$ruleset_filepath" - if [ $? -eq 0 ]; then - config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") - config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") - case "$type" in - domains) - config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") - ;; - subnets) ;; - *) log "Unsupported remote rule set type: $type" "error" ;; - esac - fi -} - configure_community_list_handler() { local tag="$1" local section="$2" @@ -941,69 +915,82 @@ configure_community_list_handler() { config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") } -configure_user_domain_or_subnets_list() { +prepare_source_ruleset() { local section="$1" - local type="$2" + local name="$2" + local type="$3" + local route_rule_tag="$4" - local items ruleset_tag ruleset_filename ruleset_filepath json_array - case "$type" in - domains) - local user_domain_list_type - config_get user_domain_list_type "$section" "user_domain_list_type" - case "$user_domain_list_type" in - dynamic) config_get items "$section" "user_domains" ;; - text) config_get items "$section" "user_domains_text" ;; - esac - ;; - subnets) - local user_subnet_list_type - config_get user_subnet_list_type "$section" "user_subnet_list_type" - case "$user_subnet_list_type" in - dynamic) config_get items "$section" "user_subnets" ;; - text) config_get items "$section" "user_subnets_text" ;; - esac - ;; - esac - - ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") - ruleset_filename="$ruleset_tag.json" - ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename" - items="$(parse_domain_or_subnet_string_to_commas_string "$items" "$type")" - json_array="$(comma_string_to_json_array "$items")" - case "$type" in - domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;; - subnets) - patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" - nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items" - ;; - esac -} - -configure_local_domain_or_subnet_lists() { - local section="$1" - local type="$2" - local route_rule_tag="$3" - - local ruleset_tag ruleset_filepath - ruleset_tag="$(get_ruleset_tag "$section" "local" "$type")" + log "Preparing a $name $type rule set for '$section' section" "debug" + ruleset_tag=$(get_ruleset_tag "$section" "$name" "$type") ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" create_source_rule_set "$ruleset_filepath" - config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") - config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") - - case "$type" in - domains) - config_list_foreach "$section" "local_domain_lists" import_local_domain_list "$ruleset_filepath" - config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") + case $? in + 0) + config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath") + config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag") + case "$type" in + domains) + config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag") + ;; + subnets) ;; + *) + log "Unsupported remote rule set type: $type" "error" + return 1 + ;; + esac ;; - subnets) - config_list_foreach "$section" "local_subnet_lists" import_local_subnets_list "$ruleset_filepath" - ;; - *) log "Unsupported local rule set type: $type" "error" ;; + 3) log "Source rule set $ruleset_filepath already exists, skipping." "debug" ;; esac } -import_local_domain_list() { +configure_user_domain_list() { + local section="$1" + local route_rule_tag="$2" + + prepare_source_ruleset "$section" "user" "domains" "$route_rule_tag" + + local user_domain_list_type items json_array + config_get user_domain_list_type "$section" "user_domain_list_type" + case "$user_domain_list_type" in + dynamic) config_get items "$section" "user_domains" ;; + text) config_get items "$section" "user_domains_text" ;; + esac + + items="$(parse_domain_or_subnet_string_to_commas_string "$items" "domains")" + json_array="$(comma_string_to_json_array "$items")" + patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" +} + +configure_user_subnet_list() { + local section="$1" + local route_rule_tag="$2" + + prepare_source_ruleset "$section" "user" "subnets" "$route_rule_tag" + + local user_subnet_list_type items json_array + config_get user_subnet_list_type "$section" "user_subnet_list_type" + case "$user_subnet_list_type" in + dynamic) config_get items "$section" "user_subnets" ;; + text) config_get items "$section" "user_subnets_text" ;; + esac + + items="$(parse_domain_or_subnet_string_to_commas_string "$items" "subnets")" + json_array="$(comma_string_to_json_array "$items")" + patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array" + nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items" +} + +configure_local_domain_lists() { + local section="$1" + local route_rule_tag="$2" + + prepare_source_ruleset "$section" "local" "domains" "$route_rule_tag" + + config_list_foreach "$section" "local_domain_lists" import_local_domain_list_handler "$ruleset_filepath" +} + +import_local_domain_list_handler() { local local_domain_list_filepath="$1" local ruleset_filepath="$2" @@ -1012,15 +999,19 @@ import_local_domain_list() { return 1 fi - if ! file_exists "$ruleset_filepath"; then - log "Target ruleset file $ruleset_filepath not found" "error" - return 1 - fi - import_plain_domain_list_to_local_source_ruleset_chunked "$local_domain_list_filepath" "$ruleset_filepath" } -import_local_subnets_list() { +configure_local_subnet_lists() { + local section="$1" + local route_rule_tag="$2" + + prepare_source_ruleset "$section" "local" "subnets" "$route_rule_tag" + + config_list_foreach "$section" "local_subnet_lists" import_local_subnets_list_handler "$ruleset_filepath" +} + +import_local_subnets_list_handler() { local local_subnet_list_filepath="$1" local ruleset_filepath="$2" @@ -1029,11 +1020,6 @@ import_local_subnets_list() { return 1 fi - if ! file_exists "$ruleset_filepath"; then - log "Target ruleset file $ruleset_filepath not found" "error" - return 1 - fi - import_plain_subnet_list_to_local_source_ruleset_chunked "$local_subnet_list_filepath" "$ruleset_filepath" nft_add_set_elements_from_file_chunked "$local_subnet_list_filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" } @@ -1046,9 +1032,10 @@ configure_remote_domain_or_subnet_list_handler() { local file_extension file_extension=$(url_get_file_extension "$url") + log "Detected file extension: '$file_extension'" "debug" case "$file_extension" in json | srs) - log "Detected file extension: '$file_extension' → proceeding with processing" "debug" + log "Creating a remote $type ruleset from the source URL" "info" local basename ruleset_tag format detour update_interval basename=$(url_get_basename "$url") ruleset_tag=$(get_ruleset_tag "$section" "$basename" "remote-$type") @@ -1067,7 +1054,7 @@ configure_remote_domain_or_subnet_list_handler() { esac ;; *) - log "Detected file extension: '$file_extension' → no processing needed, managed on list_update" "debug" + prepare_source_ruleset "$section" "remote" "$type" "$route_rule_tag" ;; esac } @@ -1280,13 +1267,14 @@ import_domains_from_remote_domain_list_handler() { local file_extension file_extension=$(url_get_file_extension "$url") + log "Detected file extension: '$file_extension'" "debug" case "$file_extension" in json | srs) - log "Detected file extension: '$file_extension' → no update needed, sing-box manages updates" "debug" + log "No update needed - sing-box manages updates automatically." ;; *) - log "Detected file extension: '$file_extension' → proceeding with processing" "debug" - import_domains_from_remote_plain_file "$url" "$section" "domains" + log "Import domains from a remote plain-text list" + import_domains_from_remote_plain_file "$url" "$section" ;; esac } @@ -1294,7 +1282,6 @@ import_domains_from_remote_domain_list_handler() { import_domains_from_remote_plain_file() { local url="$1" local section="$2" - local type="$3" local tmpfile http_proxy_address items json_array tmpfile=$(mktemp) @@ -1308,7 +1295,7 @@ import_domains_from_remote_plain_file() { fi convert_crlf_to_lf "$tmpfile" - ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") + ruleset_tag=$(get_ruleset_tag "$section" "remote" "domains") ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" import_plain_domain_list_to_local_source_ruleset_chunked "$tmpfile" "$ruleset_filepath" @@ -1333,18 +1320,19 @@ import_subnets_from_remote_subnet_list_handler() { local file_extension file_extension="$(url_get_file_extension "$url")" + log "Detected file extension: '$file_extension'" "debug" case "$file_extension" in json) - log "Detected file extension: '$file_extension' → proceeding with processing" "debug" + log "Import subnets from a remote JSON list" "info" import_subnets_from_remote_json_file "$url" ;; srs) - log "Detected file extension: '$file_extension' → proceeding with processing" "debug" + log "Import subnets from a remote SRS list" "info" import_subnets_from_remote_srs_file "$url" ;; *) - log "Detected file extension: '$file_extension' → proceeding with processing" "debug" - import_subnets_from_remote_plain_file "$url" "$section" "subnets" + log "Import subnets from a remote plain-text list" "info" + import_subnets_from_remote_plain_file "$url" "$section" ;; esac } @@ -1397,7 +1385,6 @@ import_subnets_from_remote_srs_file() { import_subnets_from_remote_plain_file() { local url="$1" local section="$2" - local type="$3" local tmpfile http_proxy_address items json_array tmpfile=$(mktemp) @@ -1412,7 +1399,7 @@ import_subnets_from_remote_plain_file() { convert_crlf_to_lf "$tmpfile" - ruleset_tag=$(get_ruleset_tag "$section" "common" "$type") + ruleset_tag=$(get_ruleset_tag "$section" "remote" "subnets") ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_tag.json" import_plain_subnet_list_to_local_source_ruleset_chunked "$tmpfile" "$ruleset_filepath" nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" diff --git a/podkop/files/usr/lib/rulesets.sh b/podkop/files/usr/lib/rulesets.sh index 29ad2ab..c79beb2 100644 --- a/podkop/files/usr/lib/rulesets.sh +++ b/podkop/files/usr/lib/rulesets.sh @@ -12,13 +12,12 @@ get_ruleset_tag() { fi } -# Creates a new ruleset JSON file if it doesn't already exist and outputs its path. +# Creates a new ruleset JSON file if it doesn't already exist create_source_rule_set() { local ruleset_filepath="$1" if file_exists "$ruleset_filepath"; then - log "Source ruleset $ruleset_filepath already exists" "debug" - return 1 + return 3 fi jq -n '{version: 3, rules: []}' > "$ruleset_filepath" @@ -39,14 +38,22 @@ patch_source_ruleset_rules() { local key="$2" local value="$3" - local content - content="$(cat "$filepath")" + local tmpfile=$(mktemp) - echo "$content" | jq \ - --arg key "$key" \ - --argjson value "$value" ' - .rules += [{ ($key): $value }] - ' > "$filepath" + jq --arg key "$key" --argjson value "$value" \ + '( .rules | map(has($key)) | index(true) ) as $idx | + if $idx != null then + .rules[$idx][$key] = (.rules[$idx][$key] + $value | unique) + else + .rules += [{ ($key): $value }] + end' "$filepath" > "$tmpfile" + + if [ $? -ne 0 ]; then + rm -f "$tmpfile" + return 1 + fi + + mv "$tmpfile" "$filepath" } # Imports a plain domain list into a ruleset in chunks, validating domains and appending them as domain_suffix rules