fix: load large plain domain/subnet lists in chunks; move ruleset logic to rulesets.sh and nft chunker to nft.sh

This commit is contained in:
Andrey Petelin
2025-11-16 09:55:44 +05:00
parent 56829c74c8
commit 32c385b309
5 changed files with 266 additions and 201 deletions

View File

@@ -18,6 +18,7 @@ check_required_file "$PODKOP_LIB/helpers.sh"
check_required_file "$PODKOP_LIB/sing_box_config_manager.sh"
check_required_file "$PODKOP_LIB/sing_box_config_facade.sh"
check_required_file "$PODKOP_LIB/logging.sh"
check_required_file "$PODKOP_LIB/rulesets.sh"
. /lib/config/uci.sh
. /lib/functions.sh
. "$PODKOP_LIB/constants.sh"
@@ -26,6 +27,7 @@ check_required_file "$PODKOP_LIB/logging.sh"
. "$PODKOP_LIB/sing_box_config_manager.sh"
. "$PODKOP_LIB/sing_box_config_facade.sh"
. "$PODKOP_LIB/logging.sh"
. "$PODKOP_LIB/rulesets.sh"
config_load "$PODKOP_CONFIG"
@@ -907,22 +909,16 @@ prepare_common_ruleset() {
log "Preparing a common $type ruleset for '$section' section" "debug"
ruleset_tag=$(get_ruleset_tag "$section" "common" "$type")
ruleset_filename="$ruleset_tag.json"
ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename"
if file_exists "$ruleset_filepath"; then
log "Ruleset $ruleset_filepath already exists. Skipping." "debug"
else
sing_box_cm_create_local_source_ruleset "$ruleset_filepath"
config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath")
config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag")
case "$type" in
domains)
config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag")
;;
subnets) ;;
*) log "Unsupported remote rule set type: $type" "error" ;;
esac
fi
ruleset_filepath=$(create_source_rule_set "$ruleset_tag")
config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath")
config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag")
case "$type" in
domains)
config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag")
;;
subnets) ;;
*) log "Unsupported remote rule set type: $type" "error" ;;
esac
}
configure_community_list_handler() {
@@ -972,9 +968,9 @@ configure_user_domain_or_subnets_list() {
items="$(parse_domain_or_subnet_string_to_commas_string "$items" "$type")"
json_array="$(comma_string_to_json_array "$items")"
case "$type" in
domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
subnets)
sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$items"
;;
esac
@@ -985,56 +981,57 @@ configure_local_domain_or_subnet_lists() {
local type="$2"
local route_rule_tag="$3"
local ruleset_tag ruleset_filename ruleset_filepath
local ruleset_tag ruleset_filepath
ruleset_tag="$(get_ruleset_tag "$section" "local" "$type")"
ruleset_filename="$ruleset_tag.json"
ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename"
sing_box_cm_create_local_source_ruleset "$ruleset_filepath"
ruleset_filepath=$(create_source_rule_set "$ruleset_tag")
config=$(sing_box_cm_add_local_ruleset "$config" "$ruleset_tag" "source" "$ruleset_filepath")
config=$(sing_box_cm_patch_route_rule "$config" "$route_rule_tag" "rule_set" "$ruleset_tag")
case "$type" in
domains)
config_list_foreach "$section" "local_domain_lists" import_local_domain_or_subnet_list "$type" \
"$section" "$ruleset_filepath"
config_list_foreach "$section" "local_domain_lists" import_local_domain_list "$ruleset_filepath"
config=$(sing_box_cm_patch_dns_route_rule "$config" "$SB_FAKEIP_DNS_RULE_TAG" "rule_set" "$ruleset_tag")
;;
subnets)
config_list_foreach "$section" "local_subnet_lists" import_local_domain_or_subnet_list "$type" \
"$section" "$ruleset_filepath"
config_list_foreach "$section" "local_subnet_lists" import_local_subnets_list "$ruleset_filepath"
;;
*) log "Unsupported local rule set type: $type" "error" ;;
esac
}
import_local_domain_or_subnet_list() {
local filepath="$1"
local type="$2"
local section="$3"
local ruleset_filepath="$4"
import_local_domain_list() {
local local_domain_list_filepath="$1"
local ruleset_filepath="$2"
if ! file_exists "$filepath"; then
log "File $filepath not found" "error"
if ! file_exists "$local_domain_list_filepath"; then
log "Local domain list file $local_domain_list_filepath not found" "error"
return 1
fi
local items json_array
items="$(parse_domain_or_subnet_file_to_comma_string "$filepath" "$type")"
if [ -z "$items" ]; then
log "No valid $type found in $filepath" "warn"
return 0
if ! file_exists "$ruleset_filepath"; then
log "Target ruleset file $ruleset_filepath not found" "error"
return 1
fi
json_array="$(comma_string_to_json_array "$items")"
case "$type" in
domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
subnets)
sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
nft_add_set_elements_from_file_chunked "$filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
;;
esac
import_plain_domain_list_to_local_source_ruleset_chunked "$local_domain_list_filepath" "$ruleset_filepath"
}
import_local_subnets_list() {
local local_subnet_list_filepath="$1"
local ruleset_filepath="$2"
if ! file_exists "$local_subnet_list_filepath"; then
log "Local subnet list file $local_subnet_list_filepath not found" "error"
return 1
fi
if ! file_exists "$ruleset_filepath"; then
log "Target ruleset file $ruleset_filepath not found" "error"
return 1
fi
import_plain_subnet_list_to_local_source_ruleset_chunked "$local_subnet_list_filepath" "$ruleset_filepath"
nft_add_set_elements_from_file_chunked "$local_subnet_list_filepath" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
}
configure_remote_domain_or_subnet_list_handler() {
@@ -1353,9 +1350,9 @@ import_domains_or_subnets_from_remote_file() {
ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename"
json_array="$(comma_string_to_json_array "$items")"
case "$type" in
domains) sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
domains) patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array" ;;
subnets)
sing_box_cm_patch_local_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
nft_add_set_elements_from_file_chunked "$tmpfile" "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME"
;;
esac
@@ -1398,8 +1395,8 @@ import_subnets_from_remote_srs_file() {
return 1
fi
if ! decompile_srs_file "$binary_tmpfile" "$json_tmpfile"; then
log "Failed to decompile SRS file" "error"
if ! decompile_binary_ruleset "$binary_tmpfile" "$json_tmpfile"; then
log "Failed to decompile binary rule set file" "error"
return 1
fi
@@ -1516,46 +1513,6 @@ nft_list_all_traffic_from_ip() {
fi
}
nft_add_set_elements_from_file_chunked() {
local filepath="$1"
local nft_table_name="$2"
local nft_set_name="$3"
local chunk_size="${4:-5000}"
local array count
count=0
while IFS= read -r line; do
line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
[ -z "$line" ] && continue
if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then
log "'$line' is not IPv4 or IPv4 CIDR" "debug"
continue
fi
if [ -z "$array" ]; then
array="$line"
else
array="$array,$line"
fi
count=$((count + 1))
if [ "$count" = "$chunk_size" ]; then
log "Adding $count elements to nft set $nft_set_name" "debug"
nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
array=""
count=0
fi
done < "$filepath"
if [ -n "$array" ]; then
log "Adding $count elements to nft set $nft_set_name" "debug"
nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
fi
}
# Diagnotics
check_proxy() {
local sing_box_config_path

View File

@@ -105,37 +105,6 @@ get_domain_resolver_tag() {
echo "$section-$postfix"
}
# Constructs and returns a ruleset tag using section, name, optional type, and a fixed postfix
get_ruleset_tag() {
local section="$1"
local name="$2"
local type="$3"
local postfix="ruleset"
if [ -n "$type" ]; then
echo "$section-$name-$type-$postfix"
else
echo "$section-$name-$postfix"
fi
}
# Determines the ruleset format based on the file extension (json → source, srs → binary)
get_ruleset_format_by_file_extension() {
local file_extension="$1"
local format
case "$file_extension" in
json) format="source" ;;
srs) format="binary" ;;
*)
log "Unsupported file extension: .$file_extension" "error"
return 1
;;
esac
echo "$format"
}
# Converts a comma-separated string into a JSON array string
comma_string_to_json_array() {
local input="$1"
@@ -300,25 +269,6 @@ convert_crlf_to_lf() {
fi
}
# Decompiles a sing-box SRS binary file into a JSON ruleset file
decompile_srs_file() {
local binary_filepath="$1"
local output_filepath="$2"
log "Decompiling $binary_filepath to $output_filepath" "debug"
if ! file_exists "$binary_filepath"; then
log "File $binary_filepath not found" "error"
return 1
fi
sing-box rule-set decompile "$binary_filepath" -o "$output_filepath"
if [[ $? -ne 0 ]]; then
log "Decompilation command failed for $binary_filepath" "error"
return 1
fi
}
#######################################
# Parses a whitespace-separated string, validates items as either domains
# or IPv4 addresses/subnets, and returns a comma-separated string of valid items.
@@ -388,17 +338,3 @@ parse_domain_or_subnet_file_to_comma_string() {
echo "$result"
}
# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file.
extract_ip_cidr_from_json_ruleset_to_file() {
local json_file="$1"
local output_file="$2"
if [ ! -f "$json_file" ]; then
log "JSON file not found: $json_file" "error"
return 1
fi
log "Extracting ip_cidr entries from $json_file to $output_file" "debug"
jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file"
}

View File

@@ -28,3 +28,43 @@ nft_add_set_elements() {
nft add element inet "$table" "$set" "{ $elements }"
}
nft_add_set_elements_from_file_chunked() {
local filepath="$1"
local nft_table_name="$2"
local nft_set_name="$3"
local chunk_size="${4:-5000}"
local array count
count=0
while IFS= read -r line; do
line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
[ -z "$line" ] && continue
if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then
log "'$line' is not IPv4 or IPv4 CIDR" "debug"
continue
fi
if [ -z "$array" ]; then
array="$line"
else
array="$array,$line"
fi
count=$((count + 1))
if [ "$count" = "$chunk_size" ]; then
log "Adding $count elements to nft set $nft_set_name" "debug"
nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
array=""
count=0
fi
done < "$filepath"
if [ -n "$array" ]; then
log "Adding $count elements to nft set $nft_set_name" "debug"
nft_add_set_elements "$nft_table_name" "$nft_set_name" "$array"
fi
}

View File

@@ -0,0 +1,177 @@
# Constructs and returns a ruleset tag using section, name, optional type, and a fixed postfix
get_ruleset_tag() {
local section="$1"
local name="$2"
local type="$3"
local postfix="ruleset"
if [ -n "$type" ]; then
echo "$section-$name-$type-$postfix"
else
echo "$section-$name-$postfix"
fi
}
# Creates a new ruleset JSON file if it doesn't already exist and outputs its path.
create_source_rule_set() {
local ruleset_name="$1"
ruleset_filename="$ruleset_name.json"
ruleset_filepath="$TMP_RULESET_FOLDER/$ruleset_filename"
if file_exists "$ruleset_filepath"; then
log "Ruleset $ruleset_filepath already exists. Skipping." "debug"
return 0
fi
jq -n '{version: 3, rules: []}' > "$ruleset_filepath"
echo "$ruleset_filepath"
}
#######################################
# Patch a source ruleset JSON file for sing-box by appending a new ruleset object containing the provided key
# and value.
# Arguments:
# filepath: path to the JSON file to patch
# key: the ruleset key to insert (e.g., "ip_cidr")
# value: a JSON array of values to assign to the key
# Example:
# patch_source_ruleset_rules "/tmp/sing-box/ruleset.json" "ip_cidr" '["1.1.1.1","2.2.2.2"]'
#######################################
patch_source_ruleset_rules() {
local filepath="$1"
local key="$2"
local value="$3"
local content
content="$(cat "$filepath")"
echo "$content" | jq \
--arg key "$key" \
--argjson value "$value" '
.rules += [{ ($key): $value }]
' > "$filepath"
}
# Imports a plain domain list into a ruleset in chunks, validating domains and appending them as domain_suffix rules
import_plain_domain_list_to_local_source_ruleset_chunked() {
local plain_list_filepath="$1"
local ruleset_filepath="$2"
local chunk_size="${3:-5000}"
local array count json_array
count=0
while IFS= read -r line; do
line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
[ -z "$line" ] && continue
if ! is_domain_suffix "$line"; then
log "'$line' is not a valid domain" "debug"
continue
fi
if [ -z "$array" ]; then
array="$line"
else
array="$array,$line"
fi
count=$((count + 1))
if [ "$count" = "$chunk_size" ]; then
log "Adding $count elements to rule set at $ruleset_filepath" "debug"
json_array="$(comma_string_to_json_array "$array")"
patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array"
array=""
count=0
fi
done < "$plain_list_filepath"
if [ -n "$array" ]; then
log "Adding $count elements to rule set at $ruleset_filepath" "debug"
json_array="$(comma_string_to_json_array "$array")"
patch_source_ruleset_rules "$ruleset_filepath" "domain_suffix" "$json_array"
fi
}
# Imports a plain IPv4/CIDR list into a ruleset in chunks, validating entries and appending them as ip_cidr rules
import_plain_subnet_list_to_local_source_ruleset_chunked() {
local plain_list_filepath="$1"
local ruleset_filepath="$2"
local chunk_size="${3:-5000}"
local array count json_array
count=0
while IFS= read -r line; do
line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
[ -z "$line" ] && continue
if ! is_ipv4 "$line" && ! is_ipv4_cidr "$line"; then
log "'$line' is not IPv4 or IPv4 CIDR" "debug"
continue
fi
if [ -z "$array" ]; then
array="$line"
else
array="$array,$line"
fi
count=$((count + 1))
if [ "$count" = "$chunk_size" ]; then
log "Adding $count elements to ruleset at $ruleset_filepath" "debug"
json_array="$(comma_string_to_json_array "$array")"
patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
array=""
count=0
fi
done < "$plain_list_filepath"
if [ -n "$array" ]; then
log "Adding $count elements to ruleset at $ruleset_filepath" "debug"
json_array="$(comma_string_to_json_array "$array")"
patch_source_ruleset_rules "$ruleset_filepath" "ip_cidr" "$json_array"
fi
}
# Determines the ruleset format based on the file extension (json → source, srs → binary)
get_ruleset_format_by_file_extension() {
local file_extension="$1"
local format
case "$file_extension" in
json) format="source" ;;
srs) format="binary" ;;
*)
log "Unsupported file extension: .$file_extension" "error"
return 1
;;
esac
echo "$format"
}
# Decompiles a sing-box SRS binary file into a JSON ruleset file
decompile_binary_ruleset() {
local binary_filepath="$1"
local output_filepath="$2"
log "Decompiling $binary_filepath to $output_filepath" "debug"
sing-box rule-set decompile "$binary_filepath" -o "$output_filepath"
if [[ $? -ne 0 ]]; then
log "Decompilation command failed for $binary_filepath" "error"
return 1
fi
}
# Extracts all ip_cidr entries from a JSON ruleset file and writes them to an output file.
extract_ip_cidr_from_json_ruleset_to_file() {
local json_file="$1"
local output_file="$2"
log "Extracting ip_cidr entries from $json_file to $output_file" "debug"
jq -r '.rules[].ip_cidr[]' "$json_file" > "$output_file"
}

View File

@@ -1365,51 +1365,6 @@ sing_box_cm_configure_clash_api() {
+ (if $secret != "" then { secret: $secret } else {} end)'
}
#######################################
# Create a local source ruleset JSON file for sing-box.
# Arguments:
# filepath: path to the JSON file to create
# Example:
# sing_box_cm_create_local_source_ruleset "/tmp/sing-box/ruleset.json"
#######################################
sing_box_cm_create_local_source_ruleset() {
local filepath="$1"
jq -n '{version: 3, rules: []}' > "$filepath"
}
#######################################
# Patch a local source ruleset JSON file for sing-box by adding unique! values to a given key.
# Arguments:
# filepath: path to the JSON file to patch
# key: the ruleset key to update (e.g., "ip_cidr")
# value: a JSON array of values to add to the key
# Example:
# sing_box_cm_patch_local_source_ruleset_rules "/tmp/sing-box/ruleset.json" "ip_cidr" '["1.1.1.1","2.2.2.2"]'
#######################################
sing_box_cm_patch_local_source_ruleset_rules() {
local filepath="$1"
local key="$2"
local value="$3"
value=$(_normalize_arg "$value")
local content
content="$(cat "$filepath")"
echo "$content" | jq \
--arg key "$key" \
--argjson value "$value" '
([.rules[]?[$key][]] | unique) as $existing
| ($value - $existing) as $value
| if ($value | length) > 0 then
.rules += [{($key): $value}]
else
.
end
' > "$filepath"
}
#######################################
# Save a sing-box JSON configuration to a file, removing service-specific tags.
# Arguments: