Mirror/podkop
1
0
Fork 0
mirror of https://github.com/itdoginfo/podkop.git synced 2025-12-06 11:36:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #149 from ampetelin:json_srs_lists

Browse Source 
Added support for JSON and SRS
This commit is contained in:
itdoginfo
2025-08-23 18:53:37 +03:00
committed by GitHub
parent 418cdc4366 6e01e036eb
commit c49354fe38
2 changed files with 493 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.shellcheckrc Normal file
View File

@@ -0,0 +1 @@
disable=SC3036,SC3010,SC3014,SC3015,SC3020,SC3003

672
podkop/files/usr/bin/podkop
View File

@@ -1,4 +1,5 @@
#!/bin/ash
# shellcheck shell=dash
[ -r /lib/functions.sh ] && . /lib/functions.sh
[ -r /lib/config/uci.sh ] && . /lib/config/uci.sh
@@ -107,7 +108,9 @@ start_main() {
config_foreach sing_box_rule_preset
config_foreach process_domains_list_local
config_foreach process_subnet_for_section
config_foreach process_remote_ruleset_srs
config_foreach configure_community_lists
config_foreach configure_remote_domain_lists
config_foreach configure_remote_subnet_lists
config_foreach process_all_traffic_for_section
config_foreach add_cron_job
@@ -572,7 +575,7 @@ prepare_custom_ruleset() {
sing_box_rules $tag $section
sing_box_dns_rule_fakeip_section $tag $tag
log "Added 'test' rule_set to sing-box config"
log "Added $tag rule_set to sing-box config"
fi
}
@@ -619,10 +622,10 @@ list_update() {
fi
echolog "📥 Downloading and processing lists..."
config_foreach process_remote_ruleset_subnet
config_foreach process_domains_list_url
config_foreach process_subnet_for_section_remote
config_foreach import_community_subnet_lists
config_foreach import_domains_from_remote_domain_lists
config_foreach import_subnets_from_remote_subnet_lists
if [ $? -eq 0 ]; then
echolog "✅ Lists update completed successfully"
@@ -964,7 +967,7 @@ sing_box_dns_rule_fakeip() {
sing_box_dns_rule_fakeip_section() {
local rule_set=$1
echo $rule_set
log "Adding section to fakeip route rules in sing-box"
jq \
@@ -1465,13 +1468,109 @@ sing_box_ruleset_subnets_json() {
log "$subnet added to $section-custom-domains-subnets.json"
}
#######################################
# Adds a new remote ruleset to the sing-box configuration.
# https://sing-box.sagernet.org/configuration/rule-set/#__tabbed_1_3
#
# Arguments:
# tag: unique identifier for the ruleset.
# format: format of the ruleset (e.g., "source" or "binary").
# url: URL from which the ruleset can be fetched.
# update_interval: update interval for the ruleset (e.g., "1d").
# detour: flag indicating whether to use a download detour ("1" or "0").
#
# Outputs:
# Modifies the sing-box configuration file by appending a new ruleset entry.
#
# Returns:
# None. Always returns 0. If a ruleset with the same tag exists, it is skipped.
#######################################
sing_box_config_add_remote_ruleset() {
local tag=$1
local format=$2
local url=$3
local update_interval=$4
local detour=$5
local tag_exists
tag_exists=$(jq -r --arg tag "$tag" '
.route.rule_set[]? | select(.tag == $tag) | .tag
' "$SING_BOX_CONFIG")
if [[ -n "$tag_exists" ]]; then
log "Ruleset with tag $tag already exists. Skipping addition."
else
jq \
--arg tag "$tag" \
--arg format "$format" \
--arg url "$url" \
--arg update_interval "$update_interval" \
--arg detour "$detour" \
'
.route.rule_set += [
(
{
"tag": $tag,
"type": "remote",
"format": $format,
"url": $url,
"update_interval": $update_interval
} +
(if $detour == "1" then {"download_detour": "main"} else {} end)
)
]' "$SING_BOX_CONFIG" | build_sing_box_config
log "Added new remote ruleset with tag $tag"
fi
}
#######################################
# Adds a remote ruleset to the sing-box configuration and applies route and dns rules.
#
# Arguments:
# url: remote ruleset URL.
# section: configuration section where rules will be applied.
# ruleset_content_type: Type of ruleset content (e.g., "domains" or "subnets").
#
# Returns:
# 0 on success, non-zero if the file extension is unsupported.
#######################################
sing_box_add_remote_ruleset_and_rules() {
local url="$1"
local section="$2"
local ruleset_content_type="$3"
local tag
local format
local update_interval='1d'
local detour
case "$(get_url_file_extension "$url")" in
json) format="source" ;;
srs) format="binary" ;;
*)
log "Unsupported file extension: .$file_extension"
return 1
;;
esac
tag=$(get_ruleset_tag_from_url "$url" "$section-remote-$ruleset_content_type")
config_get_bool detour "main" "detour" "0"
sing_box_config_add_remote_ruleset "$tag" "$format" "$url" "$update_interval" "$detour"
sing_box_rules "$tag" "$section"
if [[ "$ruleset_content_type" = "domains" ]]; then
sing_box_dns_rule_fakeip_section "$tag"
fi
}
process_domains_for_section() {
local section="$1"
config_get custom_domains_list_type "$section" "custom_domains_list_type" "disabled"
if [ "$custom_domains_list_type" != "disabled" ]; then
log "Adding a custom domains list for section $section"
log "Adding a custom domains list for $section section"
if [ "$custom_domains_list_type" = "dynamic" ]; then
# Handle list domains from custom_domains
config_list_foreach "$section" custom_domains "sing_box_ruleset_domains" "$section"
@@ -1483,96 +1582,6 @@ process_domains_for_section() {
fi
}
sing_box_ruleset_remote() {
local tag=$1
local type=$2
local update_interval=$3
local detour=$4
url="$SRS_MAIN_URL/$tag.srs"
local tag_exists=$(jq -r --arg tag "$tag" '
.route.rule_set[]? | select(.tag == $tag) | .tag
' "$SING_BOX_CONFIG")
if [[ -n "$tag_exists" ]]; then
log "Ruleset with tag $tag already exists. Skipping addition."
else
jq \
--arg tag "$tag" \
--arg type "$type" \
--arg url "$url" \
--arg update_interval "$update_interval" \
--arg detour "$detour" \
'
.route.rule_set += [
(
{
"tag": $tag,
"type": $type,
"format": "binary",
"url": $url,
"update_interval": $update_interval
} +
(if $detour == "1" then {"download_detour": "main"} else {} end)
)
]' "$SING_BOX_CONFIG" | build_sing_box_config
log "Added new ruleset with tag $tag"
fi
}
list_subnets_download() {
local service="$1"
local table="PodkopTable"
case "$service" in
"twitter")
URL=$SUBNETS_TWITTER
;;
"meta")
URL=$SUBNETS_META
;;
"telegram")
URL=$SUBNETS_TELERAM
;;
"cloudflare")
URL=$SUBNETS_CLOUDFLARE
;;
"hetzner")
URL=$SUBNETS_HETZNER
;;
"ovh")
URL=$SUBNETS_OVH
;;
"discord")
URL=$SUBNETS_DISCORD
nft add set inet $table podkop_discord_subnets { type ipv4_addr\; flags interval\; auto-merge\; }
nft add rule inet $table mangle iifname "$SRC_INTERFACE" ip daddr @podkop_discord_subnets udp dport { 50000-65535 } meta mark set 0x105 counter
;;
*)
return
;;
esac
local filename=$(basename "$URL")
config_get_bool detour "main" "detour" "0"
if [ "$detour" -eq 1 ]; then
http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "/tmp/podkop/$filename" "$URL"
else
wget -O "/tmp/podkop/$filename" "$URL"
fi
while IFS= read -r subnet; do
if [ "$service" = "discord" ]; then
nft add element inet $table podkop_discord_subnets { $subnet }
else
nft add element inet $table podkop_subnets { $subnet }
fi
done <"/tmp/podkop/$filename"
}
sing_box_rules() {
log "Configure rule in sing-box"
local rule_set="$1"
@@ -1648,29 +1657,11 @@ sing_box_quic_reject() {
fi
}
process_remote_ruleset_srs() {
config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
if [ "$domain_list_enabled" -eq 1 ]; then
config_get_bool detour "main" "detour" "0"
log "Adding a srs list for $section"
config_list_foreach "$section" domain_list "sing_box_ruleset_remote" "remote" "1d" "$detour"
fi
}
process_remote_ruleset_subnet() {
config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
if [ "$domain_list_enabled" -eq 1 ]; then
log "Adding a srs list for $section"
config_list_foreach "$section" domain_list "list_subnets_download" "$section" "$domain_list"
fi
}
# TODO(ampetelin): function needs refactoring
sing_box_rule_preset() {
config_get custom_domains_list_type "$section" "custom_domains_list_type"
config_get custom_subnets_list_enabled "$section" "custom_subnets_list_enabled"
config_get custom_local_domains_list_enabled "$section" "custom_local_domains_list_enabled"
# config_get custom_download_domains_list_enabled "$section" "custom_download_domains_list_enabled"
# config_get custom_download_subnets_list_enabled "$section" "custom_download_subnets_list_enabled"
if [ "$custom_domains_list_type" != "disabled" ] || [ "$custom_subnets_list_enabled" != "disabled" ] ||
[ "$custom_local_domains_list_enabled" = "1" ]; then
@@ -1715,46 +1706,12 @@ process_domains_list_local() {
fi
}
list_custom_url_domains_create() {
local section="$2"
local URL="$1"
local filename=$(basename "$URL")
local filepath="/tmp/podkop/${filename}"
config_get_bool detour "main" "detour" "0"
if [ "$detour" -eq 1 ]; then
http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "$filepath" "$URL"
else
wget -O "$filepath" "$URL"
fi
if grep -q $'\r' "$filepath"; then
log "$filename has Windows line endings (CRLF). Converting to Unix (LF)"
sed -i 's/\r$//' "$filepath"
fi
while IFS= read -r domain; do
log "From downloaded file: $domain"
sing_box_ruleset_domains_json $domain $section
done <"$filepath"
}
process_domains_list_url() {
local section="$1"
config_get custom_download_domains_list_enabled "$section" "custom_download_domains_list_enabled"
if [ "$custom_download_domains_list_enabled" -eq 1 ]; then
log "Adding a custom domains list from URL in $section"
config_list_foreach "$section" "custom_download_domains" list_custom_url_domains_create "$section"
fi
}
process_subnet_for_section() {
local section="$1"
config_get custom_subnets_list_enabled "$section" "custom_subnets_list_enabled" "disabled"
if [ "$custom_subnets_list_enabled" != "disabled" ]; then
log "Adding a custom subnet list for section $section"
log "Adding a custom subnet list for $section section"
if [ "$custom_subnets_list_enabled" = "dynamic" ]; then
# Handle list domains from custom_domains
config_list_foreach "$section" custom_subnets "sing_box_ruleset_subnets" "$section"
@@ -1766,52 +1723,315 @@ process_subnet_for_section() {
fi
}
list_custom_url_subnets_create() {
local section="$2"
local URL="$1"
local filename=$(basename "$URL")
local filepath="/tmp/podkop/${filename}"
config_get_bool detour "main" "detour" "0"
if [ "$detour" -eq 1 ]; then
http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "$filepath" "$URL"
else
wget -O "$filepath" "$URL"
configure_community_lists() {
config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
if [ "$domain_list_enabled" -eq 1 ]; then
log "Configuring community lists for $section section"
config_list_foreach "$section" domain_list configure_community_list_handler
fi
if grep -q $'\r' "$filepath"; then
log "$filename has Windows line endings (CRLF). Converting to Unix (LF)"
sed -i 's/\r$//' "$filepath"
fi
while IFS= read -r subnet; do
log "From local file: $subnet"
sing_box_ruleset_subnets_json $subnet $section
nft add element inet PodkopTable podkop_subnets { $subnet }
done <"$filepath"
}
process_subnet_for_section_remote() {
configure_community_list_handler() {
local tag=$1
local format="binary"
local update_interval="1d"
config_get_bool detour "main" "detour" "0"
local url="$SRS_MAIN_URL/$tag.srs"
sing_box_config_add_remote_ruleset "$tag" "$format" "$url" "$update_interval" "$detour"
}
configure_remote_domain_lists() {
local section="$1"
config_get custom_download_domains_list_enabled "$section" custom_download_domains_list_enabled
if [ "$custom_download_domains_list_enabled" -eq 1 ]; then
log "Configuring remote domain lists for $section section"
config_list_foreach "$section" custom_download_domains configure_remote_domain_list_handler "$section"
fi
}
configure_remote_domain_list_handler() {
local url="$1"
local section="$2"
log "Configuring remote domain list from URL: $url"
local file_extension
file_extension=$(get_url_file_extension "$url")
case "$file_extension" in
lst)
log "Detected file extension: .$file_extension → no processing needed, managed on list_update"
;;
json|srs)
log "Detected file extension: .$file_extension → proceeding with processing"
sing_box_add_remote_ruleset_and_rules "$url" "$section" "domains"
;;
*)
log "Detected file extension: .$file_extension → unsupported, skipping"
return 1
;;
esac
}
configure_remote_subnet_lists() {
local section="$1"
config_get custom_download_subnets_list_enabled "$section" "custom_download_subnets_list_enabled" "disabled"
config_get custom_download_subnets_list_enabled "$section" custom_download_subnets_list_enabled disabled
if [ "$custom_download_subnets_list_enabled" -eq "1" ]; then
log "Adding a custom SUBNET list from URL in $section"
config_list_foreach "$section" "custom_download_subnets" list_custom_url_subnets_create "$section"
log "Configuring remote subnet lists for $section section"
config_list_foreach "$section" custom_download_subnets configure_remote_subnet_list_handler "$section"
fi
}
configure_remote_subnet_list_handler() {
local url="$1"
local section="$2"
log "Configuring remote subnet list from URL: $url"
local file_extension
file_extension=$(get_url_file_extension "$url")
case "$file_extension" in
lst)
log "Detected file extension: .$file_extension → no processing needed, managed on list_update"
;;
json|srs)
log "Detected file extension: .$file_extension → proceeding with processing"
sing_box_add_remote_ruleset_and_rules "$url" "$section" "subnets"
;;
*)
log "Detected file extension: .$file_extension → unsupported, skipping"
return 1
;;
esac
}
process_all_traffic_for_section() {
local section="$1"
config_get all_traffic_from_ip_enabled "$section" "all_traffic_from_ip_enabled"
if [ "$all_traffic_from_ip_enabled" -eq "1" ]; then
log "Adding an IP to redirect all traffic"
config_list_foreach $section all_traffic_ip list_all_traffic_from_ip
config_list_foreach $section all_traffic_ip nft_list_all_traffic_from_ip
config_list_foreach $section all_traffic_ip sing_box_rules_source_ip_cidr $all_traffic_ip $section
fi
}
import_community_subnet_lists() {
config_get_bool domain_list_enabled "$section" "domain_list_enabled" "0"
if [ "$domain_list_enabled" -eq 1 ]; then
log "Importing community subnet lists for $section section"
config_list_foreach "$section" domain_list import_community_service_subnet_list_handler
fi
}
import_community_service_subnet_list_handler() {
local service="$1"
local table="PodkopTable"
case "$service" in
"twitter")
URL=$SUBNETS_TWITTER
;;
"meta")
URL=$SUBNETS_META
;;
"telegram")
URL=$SUBNETS_TELERAM
;;
"cloudflare")
URL=$SUBNETS_CLOUDFLARE
;;
"hetzner")
URL=$SUBNETS_HETZNER
;;
"ovh")
URL=$SUBNETS_OVH
;;
"discord")
URL=$SUBNETS_DISCORD
nft add set inet $table podkop_discord_subnets { type ipv4_addr\; flags interval\; auto-merge\; }
nft add rule inet $table mangle iifname "$SRC_INTERFACE" ip daddr @podkop_discord_subnets udp dport { 50000-65535 } meta mark set 0x105 counter
;;
*)
return
;;
esac
local filename=$(basename "$URL")
config_get_bool detour "main" "detour" "0"
if [ "$detour" -eq 1 ]; then
http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "/tmp/podkop/$filename" "$URL"
else
wget -O "/tmp/podkop/$filename" "$URL"
fi
while IFS= read -r subnet; do
if [ "$service" = "discord" ]; then
nft add element inet $table podkop_discord_subnets { $subnet }
else
nft add element inet $table podkop_subnets { $subnet }
fi
done <"/tmp/podkop/$filename"
}
import_domains_from_remote_domain_lists() {
local section="$1"
config_get custom_download_domains_list_enabled "$section" custom_download_domains_list_enabled
if [ "$custom_download_domains_list_enabled" -eq 1 ]; then
log "Importing domains from remote domain lists for $section section"
config_list_foreach "$section" custom_download_domains import_domains_from_remote_domain_list_handler "$section"
fi
}
import_domains_from_remote_domain_list_handler() {
local url="$1"
local section="$2"
log "Importing domains from URL: $url"
local file_extension
file_extension=$(get_url_file_extension "$url")
case "$file_extension" in
lst)
log "Detected file extension: .$file_extension → proceeding with processing"
import_domains_from_remote_lst_file "$url" "$section"
;;
json|srs)
log "Detected file extension: .$file_extension → no update needed, sing-box manages updates"
;;
*)
log "Detected file extension: .$file_extension → unsupported, skipping"
return 1
;;
esac
}
import_domains_from_remote_lst_file() {
local url="$1"
local section="$2"
local filename
filename=$(basename "$url")
local filepath="/tmp/podkop/${filename}"
download_to_tempfile "$url" "$filepath"
while IFS= read -r domain; do
sing_box_ruleset_domains_json $domain $section
done <"$filepath"
rm -f "$filepath"
}
import_subnets_from_remote_subnet_lists() {
local section="$1"
config_get custom_download_subnets_list_enabled "$section" custom_download_subnets_list_enabled disabled
if [ "$custom_download_subnets_list_enabled" -eq "1" ]; then
log "Importing subnets from remote subnet lists for $section section"
config_list_foreach "$section" custom_download_subnets import_subnets_from_remote_subnet_list_handler "$section"
fi
}
import_subnets_from_remote_subnet_list_handler() {
local url="$1"
local section="$2"
log "Importing subnets from URL: $url"
local file_extension
file_extension=$(get_url_file_extension "$url")
case "$file_extension" in
lst)
log "Detected file extension: .$file_extension → proceeding with processing"
import_subnets_from_remote_lst_file "$url" "$section"
;;
json)
log "Detected file extension: .$file_extension → proceeding with processing"
import_subnets_from_remote_json_file "$url"
;;
srs)
log "Detected file extension: .$file_extension → proceeding with processing"
import_subnets_from_remote_srs_file "$url"
;;
*)
log "Detected file extension: .$file_extension → unsupported, skipping"
return 1
;;
esac
}
import_subnets_from_remote_lst_file() {
local url="$1"
local section="$2"
local filename
filename=$(basename "$url")
local filepath="/tmp/podkop/${filename}"
download_to_tempfile "$url" "$filepath"
while IFS= read -r subnet; do
sing_box_ruleset_subnets_json "$subnet" "$section"
nft_add_podkop_subnet "$subnet"
done <"$filepath"
rm -f "$filepath"
}
import_subnets_from_remote_json_file() {
local url="$1"
download_to_stream "$url" | jq -r '.rules[].ip_cidr[]?' | while read -r subnet; do
nft_add_podkop_subnet "$subnet"
done
}
import_subnets_from_remote_srs_file() {
local url="$1"
local filename
filename=$(basename "$url")
local binary_filepath="/tmp/podkop/${filename}"
local json_filepath="/tmp/podkop/decompiled-${filename%%.*}.json"
download_to_tempfile "$url" "$binary_filepath"
if ! decompile_srs_file "$binary_filepath" "$json_filepath"; then
return 1
fi
jq -r '.rules[].ip_cidr[]' "$json_filepath" | while read -r subnet; do
nft_add_podkop_subnet "$subnet"
done
rm -f "$binary_filepath" "$json_filepath"
}
# Decompiles a sing-box SRS binary file into a JSON ruleset file
decompile_srs_file() {
local binary_filepath="$1"
local output_filepath="$2"
log "Decompiling $binary_filepath to $output_filepath"
if ! file_exists "$binary_filepath"; then
log "File $binary_filepath not found"
return 1
fi
sing-box rule-set decompile "$binary_filepath" -o "$output_filepath"
if [[ $? -ne 0 ]]; then
log "Decompilation command failed for $binary_filepath"
return 1
fi
}
sing_box_rules_source_ip_cidr() {
log "Configure source_ip_cidr rule in sing-box"
local source_ip_cidr="$1"
@@ -1865,7 +2085,7 @@ detour_mixed() {
}
## nftables
list_all_traffic_from_ip() {
nft_list_all_traffic_from_ip() {
local ip="$1"
local table="PodkopTable"
@@ -1890,6 +2110,17 @@ list_all_traffic_from_ip() {
fi
}
# Adds an IPv4 subnet to nftables firewall set
nft_add_podkop_subnet() {
local subnet="$1"
if is_ipv4_cidr "$subnet"; then
nft add element inet PodkopTable podkop_subnets { "$subnet" }
else
log "Invalid subnet format. $subnet is not a valid IPv4 CIDR"
fi
}
# Diagnotics
check_proxy() {
if ! command -v sing-box >/dev/null 2>&1; then
@@ -2602,6 +2833,87 @@ global_check() {
fi
}
# Download URL content directly
download_to_stream() {
local url="$1"
config_get_bool detour "main" "detour" "0"
if [ "$detour" -eq 1 ]; then
http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -qO- "$url" | sed 's/\r$//'
else
wget -qO- "$url" | sed 's/\r$//'
fi
}
# Download URL to temporary file
download_to_tempfile() {
local url="$1"
local filepath="$2"
config_get_bool detour "main" "detour" "0"
if [ "$detour" -eq 1 ]; then
http_proxy="http://127.0.0.1:4534" https_proxy="http://127.0.0.1:4534" wget -O "$filepath" "$url"
else
wget -O "$filepath" "$url"
fi
if grep -q $'\r' "$filepath"; then
log "$filename has Windows line endings (CRLF). Converting to Unix (LF)"
sed -i 's/\r$//' "$filepath"
fi
}
# helper function
# check if file exists
file_exists() {
local filepath="$1"
if [[ -f "$filepath" ]]; then
return 0 # success
else
return 1 # failure
fi
}
# extracts file extension from URL
get_url_file_extension() {
local url="$1"
local file_extension="${url##*.}"
echo "$file_extension"
}
# extracts file extension from URL
get_ruleset_tag_from_url() {
local url="$1"
local prefix="${2:-}"
local postfix="${3:-}"
local filename="${url##*/}"
local basename="${filename%%.*}"
local tag="$basename"
if [ -n "$prefix" ]; then
tag="${prefix}-${tag}"
fi
if [ -n "$postfix" ]; then
tag="${tag}-${postfix}"
fi
echo "$tag"
}
# check if string is valid IPv4 with CIDR mask
is_ipv4_cidr() {
local ip="$1"
local regex="^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}(\/(3[0-2]|2[0-9]|1[0-9]|[0-9]))$"
[[ $ip =~ $regex ]]
}
show_help() {
cat << EOF
Usage: $0 COMMAND