From fe84b3154f68520fb81d006e5465591465c06f7e Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Wed, 29 Oct 2025 21:36:46 +0500 Subject: [PATCH 1/3] fix: convert Windows CRLF line endings to LF for downloaded files --- podkop/files/usr/bin/podkop | 1 + podkop/files/usr/lib/helpers.sh | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/podkop/files/usr/bin/podkop b/podkop/files/usr/bin/podkop index 0f4f2a4..6534f6a 100755 --- a/podkop/files/usr/bin/podkop +++ b/podkop/files/usr/bin/podkop @@ -1307,6 +1307,7 @@ import_domains_or_subnets_from_remote_file() { http_proxy_address="$(get_service_proxy_address)" download_to_file "$url" "$tmpfile" "$http_proxy_address" + convert_crlf_to_lf "$tmpfile" if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then log "Download $url list failed" "error" diff --git a/podkop/files/usr/lib/helpers.sh b/podkop/files/usr/lib/helpers.sh index 7af83f2..5801fec 100644 --- a/podkop/files/usr/lib/helpers.sh +++ b/podkop/files/usr/lib/helpers.sh @@ -305,10 +305,17 @@ download_to_file() { log "Attempt $attempt/$retries to download $url failed" "warn" sleep "$wait" done +} + +# Converts Windows-style line endings (CRLF) to Unix-style (LF) +convert_crlf_to_lf() { + local filepath="$1" if grep -q $'\r' "$filepath"; then - log "Downloaded file has Windows line endings (CRLF). Converting to Unix (LF)" - sed -i 's/\r$//' "$filepath" + log "Downloaded file has Windows line endings (CRLF). Converting to Unix (LF)" "debug" + local tmpfile + tmpfile=$(mktemp) + tr -d '\r' < "$filepath" > "$tmpfile" && mv "$tmpfile" "$filepath" fi } From f168fb7e3153c908d739079d63ef5441250b92d4 Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Wed, 29 Oct 2025 21:52:44 +0500 Subject: [PATCH 2/3] refactor: fetch remote JSON to temp files and parse ip_cidr into subnets; remove download_to_stream --- podkop/files/usr/bin/podkop | 14 ++++++++------ podkop/files/usr/lib/helpers.sh | 19 ------------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/podkop/files/usr/bin/podkop b/podkop/files/usr/bin/podkop index 6534f6a..68577bf 100755 --- a/podkop/files/usr/bin/podkop +++ b/podkop/files/usr/bin/podkop @@ -1337,19 +1337,21 @@ import_domains_or_subnets_from_remote_file() { import_subnets_from_remote_json_file() { local url="$1" - local tmpfile subnets http_proxy_address - tmpfile="$(mktemp)" + local json_tmpfile subnets_tmpfile subnets http_proxy_address + json_tmpfile="$(mktemp)" + subnets_tmpfile="$(mktemp)" http_proxy_address="$(get_service_proxy_address)" - download_to_stream "$url" "$http_proxy_address" | jq -r '.rules[].ip_cidr[]?' > "$tmpfile" + download_to_file "$url" "$json_tmpfile" "$http_proxy_address" - if [ $? -ne 0 ] || [ ! -s "$tmpfile" ]; then + if [ $? -ne 0 ] || [ ! -s "$json_tmpfile" ]; then log "Download $url list failed" "error" return 1 fi - subnets="$(parse_domain_or_subnet_file_to_comma_string "$tmpfile" "subnets")" - rm -f "$tmpfile" + jq -r '.rules[].ip_cidr[]' "$json_tmpfile" > "$subnets_tmpfile" + subnets="$(parse_domain_or_subnet_file_to_comma_string "$subnets_tmpfile" "subnets")" + rm -f "$json_tmpfile" "$subnets_tmpfile" nft_add_set_elements "$NFT_TABLE_NAME" "$NFT_COMMON_SET_NAME" "$subnets" } diff --git a/podkop/files/usr/lib/helpers.sh b/podkop/files/usr/lib/helpers.sh index 5801fec..8957711 100644 --- a/podkop/files/usr/lib/helpers.sh +++ b/podkop/files/usr/lib/helpers.sh @@ -268,25 +268,6 @@ migration_rename_config_key() { fi } -# Download URL content directly -download_to_stream() { - local url="$1" - local http_proxy_address="$2" - local retries="${3:-3}" - local wait="${4:-2}" - - for attempt in $(seq 1 "$retries"); do - if [ -n "$http_proxy_address" ]; then - http_proxy="http://$http_proxy_address" https_proxy="http://$http_proxy_address" wget -qO- "$url" | sed 's/\r$//' && break - else - wget -qO- "$url" | sed 's/\r$//' && break - fi - - log "Attempt $attempt/$retries to download $url failed" "warn" - sleep "$wait" - done -} - # Download URL to file download_to_file() { local url="$1" From 23cbe7be4a439c80fc23798fc69453abec0018e3 Mon Sep 17 00:00:00 2001 From: Andrey Petelin Date: Wed, 29 Oct 2025 22:11:29 +0500 Subject: [PATCH 3/3] fix: include filename in log and remove temp file on CRLF-to-LF conversion --- podkop/files/usr/lib/helpers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/podkop/files/usr/lib/helpers.sh b/podkop/files/usr/lib/helpers.sh index 8957711..4c005f6 100644 --- a/podkop/files/usr/lib/helpers.sh +++ b/podkop/files/usr/lib/helpers.sh @@ -293,10 +293,10 @@ convert_crlf_to_lf() { local filepath="$1" if grep -q $'\r' "$filepath"; then - log "Downloaded file has Windows line endings (CRLF). Converting to Unix (LF)" "debug" + log "File '$filepath' contains CRLF line endings. Converting to LF..." "debug" local tmpfile tmpfile=$(mktemp) - tr -d '\r' < "$filepath" > "$tmpfile" && mv "$tmpfile" "$filepath" + tr -d '\r' < "$filepath" > "$tmpfile" && mv "$tmpfile" "$filepath" || rm -f "$tmpfile" fi }