Support for dynamic ip ranges from urls

- Adds ipranges command to fetch ip ranges from Cloudfront and Cloudflare
- Write the ipranges file on docker start
- Support disabling ipv4 as well as ipv6 now
- Prevent disabling both

backend/Taskfile.yml

@@ -32,6 +32,9 @@ tasks:
               silent: true
             - cmd: go build -buildvcs=false -ldflags="-X main.commit={{.GIT_COMMIT}} -X main.version={{.VERSION}}" -o ../dist/bin/server ./cmd/server/main.go
               silent: true
+            - cmd: go build -buildvcs=false -ldflags="-X main.commit={{.GIT_COMMIT}} -X main.version={{.VERSION}}" -o ../dist/bin/ipranges ./cmd/ipranges/main.go
+              silent: true
+            - cmd: rm -f /etc/nginx/conf.d/include/ipranges.conf && /app/dist/bin/ipranges > /etc/nginx/conf.d/include/ipranges.conf
             - task: lint
         vars:
             GIT_COMMIT:

backend/cmd/ipranges/main.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	"npm/internal/config"
+	"npm/internal/model"
+
+	"github.com/rotisserie/eris"
+)
+
+var commit string
+var version string
+var sentryDSN string
+
+var cloudfrontURL = "https://ip-ranges.amazonaws.com/ip-ranges.json"
+var cloudflare4URL = "https://www.cloudflare.com/ips-v4"
+var cloudflare6URL = "https://www.cloudflare.com/ips-v6"
+
+func main() {
+	config.InitArgs(&version, &commit)
+	if err := config.InitIPRanges(&version, &commit, &sentryDSN); err != nil {
+		fmt.Printf("# Config ERROR: %v\n", err)
+		os.Exit(1)
+	}
+
+	exitCode := 0
+
+	// Cloudfront
+	fmt.Printf("# Cloudfront Ranges from: %s\n", cloudfrontURL)
+	if ranges, err := parseCloudfront(); err == nil {
+		for _, item := range ranges {
+			fmt.Printf("set_real_ip_from %s;\n", item)
+		}
+	} else {
+		fmt.Printf("# ERROR: %v\n", err)
+	}
+
+	// Cloudflare ipv4
+	if !config.Configuration.DisableIPV4 {
+		fmt.Printf("\n# Cloudflare Ranges from: %s\n", cloudflare4URL)
+		if ranges, err := parseCloudflare(cloudflare4URL); err == nil {
+			for _, item := range ranges {
+				fmt.Printf("set_real_ip_from %s;\n", item)
+			}
+		} else {
+			fmt.Printf("# ERROR: %v\n", err)
+		}
+	}
+
+	// Cloudflare ipv6
+	if !config.Configuration.DisableIPV6 {
+		fmt.Printf("\n# Cloudflare Ranges from: %s\n", cloudflare6URL)
+		if ranges, err := parseCloudflare(cloudflare6URL); err == nil {
+			for _, item := range ranges {
+				fmt.Printf("set_real_ip_from %s;\n", item)
+			}
+		} else {
+			fmt.Printf("# ERROR: %v\n", err)
+		}
+	}
+
+	// Done
+	os.Exit(exitCode)
+}
+
+func parseCloudfront() ([]string, error) {
+	// nolint: gosec
+	resp, err := http.Get(cloudfrontURL)
+	if err != nil {
+		return nil, eris.Wrapf(err, "Failed to download Cloudfront IP Ranges from %s", cloudfrontURL)
+	}
+
+	// nolint: errcheck, gosec
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, eris.Wrapf(err, "Failed to read Cloudfront IP Ranges body")
+	}
+
+	var result model.CloudfrontIPRanges
+	if err := json.Unmarshal(body, &result); err != nil {
+		return nil, eris.Wrapf(err, "Failed to unmarshal Cloudfront IP Ranges file")
+	}
+
+	ranges := make([]string, 0)
+	if !config.Configuration.DisableIPV4 {
+		for _, item := range result.IPV4Prefixes {
+			ranges = append(ranges, item.Value)
+		}
+	}
+	if !config.Configuration.DisableIPV6 {
+		for _, item := range result.IPV6Prefixes {
+			ranges = append(ranges, item.Value)
+		}
+	}
+
+	return ranges, nil
+}
+
+func parseCloudflare(url string) ([]string, error) {
+	// nolint: gosec
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, eris.Wrapf(err, "Failed to download Cloudflare IP Ranges from %s", url)
+	}
+
+	// nolint: errcheck, gosec
+	defer resp.Body.Close()
+
+	scanner := bufio.NewScanner(resp.Body)
+	scanner.Split(bufio.ScanLines)
+
+	ranges := make([]string, 0)
+	for scanner.Scan() {
+		if scanner.Text() != "" {
+			ranges = append(ranges, scanner.Text())
+		}
+	}
+	return ranges, nil
+}

backend/internal/config/config.go

@@ -29,6 +29,16 @@ func Init(version, commit, sentryDSN *string) {
 	loadKeys()
 }
 
+// InitIPRanges will initialise the config for the ipranges command
+func InitIPRanges(version, commit, sentryDSN *string) error {
+	ErrorReporting = true
+	Version = *version
+	Commit = *commit
+	err := envconfig.InitWithPrefix(&Configuration, "NPM")
+	initLogger(*sentryDSN)
+	return err
+}
+
 // Init initialises the Log object and return it
 func initLogger(sentryDSN string) {
 	// this removes timestamp prefixes from logs

backend/internal/config/vars.go

@@ -38,9 +38,11 @@ type acmesh struct {
 
 // Configuration is the main configuration object
 var Configuration struct {
-	DataFolder string `json:"data_folder" envconfig:"optional,default=/data"`
-	Acmesh     acmesh `json:"acmesh"`
-	Log        log    `json:"log"`
+	DataFolder  string `json:"data_folder" envconfig:"optional,default=/data"`
+	DisableIPV4 bool   `json:"disable_ipv4" envconfig:"optional"`
+	DisableIPV6 bool   `json:"disable_ipv6" envconfig:"optional"`
+	Acmesh      acmesh `json:"acmesh"`
+	Log         log    `json:"log"`
 }
 
 // GetWellknown returns the well known path

backend/internal/model/cloudfrontranges.go

@@ -0,0 +1,17 @@
+package model
+
+// CloudfrontIPRangePrefix is used within config for cloudfront
+type CloudfrontIPRangeV4Prefix struct {
+	Value string `json:"ip_prefix"`
+}
+
+// CloudfrontIPRangeV6Prefix is used within config for cloudfront
+type CloudfrontIPRangeV6Prefix struct {
+	Value string `json:"ipv6_prefix"`
+}
+
+// CloudfrontIPRanges is the main config for cloudfront
+type CloudfrontIPRanges struct {
+	IPV4Prefixes []CloudfrontIPRangeV4Prefix `json:"prefixes"`
+	IPV6Prefixes []CloudfrontIPRangeV6Prefix `json:"ipv6_prefixes"`
+}

backend/internal/nginx/control.go

@@ -39,8 +39,8 @@ func ConfigureHost(h host.Model) error {
 		Certificate: certificateTemplate,
 		ConfDir:     fmt.Sprintf("%s/nginx/hosts", config.Configuration.DataFolder),
 		Config: Config{ // todo
-			Ipv4: true,
-			Ipv6: false,
+			Ipv4: !config.Configuration.DisableIPV4,
+			Ipv6: !config.Configuration.DisableIPV6,
 		},
 		DataDir:  config.Configuration.DataFolder,
 		Host:     h.GetTemplate(),

docker/Dockerfile

@@ -36,6 +36,7 @@ RUN mkdir -p /dist \
 FROM jc21/nginx-full:acmesh AS final
 
 COPY --from=gobuild /dist/server /app/bin/server
+COPY --from=gobuild /dist/ipranges /app/bin/ipranges
 # these certs are used for testing in CI
 COPY --from=pebbleca /test/certs/pebble.minica.pem /etc/ssl/certs/pebble.minica.pem
 COPY --from=testca /home/step/certs/root_ca.crt /etc/ssl/certs/NginxProxyManager.crt

docker/docker-compose.ci.yml

@@ -7,7 +7,7 @@ services:
     environment:
       - NPM_LOG_LEVEL=debug
       - NPM_LOG_FORMAT=json
-      - DISABLE_IPV6=true
+      - NPM_DISABLE_IPV6=true
     volumes:
       - '/etc/localtime:/etc/localtime:ro'
       - npm_data_ci:/data

docker/rootfs/bin/common.sh

@@ -31,6 +31,10 @@ log_info () {
 	echo -e "${BLUE}❯ ${CYAN}$1${RESET}"
 }
 
+log_warn () {
+	echo -e "${BLUE}❯ ${YELLOW}WARNING: $1${RESET}"
+}
+
 log_error () {
 	echo -e "${RED}❯ $1${RESET}"
 }
@@ -52,7 +56,8 @@ get_group_id () {
 
 # param $1: value
 is_true () {
-	if [ "$1" == 'true' ] || [ "$1" == 'on' ] || [ "$1" == '1' ] || [ "$1" == 'yes' ]; then
+	VAL=$(echo "${1:-}" | tr '[:upper:]' '[:lower:]')
+	if [ "$VAL" == 'true' ] || [ "$VAL" == 'on' ] || [ "$VAL" == '1' ] || [ "$VAL" == 'yes' ]; then
 		echo '1'
 	else
 		echo '0'

docker/rootfs/etc/nginx/conf.d/include/ip_ranges.conf

@@ -1,2 +0,0 @@
-# This should be left blank is it is populated programatically
-# by the application backend.

docker/rootfs/etc/nginx/nginx.conf

@@ -60,7 +60,7 @@ http {
 	set_real_ip_from 172.16.0.0/12; # Includes Docker subnet
 	set_real_ip_from 192.168.0.0/16;
 	# NPM generated CDN ip ranges:
-	include conf.d/include/ip_ranges.conf;
+	include conf.d/include/ipranges.conf;
 	# always put the following 2 lines after ip subnets:
 	real_ip_header X-Real-IP;
 	real_ip_recursive on;

docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/00-all.sh

@@ -17,6 +17,6 @@ fi
 . /etc/s6-overlay/s6-rc.d/prepare/20-paths.sh
 . /etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh
 . /etc/s6-overlay/s6-rc.d/prepare/40-dynamic.sh
-. /etc/s6-overlay/s6-rc.d/prepare/50-ipv6.sh
+. /etc/s6-overlay/s6-rc.d/prepare/50-ipv46.sh
 . /etc/s6-overlay/s6-rc.d/prepare/60-fail2ban.sh
 . /etc/s6-overlay/s6-rc.d/prepare/90-banner.sh

docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/40-dynamic.sh

@@ -5,11 +5,9 @@ set -e
 
 log_info 'Dynamic resolvers ...'
 
-DISABLE_IPV6=$(echo "${DISABLE_IPV6:-}" | tr '[:upper:]' '[:lower:]')
-
 # Dynamically generate resolvers file, if resolver is IPv6, enclose in `[]`
 # thanks @tfmm
-if [ "$(is_true "$DISABLE_IPV6")" = '1' ]; then
+if [ "$(is_true "$NPM_DISABLE_IPV6")" = '1' ]; then
 	echo resolver "$(awk 'BEGIN{ORS=" "} $1=="nameserver" { sub(/%.*$/,"",$2); print ($2 ~ ":")? "["$2"]": $2}' /etc/resolv.conf) ipv6=off valid=10s;" > /etc/nginx/conf.d/include/resolvers.conf
 else
 	echo resolver "$(awk 'BEGIN{ORS=" "} $1=="nameserver" { sub(/%.*$/,"",$2); print ($2 ~ ":")? "["$2"]": $2}' /etc/resolv.conf) valid=10s;" > /etc/nginx/conf.d/include/resolvers.conf
@@ -17,3 +15,20 @@ fi
 
 # Fire off acme.sh wrapper script to "install" itself if required
 acme.sh -h > /dev/null 2>&1
+
+# Generate IP Ranges from online CDN services
+# continue on error, as this could be due to network errors
+# and can be attempted again with a docker restart
+rm -rf /etc/nginx/conf.d/include/ipranges.conf
+set +e
+RC=0
+if [ "$(is_true "$DEVELOPMENT")" = '1' ]; then
+	echo '# ignored in development mode' > /etc/nginx/conf.d/include/ipranges.conf
+else
+	/app/bin/ipranges > /etc/nginx/conf.d/include/ipranges.conf
+	RC=$?
+fi
+if [ "$RC" != '0' ]; then
+	log_warn 'Generation of IP Ranges file has an error. Check output of /etc/nginx/conf.d/include/ipranges.conf for more information.'
+fi
+set -e

docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/50-ipv46.sh

@@ -0,0 +1,58 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+# This command reads the `NPM_DISABLE_IPV4` and `NPM_DISABLE_IPV6`` env vars and will either enable
+# or disable ipv6 in all nginx configs based on this setting.
+
+set -e
+
+log_info 'IPv4/IPv6 ...'
+
+DIS_4=$(is_true "$NPM_DISABLE_IPV4")
+DIS_6=$(is_true "$NPM_DISABLE_IPV6")
+
+# Ensure someone didn't misconfigure the settings
+if [ "$DIS_4" = "1" ] && [ "$DIS_6" = "1" ]; then
+	log_fatal 'NPM_DISABLE_IPV4 and NPM_DISABLE_IPV6 cannot both be set!'
+fi
+
+process_folder () {
+	FILES=$(find "$1" -type f -name "*.conf")
+	SED_REGEX=
+
+	# IPV4 ...
+	if [ "$DIS_4" = "1" ]; then
+		echo "Disabling IPV4 in hosts in: $1"
+		SED_REGEX='s/^([^#]*)listen ([0-9]+)/\1#listen \2/g'
+	else
+		echo "Enabling IPV4 in hosts in: $1"
+		SED_REGEX='s/^(\s*)#listen ([0-9]+)/\1listen \2/g'
+	fi
+
+	for FILE in $FILES
+	do
+		echo "  - ${FILE}"
+		sed -E -i "$SED_REGEX" "$FILE" || true
+	done
+
+	# IPV6 ...
+	if [ "$DIS_6" = "1" ]; then
+		echo "Disabling IPV6 in hosts in: $1"
+		SED_REGEX='s/^([^#]*)listen \[::\]/\1#listen [::]/g'
+	else
+		echo "Enabling IPV6 in hosts in: $1"
+		SED_REGEX='s/^(\s*)#listen \[::\]/\1listen [::]/g'
+	fi
+
+	for FILE in $FILES
+	do
+		echo "  - ${FILE}"
+		sed -E -i "$SED_REGEX" "$FILE" || true
+	done
+
+	# ensure the files are still owned by the npm user
+	chown -R "$PUID:$PGID" "$1"
+}
+
+process_folder /etc/nginx/conf.d
+process_folder /data/nginx

docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/50-ipv6.sh

@@ -1,39 +0,0 @@
-#!/command/with-contenv bash
-# shellcheck shell=bash
-
-# This command reads the `DISABLE_IPV6` env var and will either enable
-# or disable ipv6 in all nginx configs based on this setting.
-
-set -e
-
-log_info 'IPv6 ...'
-
-# Lowercase
-DISABLE_IPV6=$(echo "${DISABLE_IPV6:-}" | tr '[:upper:]' '[:lower:]')
-
-process_folder () {
-	FILES=$(find "$1" -type f -name "*.conf")
-	SED_REGEX=
-
-	if [ "$DISABLE_IPV6" == "true" ] || [ "$DISABLE_IPV6" == "on" ] || [ "$DISABLE_IPV6" == "1" ] || [ "$DISABLE_IPV6" == "yes" ]; then
-		# IPV6 is disabled
-		echo "Disabling IPV6 in hosts in: $1"
-		SED_REGEX='s/^([^#]*)listen \[::\]/\1#listen [::]/g'
-	else
-		# IPV6 is enabled
-		echo "Enabling IPV6 in hosts in: $1"
-		SED_REGEX='s/^(\s*)#listen \[::\]/\1listen [::]/g'
-	fi
-
-	for FILE in $FILES
-	do
-		echo "  - ${FILE}"
-		sed -E -i "$SED_REGEX" "$FILE" || true
-	done
-
-	# ensure the files are still owned by the npm user
-	chown -R "$PUID:$PGID" "$1"
-}
-
-process_folder /etc/nginx/conf.d
-process_folder /data/nginx

docs/advanced-config/README.md

@@ -88,7 +88,7 @@ services:
       # and remove all DB_MYSQL_* lines above
       # DB_SQLITE_FILE: "/data/database.sqlite"
       # Uncomment this if IPv6 is not enabled on your host
-      # DISABLE_IPV6: 'true'
+      # NPM_DISABLE_IPV6: 'true'
     volumes:
       - ./data:/data
       - ./letsencrypt:/etc/letsencrypt
@@ -124,7 +124,7 @@ The easy fix is to add a Docker environment variable to the Nginx Proxy Manager
 
 ```yml
     environment:
-      DISABLE_IPV6: 'true'
+      NPM_DISABLE_IPV6: 'true'
 ```
 
 

docs/setup/README.md

@@ -23,7 +23,7 @@ services:
       - PUID=1000
       - PGID=1000
       # Uncomment this if IPv6 is not enabled on your host
-      # DISABLE_IPV6: 'true'
+      # NPM_DISABLE_IPV6: 'true'
     volumes:
       - ./data:/data
 ```

scripts/go-multiarch-wrapper

@@ -20,15 +20,24 @@ case ${TARGETPLATFORM:-} in
 		;;
 esac
 
-echo -e "${BLUE}❯ ${CYAN}Building binary for ${YELLOW}${GOARCH} (${TARGETPLATFORM:-})${RESET}"
+echo -e "${BLUE}❯ ${CYAN}Building binaries for ${YELLOW}${GOARCH} (${TARGETPLATFORM:-})${RESET}"
 
+# server
 go build \
 	-buildvcs=false \
 	-ldflags "-w -s -X main.commit=${BUILD_COMMIT:-notset} -X main.version=${BUILD_VERSION} -X main.sentryDSN=${SENTRY_DSN:-}" \
 	-o "${1:-/dist/server}" \
 	./cmd/server
 
-# test binary
-/dist/server --version
+# ipranges
+go build \
+	-buildvcs=false \
+	-ldflags "-w -s -X main.commit=${BUILD_COMMIT:-notset} -X main.version=${BUILD_VERSION} -X main.sentryDSN=${SENTRY_DSN:-}" \
+	-o "${1:-/dist/ipranges}" \
+	./cmd/ipranges
 
-echo -e "${BLUE}❯ ${CYAN}Build binary complete${RESET}"
+# test binaries
+/dist/server --version
+/dist/ipranges --version
+
+echo -e "${BLUE}❯ ${CYAN}Build binaries complete${RESET}"