diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 4ca156b5..6202b924 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -29,7 +29,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && npm install --location=global yarn \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /etc/fail2ban \ - && go install github.com/xo/usql@master + && go install github.com/xo/usql@v0.13.12 # Task RUN cd /usr \ diff --git a/docker/rootfs/bin/common.sh b/docker/rootfs/bin/common.sh new file mode 100644 index 00000000..413cd91b --- /dev/null +++ b/docker/rootfs/bin/common.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e + +CYAN='\E[1;36m' +BLUE='\E[1;34m' +YELLOW='\E[1;33m' +RED='\E[1;31m' +RESET='\E[0m' +export CYAN BLUE YELLOW RED RESET + +PUID=${PUID:-0} +PGID=${PGID:-0} + +if [[ "$PUID" -ne '0' ]] && [ "$PGID" = '0' ]; then + # set group id to same as user id, + # the user probably forgot to specify the group id and + # it would be rediculous to intentionally use the root group + # for a non-root user + PGID=$PUID +fi + +export PUID PGID + +log_info () { + echo -e "${BLUE}❯ ${CYAN}$1${RESET}" +} + +log_error () { + echo -e "${RED}❯ $1${RESET}" +} + +log_fatal () { + echo -e "${RED}--------------------------------------${RESET}" + echo -e "${RED}ERROR: $1${RESET}" + echo -e "${RED}--------------------------------------${RESET}" + /run/s6/basedir/bin/halt + exit 1 +} + +disable_ipv6 () { + if [ "$DISABLE_IPV6" == 'true' ] || [ "$DISABLE_IPV6" == 'on' ] || [ "$DISABLE_IPV6" == '1' ] || [ "$DISABLE_IPV6" == 'yes' ]; then + echo '1' + else + echo '0' + fi +} diff --git a/docker/rootfs/etc/nginx/conf.d/include/ip_ranges.conf b/docker/rootfs/etc/nginx/conf.d/include/ip_ranges.conf new file mode 100644 index 00000000..34249325 --- /dev/null +++ b/docker/rootfs/etc/nginx/conf.d/include/ip_ranges.conf @@ -0,0 +1,2 @@ +# This should be left blank is it is populated programatically +# by the application backend. diff --git a/docker/rootfs/etc/nginx/nginx.conf b/docker/rootfs/etc/nginx/nginx.conf index 87118ad9..faf5c4c7 100644 --- a/docker/rootfs/etc/nginx/nginx.conf +++ b/docker/rootfs/etc/nginx/nginx.conf @@ -1,7 +1,7 @@ # run nginx in foreground daemon off; -#user npmuser; pid /run/nginx/nginx.pid; +user npmuser; # Set number of worker processes automatically based on number of CPU cores. worker_processes auto; @@ -9,7 +9,7 @@ worker_processes auto; # Enables the use of JIT for regular expressions to speed-up their processing. pcre_jit on; -error_log /data/logs/error.log warn; +error_log /data/logs/fallback_error.log warn; # Includes files with directives to load dynamic modules. include /etc/nginx/modules/*.conf; @@ -41,7 +41,12 @@ http { proxy_cache_path /var/lib/nginx/cache/private levels=1:2 keys_zone=private-cache:5m max_size=1024m; log_format proxy '[$time_local] $upstream_cache_status $upstream_status $status - $request_method $scheme $host "$request_uri" [Client $remote_addr] [Length $body_bytes_sent] [Gzip $gzip_ratio] "$http_user_agent" "$http_referer"'; - access_log /data/logs/default.log proxy; + log_format standard '[$time_local] $status - $request_method $scheme $host "$request_uri" [Client $remote_addr] [Length $body_bytes_sent] [Gzip $gzip_ratio] "$http_user_agent" "$http_referer"'; + + access_log /data/logs/fallback_access.log proxy; + + # Dynamically generated resolvers file + include /etc/nginx/conf.d/include/resolvers.conf; # Default upstream scheme map $host $forward_scheme { @@ -49,18 +54,38 @@ http { } # Real IP Determination - # Docker subnet: - set_real_ip_from 172.0.0.0/8; + + # Local subnets: + set_real_ip_from 10.0.0.0/8; + set_real_ip_from 172.16.0.0/12; # Includes Docker subnet + set_real_ip_from 192.168.0.0/16; # NPM generated CDN ip ranges: - #include conf.d/include/ip_ranges.conf; + include conf.d/include/ip_ranges.conf; # always put the following 2 lines after ip subnets: - real_ip_header X-Forwarded-For; + real_ip_header X-Real-IP; real_ip_recursive on; + # Custom + include /data/nginx/custom/http_top[.]conf; + # Files generated by NPM include /etc/nginx/conf.d/*.conf; include /data/nginx/default_host/*.conf; include /data/nginx/upstreams/*.conf; include /data/nginx/hosts/*.conf; include /data/nginx/streams/*.conf; + + # Custom + include /data/nginx/custom/http[.]conf; } + +stream { + # Files generated by NPM + include /data/nginx/stream/*.conf; + + # Custom + include /data/nginx/custom/stream[.]conf; +} + +# Custom +include /data/nginx/custom/root[.]conf; diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run index ceebed59..0098ff66 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run @@ -3,7 +3,9 @@ set -e -echo "❯ Starting backend ..." +. /bin/common.sh + +log_info 'Starting backend ...' if [ "$DEVELOPMENT" == "true" ]; then HOME=/tmp/npmuserhome diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/fail2ban/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/fail2ban/run index 261fea18..a1b25380 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/fail2ban/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/fail2ban/run @@ -1,5 +1,7 @@ #!/command/with-contenv bash # shellcheck shell=bash -echo "❯ Starting fail2ban ..." +. /bin/common.sh + +log_info 'Starting fail2ban ...' exec /usr/bin/fail2ban-client -c /fail2ban -x -vv -f start diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/frontend/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/frontend/run index d2d9bb2c..5911a987 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/frontend/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/frontend/run @@ -10,7 +10,13 @@ if [ "$DEVELOPMENT" == "true" ]; then HOME=/tmp/npmuserhome export CI export HOME + + . /bin/common.sh cd /app/frontend || exit 1 + HOME=/tmp/npmuserhome + export HOME + + log_info 'Starting frontend ...' s6-setuidgid npmuser yarn install exec s6-setuidgid npmuser yarn start else diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run index 13b48395..fa8c1fc5 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run @@ -3,5 +3,7 @@ set -e -echo "❯ Starting nginx ..." +. /bin/common.sh + +log_info 'Starting nginx ...' exec s6-setuidgid npmuser nginx diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/00-all.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/00-all.sh new file mode 100755 index 00000000..e0b7eed4 --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/00-all.sh @@ -0,0 +1,18 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +set -e + +. /bin/common.sh + +if [ "$(id -u)" != "0" ]; then + log_fatal "This docker container must be run as root, do not specify a user.\nYou can specify PUID and PGID env vars to run processes as that user and group after initialization." +fi + +. /etc/s6-overlay/s6-rc.d/prepare/10-npmuser.sh +. /etc/s6-overlay/s6-rc.d/prepare/20-paths.sh +. /etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh +. /etc/s6-overlay/s6-rc.d/prepare/40-dynamic.sh +. /etc/s6-overlay/s6-rc.d/prepare/50-ipv6.sh +. /etc/s6-overlay/s6-rc.d/prepare/60-fail2ban.sh +. /etc/s6-overlay/s6-rc.d/prepare/90-banner.sh diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/10-npmuser.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/10-npmuser.sh new file mode 100755 index 00000000..c5cf5435 --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/10-npmuser.sh @@ -0,0 +1,20 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +set -e + +log_info 'Configuring npmuser ...' + +if id -u npmuser; then + # user already exists + usermod -u "$PUID" npmuser || exit 1 +else + # Add npmuser user + useradd -o -u "$PUID" -U -d /tmp/npmuserhome -s /bin/false npmuser || exit 1 +fi + +usermod -G "$PGID" npmuser || exit 1 +groupmod -o -g "$PGID" npmuser || exit 1 +# Home for npmuser +mkdir -p /tmp/npmuserhome +chown -R "$PUID:$PGID" /tmp/npmuserhome diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/20-paths.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/20-paths.sh new file mode 100755 index 00000000..285492c0 --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/20-paths.sh @@ -0,0 +1,24 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +set -e + +log_info 'Checking paths ...' + +# Ensure /data is mounted +if [ ! -d '/data' ]; then + log_fatal '/data is not mounted! Check your docker configuration.' +fi + +# Create required folders +mkdir -p /tmp/nginx/body \ + /run/nginx \ + /var/log/nginx \ + /var/lib/nginx/cache/public \ + /var/lib/nginx/cache/private \ + /var/cache/nginx/proxy_temp \ + /data/logs + +touch /var/log/nginx/error.log || true +chmod 777 /var/log/nginx/error.log || true +chmod -R 777 /var/cache/nginx || true diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh new file mode 100755 index 00000000..4a264066 --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh @@ -0,0 +1,22 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +set -e + +log_info 'Setting ownership ...' + +# root +chown root /tmp/nginx + +# npmuser +chown -R "$PUID:$PGID" /data \ + /run/nginx \ + /tmp/nginx \ + /var/cache/nginx \ + /var/lib/nginx \ + /var/log/nginx + +# Don't chown entire /etc/nginx folder as this causes crashes on some systems +chown -R "$PUID:$PGID" /etc/nginx/nginx \ + /etc/nginx/nginx.conf \ + /etc/nginx/conf.d diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/40-dynamic.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/40-dynamic.sh new file mode 100755 index 00000000..9fb43076 --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/40-dynamic.sh @@ -0,0 +1,19 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +set -e + +log_info 'Dynamic resolvers ...' + +DISABLE_IPV6=$(echo "${DISABLE_IPV6:-}" | tr '[:upper:]' '[:lower:]') + +# Dynamically generate resolvers file, if resolver is IPv6, enclose in `[]` +# thanks @tfmm +if [ "$(disable_ipv6)" == '1' ]; then + echo resolver "$(awk 'BEGIN{ORS=" "} $1=="nameserver" { sub(/%.*$/,"",$2); print ($2 ~ ":")? "["$2"]": $2}' /etc/resolv.conf) ipv6=off valid=10s;" > /etc/nginx/conf.d/include/resolvers.conf +else + echo resolver "$(awk 'BEGIN{ORS=" "} $1=="nameserver" { sub(/%.*$/,"",$2); print ($2 ~ ":")? "["$2"]": $2}' /etc/resolv.conf) valid=10s;" > /etc/nginx/conf.d/include/resolvers.conf +fi + +# Fire off acme.sh wrapper script to "install" itself if required +acme.sh -h > /dev/null 2>&1 diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/50-ipv6.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/50-ipv6.sh new file mode 100755 index 00000000..e018889d --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/50-ipv6.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This command reads the `DISABLE_IPV6` env var and will either enable +# or disable ipv6 in all nginx configs based on this setting. + +log_info 'IPv6 ...' + +# Lowercase +DISABLE_IPV6=$(echo "${DISABLE_IPV6:-}" | tr '[:upper:]' '[:lower:]') + +process_folder () { + FILES=$(find "$1" -type f -name "*.conf") + SED_REGEX= + + if [ "$(disable_ipv6)" == '1' ]; then + # IPV6 is disabled + echo "❯ Disabling IPV6 in hosts in: $1" + SED_REGEX='s/^([^#]*)listen \[::\]/\1#listen [::]/g' + else + # IPV6 is enabled + echo "❯ Enabling IPV6 in hosts in: $1" + SED_REGEX='s/^(\s*)#listen \[::\]/\1listen [::]/g' + fi + + for FILE in $FILES + do + echo " - ${FILE}" + sed -E -i "$SED_REGEX" "$FILE" || true + done + + # ensure the files are still owned by the npmuser + chown -R "$PUID:$PGID" "$1" +} + +process_folder /etc/nginx/conf.d +process_folder /data/nginx diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/60-fail2ban.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/60-fail2ban.sh new file mode 100755 index 00000000..cf994904 --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/60-fail2ban.sh @@ -0,0 +1,13 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +set -e + +log_info 'fail2ban ...' + +mkdir -p /fail2ban/{action.d,filter.d,jail.d,log} +chown -R "$PUID:$PGID" /fail2ban +mkdir -p /var/run/fail2ban \ + /data/logs/fail2ban +chown nobody:nogroup /data/logs/fail2ban +chmod 02755 /data/logs/fail2ban diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/90-banner.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/90-banner.sh new file mode 100755 index 00000000..7991ddf4 --- /dev/null +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/90-banner.sh @@ -0,0 +1,17 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +set -e + +echo " +------------------------------------- + _ _ ____ __ __ +| \ | | _ \| \/ | +| \| | |_) | |\/| | +| |\ | __/| | | | +|_| \_|_| |_| |_| +------------------------------------- +User ID: $PUID +Group ID: $PGID +------------------------------------- +" diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/script.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/script.sh deleted file mode 100755 index 23a37d15..00000000 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/script.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/command/with-contenv bash -# shellcheck shell=bash - -set -e - -DATA_PATH=/data -PUID=${PUID:-911} -PGID=${PGID:-911} - -# Ensure /data is mounted -if [ ! -d "$DATA_PATH" ]; then - echo '--------------------------------------' - echo "ERROR: $DATA_PATH is not mounted! Check your docker configuration." - echo '--------------------------------------' - /run/s6/basedir/bin/halt - exit 1 -fi - -echo "❯ Checking folder structure ..." - -# Create required folders -mkdir -p /tmp/nginx/body \ - /run/nginx \ - /var/log/nginx \ - /var/lib/nginx/cache/public \ - /var/lib/nginx/cache/private \ - /var/cache/nginx/proxy_temp \ - /data/logs -touch /var/log/nginx/error.log && chmod 777 /var/log/nginx/error.log && chmod -R 777 /var/cache/nginx -# Dynamically generate resolvers file -echo resolver "$(awk 'BEGIN{ORS=" "} $1=="nameserver" {print $2}' /etc/resolv.conf)" ";" > /etc/nginx/conf.d/include/resolvers.conf -# Fire off acme.sh wrapper script to "install" itself if required -acme.sh -h > /dev/null 2>&1 - -# Add npmuser user -echo "❯ Creating user ..." -groupmod -g 1000 users || exit 1 -useradd -u "${PUID}" -U -d /data -s /bin/false npmuser || exit 1 -usermod -G users npmuser || exit 1 -groupmod -o -g "$PGID" npmuser || exit 1 -chown -R npmuser:npmuser /data -chown -R npmuser:npmuser /run/nginx -chown -R npmuser:npmuser /etc/nginx -chown -R npmuser:npmuser /tmp/nginx -chown -R npmuser:npmuser /var/cache/nginx -chown -R npmuser:npmuser /var/lib/nginx -chown -R npmuser:npmuser /var/log/nginx -# Home for npmuser -mkdir -p /tmp/npmuserhome -chown -R npmuser:npmuser /tmp/npmuserhome - -# fail2ban configuration -mkdir -p /fail2ban/{action.d,filter.d,jail.d,log} -chown -R npmuser:npmuser /fail2ban -mkdir -p /var/run/fail2ban -mkdir -p /data/logs/fail2ban -chown nobody:nogroup /data/logs/fail2ban -chmod 02755 /data/logs/fail2ban - -echo -echo "------------------------------------- - _ _ ____ __ __ -| \ | | _ \| \/ | -| \| | |_) | |\/| | -| |\ | __/| | | | -|_| \_|_| |_| |_| -------------------------------------- -User UID: $(id -u npmuser) -User GID: $(id -g npmuser) -------------------------------------- -" diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/up b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/up index b58eed6b..896a01b6 100644 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/up +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/up @@ -1,2 +1,2 @@ # shellcheck shell=bash -/etc/s6-overlay/s6-rc.d/prepare/script.sh +/etc/s6-overlay/s6-rc.d/prepare/00-all.sh