Fetch ip ranges for CDN servers, Cloudfront and Cloudfare

This commit is contained in:
Jamie Curnow 2019-01-03 17:04:53 +10:00
parent 9fcd32c2ca
commit 3836f7c40a
6 changed files with 355 additions and 8 deletions

View File

@ -1,3 +1,6 @@
# Generated IP Ranges for safe real IP determination
include conf.d/include/ip_ranges.conf;
# Healthcheck Host which proxies to the Manager,
# thus the healthcheck ensures both services are running
server {

View File

@ -0,0 +1,174 @@
set_real_ip_from 13.124.199.0/24;
set_real_ip_from 34.226.14.0/24;
set_real_ip_from 52.124.128.0/17;
set_real_ip_from 54.230.0.0/16;
set_real_ip_from 54.239.128.0/18;
set_real_ip_from 52.82.128.0/19;
set_real_ip_from 99.84.0.0/16;
set_real_ip_from 52.15.127.128/26;
set_real_ip_from 35.158.136.0/24;
set_real_ip_from 52.57.254.0/24;
set_real_ip_from 18.216.170.128/25;
set_real_ip_from 13.54.63.128/26;
set_real_ip_from 13.59.250.0/26;
set_real_ip_from 13.210.67.128/26;
set_real_ip_from 35.167.191.128/26;
set_real_ip_from 52.47.139.0/24;
set_real_ip_from 52.199.127.192/26;
set_real_ip_from 52.212.248.0/26;
set_real_ip_from 205.251.192.0/19;
set_real_ip_from 52.66.194.128/26;
set_real_ip_from 54.239.192.0/19;
set_real_ip_from 70.132.0.0/18;
set_real_ip_from 13.32.0.0/15;
set_real_ip_from 13.113.203.0/24;
set_real_ip_from 34.195.252.0/24;
set_real_ip_from 35.162.63.192/26;
set_real_ip_from 34.223.12.224/27;
set_real_ip_from 13.35.0.0/16;
set_real_ip_from 204.246.172.0/23;
set_real_ip_from 204.246.164.0/22;
set_real_ip_from 52.56.127.0/25;
set_real_ip_from 204.246.168.0/22;
set_real_ip_from 13.228.69.0/24;
set_real_ip_from 34.216.51.0/25;
set_real_ip_from 71.152.0.0/17;
set_real_ip_from 216.137.32.0/19;
set_real_ip_from 205.251.249.0/24;
set_real_ip_from 99.86.0.0/16;
set_real_ip_from 52.46.0.0/18;
set_real_ip_from 52.84.0.0/15;
set_real_ip_from 54.233.255.128/26;
set_real_ip_from 64.252.64.0/18;
set_real_ip_from 52.52.191.128/26;
set_real_ip_from 204.246.174.0/23;
set_real_ip_from 64.252.128.0/18;
set_real_ip_from 205.251.254.0/24;
set_real_ip_from 143.204.0.0/16;
set_real_ip_from 205.251.252.0/23;
set_real_ip_from 52.78.247.128/26;
set_real_ip_from 204.246.176.0/20;
set_real_ip_from 52.220.191.0/26;
set_real_ip_from 13.249.0.0/16;
set_real_ip_from 54.240.128.0/18;
set_real_ip_from 205.251.250.0/23;
set_real_ip_from 52.222.128.0/17;
set_real_ip_from 54.182.0.0/16;
set_real_ip_from 54.192.0.0/16;
set_real_ip_from 34.232.163.208/29;
set_real_ip_from 2600:9000:eee::/48;
set_real_ip_from 2600:9000:4000::/36;
set_real_ip_from 2600:9000:3000::/36;
set_real_ip_from 2600:9000:f000::/36;
set_real_ip_from 2600:9000:fff::/48;
set_real_ip_from 2600:9000:2000::/36;
set_real_ip_from 2600:9000:1000::/36;
set_real_ip_from 2600:9000:ddd::/48;
set_real_ip_from 2600:9000:5300::/40;
set_real_ip_from 103.21.244.0/22;
set_real_ip_from 103.22.200.0/22;
set_real_ip_from 103.31.4.0/22;
set_real_ip_from 104.16.0.0/12;
set_real_ip_from 108.162.192.0/18;
set_real_ip_from 131.0.72.0/22;
set_real_ip_from 141.101.64.0/18;
set_real_ip_from 162.158.0.0/15;
set_real_ip_from 172.64.0.0/13;
set_real_ip_from 173.245.48.0/20;
set_real_ip_from 188.114.96.0/20;
set_real_ip_from 190.93.240.0/20;
set_real_ip_from 197.234.240.0/22;
set_real_ip_from 198.41.128.0/17;
set_real_ip_from 2400:cb00::/32;
set_real_ip_from 2405:b500::/32;
set_real_ip_from 2606:4700::/32;
set_real_ip_from 2803:f800::/32;
set_real_ip_from 2c0f:f248::/32;
set_real_ip_from 2a06:98c0::/29;

View File

@ -11,6 +11,7 @@ function appStart () {
const app = require('./app');
const apiValidator = require('./lib/validator/api');
const internalCertificate = require('./internal/certificate');
const internalIpRanges = require('./internal/ip_ranges');
return migrate.latest()
.then(setup)
@ -18,9 +19,11 @@ function appStart () {
.then(() => {
return apiValidator.loadSchemas;
})
.then(internalIpRanges.fetch)
.then(() => {
internalCertificate.initTimer();
internalIpRanges.initTimer();
const server = app.listen(81, () => {
logger.info('PID ' + process.pid + ' listening on port 81 ...');

View File

@ -0,0 +1,163 @@
'use strict';
const https = require('https');
const fs = require('fs');
const _ = require('lodash');
const logger = require('../logger').ip_ranges;
const debug_mode = process.env.NODE_ENV !== 'production';
const error = require('../lib/error');
const internalNginx = require('./nginx');
const Liquid = require('liquidjs');
const CLOUDFRONT_URL = 'https://ip-ranges.amazonaws.com/ip-ranges.json';
const CLOUDFARE_V4_URL = 'https://www.cloudflare.com/ips-v4';
const CLOUDFARE_V6_URL = 'https://www.cloudflare.com/ips-v6';
const internalIpRanges = {
interval_timeout: 1000 * 60 * 60 * 6, // 6 hours
interval: null,
interval_processing: false,
iteration_count: 0,
initTimer: () => {
logger.info('IP Ranges Renewal Timer initialized');
internalIpRanges.interval = setInterval(internalIpRanges.fetch, internalIpRanges.interval_timeout);
},
fetchUrl: url => {
return new Promise((resolve, reject) => {
logger.info('Fetching ' + url);
return https.get(url, res => {
res.setEncoding('utf8');
let raw_data = '';
res.on('data', chunk => {
raw_data += chunk;
});
res.on('end', () => {
resolve(raw_data);
});
}).on('error', err => {
reject(err);
});
});
},
/**
* Triggered at startup and then later by a timer, this will fetch the ip ranges from services and apply them to nginx.
*/
fetch: () => {
if (!internalIpRanges.interval_processing) {
internalIpRanges.interval_processing = true;
logger.info('Fetching IP Ranges from online services...');
let ip_ranges = [];
return internalIpRanges.fetchUrl(CLOUDFRONT_URL)
.then(cloudfront_data => {
let data = JSON.parse(cloudfront_data);
if (data && typeof data.prefixes !== 'undefined') {
data.prefixes.map(item => {
if (item.service === 'CLOUDFRONT') {
ip_ranges.push(item.ip_prefix);
}
});
}
if (data && typeof data.ipv6_prefixes !== 'undefined') {
data.ipv6_prefixes.map(item => {
if (item.service === 'CLOUDFRONT') {
ip_ranges.push(item.ipv6_prefix);
}
});
}
})
.then(() => {
return internalIpRanges.fetchUrl(CLOUDFARE_V4_URL);
})
.then(cloudfare_data => {
let items = cloudfare_data.split('\n');
ip_ranges = [... ip_ranges, ... items];
})
.then(() => {
return internalIpRanges.fetchUrl(CLOUDFARE_V6_URL);
})
.then(cloudfare_data => {
let items = cloudfare_data.split('\n');
ip_ranges = [... ip_ranges, ... items];
})
.then(() => {
let clean_ip_ranges = [];
ip_ranges.map(range => {
if (range) {
clean_ip_ranges.push(range);
}
});
return internalIpRanges.generateConfig(clean_ip_ranges)
.then(() => {
if (internalIpRanges.iteration_count) {
// Reload nginx
return internalNginx.reload();
}
});
})
.then(() => {
internalIpRanges.interval_processing = false;
internalIpRanges.iteration_count++;
})
.catch(err => {
logger.error(err.message);
internalIpRanges.interval_processing = false;
});
}
},
/**
* @param {Array} ip_ranges
* @returns {Promise}
*/
generateConfig: (ip_ranges) => {
if (debug_mode) {
logger.info('Generating IP Ranges Config', ip_ranges);
}
let renderEngine = Liquid({
root: __dirname + '/../templates/'
});
return new Promise((resolve, reject) => {
let template = null;
let filename = '/etc/nginx/conf.d/include/ip_ranges.conf';
try {
template = fs.readFileSync(__dirname + '/../templates/ip_ranges.conf', {encoding: 'utf8'});
} catch (err) {
reject(new error.ConfigurationError(err.message));
return;
}
renderEngine
.parseAndRender(template, {ip_ranges: ip_ranges})
.then(config_text => {
fs.writeFileSync(filename, config_text, {encoding: 'utf8'});
if (debug_mode) {
logger.debug('Wrote config:', filename, config_text);
}
resolve(true);
})
.catch(err => {
if (debug_mode) {
logger.warn('Could not write ' + filename + ':', err.message);
}
reject(new error.ConfigurationError(err.message));
});
});
}
};
module.exports = internalIpRanges;

View File

@ -1,12 +1,13 @@
const {Signale} = require('signale');
module.exports = {
global: new Signale({scope: 'Global '}),
migrate: new Signale({scope: 'Migrate '}),
express: new Signale({scope: 'Express '}),
access: new Signale({scope: 'Access '}),
nginx: new Signale({scope: 'Nginx '}),
ssl: new Signale({scope: 'SSL '}),
import: new Signale({scope: 'Importer'}),
setup: new Signale({scope: 'Setup '})
global: new Signale({scope: 'Global '}),
migrate: new Signale({scope: 'Migrate '}),
express: new Signale({scope: 'Express '}),
access: new Signale({scope: 'Access '}),
nginx: new Signale({scope: 'Nginx '}),
ssl: new Signale({scope: 'SSL '}),
import: new Signale({scope: 'Importer '}),
setup: new Signale({scope: 'Setup '}),
ip_ranges: new Signale({scope: 'IP Ranges'})
};

View File

@ -0,0 +1,3 @@
{% for range in ip_ranges %}
set_real_ip_from {{ range }};
{% endfor %}