Fetch ip ranges for CDN servers, Cloudfront and Cloudfare

rootfs/etc/nginx/conf.d/default.conf

@@ -1,3 +1,6 @@
+# Generated IP Ranges for safe real IP determination
+include conf.d/include/ip_ranges.conf;
+
 # Healthcheck Host which proxies to the Manager,
 # thus the healthcheck ensures both services are running
 server {

rootfs/etc/nginx/conf.d/include/ip_ranges.conf

@@ -0,0 +1,174 @@
+
+set_real_ip_from 13.124.199.0/24;
+
+set_real_ip_from 34.226.14.0/24;
+
+set_real_ip_from 52.124.128.0/17;
+
+set_real_ip_from 54.230.0.0/16;
+
+set_real_ip_from 54.239.128.0/18;
+
+set_real_ip_from 52.82.128.0/19;
+
+set_real_ip_from 99.84.0.0/16;
+
+set_real_ip_from 52.15.127.128/26;
+
+set_real_ip_from 35.158.136.0/24;
+
+set_real_ip_from 52.57.254.0/24;
+
+set_real_ip_from 18.216.170.128/25;
+
+set_real_ip_from 13.54.63.128/26;
+
+set_real_ip_from 13.59.250.0/26;
+
+set_real_ip_from 13.210.67.128/26;
+
+set_real_ip_from 35.167.191.128/26;
+
+set_real_ip_from 52.47.139.0/24;
+
+set_real_ip_from 52.199.127.192/26;
+
+set_real_ip_from 52.212.248.0/26;
+
+set_real_ip_from 205.251.192.0/19;
+
+set_real_ip_from 52.66.194.128/26;
+
+set_real_ip_from 54.239.192.0/19;
+
+set_real_ip_from 70.132.0.0/18;
+
+set_real_ip_from 13.32.0.0/15;
+
+set_real_ip_from 13.113.203.0/24;
+
+set_real_ip_from 34.195.252.0/24;
+
+set_real_ip_from 35.162.63.192/26;
+
+set_real_ip_from 34.223.12.224/27;
+
+set_real_ip_from 13.35.0.0/16;
+
+set_real_ip_from 204.246.172.0/23;
+
+set_real_ip_from 204.246.164.0/22;
+
+set_real_ip_from 52.56.127.0/25;
+
+set_real_ip_from 204.246.168.0/22;
+
+set_real_ip_from 13.228.69.0/24;
+
+set_real_ip_from 34.216.51.0/25;
+
+set_real_ip_from 71.152.0.0/17;
+
+set_real_ip_from 216.137.32.0/19;
+
+set_real_ip_from 205.251.249.0/24;
+
+set_real_ip_from 99.86.0.0/16;
+
+set_real_ip_from 52.46.0.0/18;
+
+set_real_ip_from 52.84.0.0/15;
+
+set_real_ip_from 54.233.255.128/26;
+
+set_real_ip_from 64.252.64.0/18;
+
+set_real_ip_from 52.52.191.128/26;
+
+set_real_ip_from 204.246.174.0/23;
+
+set_real_ip_from 64.252.128.0/18;
+
+set_real_ip_from 205.251.254.0/24;
+
+set_real_ip_from 143.204.0.0/16;
+
+set_real_ip_from 205.251.252.0/23;
+
+set_real_ip_from 52.78.247.128/26;
+
+set_real_ip_from 204.246.176.0/20;
+
+set_real_ip_from 52.220.191.0/26;
+
+set_real_ip_from 13.249.0.0/16;
+
+set_real_ip_from 54.240.128.0/18;
+
+set_real_ip_from 205.251.250.0/23;
+
+set_real_ip_from 52.222.128.0/17;
+
+set_real_ip_from 54.182.0.0/16;
+
+set_real_ip_from 54.192.0.0/16;
+
+set_real_ip_from 34.232.163.208/29;
+
+set_real_ip_from 2600:9000:eee::/48;
+
+set_real_ip_from 2600:9000:4000::/36;
+
+set_real_ip_from 2600:9000:3000::/36;
+
+set_real_ip_from 2600:9000:f000::/36;
+
+set_real_ip_from 2600:9000:fff::/48;
+
+set_real_ip_from 2600:9000:2000::/36;
+
+set_real_ip_from 2600:9000:1000::/36;
+
+set_real_ip_from 2600:9000:ddd::/48;
+
+set_real_ip_from 2600:9000:5300::/40;
+
+set_real_ip_from 103.21.244.0/22;
+
+set_real_ip_from 103.22.200.0/22;
+
+set_real_ip_from 103.31.4.0/22;
+
+set_real_ip_from 104.16.0.0/12;
+
+set_real_ip_from 108.162.192.0/18;
+
+set_real_ip_from 131.0.72.0/22;
+
+set_real_ip_from 141.101.64.0/18;
+
+set_real_ip_from 162.158.0.0/15;
+
+set_real_ip_from 172.64.0.0/13;
+
+set_real_ip_from 173.245.48.0/20;
+
+set_real_ip_from 188.114.96.0/20;
+
+set_real_ip_from 190.93.240.0/20;
+
+set_real_ip_from 197.234.240.0/22;
+
+set_real_ip_from 198.41.128.0/17;
+
+set_real_ip_from 2400:cb00::/32;
+
+set_real_ip_from 2405:b500::/32;
+
+set_real_ip_from 2606:4700::/32;
+
+set_real_ip_from 2803:f800::/32;
+
+set_real_ip_from 2c0f:f248::/32;
+
+set_real_ip_from 2a06:98c0::/29;

src/backend/index.js

@@ -11,6 +11,7 @@ function appStart () {
     const app                 = require('./app');
     const apiValidator        = require('./lib/validator/api');
     const internalCertificate = require('./internal/certificate');
+    const internalIpRanges    = require('./internal/ip_ranges');
 
     return migrate.latest()
         .then(setup)
@@ -18,9 +19,11 @@ function appStart () {
         .then(() => {
             return apiValidator.loadSchemas;
         })
+        .then(internalIpRanges.fetch)
         .then(() => {
 
             internalCertificate.initTimer();
+            internalIpRanges.initTimer();
 
             const server = app.listen(81, () => {
                 logger.info('PID ' + process.pid + ' listening on port 81 ...');

src/backend/internal/ip_ranges.js

@@ -0,0 +1,163 @@
+'use strict';
+
+const https         = require('https');
+const fs            = require('fs');
+const _             = require('lodash');
+const logger        = require('../logger').ip_ranges;
+const debug_mode    = process.env.NODE_ENV !== 'production';
+const error         = require('../lib/error');
+const internalNginx = require('./nginx');
+const Liquid        = require('liquidjs');
+
+const CLOUDFRONT_URL   = 'https://ip-ranges.amazonaws.com/ip-ranges.json';
+const CLOUDFARE_V4_URL = 'https://www.cloudflare.com/ips-v4';
+const CLOUDFARE_V6_URL = 'https://www.cloudflare.com/ips-v6';
+
+const internalIpRanges = {
+
+    interval_timeout:    1000 * 60 * 60 * 6, // 6 hours
+    interval:            null,
+    interval_processing: false,
+    iteration_count:     0,
+
+    initTimer: () => {
+        logger.info('IP Ranges Renewal Timer initialized');
+        internalIpRanges.interval = setInterval(internalIpRanges.fetch, internalIpRanges.interval_timeout);
+    },
+
+    fetchUrl: url => {
+        return new Promise((resolve, reject) => {
+            logger.info('Fetching ' + url);
+            return https.get(url, res => {
+                res.setEncoding('utf8');
+                let raw_data = '';
+                res.on('data', chunk => {
+                    raw_data += chunk;
+                });
+
+                res.on('end', () => {
+                    resolve(raw_data);
+                });
+            }).on('error', err => {
+                reject(err);
+            });
+        });
+    },
+
+    /**
+     * Triggered at startup and then later by a timer, this will fetch the ip ranges from services and apply them to nginx.
+     */
+    fetch: () => {
+        if (!internalIpRanges.interval_processing) {
+            internalIpRanges.interval_processing = true;
+            logger.info('Fetching IP Ranges from online services...');
+
+            let ip_ranges = [];
+
+            return internalIpRanges.fetchUrl(CLOUDFRONT_URL)
+                .then(cloudfront_data => {
+                    let data = JSON.parse(cloudfront_data);
+
+                    if (data && typeof data.prefixes !== 'undefined') {
+                        data.prefixes.map(item => {
+                            if (item.service === 'CLOUDFRONT') {
+                                ip_ranges.push(item.ip_prefix);
+                            }
+                        });
+                    }
+
+                    if (data && typeof data.ipv6_prefixes !== 'undefined') {
+                        data.ipv6_prefixes.map(item => {
+                            if (item.service === 'CLOUDFRONT') {
+                                ip_ranges.push(item.ipv6_prefix);
+                            }
+                        });
+                    }
+                })
+                .then(() => {
+                    return internalIpRanges.fetchUrl(CLOUDFARE_V4_URL);
+                })
+                .then(cloudfare_data => {
+                    let items = cloudfare_data.split('\n');
+                    ip_ranges = [... ip_ranges, ... items];
+                })
+                .then(() => {
+                    return internalIpRanges.fetchUrl(CLOUDFARE_V6_URL);
+                })
+                .then(cloudfare_data => {
+                    let items = cloudfare_data.split('\n');
+                    ip_ranges = [... ip_ranges, ... items];
+                })
+                .then(() => {
+                    let clean_ip_ranges = [];
+                    ip_ranges.map(range => {
+                        if (range) {
+                            clean_ip_ranges.push(range);
+                        }
+                    });
+
+                    return internalIpRanges.generateConfig(clean_ip_ranges)
+                        .then(() => {
+                            if (internalIpRanges.iteration_count) {
+                                // Reload nginx
+                                return internalNginx.reload();
+                            }
+                        });
+                })
+                .then(() => {
+                    internalIpRanges.interval_processing = false;
+                    internalIpRanges.iteration_count++;
+                })
+                .catch(err => {
+                    logger.error(err.message);
+                    internalIpRanges.interval_processing = false;
+                });
+        }
+    },
+
+    /**
+     * @param   {Array}  ip_ranges
+     * @returns {Promise}
+     */
+    generateConfig: (ip_ranges) => {
+        if (debug_mode) {
+            logger.info('Generating IP Ranges Config', ip_ranges);
+        }
+
+        let renderEngine = Liquid({
+            root: __dirname + '/../templates/'
+        });
+
+        return new Promise((resolve, reject) => {
+            let template = null;
+            let filename = '/etc/nginx/conf.d/include/ip_ranges.conf';
+            try {
+                template = fs.readFileSync(__dirname + '/../templates/ip_ranges.conf', {encoding: 'utf8'});
+            } catch (err) {
+                reject(new error.ConfigurationError(err.message));
+                return;
+            }
+
+            renderEngine
+                .parseAndRender(template, {ip_ranges: ip_ranges})
+                .then(config_text => {
+                    fs.writeFileSync(filename, config_text, {encoding: 'utf8'});
+
+                    if (debug_mode) {
+                        logger.debug('Wrote config:', filename, config_text);
+                    }
+
+                    resolve(true);
+                })
+                .catch(err => {
+                    if (debug_mode) {
+                        logger.warn('Could not write ' + filename + ':', err.message);
+                    }
+
+                    reject(new error.ConfigurationError(err.message));
+                });
+        });
+    }
+};
+
+module.exports = internalIpRanges;

src/backend/logger.js

@@ -7,6 +7,7 @@ module.exports = {
     access:    new Signale({scope: 'Access   '}),
     nginx:     new Signale({scope: 'Nginx    '}),
     ssl:       new Signale({scope: 'SSL      '}),
-    import:  new Signale({scope: 'Importer'}),
+    import:    new Signale({scope: 'Importer '}),
-    setup:   new Signale({scope: 'Setup   '})
+    setup:     new Signale({scope: 'Setup    '}),
+    ip_ranges: new Signale({scope: 'IP Ranges'})
 };

src/backend/templates/ip_ranges.conf

@@ -0,0 +1,3 @@
+{% for range in ip_ranges %}
+set_real_ip_from {{ range }};
+{% endfor %}