certificates work

rootfs/etc/nginx/nginx.conf

@@ -54,6 +54,7 @@ http {
   include /data/nginx/proxy_host/*.conf;
   include /data/nginx/redirection_host/*.conf;
   include /data/nginx/dead_host/*.conf;
+  include /data/nginx/temp/*.conf;
 }
 
 stream {

rootfs/etc/services.d/nginx/run

@@ -3,7 +3,7 @@
 mkdir -p /tmp/nginx/body \
   /var/log/nginx \
   /data/{nginx,logs,access} \
-  /data/nginx/{proxy_host,redirection_host,stream,dead_host} \
+  /data/nginx/{proxy_host,redirection_host,stream,dead_host,temp} \
   /var/lib/nginx/cache/{public,private}
 
 touch /var/log/nginx/error.log && chmod 777 /var/log/nginx/error.log

src/backend/internal/certificate.js

@@ -10,6 +10,8 @@ const tempWrite        = require('temp-write');
 const utils            = require('../lib/utils');
 const moment           = require('moment');
 const debug_mode       = process.env.NODE_ENV !== 'production';
+const internalNginx    = require('./nginx');
+const internalHost     = require('./host');
 const certbot_command  = '/usr/bin/certbot';
 
 function omissions () {
@@ -32,8 +34,6 @@ const internalCertificate = {
      * Triggered by a timer, this will check for expiring hosts and renew their ssl certs if required
      */
     processExpiringHosts: () => {
-        let internalNginx = require('./nginx');
-
         if (!internalCertificate.interval_processing) {
             internalCertificate.interval_processing = true;
             logger.info('Renewing SSL certs close to expiry...');
@@ -75,18 +75,94 @@ const internalCertificate = {
                     .omit(omissions())
                     .insertAndFetch(data);
             })
-            .then(row => {
+            .then(certificate => {
-                data.meta = _.assign({}, data.meta || {}, row.meta);
+                if (certificate.provider === 'letsencrypt') {
+                    // Request a new Cert from LE. Let the fun begin.
+
+                    // 1. Find out any hosts that are using any of the hostnames in this cert
+                    // 2. Disable them in nginx temporarily
+                    // 3. Generate the LE config
+                    // 4. Request cert
+                    // 5. Remove LE config
+                    // 6. Re-instate previously disabled hosts
+
+                    // 1. Find out any hosts that are using any of the hostnames in this cert
+                    return internalHost.getHostsWithDomains(certificate.domain_names)
+                        .then(in_use_result => {
+                            // 2. Disable them in nginx temporarily
+                            return internalCertificate.disableInUseHosts(in_use_result)
+                                .then(() => {
+                                    return in_use_result;
+                                });
+                        })
+                        .then(in_use_result => {
+                            // 3. Generate the LE config
+                            return internalNginx.generateLetsEncryptRequestConfig(certificate)
+                                .then(internalNginx.reload)
+                                .then(() => {
+                                    // 4. Request cert
+                                    return internalCertificate.requestLetsEncryptSsl(certificate);
+                                })
+                                .then(() => {
+                                    // 5. Remove LE config
+                                    return internalNginx.deleteLetsEncryptRequestConfig(certificate);
+                                })
+                                .then(internalNginx.reload)
+                                .then(() => {
+                                    // 6. Re-instate previously disabled hosts
+                                    return internalCertificate.enableInUseHosts(in_use_result);
+                                })
+                                .then(() => {
+                                    return certificate;
+                                })
+                                .catch(err => {
+                                    // In the event of failure, revert things and throw err back
+                                    return internalNginx.deleteLetsEncryptRequestConfig(certificate)
+                                        .then(() => {
+                                            return internalCertificate.enableInUseHosts(in_use_result);
+                                        })
+                                        .then(internalNginx.reload)
+                                        .then(() => {
+                                            throw err;
+                                        });
+                                });
+                        })
+                        .then(() => {
+                            // At this point, the letsencrypt cert should exist on disk.
+                            // Lets get the expiry date from the file and update the row silently
+                            return internalCertificate.getCertificateInfoFromFile('/etc/letsencrypt/live/npm-' + certificate.id + '/fullchain.pem')
+                                .then(cert_info => {
+                                    return certificateModel
+                                        .query()
+                                        .patchAndFetchById(certificate.id, {
+                                            expires_on: certificateModel.raw('FROM_UNIXTIME(' + cert_info.dates.to + ')')
+                                        })
+                                        .then(saved_row => {
+                                            // Add cert data for audit log
+                                            saved_row.meta = _.assign({}, saved_row.meta, {
+                                                letsencrypt_certificate: cert_info
+                                            });
+
+                                            return saved_row;
+                                        });
+                                });
+                        });
+                } else {
+                    return certificate;
+                }
+            }).then(certificate => {
+
+                data.meta = _.assign({}, data.meta || {}, certificate.meta);
 
                 // Add to audit log
                 return internalAuditLog.add(access, {
                     action:      'created',
                     object_type: 'certificate',
-                    object_id:   row.id,
+                    object_id:   certificate.id,
                     meta:        data
                 })
                     .then(() => {
-                        return row;
+                        return certificate;
                     });
             });
     },
@@ -114,7 +190,6 @@ const internalCertificate = {
                     .query()
                     .omit(omissions())
                     .patchAndFetchById(row.id, data)
-                    .debug()
                     .then(saved_row => {
                         saved_row.meta = internalCertificate.cleanMeta(saved_row.meta);
                         data.meta      = internalCertificate.cleanMeta(data.meta);
@@ -221,6 +296,12 @@ const internalCertificate = {
                             object_id:   row.id,
                             meta:        _.omit(row, omissions())
                         });
+                    })
+                    .then(() => {
+                        if (row.provider === 'letsencrypt') {
+                            // Revoke the cert
+                            return internalCertificate.revokeLetsEncryptSsl(row);
+                        }
                     });
             })
             .then(() => {
@@ -229,7 +310,7 @@ const internalCertificate = {
     },
 
     /**
-     * All Lists
+     * All Certs
      *
      * @param   {Access}  access
      * @param   {Array}   [expand]
@@ -381,6 +462,7 @@ const internalCertificate = {
                             }
                         });
 
+                        // TODO: This uses a mysql only raw function that won't translate to postgres
                         return internalCertificate.update(access, {
                             id:           data.id,
                             expires_on:   certificateModel.raw('FROM_UNIXTIME(' + validations.certificate.dates.to + ')'),
@@ -428,9 +510,28 @@ const internalCertificate = {
     getCertificateInfo: (certificate, throw_expired) => {
         return tempWrite(certificate, '/tmp')
             .then(filepath => {
+                return internalCertificate.getCertificateInfoFromFile(filepath, throw_expired)
+                    .then(cert_data => {
+                        fs.unlinkSync(filepath);
+                        return cert_data;
+                    }).catch(err => {
+                        fs.unlinkSync(filepath);
+                        throw err;
+                    });
+            });
+    },
+
+    /**
+     * Uses the openssl command to both validate and get info out of the certificate.
+     * It will save the file to disk first, then run commands on it, then delete the file.
+     *
+     * @param {String}  certificate_file The file location on disk
+     * @param {Boolean} [throw_expired]  Throw when the certificate is out of date
+     */
+    getCertificateInfoFromFile: (certificate_file, throw_expired) => {
         let cert_data = {};
 
-                return utils.exec('openssl x509 -in ' + filepath + ' -subject -noout')
+        return utils.exec('openssl x509 -in ' + certificate_file + ' -subject -noout')
             .then(result => {
                 // subject=CN = something.example.com
                 let regex = /(?:subject=)?[^=]+=\s+(\S+)/gim;
@@ -443,7 +544,7 @@ const internalCertificate = {
                 cert_data['cn'] = match[1];
             })
             .then(() => {
-                        return utils.exec('openssl x509 -in ' + filepath + ' -issuer -noout');
+                return utils.exec('openssl x509 -in ' + certificate_file + ' -issuer -noout');
             })
             .then(result => {
                 // issuer=C = US, O = Let's Encrypt, CN = Let's Encrypt Authority X3
@@ -457,7 +558,7 @@ const internalCertificate = {
                 cert_data['issuer'] = match[1];
             })
             .then(() => {
-                        return utils.exec('openssl x509 -in ' + filepath + ' -dates -noout');
+                return utils.exec('openssl x509 -in ' + certificate_file + ' -dates -noout');
             })
             .then(result => {
                 // notBefore=Jul 14 04:04:29 2018 GMT
@@ -493,15 +594,11 @@ const internalCertificate = {
                     from: valid_from,
                     to:   valid_to
                 };
-                    })
+
-                    .then(() => {
-                        fs.unlinkSync(filepath);
                 return cert_data;
             }).catch(err => {
-                        fs.unlinkSync(filepath);
                 throw new error.ValidationError('Certificate is not valid (' + err.message + ')', err);
             });
-            });
     },
 
     /**
@@ -521,6 +618,7 @@ const internalCertificate = {
                 }
             }
         });
+
         return meta;
     },
 
@@ -531,13 +629,19 @@ const internalCertificate = {
     requestLetsEncryptSsl: certificate => {
         logger.info('Requesting Let\'sEncrypt certificates for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', '));
 
-        return utils.exec(certbot_command + ' certonly --cert-name "npm-' + certificate.id + '" --agree-tos ' +
+        let cmd = certbot_command + ' certonly --cert-name "npm-' + certificate.id + '" --agree-tos ' +
             '--email "' + certificate.meta.letsencrypt_email + '" ' +
             '--preferred-challenges "http" ' +
             '-n -a webroot -d "' + certificate.domain_names.join(',') + '" ' +
-            (debug_mode ? '--staging' : ''))
+            (debug_mode ? '--staging' : '');
+
+        if (debug_mode) {
+            logger.info('Command:', cmd);
+        }
+
+        return utils.exec(cmd)
             .then(result => {
-                logger.info(result);
+                logger.success(result);
                 return result;
             });
     },
@@ -549,13 +653,49 @@ const internalCertificate = {
     renewLetsEncryptSsl: certificate => {
         logger.info('Renewing Let\'sEncrypt certificates for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', '));
 
-        return utils.exec(certbot_command + ' renew -n --force-renewal --disable-hook-validation --cert-name "npm-' + certificate.id + '" ' + (debug_mode ? '--staging' : ''))
+        let cmd = certbot_command + ' renew -n --force-renewal --disable-hook-validation --cert-name "npm-' + certificate.id + '" ' + (debug_mode ? '--staging' : '');
+
+        if (debug_mode) {
+            logger.info('Command:', cmd);
+        }
+
+        return utils.exec(cmd)
             .then(result => {
                 logger.info(result);
                 return result;
             });
     },
 
+    /**
+     * @param   {Object}  certificate    the certificate row
+     * @param   {Boolean} [throw_errors]
+     * @returns {Promise}
+     */
+    revokeLetsEncryptSsl: (certificate, throw_errors) => {
+        logger.info('Revoking Let\'sEncrypt certificates for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', '));
+
+        let cmd = certbot_command + ' revoke --cert-path "/etc/letsencrypt/live/npm-' + certificate.id + '/fullchain.pem" ' + (debug_mode ? '--staging' : '');
+
+        if (debug_mode) {
+            logger.info('Command:', cmd);
+        }
+
+        return utils.exec(cmd)
+            .then(result => {
+                logger.info(result);
+                return result;
+            })
+            .catch(err => {
+                if (debug_mode) {
+                    logger.error(err.message);
+                }
+
+                if (throw_errors) {
+                    throw err;
+                }
+            });
+    },
+
     /**
      * @param   {Object}  certificate
      * @returns {Boolean}
@@ -564,6 +704,66 @@ const internalCertificate = {
         let le_path = '/etc/letsencrypt/live/npm-' + certificate.id;
 
         return fs.existsSync(le_path + '/fullchain.pem') && fs.existsSync(le_path + '/privkey.pem');
+    },
+
+    /**
+     * @param {Object}  in_use_result
+     * @param {Integer} in_use_result.total_count
+     * @param {Array}   in_use_result.proxy_hosts
+     * @param {Array}   in_use_result.redirection_hosts
+     * @param {Array}   in_use_result.dead_hosts
+     */
+    disableInUseHosts: in_use_result => {
+        if (in_use_result.total_count) {
+            let promises = [];
+
+            if (in_use_result.proxy_hosts.length) {
+                promises.push(internalNginx.bulkDeleteConfigs('proxy_host', in_use_result.proxy_hosts));
+            }
+
+            if (in_use_result.redirection_hosts.length) {
+                promises.push(internalNginx.bulkDeleteConfigs('redirection_host', in_use_result.redirection_hosts));
+            }
+
+            if (in_use_result.dead_hosts.length) {
+                promises.push(internalNginx.bulkDeleteConfigs('dead_host', in_use_result.dead_hosts));
+            }
+
+            return Promise.all(promises);
+
+        } else {
+            return Promise.resolve();
+        }
+    },
+
+    /**
+     * @param {Object}  in_use_result
+     * @param {Integer} in_use_result.total_count
+     * @param {Array}   in_use_result.proxy_hosts
+     * @param {Array}   in_use_result.redirection_hosts
+     * @param {Array}   in_use_result.dead_hosts
+     */
+    enableInUseHosts: in_use_result => {
+        if (in_use_result.total_count) {
+            let promises = [];
+
+            if (in_use_result.proxy_hosts.length) {
+                promises.push(internalNginx.bulkGenerateConfigs('proxy_host', in_use_result.proxy_hosts));
+            }
+
+            if (in_use_result.redirection_hosts.length) {
+                promises.push(internalNginx.bulkGenerateConfigs('redirection_host', in_use_result.redirection_hosts));
+            }
+
+            if (in_use_result.dead_hosts.length) {
+                promises.push(internalNginx.bulkGenerateConfigs('dead_host', in_use_result.dead_hosts));
+            }
+
+            return Promise.all(promises);
+
+        } else {
+            return Promise.resolve();
+        }
     }
 };
 

src/backend/internal/host.js

@@ -6,6 +6,57 @@ const deadHostModel        = require('../models/dead_host');
 
 const internalHost = {
 
+    /**
+     * This returns all the host types with any domain listed in the provided domain_names array.
+     * This is used by the certificates to temporarily disable any host that is using the domain
+     *
+     * @param   {Array}  domain_names
+     * @returns {Promise}
+     */
+    getHostsWithDomains: function (domain_names) {
+        let promises = [
+            proxyHostModel
+                .query()
+                .where('is_deleted', 0),
+            redirectionHostModel
+                .query()
+                .where('is_deleted', 0),
+            deadHostModel
+                .query()
+                .where('is_deleted', 0)
+        ];
+
+        return Promise.all(promises)
+            .then(promises_results => {
+                let response_object = {
+                    total_count:       0,
+                    dead_hosts:        [],
+                    proxy_hosts:       [],
+                    redirection_hosts: []
+                };
+
+                if (promises_results[0]) {
+                    // Proxy Hosts
+                    response_object.proxy_hosts = internalHost._getHostsWithDomains(promises_results[0], domain_names);
+                    response_object.total_count += response_object.proxy_hosts.length;
+                }
+
+                if (promises_results[1]) {
+                    // Redirection Hosts
+                    response_object.redirection_hosts = internalHost._getHostsWithDomains(promises_results[1], domain_names);
+                    response_object.total_count += response_object.redirection_hosts.length;
+                }
+
+                if (promises_results[1]) {
+                    // Dead Hosts
+                    response_object.dead_hosts = internalHost._getHostsWithDomains(promises_results[2], domain_names);
+                    response_object.total_count += response_object.dead_hosts.length;
+                }
+
+                return response_object;
+            });
+    },
+
     /**
      * Internal use only, checks to see if the domain is already taken by any other record
      *
@@ -87,6 +138,37 @@ const internalHost = {
         }
 
         return is_taken;
+    },
+
+    /**
+     * Private call only
+     *
+     * @param   {Array}   hosts
+     * @param   {Array}   domain_names
+     * @returns {Array}
+     */
+    _getHostsWithDomains: function (hosts, domain_names) {
+        let response = [];
+
+        if (hosts && hosts.length) {
+            hosts.map(function (host) {
+                let host_matches = false;
+
+                domain_names.map(function (domain_name) {
+                    host.domain_names.map(function (host_domain_name) {
+                        if (domain_name.toLowerCase() === host_domain_name.toLowerCase()) {
+                            host_matches = true;
+                        }
+                    });
+                });
+
+                if (host_matches) {
+                    response.push(host);
+                }
+            });
+        }
+
+        return response;
     }
 
 };

src/backend/internal/nginx.js

@@ -6,7 +6,6 @@ const Liquid              = require('liquidjs');
 const logger     = require('../logger').nginx;
 const utils      = require('../lib/utils');
 const error      = require('../lib/error');
-const internalCertificate = require('./certificate');
 const debug_mode = process.env.NODE_ENV !== 'production';
 
 const internalNginx = {
@@ -120,7 +119,7 @@ const internalNginx = {
         }
 
         let renderEngine = Liquid({
-            root: __dirname + '/../templates/',
+            root: __dirname + '/../templates/'
         });
 
         return new Promise((resolve, reject) => {
@@ -154,6 +153,85 @@ const internalNginx = {
         });
     },
 
+    /**
+     * This generates a temporary nginx config listening on port 80 for the domain names listed
+     * in the certificate setup. It allows the letsencrypt acme challenge to be requested by letsencrypt
+     * when requesting a certificate without having a hostname set up already.
+     *
+     * @param   {Object}  certificate
+     * @returns {Promise}
+     */
+    generateLetsEncryptRequestConfig: certificate => {
+        if (debug_mode) {
+            logger.info('Generating LetsEncrypt Request Config:', certificate);
+        }
+
+        let renderEngine = Liquid({
+            root: __dirname + '/../templates/'
+        });
+
+        return new Promise((resolve, reject) => {
+            let template = null;
+            let filename = '/data/nginx/temp/letsencrypt_' + certificate.id + '.conf';
+            try {
+                template = fs.readFileSync(__dirname + '/../templates/letsencrypt-request.conf', {encoding: 'utf8'});
+            } catch (err) {
+                reject(new error.ConfigurationError(err.message));
+                return;
+            }
+
+            renderEngine
+                .parseAndRender(template, certificate)
+                .then(config_text => {
+                    fs.writeFileSync(filename, config_text, {encoding: 'utf8'});
+
+                    if (debug_mode) {
+                        logger.success('Wrote config:', filename, config_text);
+                    }
+
+                    resolve(true);
+                })
+                .catch(err => {
+                    if (debug_mode) {
+                        logger.warn('Could not write ' + filename + ':', err.message);
+                    }
+
+                    reject(new error.ConfigurationError(err.message));
+                });
+        });
+    },
+
+    /**
+     * This removes the temporary nginx config file generated by `generateLetsEncryptRequestConfig`
+     *
+     * @param   {Object}  certificate
+     * @param   {Boolean} [throw_errors]
+     * @returns {Promise}
+     */
+    deleteLetsEncryptRequestConfig: (certificate, throw_errors) => {
+        return new Promise((resolve, reject) => {
+            try {
+                let config_file = '/data/nginx/temp/letsencrypt_' + certificate.id + '.conf';
+
+                if (debug_mode) {
+                    logger.warn('Deleting nginx config: ' + config_file);
+                }
+
+                fs.unlinkSync(config_file);
+            } catch (err) {
+                if (debug_mode) {
+                    logger.warn('Could not delete config:', err.message);
+                }
+
+                if (throw_errors) {
+                    reject(err);
+                }
+            }
+
+            resolve();
+        });
+    },
+
     /**
      * @param   {String}  host_type
      * @param   {Object}  host
@@ -184,6 +262,35 @@ const internalNginx = {
 
             resolve();
         });
+    },
+
+    /**
+     * @param   {String}  host_type
+     * @param   {Array}   hosts
+     * @returns {Promise}
+     */
+    bulkGenerateConfigs: (host_type, hosts) => {
+        let promises = [];
+        hosts.map(function (host) {
+            promises.push(internalNginx.generateConfig(host_type, host));
+        });
+
+        return Promise.all(promises);
+    },
+
+    /**
+     * @param   {String}  host_type
+     * @param   {Array}   hosts
+     * @param   {Boolean} [throw_errors]
+     * @returns {Promise}
+     */
+    bulkDeleteConfigs: (host_type, hosts, throw_errors) => {
+        let promises = [];
+        hosts.map(function (host) {
+            promises.push(internalNginx.deleteConfig(host_type, host, throw_errors));
+        });
+
+        return Promise.all(promises);
     }
 };
 

src/backend/templates/_assets.conf

@@ -1,4 +1,4 @@
 {% if caching_enabled == 1 or caching_enabled == true -%}
   # Asset Caching
   include conf.d/include/assets.conf;
-{%- endif %}
+{% endif %}

src/backend/templates/_certificates.conf

@@ -1,12 +1,10 @@
-{%- if certificate and certificate_id > 0 -%}
+{% if certificate and certificate_id > 0 -%}
-{%- if certificate.provider == "letsencrypt" %}
+{% if certificate.provider == "letsencrypt" %}
   # Let's Encrypt SSL
   include conf.d/include/letsencrypt-acme-challenge.conf;
   include conf.d/include/ssl-ciphers.conf;
-  ssl_certificate /etc/letsencrypt/live/npm-{{ certificate.id }}/fullchain.pem;
+  ssl_certificate /etc/letsencrypt/live/npm-{{ certificate_id }}/fullchain.pem;
-  ssl_certificate_key /etc/letsencrypt/live/npm-{{ certificate.id }}/privkey.pem;
+  ssl_certificate_key /etc/letsencrypt/live/npm-{{ certificate_id }}/privkey.pem;
-{%- endif -%}
+{% endif %}
-
   # TODO: Custom SSL paths
-
+{% endif %}
-{%- endif %}

src/backend/templates/_exploits.conf

@@ -1,4 +1,4 @@
-{% if block_exploits == 1 or block_exploits == true -%}
+{% if block_exploits == 1 or block_exploits == true %}
   # Block Exploits
   include conf.d/include/block-exploits.conf;
-{%- endif -%}
+{% endif %}

src/backend/templates/_forced_ssl.conf

@@ -1,6 +1,6 @@
-{%- if certificate and certificate_id > 0 -%}
+{% if certificate and certificate_id > 0 -%}
-{%- if ssl_forced == 1 or ssl_forced == true -%}
+{% if ssl_forced == 1 or ssl_forced == true %}
     # Force SSL
     include conf.d/include/force-ssl.conf;
-{%- endif -%}
+{% endif %}
-{%- endif %}
+{% endif %}

src/backend/templates/_listen.conf

@@ -1,5 +1,5 @@
   listen 80;
-{%- if certificate -%}
+{% if certificate -%}
   listen 443 ssl;
-{%- endif %}
+{% endif %}
   server_name {{ domain_names | join: " " }};

src/backend/templates/letsencrypt-request.conf

@@ -0,0 +1,14 @@
+{% include "_header_comment.conf" %}
+
+server {
+  listen 80;
+  server_name {{ domain_names | join: " " }};
+
+  access_log /data/logs/letsencrypt-requests.log proxy;
+
+  include conf.d/include/letsencrypt-acme-challenge.conf;
+
+  location / {
+    return 404;
+  }
+}