From db23c9a52f129db1e21f077f270764877921fc05 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Thu, 18 Jan 2024 12:26:55 +1000 Subject: [PATCH] Refactor certbot plugins install - Added a script to install every single plugin, used in development and debugging - Improved certbot plugin install commands - Adjusted some version for plugins to install properly - It's noted that some plugins require deps that do not match other plugins, however these use cases should be extremely rare --- backend/internal/certificate.js | 62 +- backend/lib/certbot.js | 46 ++ backend/lib/error.js | 11 +- backend/lib/utils.js | 27 +- backend/logger.js | 1 + backend/scripts/install-certbot-plugins | 49 ++ .../s6-rc.d/prepare/30-ownership.sh | 1 + frontend/js/app/nginx/certificates/form.ejs | 48 +- frontend/js/app/nginx/certificates/form.js | 16 +- .../js/app/nginx/certificates/list/item.ejs | 4 +- frontend/js/app/nginx/dead/form.ejs | 46 +- frontend/js/app/nginx/proxy/form.ejs | 46 +- frontend/js/app/nginx/redirection/form.ejs | 46 +- global/README.md | 21 + global/certbot-dns-plugins.js | 615 ------------------ global/certbot-dns-plugins.json | 426 ++++++++++++ global/test.js | 3 + 17 files changed, 702 insertions(+), 766 deletions(-) create mode 100644 backend/lib/certbot.js create mode 100755 backend/scripts/install-certbot-plugins create mode 100644 global/README.md delete mode 100644 global/certbot-dns-plugins.js create mode 100644 global/certbot-dns-plugins.json create mode 100644 global/test.js diff --git a/backend/internal/certificate.js b/backend/internal/certificate.js index 8a917ef1..60337049 100644 --- a/backend/internal/certificate.js +++ b/backend/internal/certificate.js @@ -9,10 +9,11 @@ const error = require('../lib/error'); const utils = require('../lib/utils'); const certificateModel = require('../models/certificate'); const tokenModel = require('../models/token'); -const dnsPlugins = require('../global/certbot-dns-plugins'); +const dnsPlugins = require('../global/certbot-dns-plugins.json'); const internalAuditLog = require('./audit-log'); const internalNginx = require('./nginx'); const internalHost = require('./host'); +const certbot = require('../lib/certbot'); const archiver = require('archiver'); const path = require('path'); const { isArray } = require('lodash'); @@ -849,26 +850,20 @@ const internalCertificate = { /** * @param {Object} certificate the certificate row - * @param {String} dns_provider the dns provider name (key used in `certbot-dns-plugins.js`) + * @param {String} dns_provider the dns provider name (key used in `certbot-dns-plugins.json`) * @param {String | null} credentials the content of this providers credentials file - * @param {String} propagation_seconds the cloudflare api token + * @param {String} propagation_seconds * @returns {Promise} */ - requestLetsEncryptSslWithDnsChallenge: (certificate) => { - const dns_plugin = dnsPlugins[certificate.meta.dns_provider]; - - if (!dns_plugin) { - throw Error(`Unknown DNS provider '${certificate.meta.dns_provider}'`); - } - - logger.info(`Requesting Let'sEncrypt certificates via ${dns_plugin.display_name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`); + requestLetsEncryptSslWithDnsChallenge: async (certificate) => { + await certbot.installPlugin(certificate.meta.dns_provider); + const dnsPlugin = dnsPlugins[certificate.meta.dns_provider]; + logger.info(`Requesting Let'sEncrypt certificates via ${dnsPlugin.name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`); const credentialsLocation = '/etc/letsencrypt/credentials/credentials-' + certificate.id; // Escape single quotes and backslashes const escapedCredentials = certificate.meta.dns_provider_credentials.replaceAll('\'', '\\\'').replaceAll('\\', '\\\\'); const credentialsCmd = 'mkdir -p /etc/letsencrypt/credentials 2> /dev/null; echo \'' + escapedCredentials + '\' > \'' + credentialsLocation + '\' && chmod 600 \'' + credentialsLocation + '\''; - // we call `. /opt/certbot/bin/activate` (`.` is alternative to `source` in dash) to access certbot venv - const prepareCmd = '. /opt/certbot/bin/activate && pip install --no-cache-dir ' + dns_plugin.package_name + (dns_plugin.version_requirement || '') + ' ' + dns_plugin.dependencies + ' && deactivate'; // Whether the plugin has a ---credentials argument const hasConfigArg = certificate.meta.dns_provider !== 'route53'; @@ -881,15 +876,15 @@ const internalCertificate = { '--agree-tos ' + '--email "' + certificate.meta.letsencrypt_email + '" ' + '--domains "' + certificate.domain_names.join(',') + '" ' + - '--authenticator ' + dns_plugin.full_plugin_name + ' ' + + '--authenticator ' + dnsPlugin.full_plugin_name + ' ' + ( hasConfigArg - ? '--' + dns_plugin.full_plugin_name + '-credentials "' + credentialsLocation + '"' + ? '--' + dnsPlugin.full_plugin_name + '-credentials "' + credentialsLocation + '"' : '' ) + ( certificate.meta.propagation_seconds !== undefined - ? ' --' + dns_plugin.full_plugin_name + '-propagation-seconds ' + certificate.meta.propagation_seconds + ? ' --' + dnsPlugin.full_plugin_name + '-propagation-seconds ' + certificate.meta.propagation_seconds : '' ) + (letsencryptStaging ? ' --staging' : ''); @@ -903,24 +898,19 @@ const internalCertificate = { mainCmd = mainCmd + ' --dns-duckdns-no-txt-restore'; } - logger.info('Command:', `${credentialsCmd} && ${prepareCmd} && ${mainCmd}`); + logger.info('Command:', `${credentialsCmd} && && ${mainCmd}`); - return utils.exec(credentialsCmd) - .then(() => { - return utils.exec(prepareCmd) - .then(() => { - return utils.exec(mainCmd) - .then(async (result) => { - logger.info(result); - return result; - }); - }); - }).catch(async (err) => { - // Don't fail if file does not exist - const delete_credentialsCmd = `rm -f '${credentialsLocation}' || true`; - await utils.exec(delete_credentialsCmd); - throw err; - }); + try { + await utils.exec(credentialsCmd); + const result = await utils.exec(mainCmd); + logger.info(result); + return result; + } catch (err) { + // Don't fail if file does not exist + const delete_credentialsCmd = `rm -f '${credentialsLocation}' || true`; + await utils.exec(delete_credentialsCmd); + throw err; + } }, @@ -999,13 +989,13 @@ const internalCertificate = { * @returns {Promise} */ renewLetsEncryptSslWithDnsChallenge: (certificate) => { - const dns_plugin = dnsPlugins[certificate.meta.dns_provider]; + const dnsPlugin = dnsPlugins[certificate.meta.dns_provider]; - if (!dns_plugin) { + if (!dnsPlugin) { throw Error(`Unknown DNS provider '${certificate.meta.dns_provider}'`); } - logger.info(`Renewing Let'sEncrypt certificates via ${dns_plugin.display_name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`); + logger.info(`Renewing Let'sEncrypt certificates via ${dnsPlugin.name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`); let mainCmd = certbotCommand + ' renew --force-renewal ' + '--config "' + letsencryptConfig + '" ' + diff --git a/backend/lib/certbot.js b/backend/lib/certbot.js new file mode 100644 index 00000000..c9d6c488 --- /dev/null +++ b/backend/lib/certbot.js @@ -0,0 +1,46 @@ +const dnsPlugins = require('../global/certbot-dns-plugins.json'); +const utils = require('./utils'); +const error = require('./error'); +const logger = require('../logger').certbot; + +// const letsencryptStaging = config.useLetsencryptStaging(); +// const letsencryptConfig = '/etc/letsencrypt.ini'; +// const certbotCommand = 'certbot'; + +// const acmeVersion = '1.32.0'; +const CERTBOT_VERSION_REPLACEMENT = '$(certbot --version | grep -Eo \'[0-9](\\.[0-9]+)+\')'; + +const certbot = { + + /** + * Installs a cerbot plugin given the key for the object from + * ../global/certbot-dns-plugins.json + * + * @param {string} pluginKey + * @returns {Object} + */ + installPlugin: async function (pluginKey) { + if (typeof dnsPlugins[pluginKey] === 'undefined') { + // throw Error(`Certbot plugin ${pluginKey} not found`); + throw new error.ItemNotFoundError(pluginKey); + } + + const plugin = dnsPlugins[pluginKey]; + logger.start(`Installing ${pluginKey}...`); + + plugin.version = plugin.version.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT); + plugin.dependencies = plugin.dependencies.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT); + + const cmd = '. /opt/certbot/bin/activate && pip install --no-cache-dir ' + plugin.dependencies + ' ' + plugin.package_name + plugin.version + ' ' + ' && deactivate'; + return utils.exec(cmd) + .then((result) => { + logger.complete(`Installed ${pluginKey}`); + return result; + }) + .catch((err) => { + throw err; + }); + }, +}; + +module.exports = certbot; diff --git a/backend/lib/error.js b/backend/lib/error.js index 9e456f05..413d6a7d 100644 --- a/backend/lib/error.js +++ b/backend/lib/error.js @@ -82,7 +82,16 @@ module.exports = { this.message = message; this.public = false; this.status = 400; - } + }, + + CommandError: function (stdErr, code, previous) { + Error.captureStackTrace(this, this.constructor); + this.name = this.constructor.name; + this.previous = previous; + this.message = stdErr; + this.code = code; + this.public = false; + }, }; _.forEach(module.exports, function (error) { diff --git a/backend/lib/utils.js b/backend/lib/utils.js index 2a184ee1..bcdb3341 100644 --- a/backend/lib/utils.js +++ b/backend/lib/utils.js @@ -3,23 +3,27 @@ const exec = require('child_process').exec; const execFile = require('child_process').execFile; const { Liquid } = require('liquidjs'); const logger = require('../logger').global; +const error = require('./error'); module.exports = { - /** - * @param {String} cmd - * @returns {Promise} - */ - exec: function (cmd) { - return new Promise((resolve, reject) => { - exec(cmd, function (err, stdout, /*stderr*/) { - if (err && typeof err === 'object') { - reject(err); + exec: async function(cmd, options = {}) { + logger.debug('CMD:', cmd); + + const { stdout, stderr } = await new Promise((resolve, reject) => { + const child = exec(cmd, options, (isError, stdout, stderr) => { + if (isError) { + reject(new error.CommandError(stderr, isError)); } else { - resolve(stdout.trim()); + resolve({ stdout, stderr }); } }); + + child.on('error', (e) => { + reject(new error.CommandError(stderr, 1, e)); + }); }); + return stdout; }, /** @@ -28,7 +32,8 @@ module.exports = { * @returns {Promise} */ execFile: function (cmd, args) { - logger.debug('CMD: ' + cmd + ' ' + (args ? args.join(' ') : '')); + // logger.debug('CMD: ' + cmd + ' ' + (args ? args.join(' ') : '')); + return new Promise((resolve, reject) => { execFile(cmd, args, function (err, stdout, /*stderr*/) { if (err && typeof err === 'object') { diff --git a/backend/logger.js b/backend/logger.js index 680af6d5..0ebb07c5 100644 --- a/backend/logger.js +++ b/backend/logger.js @@ -7,6 +7,7 @@ module.exports = { access: new Signale({scope: 'Access '}), nginx: new Signale({scope: 'Nginx '}), ssl: new Signale({scope: 'SSL '}), + certbot: new Signale({scope: 'Certbot '}), import: new Signale({scope: 'Importer '}), setup: new Signale({scope: 'Setup '}), ip_ranges: new Signale({scope: 'IP Ranges'}) diff --git a/backend/scripts/install-certbot-plugins b/backend/scripts/install-certbot-plugins new file mode 100755 index 00000000..bf995410 --- /dev/null +++ b/backend/scripts/install-certbot-plugins @@ -0,0 +1,49 @@ +#!/usr/bin/node + +// Usage: +// Install all plugins defined in `certbot-dns-plugins.json`: +// ./install-certbot-plugins +// Install one or more specific plugins: +// ./install-certbot-plugins route53 cloudflare +// +// Usage with a running docker container: +// docker exec npm_core /command/s6-setuidgid 1000:1000 bash -c "/app/scripts/install-certbot-plugins" +// + +const dnsPlugins = require('../global/certbot-dns-plugins.json'); +const certbot = require('../lib/certbot'); +const logger = require('../logger').certbot; +const batchflow = require('batchflow'); + +let hasErrors = false; +let failingPlugins = []; + +let pluginKeys = Object.keys(dnsPlugins); +if (process.argv.length > 2) { + pluginKeys = process.argv.slice(2); +} + +batchflow(pluginKeys).sequential() + .each((i, pluginKey, next) => { + certbot.installPlugin(pluginKey) + .then(() => { + next(); + }) + .catch((err) => { + hasErrors = true; + failingPlugins.push(pluginKey); + next(err); + }); + }) + .error((err) => { + logger.error(err.message); + }) + .end(() => { + if (hasErrors) { + logger.error('Some plugins failed to install. Please check the logs above. Failing plugins: ' + '\n - ' + failingPlugins.join('\n - ')); + process.exit(1); + } else { + logger.complete('Plugins installed successfully'); + process.exit(0); + } + }); diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh index fa4e262c..27a1fe24 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/prepare/30-ownership.sh @@ -24,4 +24,5 @@ chown -R "$PUID:$PGID" /etc/nginx/nginx.conf chown -R "$PUID:$PGID" /etc/nginx/conf.d # Prevents errors when installing python certbot plugins when non-root +chown "$PUID:$PGID" /opt/certbot /opt/certbot/bin chown -R "$PUID:$PGID" /opt/certbot/lib/python*/site-packages diff --git a/frontend/js/app/nginx/certificates/form.ejs b/frontend/js/app/nginx/certificates/form.ejs index 7fc12785..6adb4b01 100644 --- a/frontend/js/app/nginx/certificates/form.ejs +++ b/frontend/js/app/nginx/certificates/form.ejs @@ -22,7 +22,7 @@
- + <%- i18n('certificates', 'reachability-info') %>
@@ -38,11 +38,11 @@