- Updated objection, knex, liquidjs, signale and sqlite3 packages

- Changes for objection migration
- Moved common access template code to an include
- Fixed access rules configuration generation

backend/internal/access-list.js

@@ -3,13 +3,13 @@ const fs                    = require('fs');
 const batchflow             = require('batchflow');
 const logger                = require('../logger').access;
 const error                 = require('../lib/error');
+const utils                 = require('../lib/utils');
 const accessListModel       = require('../models/access_list');
 const accessListAuthModel   = require('../models/access_list_auth');
 const accessListClientModel = require('../models/access_list_client');
 const proxyHostModel        = require('../models/proxy_host');
 const internalAuditLog      = require('./audit-log');
 const internalNginx         = require('./nginx');
-const utils                 = require('../lib/utils');
 
 function omissions () {
 	return ['is_deleted'];
@@ -27,13 +27,13 @@ const internalAccessList = {
 			.then((/*access_data*/) => {
 				return accessListModel
 					.query()
-					.omit(omissions())
 					.insertAndFetch({
 						name:          data.name,
 						satisfy_any:   data.satisfy_any,
 						pass_auth:     data.pass_auth,
 						owner_user_id: access.token.getUserId(1)
-					});
+					})
+					.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
 				data.id = row.id;
@@ -256,35 +256,31 @@ const internalAccessList = {
 					.joinRaw('LEFT JOIN `proxy_host` ON `proxy_host`.`access_list_id` = `access_list`.`id` AND `proxy_host`.`is_deleted` = 0')
 					.where('access_list.is_deleted', 0)
 					.andWhere('access_list.id', data.id)
-					.allowEager('[owner,items,clients,proxy_hosts.[certificate,access_list.[clients,items]]]')
-					.omit(['access_list.is_deleted'])
+					.allowGraph('[owner,items,clients,proxy_hosts.[certificate,access_list.[clients,items]]]')
 					.first();
 
 				if (access_data.permission_visibility !== 'all') {
 					query.andWhere('access_list.owner_user_id', access.token.getUserId(1));
 				}
 
-				// Custom omissions
-				if (typeof data.omit !== 'undefined' && data.omit !== null) {
-					query.omit(data.omit);
-				}
-
 				if (typeof data.expand !== 'undefined' && data.expand !== null) {
-					query.eager('[' + data.expand.join(', ') + ']');
+					query.withGraphFetched('[' + data.expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
-				if (row) {
+				if (!row) {
+					throw new error.ItemNotFoundError(data.id);
+				}
 				if (!skip_masking && typeof row.items !== 'undefined' && row.items) {
 					row = internalAccessList.maskItems(row);
 				}
-
-					return _.omit(row, omissions());
-				} else {
-					throw new error.ItemNotFoundError(data.id);
+				// Custom omissions
+				if (typeof data.omit !== 'undefined' && data.omit !== null) {
+					row = _.omit(row, data.omit);
 				}
+				return row;
 			});
 	},
 
@@ -381,8 +377,7 @@ const internalAccessList = {
 					.joinRaw('LEFT JOIN `proxy_host` ON `proxy_host`.`access_list_id` = `access_list`.`id` AND `proxy_host`.`is_deleted` = 0')
 					.where('access_list.is_deleted', 0)
 					.groupBy('access_list.id')
-					.omit(['access_list.is_deleted'])
-					.allowEager('[owner,items,clients]')
+					.allowGraph('[owner,items,clients]')
 					.orderBy('access_list.name', 'ASC');
 
 				if (access_data.permission_visibility !== 'all') {
@@ -397,10 +392,10 @@ const internalAccessList = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRows(omissions()));
 			})
 			.then((rows) => {
 				if (rows) {

backend/internal/audit-log.js

@@ -19,7 +19,7 @@ const internalAuditLog = {
 					.orderBy('created_on', 'DESC')
 					.orderBy('id', 'DESC')
 					.limit(100)
-					.allowEager('[user]');
+					.allowGraph('[user]');
 
 				// Query is used for searching
 				if (typeof search_query === 'string') {
@@ -29,7 +29,7 @@ const internalAuditLog = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
 				return query;

backend/internal/certificate.js

@@ -121,8 +121,8 @@ const internalCertificate = {
 
 				return certificateModel
 					.query()
-					.omit(omissions())
-					.insertAndFetch(data);
+					.insertAndFetch(data)
+					.then(utils.omitRow(omissions()));
 			})
 			.then((certificate) => {
 				if (certificate.provider === 'letsencrypt') {
@@ -269,8 +269,8 @@ const internalCertificate = {
 
 				return certificateModel
 					.query()
-					.omit(omissions())
 					.patchAndFetchById(row.id, data)
+					.then(utils.omitRow(omissions()))
 					.then((saved_row) => {
 						saved_row.meta = internalCertificate.cleanMeta(saved_row.meta);
 						data.meta      = internalCertificate.cleanMeta(data.meta);
@@ -288,7 +288,7 @@ const internalCertificate = {
 							meta:        _.omit(data, ['expires_on']) // this prevents json circular reference because expires_on might be raw
 						})
 							.then(() => {
-								return _.omit(saved_row, omissions());
+								return saved_row;
 							});
 					});
 			});
@@ -313,30 +313,28 @@ const internalCertificate = {
 					.query()
 					.where('is_deleted', 0)
 					.andWhere('id', data.id)
-					.allowEager('[owner]')
+					.allowGraph('[owner]')
 					.first();
 
 				if (access_data.permission_visibility !== 'all') {
 					query.andWhere('owner_user_id', access.token.getUserId(1));
 				}
 
-				// Custom omissions
-				if (typeof data.omit !== 'undefined' && data.omit !== null) {
-					query.omit(data.omit);
-				}
-
 				if (typeof data.expand !== 'undefined' && data.expand !== null) {
-					query.eager('[' + data.expand.join(', ') + ']');
+					query.withGraphFetched('[' + data.expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
-				if (row) {
-					return _.omit(row, omissions());
-				} else {
+				if (!row) {
 					throw new error.ItemNotFoundError(data.id);
 				}
+				// Custom omissions
+				if (typeof data.omit !== 'undefined' && data.omit !== null) {
+					row = _.omit(row, data.omit);
+				}
+				return row;
 			});
 	},
 
@@ -466,8 +464,7 @@ const internalCertificate = {
 					.query()
 					.where('is_deleted', 0)
 					.groupBy('id')
-					.omit(['is_deleted'])
-					.allowEager('[owner]')
+					.allowGraph('[owner]')
 					.orderBy('nice_name', 'ASC');
 
 				if (access_data.permission_visibility !== 'all') {
@@ -482,10 +479,10 @@ const internalCertificate = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRows(omissions()));
 			});
 	},
 
@@ -662,7 +659,6 @@ const internalCertificate = {
 							meta:         _.clone(row.meta) // Prevent the update method from changing this value that we'll use later
 						})
 							.then((certificate) => {
-								console.log('ROWMETA:', row.meta);
 								certificate.meta = row.meta;
 								return internalCertificate.writeCustomCert(certificate);
 							});

backend/internal/dead-host.js

@@ -1,5 +1,6 @@
 const _                   = require('lodash');
 const error               = require('../lib/error');
+const utils               = require('../lib/utils');
 const deadHostModel       = require('../models/dead_host');
 const internalHost        = require('./host');
 const internalNginx       = require('./nginx');
@@ -49,8 +50,8 @@ const internalDeadHost = {
 
 				return deadHostModel
 					.query()
-					.omit(omissions())
-					.insertAndFetch(data);
+					.insertAndFetch(data)
+					.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
 				if (create_certificate) {
@@ -218,31 +219,28 @@ const internalDeadHost = {
 					.query()
 					.where('is_deleted', 0)
 					.andWhere('id', data.id)
-					.allowEager('[owner,certificate]')
+					.allowGraph('[owner,certificate]')
 					.first();
 
 				if (access_data.permission_visibility !== 'all') {
 					query.andWhere('owner_user_id', access.token.getUserId(1));
 				}
 
-				// Custom omissions
-				if (typeof data.omit !== 'undefined' && data.omit !== null) {
-					query.omit(data.omit);
-				}
-
 				if (typeof data.expand !== 'undefined' && data.expand !== null) {
-					query.eager('[' + data.expand.join(', ') + ']');
+					query.withGraphFetched('[' + data.expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
-				if (row) {
-					row = internalHost.cleanRowCertificateMeta(row);
-					return _.omit(row, omissions());
-				} else {
+				if (!row) {
 					throw new error.ItemNotFoundError(data.id);
 				}
+				// Custom omissions
+				if (typeof data.omit !== 'undefined' && data.omit !== null) {
+					row = _.omit(row, data.omit);
+				}
+				return row;
 			});
 	},
 
@@ -404,8 +402,7 @@ const internalDeadHost = {
 					.query()
 					.where('is_deleted', 0)
 					.groupBy('id')
-					.omit(['is_deleted'])
-					.allowEager('[owner,certificate]')
+					.allowGraph('[owner,certificate]')
 					.orderBy('domain_names', 'ASC');
 
 				if (access_data.permission_visibility !== 'all') {
@@ -420,10 +417,10 @@ const internalDeadHost = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRows(omissions()));
 			})
 			.then((rows) => {
 				if (typeof expand !== 'undefined' && expand !== null && expand.indexOf('certificate') !== -1) {

backend/internal/ip_ranges.js

@@ -2,8 +2,8 @@ const https         = require('https');
 const fs            = require('fs');
 const logger        = require('../logger').ip_ranges;
 const error         = require('../lib/error');
+const utils         = require('../lib/utils');
 const internalNginx = require('./nginx');
-const { Liquid }    = require('liquidjs');
 
 const CLOUDFRONT_URL   = 'https://ip-ranges.amazonaws.com/ip-ranges.json';
 const CLOUDFARE_V4_URL = 'https://www.cloudflare.com/ips-v4';
@@ -119,10 +119,7 @@ const internalIpRanges = {
 	 * @returns {Promise}
 	 */
 	generateConfig: (ip_ranges) => {
-		let renderEngine = new Liquid({
-			root: __dirname + '/../templates/'
-		});
-
+		const renderEngine = utils.getRenderEngine();
 		return new Promise((resolve, reject) => {
 			let template = null;
 			let filename = '/etc/nginx/conf.d/include/ip_ranges.conf';

backend/internal/nginx.js

@@ -3,7 +3,6 @@ const fs         = require('fs');
 const logger     = require('../logger').nginx;
 const utils      = require('../lib/utils');
 const error      = require('../lib/error');
-const { Liquid } = require('liquidjs');
 const debug_mode = process.env.NODE_ENV !== 'production' || !!process.env.DEBUG;
 
 const internalNginx = {
@@ -138,8 +137,6 @@ const internalNginx = {
 	 * @returns {Promise}
 	 */
 	renderLocations: (host) => {
-
-		//logger.info('host = ' + JSON.stringify(host, null, 2));
 		return new Promise((resolve, reject) => {
 			let template;
 
@@ -150,9 +147,7 @@ const internalNginx = {
 				return;
 			}
 
-			let renderer          = new Liquid({
-				root: __dirname + '/../templates/'
-			});
+			const renderEngine = utils.getRenderEngine();
 			let renderedLocations = '';
 
 			const locationRendering = async () => {
@@ -170,10 +165,8 @@ const internalNginx = {
 						locationCopy.forward_path = `/${splitted.join('/')}`;
 					}
 
-					//logger.info('locationCopy = ' + JSON.stringify(locationCopy, null, 2));
-
 					// eslint-disable-next-line
-					renderedLocations += await renderer.parseAndRender(template, locationCopy);
+					renderedLocations += await renderEngine.parseAndRender(template, locationCopy);
 				}
 
 			};
@@ -195,11 +188,7 @@ const internalNginx = {
 			logger.info('Generating ' + nice_host_type + ' Config:', JSON.stringify(host, null, 2));
 		}
 
-		// logger.info('host = ' + JSON.stringify(host, null, 2));
-
-		let renderEngine = new Liquid({
-			root: __dirname + '/../templates/'
-		});
+		const renderEngine = utils.getRenderEngine();
 
 		return new Promise((resolve, reject) => {
 			let template = null;
@@ -283,9 +272,7 @@ const internalNginx = {
 			logger.info('Generating LetsEncrypt Request Config:', certificate);
 		}
 
-		let renderEngine = new Liquid({
-			root: __dirname + '/../templates/'
-		});
+		const renderEngine = utils.getRenderEngine();
 
 		return new Promise((resolve, reject) => {
 			let template = null;

backend/internal/proxy-host.js

@@ -1,5 +1,6 @@
 const _                   = require('lodash');
 const error               = require('../lib/error');
+const utils               = require('../lib/utils');
 const proxyHostModel      = require('../models/proxy_host');
 const internalHost        = require('./host');
 const internalNginx       = require('./nginx');
@@ -49,8 +50,8 @@ const internalProxyHost = {
 
 				return proxyHostModel
 					.query()
-					.omit(omissions())
-					.insertAndFetch(data);
+					.insertAndFetch(data)
+					.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
 				if (create_certificate) {
@@ -170,6 +171,7 @@ const internalProxyHost = {
 					.query()
 					.where({id: data.id})
 					.patch(data)
+					.then(utils.omitRow(omissions()))
 					.then((saved_row) => {
 						// Add to audit log
 						return internalAuditLog.add(access, {
@@ -179,7 +181,7 @@ const internalProxyHost = {
 							meta:        data
 						})
 							.then(() => {
-								return _.omit(saved_row, omissions());
+								return saved_row;
 							});
 					});
 			})
@@ -223,31 +225,29 @@ const internalProxyHost = {
 					.query()
 					.where('is_deleted', 0)
 					.andWhere('id', data.id)
-					.allowEager('[owner,access_list,access_list.[clients,items],certificate]')
+					.allowGraph('[owner,access_list,access_list.[clients,items],certificate]')
 					.first();
 
 				if (access_data.permission_visibility !== 'all') {
 					query.andWhere('owner_user_id', access.token.getUserId(1));
 				}
 
-				// Custom omissions
-				if (typeof data.omit !== 'undefined' && data.omit !== null) {
-					query.omit(data.omit);
-				}
-
 				if (typeof data.expand !== 'undefined' && data.expand !== null) {
-					query.eager('[' + data.expand.join(', ') + ']');
+					query.withGraphFetched('[' + data.expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
-				if (row) {
-					row = internalHost.cleanRowCertificateMeta(row);
-					return _.omit(row, omissions());
-				} else {
+				if (!row) {
 					throw new error.ItemNotFoundError(data.id);
 				}
+				row = internalHost.cleanRowCertificateMeta(row);
+				// Custom omissions
+				if (typeof data.omit !== 'undefined' && data.omit !== null) {
+					row = _.omit(row, data.omit);
+				}
+				return row;
 			});
 	},
 
@@ -409,8 +409,7 @@ const internalProxyHost = {
 					.query()
 					.where('is_deleted', 0)
 					.groupBy('id')
-					.omit(['is_deleted'])
-					.allowEager('[owner,access_list,certificate]')
+					.allowGraph('[owner,access_list,certificate]')
 					.orderBy('domain_names', 'ASC');
 
 				if (access_data.permission_visibility !== 'all') {
@@ -425,10 +424,10 @@ const internalProxyHost = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRows(omissions()));
 			})
 			.then((rows) => {
 				if (typeof expand !== 'undefined' && expand !== null && expand.indexOf('certificate') !== -1) {

backend/internal/redirection-host.js

@@ -1,5 +1,6 @@
 const _                    = require('lodash');
 const error                = require('../lib/error');
+const utils                = require('../lib/utils');
 const redirectionHostModel = require('../models/redirection_host');
 const internalHost         = require('./host');
 const internalNginx        = require('./nginx');
@@ -49,8 +50,8 @@ const internalRedirectionHost = {
 
 				return redirectionHostModel
 					.query()
-					.omit(omissions())
-					.insertAndFetch(data);
+					.insertAndFetch(data)
+					.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
 				if (create_certificate) {
@@ -65,9 +66,8 @@ const internalRedirectionHost = {
 						.then(() => {
 							return row;
 						});
-				} else {
-					return row;
 				}
+				return row;
 			})
 			.then((row) => {
 				// re-fetch with cert
@@ -218,31 +218,29 @@ const internalRedirectionHost = {
 					.query()
 					.where('is_deleted', 0)
 					.andWhere('id', data.id)
-					.allowEager('[owner,certificate]')
+					.allowGraph('[owner,certificate]')
 					.first();
 
 				if (access_data.permission_visibility !== 'all') {
 					query.andWhere('owner_user_id', access.token.getUserId(1));
 				}
 
-				// Custom omissions
-				if (typeof data.omit !== 'undefined' && data.omit !== null) {
-					query.omit(data.omit);
-				}
-
 				if (typeof data.expand !== 'undefined' && data.expand !== null) {
-					query.eager('[' + data.expand.join(', ') + ']');
+					query.withGraphFetched('[' + data.expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
-				if (row) {
-					row = internalHost.cleanRowCertificateMeta(row);
-					return _.omit(row, omissions());
-				} else {
+				if (!row) {
 					throw new error.ItemNotFoundError(data.id);
 				}
+				row = internalHost.cleanRowCertificateMeta(row);
+				// Custom omissions
+				if (typeof data.omit !== 'undefined' && data.omit !== null) {
+					row = _.omit(row, data.omit);
+				}
+				return row;
 			});
 	},
 
@@ -404,8 +402,7 @@ const internalRedirectionHost = {
 					.query()
 					.where('is_deleted', 0)
 					.groupBy('id')
-					.omit(['is_deleted'])
-					.allowEager('[owner,certificate]')
+					.allowGraph('[owner,certificate]')
 					.orderBy('domain_names', 'ASC');
 
 				if (access_data.permission_visibility !== 'all') {
@@ -420,10 +417,10 @@ const internalRedirectionHost = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRows(omissions()));
 			})
 			.then((rows) => {
 				if (typeof expand !== 'undefined' && expand !== null && expand.indexOf('certificate') !== -1) {

backend/internal/stream.js

@@ -1,5 +1,6 @@
 const _                = require('lodash');
 const error            = require('../lib/error');
+const utils            = require('../lib/utils');
 const streamModel      = require('../models/stream');
 const internalNginx    = require('./nginx');
 const internalAuditLog = require('./audit-log');
@@ -27,8 +28,8 @@ const internalStream = {
 
 				return streamModel
 					.query()
-					.omit(omissions())
-					.insertAndFetch(data);
+					.insertAndFetch(data)
+					.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
 				// Configure nginx
@@ -71,8 +72,8 @@ const internalStream = {
 
 				return streamModel
 					.query()
-					.omit(omissions())
 					.patchAndFetchById(row.id, data)
+					.then(utils.omitRow(omissions()))
 					.then((saved_row) => {
 						return internalNginx.configure(streamModel, 'stream', saved_row)
 							.then(() => {
@@ -88,7 +89,7 @@ const internalStream = {
 							meta:        data
 						})
 							.then(() => {
-								return _.omit(saved_row, omissions());
+								return saved_row;
 							});
 					});
 			});
@@ -113,30 +114,28 @@ const internalStream = {
 					.query()
 					.where('is_deleted', 0)
 					.andWhere('id', data.id)
-					.allowEager('[owner]')
+					.allowGraph('[owner]')
 					.first();
 
 				if (access_data.permission_visibility !== 'all') {
 					query.andWhere('owner_user_id', access.token.getUserId(1));
 				}
 
-				// Custom omissions
-				if (typeof data.omit !== 'undefined' && data.omit !== null) {
-					query.omit(data.omit);
-				}
-
 				if (typeof data.expand !== 'undefined' && data.expand !== null) {
-					query.eager('[' + data.expand.join(', ') + ']');
+					query.withGraphFetched('[' + data.expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
-				if (row) {
-					return _.omit(row, omissions());
-				} else {
+				if (!row) {
 					throw new error.ItemNotFoundError(data.id);
 				}
+				// Custom omissions
+				if (typeof data.omit !== 'undefined' && data.omit !== null) {
+					row = _.omit(row, data.omit);
+				}
+				return row;
 			});
 	},
 
@@ -298,8 +297,7 @@ const internalStream = {
 					.query()
 					.where('is_deleted', 0)
 					.groupBy('id')
-					.omit(['is_deleted'])
-					.allowEager('[owner]')
+					.allowGraph('[owner]')
 					.orderBy('incoming_port', 'ASC');
 
 				if (access_data.permission_visibility !== 'all') {
@@ -314,10 +312,10 @@ const internalStream = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRows(omissions()));
 			});
 	},
 

backend/internal/user.js

@@ -1,5 +1,6 @@
 const _                   = require('lodash');
 const error               = require('../lib/error');
+const utils               = require('../lib/utils');
 const userModel           = require('../models/user');
 const userPermissionModel = require('../models/user_permission');
 const authModel           = require('../models/auth');
@@ -35,8 +36,8 @@ const internalUser = {
 
 				return userModel
 					.query()
-					.omit(omissions())
-					.insertAndFetch(data);
+					.insertAndFetch(data)
+					.then(utils.omitRow(omissions()));
 			})
 			.then((user) => {
 				if (auth) {
@@ -140,11 +141,8 @@ const internalUser = {
 
 				return userModel
 					.query()
-					.omit(omissions())
 					.patchAndFetchById(user.id, data)
-					.then((saved_user) => {
-						return _.omit(saved_user, omissions());
-					});
+					.then(utils.omitRow(omissions()));
 			})
 			.then(() => {
 				return internalUser.get(access, {id: data.id});
@@ -186,26 +184,24 @@ const internalUser = {
 					.query()
 					.where('is_deleted', 0)
 					.andWhere('id', data.id)
-					.allowEager('[permissions]')
+					.allowGraph('[permissions]')
 					.first();
 
-				// Custom omissions
-				if (typeof data.omit !== 'undefined' && data.omit !== null) {
-					query.omit(data.omit);
-				}
-
 				if (typeof data.expand !== 'undefined' && data.expand !== null) {
-					query.eager('[' + data.expand.join(', ') + ']');
+					query.withGraphFetched('[' + data.expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRow(omissions()));
 			})
 			.then((row) => {
-				if (row) {
-					return _.omit(row, omissions());
-				} else {
+				if (!row) {
 					throw new error.ItemNotFoundError(data.id);
 				}
+				// Custom omissions
+				if (typeof data.omit !== 'undefined' && data.omit !== null) {
+					row = _.omit(row, data.omit);
+				}
+				return row;
 			});
 	},
 
@@ -322,8 +318,7 @@ const internalUser = {
 					.query()
 					.where('is_deleted', 0)
 					.groupBy('id')
-					.omit(['is_deleted'])
-					.allowEager('[permissions]')
+					.allowGraph('[permissions]')
 					.orderBy('name', 'ASC');
 
 				// Query is used for searching
@@ -335,10 +330,10 @@ const internalUser = {
 				}
 
 				if (typeof expand !== 'undefined' && expand !== null) {
-					query.eager('[' + expand.join(', ') + ']');
+					query.withGraphFetched('[' + expand.join(', ') + ']');
 				}
 
-				return query;
+				return query.then(utils.omitRows(omissions()));
 			});
 	},
 

backend/lib/access.js

@@ -55,8 +55,8 @@ module.exports = function (token_string) {
 								.where('id', token_data.attrs.id)
 								.andWhere('is_deleted', 0)
 								.andWhere('is_disabled', 0)
-								.allowEager('[permissions]')
-								.eager('[permissions]')
+								.allowGraph('[permissions]')
+								.withGraphFetched('[permissions]')
 								.first()
 								.then((user) => {
 									if (user) {

backend/lib/utils.js

@@ -1,5 +1,7 @@
+const _          = require('lodash');
 const exec       = require('child_process').exec;
 const execFile   = require('child_process').execFile;
+const { Liquid } = require('liquidjs');
 const logger     = require('../logger').global;
 
 module.exports = {
@@ -36,5 +38,64 @@ module.exports = {
 				}
 			});
 		});
+	},
+
+	/**
+	 * Used in objection query builder
+	 *
+	 * @param   {Array}  omissions
+	 * @returns {Function}
+	 */
+	omitRow: function (omissions) {
+		/**
+		 * @param   {Object} row
+		 * @returns {Object}
+		 */
+		return (row) => {
+			return _.omit(row, omissions);
+		}
+	},
+
+	/**
+	 * Used in objection query builder
+	 *
+	 * @param   {Array}  omissions
+	 * @returns {Function}
+	 */
+	omitRows: function (omissions) {
+		/**
+		 * @param   {Array} rows
+		 * @returns {Object}
+		 */
+		return (rows) => {
+			rows.forEach((row, idx) => {
+				rows[idx] = _.omit(row, omissions);
+			});
+			return rows;
+		}
+	},
+
+	/**
+	 * @returns {Object} Liquid render engine
+	 */
+	getRenderEngine: function () {
+		const renderEngine = new Liquid({
+			root: __dirname + '/../templates/'
+		});
+
+		/**
+		 * nginxAccessRule expects the object given to have 2 properties:
+		 *
+		 * directive  string
+		 * address    string
+		 */
+		renderEngine.registerFilter('nginxAccessRule', (v) => {
+			if (typeof v.directive !== 'undefined' && typeof v.address !== 'undefined' && v.directive && v.address) {
+				return `${v.directive} ${v.address};`;
+			}
+			return '';
+		});
+
+		return renderEngine;
 	}
 };

backend/models/access_list.js

@@ -50,7 +50,6 @@ class AccessList extends Model {
 				},
 				modify: function (qb) {
 					qb.where('user.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted', 'email', 'roles']);
 				}
 			},
 			items: {
@@ -59,9 +58,6 @@ class AccessList extends Model {
 				join:       {
 					from: 'access_list.id',
 					to:   'access_list_auth.access_list_id'
-				},
-				modify: function (qb) {
-					qb.omit(['id', 'created_on', 'modified_on', 'access_list_id', 'meta']);
 				}
 			},
 			clients: {
@@ -70,9 +66,6 @@ class AccessList extends Model {
 				join:       {
 					from: 'access_list.id',
 					to:   'access_list_client.access_list_id'
-				},
-				modify: function (qb) {
-					qb.omit(['id', 'created_on', 'modified_on', 'access_list_id', 'meta']);
 				}
 			},
 			proxy_hosts: {
@@ -84,19 +77,10 @@ class AccessList extends Model {
 				},
 				modify: function (qb) {
 					qb.where('proxy_host.is_deleted', 0);
-					qb.omit(['is_deleted', 'meta']);
 				}
 			}
 		};
 	}
-
-	get satisfy() {
-		return this.satisfy_any ? 'satisfy any' : 'satisfy all';
-	}
-
-	get passauth() {
-		return this.pass_auth ? '' : 'proxy_set_header Authorization "";';
-	}
 }
 
 module.exports = AccessList;

backend/models/access_list_auth.js

@@ -45,7 +45,6 @@ class AccessListAuth extends Model {
 				},
 				modify: function (qb) {
 					qb.where('access_list.is_deleted', 0);
-					qb.omit(['created_on', 'modified_on', 'is_deleted', 'access_list_id']);
 				}
 			}
 		};

backend/models/access_list_client.js

@@ -45,15 +45,10 @@ class AccessListClient extends Model {
 				},
 				modify: function (qb) {
 					qb.where('access_list.is_deleted', 0);
-					qb.omit(['created_on', 'modified_on', 'is_deleted', 'access_list_id']);
 				}
 			}
 		};
 	}
-
-	get rule() {
-		return `${this.directive} ${this.address}`;
-	}
 }
 
 module.exports = AccessListClient;

backend/models/audit-log.js

@@ -43,9 +43,6 @@ class AuditLog extends Model {
 				join:       {
 					from: 'audit_log.user_id',
 					to:   'user.id'
-				},
-				modify: function (qb) {
-					qb.omit(['id', 'created_on', 'modified_on', 'roles']);
 				}
 			}
 		};

backend/models/auth.js

@@ -74,9 +74,6 @@ class Auth extends Model {
 				},
 				filter: {
 					is_deleted: 0
-				},
-				modify: function (qb) {
-					qb.omit(['is_deleted']);
 				}
 			}
 		};

backend/models/certificate.js

@@ -63,7 +63,6 @@ class Certificate extends Model {
 				},
 				modify: function (qb) {
 					qb.where('user.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted', 'email', 'roles']);
 				}
 			}
 		};

backend/models/dead_host.js

@@ -59,7 +59,6 @@ class DeadHost extends Model {
 				},
 				modify: function (qb) {
 					qb.where('user.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted', 'email', 'roles']);
 				}
 			},
 			certificate: {
@@ -71,7 +70,6 @@ class DeadHost extends Model {
 				},
 				modify: function (qb) {
 					qb.where('certificate.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted']);
 				}
 			}
 		};

backend/models/now_helper.js

@@ -6,8 +6,7 @@ Model.knex(db);
 
 module.exports = function () {
 	if (config.database.knex && config.database.knex.client === 'sqlite3') {
-		return Model.raw('datetime(\'now\',\'localtime\')');
-	} else {
-		return Model.raw('NOW()');
+		return Model.raw("datetime('now','localtime')");
 	}
+	return Model.raw('NOW()');
 };

backend/models/proxy_host.js

@@ -60,7 +60,6 @@ class ProxyHost extends Model {
 				},
 				modify: function (qb) {
 					qb.where('user.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted', 'email', 'roles']);
 				}
 			},
 			access_list: {
@@ -72,7 +71,6 @@ class ProxyHost extends Model {
 				},
 				modify: function (qb) {
 					qb.where('access_list.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted']);
 				}
 			},
 			certificate: {
@@ -84,7 +82,6 @@ class ProxyHost extends Model {
 				},
 				modify: function (qb) {
 					qb.where('certificate.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted']);
 				}
 			}
 		};

backend/models/redirection_host.js

@@ -1,3 +1,4 @@
+
 // Objection Docs:
 // http://vincit.github.io/objection.js/
 
@@ -59,7 +60,6 @@ class RedirectionHost extends Model {
 				},
 				modify: function (qb) {
 					qb.where('user.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted', 'email', 'roles']);
 				}
 			},
 			certificate: {
@@ -71,7 +71,6 @@ class RedirectionHost extends Model {
 				},
 				modify: function (qb) {
 					qb.where('certificate.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted']);
 				}
 			}
 		};

backend/models/stream.js

@@ -46,7 +46,6 @@ class Stream extends Model {
 				},
 				modify: function (qb) {
 					qb.where('user.is_deleted', 0);
-					qb.omit(['id', 'created_on', 'modified_on', 'is_deleted', 'email', 'roles']);
 				}
 			}
 		};

backend/models/token.js

@@ -83,8 +83,6 @@ module.exports = function () {
 								// Hack: some tokens out in the wild have a scope of 'all' instead of 'user'.
 								// For 30 days at least, we need to replace 'all' with user.
 								if ((typeof token_data.scope !== 'undefined' && _.indexOf(token_data.scope, 'all') !== -1)) {
-									//console.log('Warning! Replacing "all" scope with "user"');
-
 									token_data.scope = ['user'];
 								}
 

backend/models/user.js

@@ -43,9 +43,6 @@ class User extends Model {
 				join:       {
 					from: 'user.id',
 					to:   'user_permission.user_id'
-				},
-				modify: function (qb) {
-					qb.omit(['id', 'created_on', 'modified_on', 'user_id']);
 				}
 			}
 		};

backend/package.json

@@ -16,17 +16,17 @@
 		"gravatar": "^1.8.0",
 		"json-schema-ref-parser": "^8.0.0",
 		"jsonwebtoken": "^9.0.0",
-		"knex": "^2.4.0",
-		"liquidjs": "^10.0.0",
+		"knex": "2.4.2",
+		"liquidjs": "10.6.1",
 		"lodash": "^4.17.21",
 		"moment": "^2.29.4",
 		"mysql": "^2.18.1",
 		"node-rsa": "^1.0.8",
 		"nodemon": "^2.0.2",
-		"objection": "^2.2.16",
+		"objection": "3.0.1",
 		"path": "^0.12.7",
-		"signale": "^1.4.0",
-		"sqlite3": "^4.1.1",
+		"signale": "1.4.0",
+		"sqlite3": "5.1.6",
 		"temp-write": "^4.0.0"
 	},
 	"signale": {

backend/templates/_access.conf

@@ -0,0 +1,25 @@
+{% if access_list_id > 0 %}
+    {% if access_list.items.length > 0 %}
+    # Authorization
+    auth_basic            "Authorization required";
+    auth_basic_user_file  /data/access/{{ access_list_id }};
+
+    {% if access_list.pass_auth == 0 %}
+    proxy_set_header Authorization "";
+    {% endif %}
+
+    {% endif %}
+
+    # Access Rules: {{ access_list.clients | size }} total
+    {% for client in access_list.clients %}
+    {{client | nginxAccessRule}}
+    {% endfor %}
+    deny all;
+
+    # Access checks must...
+    {% if access_list.satisfy_any == 1 %}
+    satisfy any;
+    {% else %}
+    satisfy all;
+    {% endif %}
+{% endif %}

backend/templates/_location.conf

@@ -6,30 +6,9 @@
     proxy_set_header X-Real-IP		$remote_addr;
     proxy_pass       {{ forward_scheme }}://{{ forward_host }}:{{ forward_port }}{{ forward_path }};
 
-    {% if access_list_id > 0 %}
-    {% if access_list.items.length > 0 %}
-    # Authorization
-    auth_basic            "Authorization required";
-    auth_basic_user_file  /data/access/{{ access_list_id }};
- 
-    {{ access_list.passauth }}
-    {% endif %}
- 
-    # Access Rules
-    {% for client in access_list.clients %}
-    {{- client.rule -}};
-    {% endfor %}deny all;
- 
-    # Access checks must...
-    {% if access_list.satisfy %}
-    {{ access_list.satisfy }};
-    {% endif %}
- 
-    {% endif %}
-
+    {% include "_access.conf" %}
     {% include "_assets.conf" %}
     {% include "_exploits.conf" %}
-
     {% include "_forced_ssl.conf" %}
     {% include "_hsts.conf" %}
 

backend/templates/proxy_host.conf

@@ -30,27 +30,7 @@ proxy_http_version 1.1;
 
   location / {
 
-    {% if access_list_id > 0 %}
-    {% if access_list.items.length > 0 %}
-    # Authorization
-    auth_basic            "Authorization required";
-    auth_basic_user_file  /data/access/{{ access_list_id }};
-
-    {{ access_list.passauth }}
-    {% endif %}
-
-    # Access Rules
-    {% for client in access_list.clients %}
-    {{- client.rule -}};
-    {% endfor %}deny all;
-
-    # Access checks must...
-    {% if access_list.satisfy %}
-    {{ access_list.satisfy }};
-    {% endif %}
-
-    {% endif %}
-
+{% include "_access.conf" %}
 {% include "_hsts.conf" %}
 
     {% if allow_websocket_upgrade == 1 or allow_websocket_upgrade == true %}