diff --git a/TODO.md b/TODO.md index eb3e1d0..e3d4782 100644 --- a/TODO.md +++ b/TODO.md @@ -2,9 +2,12 @@ In order of importance, somewhat.. +- v1 Importer + - ssl certificates + - nginx advanced config +- Redirection host preserve path nginx configuration - Custom ssl certificate saving to disk and usage in nginx configs - Dashboard stats are caching instead of querying -- Create a nice way of importing from v1 let's encrypt certs and config data - UI Log tail - Custom Nginx Config Editor diff --git a/src/backend/importer.js b/src/backend/importer.js index 80525f1..f85d732 100644 --- a/src/backend/importer.js +++ b/src/backend/importer.js @@ -1,66 +1,398 @@ 'use strict'; -const fs = require('fs'); -const logger = require('./logger').import; -const utils = require('./lib/utils'); +const fs = require('fs'); +const logger = require('./logger').import; +const utils = require('./lib/utils'); +const batchflow = require('batchflow'); + +const internalProxyHost = require('./internal/proxy-host'); +const internalRedirectionHost = require('./internal/redirection-host'); +const internalDeadHost = require('./internal/dead-host'); +const internalNginx = require('./internal/nginx'); +const internalAccessList = require('./internal/access-list'); +const internalStream = require('./internal/stream'); + +const accessListModel = require('./models/access_list'); +const accessListAuthModel = require('./models/access_list_auth'); +const proxyHostModel = require('./models/proxy_host'); +const redirectionHostModel = require('./models/redirection_host'); +const deadHostModel = require('./models/dead_host'); +const streamModel = require('./models/stream'); module.exports = function () { + + let access_map = {}; + let certificate_map = {}; + + /** + * @param {Access} access + * @param {Object} db + * @returns {Promise} + */ + const importAccessLists = function (access, db) { + return new Promise((resolve, reject) => { + let lists = db.access.find(); + + batchflow(lists).sequential() + .each((i, list, next) => { + + importAccessList(access, list) + .then(() => { + next(); + }) + .catch(err => { + next(err); + }); + }) + .end(results => { + resolve(results); + }); + }); + }; + + /** + * @param {Access} access + * @param {Object} list + * @returns {Promise} + */ + const importAccessList = function (access, list) { + // Create the list + logger.info('Creating Access List: ' + list.name); + + return accessListModel + .query() + .insertAndFetch({ + name: list.name, + owner_user_id: 1 + }) + .then(row => { + access_map[list._id] = row.id; + + return new Promise((resolve, reject) => { + batchflow(list.items).sequential() + .each((i, item, next) => { + if (typeof item.password !== 'undefined' && item.password.length) { + logger.info('Adding to Access List: ' + item.username); + + accessListAuthModel + .query() + .insert({ + access_list_id: row.id, + username: item.username, + password: item.password + }) + .then(() => { + next(); + }) + .catch(err => { + logger.error(err); + next(err); + }); + } + }) + .error(err => { + logger.error(err); + reject(err); + }) + .end(results => { + logger.success('Finished importing Access List: ' + list.name); + resolve(results); + }); + }) + .then(() => { + return internalAccessList.get(access, { + id: row.id, + expand: ['owner', 'items'] + }, true /* <- skip masking */); + }) + .then(full_list => { + return internalAccessList.build(full_list); + }); + }); + }; + + /** + * @param {Access} access + * @returns {Promise} + */ + const importCertificates = function (access) { + // This step involves transforming the letsencrypt folder structure significantly. + + // - /etc/letsencrypt/accounts Do not touch + // - /etc/letsencrypt/archive Modify directory names + // - /etc/letsencrypt/csr Do not touch + // - /etc/letsencrypt/keys Do not touch + // - /etc/letsencrypt/live Modify directory names, modify file symlinks + // - /etc/letsencrypt/renewal Modify filenames and file content + + return new Promise((resolve, reject) => { + // TODO + resolve(); + }); + }; + + /** + * @param {Access} access + * @param {Object} db + * @returns {Promise} + */ + const importHosts = function (access, db) { + return new Promise((resolve, reject) => { + let hosts = db.hosts.find(); + + batchflow(hosts).sequential() + .each((i, host, next) => { + importHost(access, host) + .then(() => { + next(); + }) + .catch(err => { + next(err); + }); + }) + .end(results => { + resolve(results); + }); + }); + }; + + /** + * @param {Access} access + * @param {Object} host + * @returns {Promise} + */ + const importHost = function (access, host) { + // Create the list + if (typeof host.type === 'undefined') { + host.type = 'proxy'; + } + + switch (host.type) { + case 'proxy': + return importProxyHost(access, host); + case '404': + return importDeadHost(access, host); + case 'redirection': + return importRedirectionHost(access, host); + case 'stream': + return importStream(access, host); + default: + return Promise.resolve(); + } + }; + + /** + * @param {Access} access + * @param {Object} host + * @returns {Promise} + */ + const importProxyHost = function (access, host) { + logger.info('Creating Proxy Host: ' + host.hostname); + + let access_list_id = 0; + let certificate_id = 0; + let meta = {}; + + if (typeof host.letsencrypt_email !== 'undefined') { + meta.letsencrypt_email = host.letsencrypt_email; + } + + // determine access_list_id + if (typeof host.access_list_id !== 'undefined' && host.access_list_id && typeof access_map[host.access_list_id] !== 'undefined') { + access_list_id = access_map[host.access_list_id]; + } + + // determine certificate_id + if (host.ssl && typeof certificate_map[host.hostname] !== 'undefined') { + certificate_id = certificate_map[host.hostname]; + } + + // TODO: Advanced nginx config + + return proxyHostModel + .query() + .insertAndFetch({ + owner_user_id: 1, + domain_names: [host.hostname], + forward_ip: host.forward_server, + forward_port: host.forward_port, + access_list_id: access_list_id, + certificate_id: certificate_id, + ssl_forced: host.force_ssl || false, + caching_enabled: host.asset_caching || false, + block_exploits: host.block_exploits || false, + meta: meta + }) + .then(row => { + // re-fetch with cert + return internalProxyHost.get(access, { + id: row.id, + expand: ['certificate', 'owner', 'access_list'] + }); + }) + .then(row => { + // Configure nginx + return internalNginx.configure(proxyHostModel, 'proxy_host', row); + }); + }; + + /** + * @param {Access} access + * @param {Object} host + * @returns {Promise} + */ + const importDeadHost = function (access, host) { + logger.info('Creating 404 Host: ' + host.hostname); + + let certificate_id = 0; + let meta = {}; + + if (typeof host.letsencrypt_email !== 'undefined') { + meta.letsencrypt_email = host.letsencrypt_email; + } + + // determine certificate_id + if (host.ssl && typeof certificate_map[host.hostname] !== 'undefined') { + certificate_id = certificate_map[host.hostname]; + } + + // TODO: Advanced nginx config + + return deadHostModel + .query() + .insertAndFetch({ + owner_user_id: 1, + domain_names: [host.hostname], + certificate_id: certificate_id, + ssl_forced: host.force_ssl || false, + meta: meta + }) + .then(row => { + // re-fetch with cert + return internalDeadHost.get(access, { + id: row.id, + expand: ['certificate', 'owner'] + }); + }) + .then(row => { + // Configure nginx + return internalNginx.configure(deadHostModel, 'dead_host', row); + }); + }; + + /** + * @param {Access} access + * @param {Object} host + * @returns {Promise} + */ + const importRedirectionHost = function (access, host) { + logger.info('Creating Redirection Host: ' + host.hostname); + + let certificate_id = 0; + let meta = {}; + + if (typeof host.letsencrypt_email !== 'undefined') { + meta.letsencrypt_email = host.letsencrypt_email; + } + + // determine certificate_id + if (host.ssl && typeof certificate_map[host.hostname] !== 'undefined') { + certificate_id = certificate_map[host.hostname]; + } + + // TODO: Advanced nginx config + + return redirectionHostModel + .query() + .insertAndFetch({ + owner_user_id: 1, + domain_names: [host.hostname], + forward_domain_name: host.forward_host, + block_exploits: host.block_exploits || false, + certificate_id: certificate_id, + ssl_forced: host.force_ssl || false, + meta: meta + }) + .then(row => { + // re-fetch with cert + return internalRedirectionHost.get(access, { + id: row.id, + expand: ['certificate', 'owner'] + }); + }) + .then(row => { + // Configure nginx + return internalNginx.configure(redirectionHostModel, 'redirection_host', row); + }); + }; + + /** + * @param {Access} access + * @param {Object} host + * @returns {Promise} + */ + const importStream = function (access, host) { + logger.info('Creating Stream: ' + host.incoming_port); + + // TODO: Advanced nginx config + + return streamModel + .query() + .insertAndFetch({ + owner_user_id: 1, + incoming_port: host.incoming_port, + forward_ip: host.forward_server, + forwarding_port: host.forward_port, + tcp_forwarding: host.protocols.indexOf('tcp') !== -1, + udp_forwarding: host.protocols.indexOf('udp') !== -1 + }) + .then(row => { + // re-fetch with cert + return internalStream.get(access, { + id: row.id, + expand: ['owner'] + }); + }) + .then(row => { + // Configure nginx + return internalNginx.configure(streamModel, 'stream', row); + }); + }; + + /** + * Returned Promise + */ return new Promise((resolve, reject) => { if (fs.existsSync('/config') && !fs.existsSync('/config/v2-imported')) { logger.info('Beginning import from V1 ...'); - // Setup - const batchflow = require('batchflow'); - const db = require('diskdb'); - module.exports = db.connect('/config', ['hosts', 'access']); + const db = require('diskdb'); + module.exports = db.connect('/config', ['hosts', 'access']); // Create a fake access object const Access = require('./lib/access'); let access = new Access(null); + resolve(access.load(true) - .then(access => { - - - + .then(() => { // Import access lists first - let lists = db.access.find(); - lists.map(list => { - logger.warn('List:', list); - - }); - + return importAccessLists(access, db) + .then(() => { + // Then import Lets Encrypt Certificates + return importCertificates(access); + }) + .then(() => { + // then hosts + return importHosts(access, db); + }) + .then(() => { + // Write the /config/v2-imported file so we don't import again + // TODO + }); }) ); - /* - let hosts = db.hosts.find(); - hosts.map(host => { - logger.warn('Host:', host); - }); - */ - - // Looks like we need to import from version 1 - // There are numerous parts to this import: - // - // 1. The letsencrypt certificates, the need to be added to the database and files renamed - // 2. The access lists from the previous datastore - // 3. The Hosts from the previous datastore - - // get all hosts: - // resolve(db.hosts.find()); - - // get specific host: - // existing_host = db.hosts.findOne({incoming_port: payload.incoming_port}); - - // remove host: - // db.hosts.remove({hostname: payload.hostname}); - - // get all access: - // resolve(db.access.find()); - - resolve(); - } else { resolve(); } diff --git a/src/backend/internal/access-list.js b/src/backend/internal/access-list.js index 5a0b4b2..83dd2e5 100644 --- a/src/backend/internal/access-list.js +++ b/src/backend/internal/access-list.js @@ -31,7 +31,7 @@ const internalAccessList = { .omit(omissions()) .insertAndFetch({ name: data.name, - owner_user_id: access.token.get('attrs').id + owner_user_id: access.token.getUserId(1) }); }) .then(row => { @@ -198,10 +198,6 @@ const internalAccessList = { data = {}; } - if (typeof data.id === 'undefined' || !data.id) { - data.id = access.token.get('attrs').id; - } - return access.can('access_lists:get', data.id) .then(access_data => { let query = accessListModel @@ -215,7 +211,7 @@ const internalAccessList = { .first(); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('access_list.owner_user_id', access.token.getUserId(1)); } // Custom omissions @@ -340,7 +336,7 @@ const internalAccessList = { .orderBy('access_list.name', 'ASC'); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Query is used for searching diff --git a/src/backend/internal/audit-log.js b/src/backend/internal/audit-log.js index ad67d0f..543b166 100644 --- a/src/backend/internal/audit-log.js +++ b/src/backend/internal/audit-log.js @@ -56,7 +56,7 @@ const internalAuditLog = { return new Promise((resolve, reject) => { // Default the user id if (typeof data.user_id === 'undefined' || !data.user_id) { - data.user_id = access.token.get('attrs').id; + data.user_id = access.token.getUserId(1); } if (typeof data.action === 'undefined' || !data.action) { diff --git a/src/backend/internal/certificate.js b/src/backend/internal/certificate.js index 0936b04..93bcfe9 100644 --- a/src/backend/internal/certificate.js +++ b/src/backend/internal/certificate.js @@ -99,7 +99,7 @@ const internalCertificate = { create: (access, data) => { return access.can('certificates:create', data) .then(() => { - data.owner_user_id = access.token.get('attrs').id; + data.owner_user_id = access.token.getUserId(1); if (data.provider === 'letsencrypt') { data.nice_name = data.domain_names.sort().join(', '); @@ -261,10 +261,6 @@ const internalCertificate = { data = {}; } - if (typeof data.id === 'undefined' || !data.id) { - data.id = access.token.get('attrs').id; - } - return access.can('certificates:get', data.id) .then(access_data => { let query = certificateModel @@ -275,7 +271,7 @@ const internalCertificate = { .first(); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Custom omissions @@ -364,7 +360,7 @@ const internalCertificate = { .orderBy('nice_name', 'ASC'); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Query is used for searching diff --git a/src/backend/internal/dead-host.js b/src/backend/internal/dead-host.js index 455937e..e795620 100644 --- a/src/backend/internal/dead-host.js +++ b/src/backend/internal/dead-host.js @@ -46,7 +46,7 @@ const internalDeadHost = { }) .then(() => { // At this point the domains should have been checked - data.owner_user_id = access.token.get('attrs').id; + data.owner_user_id = access.token.getUserId(1); return deadHostModel .query() @@ -219,7 +219,7 @@ const internalDeadHost = { .first(); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Custom omissions @@ -307,7 +307,7 @@ const internalDeadHost = { .orderBy('domain_names', 'ASC'); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Query is used for searching diff --git a/src/backend/internal/proxy-host.js b/src/backend/internal/proxy-host.js index e3c07f7..2d1c9b2 100644 --- a/src/backend/internal/proxy-host.js +++ b/src/backend/internal/proxy-host.js @@ -46,7 +46,7 @@ const internalProxyHost = { }) .then(() => { // At this point the domains should have been checked - data.owner_user_id = access.token.get('attrs').id; + data.owner_user_id = access.token.getUserId(1); return proxyHostModel .query() @@ -220,7 +220,7 @@ const internalProxyHost = { .first(); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Custom omissions @@ -308,7 +308,7 @@ const internalProxyHost = { .orderBy('domain_names', 'ASC'); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Query is used for searching diff --git a/src/backend/internal/redirection-host.js b/src/backend/internal/redirection-host.js index 50ea3df..15df7d9 100644 --- a/src/backend/internal/redirection-host.js +++ b/src/backend/internal/redirection-host.js @@ -46,7 +46,7 @@ const internalRedirectionHost = { }) .then(() => { // At this point the domains should have been checked - data.owner_user_id = access.token.get('attrs').id; + data.owner_user_id = access.token.getUserId(1); return redirectionHostModel .query() @@ -219,7 +219,7 @@ const internalRedirectionHost = { .first(); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Custom omissions @@ -307,7 +307,7 @@ const internalRedirectionHost = { .orderBy('domain_names', 'ASC'); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Query is used for searching diff --git a/src/backend/internal/report.js b/src/backend/internal/report.js index 5124982..bd4faca 100644 --- a/src/backend/internal/report.js +++ b/src/backend/internal/report.js @@ -1,7 +1,5 @@ 'use strict'; -const _ = require('lodash'); -const error = require('../lib/error'); const internalProxyHost = require('./proxy-host'); const internalRedirectionHost = require('./redirection-host'); const internalDeadHost = require('./dead-host'); @@ -16,7 +14,7 @@ const internalReport = { getHostsReport: access => { return access.can('reports:hosts', 1) .then(access_data => { - let user_id = access.token.get('attrs').id; + let user_id = access.token.getUserId(1); let promises = [ internalProxyHost.getCount(user_id, access_data.visibility), diff --git a/src/backend/internal/stream.js b/src/backend/internal/stream.js index 27dfa15..b47b9fa 100644 --- a/src/backend/internal/stream.js +++ b/src/backend/internal/stream.js @@ -21,7 +21,7 @@ const internalStream = { return access.can('streams:create', data) .then(access_data => { // TODO: At this point the existing ports should have been checked - data.owner_user_id = access.token.get('attrs').id; + data.owner_user_id = access.token.getUserId(1); if (typeof data.meta === 'undefined') { data.meta = {}; @@ -113,7 +113,7 @@ const internalStream = { .first(); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Custom omissions @@ -201,7 +201,7 @@ const internalStream = { .orderBy('incoming_port', 'ASC'); if (access_data.permission_visibility !== 'all') { - query.andWhere('owner_user_id', access.token.get('attrs').id); + query.andWhere('owner_user_id', access.token.getUserId(1)); } // Query is used for searching diff --git a/src/backend/internal/token.js b/src/backend/internal/token.js index 6d1b8ca..5b074d3 100644 --- a/src/backend/internal/token.js +++ b/src/backend/internal/token.js @@ -98,7 +98,7 @@ module.exports = { data = data || {}; data.expiry = data.expiry || '30d'; - if (access && access.token.get('attrs').id) { + if (access && access.token.getUserId(0)) { // Create a moment of the expiry expression let expiry = helpers.parseDatePeriod(data.expiry); @@ -107,7 +107,7 @@ module.exports = { } let token_attrs = { - id: access.token.get('attrs').id + id: access.token.getUserId(0) }; // Only admins can request otherwise scoped tokens diff --git a/src/backend/internal/user.js b/src/backend/internal/user.js index 0068881..ff9853f 100644 --- a/src/backend/internal/user.js +++ b/src/backend/internal/user.js @@ -179,7 +179,7 @@ const internalUser = { } if (typeof data.id === 'undefined' || !data.id) { - data.id = access.token.get('attrs').id; + data.id = access.token.getUserId(0); } return access.can('users:get', data.id) @@ -253,7 +253,7 @@ const internalUser = { } // Make sure user can't delete themselves - if (user.id === access.token.get('attrs').id) { + if (user.id === access.token.getUserId(0)) { throw new error.PermissionError('You cannot delete yourself.'); } @@ -352,7 +352,7 @@ const internalUser = { getUserOmisionsByAccess: (access, id_requested) => { let response = []; // Admin response - if (!access.token.hasScope('admin') && access.token.get('attrs').id !== id_requested) { + if (!access.token.hasScope('admin') && access.token.getUserId(0) !== id_requested) { response = ['roles', 'is_deleted']; // Restricted response } @@ -378,7 +378,7 @@ const internalUser = { throw new error.InternalValidationError('User could not be updated, IDs do not match: ' + user.id + ' !== ' + data.id); } - if (user.id === access.token.get('attrs').id) { + if (user.id === access.token.getUserId(0)) { // they're setting their own password. Make sure their current password is correct if (typeof data.current === 'undefined' || !data.current) { throw new error.ValidationError('Current password was not supplied'); diff --git a/src/backend/models/token.js b/src/backend/models/token.js index 2440962..4a96fc4 100644 --- a/src/backend/models/token.js +++ b/src/backend/models/token.js @@ -18,7 +18,8 @@ module.exports = function () { let token_data = {}; - return { + let self = { + //return { /** * @param {Object} payload * @param {Object} [user_options] @@ -128,6 +129,21 @@ module.exports = function () { */ set: function (key, value) { token_data[key] = value; + }, + + /** + * @param [default_value] + * @returns {Integer} + */ + getUserId: default_value => { + let attrs = self.get('attrs'); + if (attrs && typeof attrs.id !== 'undefined' && attrs.id) { + return attrs.id; + } + + return default_value || 0; } }; + + return self; }; diff --git a/src/frontend/js/app/audit-log/list/item.ejs b/src/frontend/js/app/audit-log/list/item.ejs index 817680d..84743c8 100644 --- a/src/frontend/js/app/audit-log/list/item.ejs +++ b/src/frontend/js/app/audit-log/list/item.ejs @@ -39,7 +39,7 @@ items = meta.domain_names; break; case 'access-list': - %> <% + %> <% items.push(meta.name); break; case 'user': @@ -47,7 +47,7 @@ items.push(meta.name); break; case 'certificate': - %> <% + %> <% if (meta.provider === 'letsencrypt') { items = meta.domain_names; } else { diff --git a/src/frontend/js/app/nginx/certificates-list-item.ejs b/src/frontend/js/app/nginx/certificates-list-item.ejs index 89f75b2..aa4b53a 100644 --- a/src/frontend/js/app/nginx/certificates-list-item.ejs +++ b/src/frontend/js/app/nginx/certificates-list-item.ejs @@ -1,17 +1,17 @@
<% if (id === 'new') { %>
- + <%- i18n('all-hosts', 'new-cert') %>
<%- i18n('all-hosts', 'with-le') %> <% } else if (id > 0) { %>
- <%- provider === 'other' ? nice_name : domain_names.join(', ') %> + <%- provider === 'other' ? nice_name : domain_names.join(', ') %>
<%- i18n('ssl', provider) %> – Expires: <%- formatDbDate(expires_on, 'Do MMMM YYYY, h:mm a') %> <% } else { %>
- <%- i18n('all-hosts', 'none') %> + <%- i18n('all-hosts', 'none') %>
<%- i18n('all-hosts', 'no-ssl') %> <% } %> diff --git a/src/frontend/js/app/nginx/proxy/access-list-item.ejs b/src/frontend/js/app/nginx/proxy/access-list-item.ejs index 9bd17fe..9232d38 100644 --- a/src/frontend/js/app/nginx/proxy/access-list-item.ejs +++ b/src/frontend/js/app/nginx/proxy/access-list-item.ejs @@ -1,12 +1,12 @@
<% if (id > 0) { %>
- <%- name %> + <%- name %>
<%- i18n('access-lists', 'item-count', {count: items.length || 0}) %> – Created: <%- formatDbDate(created_on, 'Do MMMM YYYY, h:mm a') %> <% } else { %>
- <%- i18n('access-lists', 'public') %> + <%- i18n('access-lists', 'public') %>
<%- i18n('access-lists', 'public-sub') %> <% } %> diff --git a/src/frontend/js/app/ui/menu/main.ejs b/src/frontend/js/app/ui/menu/main.ejs index 465a73b..3363640 100644 --- a/src/frontend/js/app/ui/menu/main.ejs +++ b/src/frontend/js/app/ui/menu/main.ejs @@ -27,12 +27,12 @@ <% if (canShow('access_lists')) { %> <% } %> <% if (canShow('certificates')) { %> <% } %> <% if (isAdmin()) { %>