walnut.js/lib/worker.js

'use strict';

module.exports.create = function (webserver, xconfx, state) {
  console.log('[worker] create');
  xconfx.debug = true;
  console.log('DEBUG create worker');

  if (!state) {
    state = {};
  }

  var PromiseA = state.Promise || require('bluebird');
  var memstore;
  var sqlstores = {};
  var systemFactory = require('sqlite3-cluster/client').createClientFactory({
      dirname: xconfx.varpath
    , prefix: 'walnut+'
    //, dbname: 'config'
    , suffix: '@daplie.com'
    , ext: '.sqlite3'
    , sock: xconfx.sqlite3Sock
    , ipcKey: xconfx.ipcKey
  });
  /*
  var clientFactory = require('sqlite3-cluster/client').createClientFactory({
      algorithm: 'aes'
    , bits: 128
    , mode: 'cbc'
    , dirname: xconfx.varpath // TODO conf
    , prefix: 'com.daplie.walnut.'
    //, dbname: 'cluster'
    , suffix: ''
    , ext: '.sqlcipher'
    , sock: xconfx.sqlite3Sock
    , ipcKey: xconfx.ipcKey
  });
  */
  var cstore = require('cluster-store');

  console.log('[worker] creating data stores...');
  return PromiseA.all([
    // TODO security on memstore
    // TODO memstoreFactory.create
    cstore.create({
      sock: xconfx.memstoreSock
    , connect: xconfx.memstoreSock
      // TODO implement
    , key: xconfx.ipcKey
    }).then(function (_memstore) {
      console.log('[worker] cstore created');
      memstore = PromiseA.promisifyAll(_memstore);
      return memstore;
    })
    // TODO mark a device as lost, stolen, missing in DNS records
    // (and in turn allow other devices to lock it, turn on location reporting, etc)
  , systemFactory.create({
        init: true
      , dbname: 'config'
    }).then(function (sysdb) {
      console.log('[worker] sysdb created');
      return sysdb;
    })
  ]).then(function (args) {
    console.log('[worker] database factories created');
    memstore = args[0];
    sqlstores.config = args[1];

    var wrap = require('masterquest-sqlite3');
    var dir = [
      { tablename: 'com_daplie_walnut_config'
      , idname: 'id'
      , unique: [ 'id' ]
      , indices: [ 'createdAt', 'updatedAt' ]
      }
    , { tablename: 'com_daplie_walnut_redirects'
      , idname: 'id'      // blog.example.com:sample.net/blog
      , unique: [ 'id' ]
      , indices: [ 'createdAt', 'updatedAt' ]
      }
    ];

    function scopeMemstore(expId) {
      var scope = expId + '|';
      return {
        getAsync: function (id) {
          return memstore.getAsync(scope + id);
        }
      , setAsync: function (id, data) {
          return memstore.setAsync(scope + id, data);
        }
      , touchAsync: function (id, data) {
          return memstore.touchAsync(scope + id, data);
        }
      , destroyAsync: function (id) {
          return memstore.destroyAsync(scope + id);
        }

      // helpers
      , allAsync: function () {
          return memstore.allAsync().then(function (db) {
            return Object.keys(db).filter(function (key) {
              return 0 === key.indexOf(scope);
            }).map(function (key) {
              return db[key];
            });
          });
        }
      , lengthAsync: function () {
          return memstore.allAsync().then(function (db) {
            return Object.keys(db).filter(function (key) {
              return 0 === key.indexOf(scope);
            }).length;
          });
        }
      , clearAsync: function () {
          return memstore.allAsync().then(function (db) {
            return Object.keys(db).filter(function (key) {
              return 0 === key.indexOf(scope);
            }).map(function (key) {
              return memstore.destroyAsync(key);
            });
          }).then(function () {
            return null;
          });
        }
      };
    }

    return wrap.wrap(sqlstores.config, dir).then(function (models) {
      console.log('[worker] database wrapped');
      return models.ComDaplieWalnutConfig.find(null, { limit: 100 }).then(function (results) {
        console.log('[worker] config query complete');
        return models.ComDaplieWalnutConfig.find(null, { limit: 10000 }).then(function (redirects) {
          console.log('[worker] configuring express');
          var express = require('express-lazy');
          var app = express();
          var recase = require('connect-recase')({
            // TODO allow explicit and or default flag
            explicit: false
          , default: 'snake'
          , prefixes: ['/api']
            // TODO allow exclude
          //, exclusions: [config.oauthPrefix]
          , exceptions: {}
          //, cancelParam: 'camel'
          });
          var bootstrapApp;
          var mainApp;
          var apiDeps = {
            models: models
            // TODO don't let packages use this directly
          , Promise: PromiseA
          , dns: PromiseA.promisifyAll(require('dns'))
          , crypto: PromiseA.promisifyAll(require('crypto'))
          , fs: PromiseA.promisifyAll(require('fs'))
          , path: require('path')
          , validate: {
              isEmail: function (email) {
                return /@/.test(email) && !/\s+/.test(email);
              }
            , email: function (email) {
                if (apiDeps.validate.isEmail(email)) {
                  return null;
                }
                return new Error('invalid email address');
              }
            }
          };
          var apiFactories = {
            memstoreFactory: { create: scopeMemstore }
          , systemSqlFactory: systemFactory
          };

          var hostsmap = {};

          function log(req, res, next) {
            var hostname = (req.hostname || req.headers.host || '').split(':').shift();

            // Printing all incoming requests for debugging
            console.log('[worker/log]', req.method, hostname, req.url);

            // logging all the invalid hostnames that come here out of curiousity
            if (hostname && !hostsmap[hostname]) {
              hostsmap[hostname] = true;
              require('fs').writeFile(
                require('path').join(__dirname, '..', '..', 'var', 'hostnames', hostname)
              , hostname
              , function () {}
              );
            }

            next();
          }

          function setupMain() {
            if (xconfx.debug) { console.log('[main] setup'); }
            mainApp = express();
            require('./main').create(mainApp, xconfx, apiFactories, apiDeps, errorIfApi, errorIfAssets).then(function () {
              if (xconfx.debug) { console.log('[main] ready'); }
              // TODO process.send({});
            });
          }

          if (!bootstrapApp) {
            if (xconfx.debug) { console.log('[bootstrap] setup'); }
            if (xconfx.primaryDomain) {
              bootstrapApp = true;
              setupMain();
              return;
            }
            bootstrapApp = express();
            require('./bootstrap').create(bootstrapApp, xconfx, models).then(function () {
              if (xconfx.debug) { console.log('[bootstrap] ready'); }
              // TODO process.send({});
              setupMain();
            });
          }

          process.on('message', function (data) {
            if ('com.daplie.walnut.bootstrap' === data.type) {
              setupMain();
            }
          });

          function errorIfNotApi(req, res, next) {
            var hostname = req.hostname || req.headers.host;

            if (!/^api\.[a-z0-9\-]+/.test(hostname)) {
              res.send({ error:
               { message: "['" + hostname + req.url + "'] API access is restricted to proper 'api'-prefixed lowercase subdomains."
                   + " The HTTP 'Host' header must exist and must begin with 'api.' as in 'api.example.com'."
                   + " For development you may test with api.localhost.daplie.me (or any domain by modifying your /etc/hosts)"
               , code: 'E_NOT_API'
               , _hostname: hostname
               }
              });
              return;
            }

            next();
          }

          function errorIfNotAssets(req, res, next) {
            var hostname = req.hostname || req.headers.host;

            if (!/^assets\.[a-z0-9\-]+/.test(hostname)) {
              res.send({ error:
               { message: "['" + hostname + req.url + "'] protected asset access is restricted to proper 'asset'-prefixed lowercase subdomains."
                   + " The HTTP 'Host' header must exist and must begin with 'assets.' as in 'assets.example.com'."
                   + " For development you may test with assets.localhost.daplie.me (or any domain by modifying your /etc/hosts)"
               , code: 'E_NOT_API'
               , _hostname: hostname
               }
              });
              return;
            }

            next();
          }

          function errorIfApi(req, res, next) {
            if (!/^api\./.test(req.headers.host)) {
              next();
              return;
            }

            // has api. hostname prefix

            // doesn't have /api url prefix
            if (!/^\/api\//.test(req.url)) {
              console.log('[walnut/worker api] req.url', req.url);
              res.send({ error: { message: "missing /api/ url prefix" } });
              return;
            }

            res.send({ error: { code: 'E_NO_IMPL', message: "API not implemented" } });
          }

          function errorIfAssets(req, res, next) {
            if (!/^assets\./.test(req.headers.host)) {
              next();
              return;
            }

            // has api. hostname prefix

            // doesn't have /api url prefix
            if (!/^\/assets\//.test(req.url)) {
              console.log('[walnut/worker assets] req.url', req.url);
              res.send({ error: { message: "missing /assets/ url prefix" } });
              return;
            }

            res.send({ error: { code: 'E_NO_IMPL', message: "assets handler not implemented" } });
          }

          app.disable('x-powered-by');
          app.use('/', log);
          app.use('/api', require('body-parser').json({
            strict: true // only objects and arrays
          , inflate: true
            // limited to due performance issues with JSON.parse and JSON.stringify
            // http://josh.zeigler.us/technology/web-development/how-big-is-too-big-for-json/
          //, limit: 128 * 1024
          , limit: 1.5 * 1024 * 1024
          , reviver: undefined
          , type: 'json'
          , verify: undefined
          }));
          app.use('/api', recase);

          var cookieParser = require('cookie-parser'); // signing is done in JWT

          app.set('trust proxy', ['loopback', 'linklocal', 'uniquelocal']);
          app.use('/api', errorIfNotApi);
          app.use('/assets', /*errorIfNotAssets,*/ cookieParser()); // serializer { path: '/assets', httpOnly: true, sameSite: true/*, domain: assets.example.com*/ }
          app.use('/', function (req, res) {
            if (!(req.encrypted || req.secure)) {
              // did not come from https
              if (/\.(appcache|manifest)\b/.test(req.url)) {
                require('./unbrick-appcache').unbrick(req, res);
                return;
              }
              console.log('[lib/worker] unencrypted:', req.headers);
              res.end("Connection is not encrypted. That's no bueno or, as we say in Hungarian, nem szabad!");
              return;
            }

            // TODO check https://letsencrypt.status.io to see if https certification is not available

            if (mainApp) {
              mainApp(req, res);
              return;
            }
            else {
              bootstrapApp(req, res);
              return;
            }
          });

          return app;
        });
      });
    });
  });
};