coolaj86/digd.js
2
1
Fork 1
Code Issues 7 Pull Requests 2 Releases Activity

WIP: split the engine and the store via an API

Browse Source
This commit is contained in:
Aaron Madsen 2018-11-21 10:53:54 -07:00
parent a4a3a89fd2
commit 8eec24c555
6 changed files with 310 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
node_modules
.*.sw*
local-db.js

4
bin/digd.js
View File

@ -387,7 +387,7 @@ cli.main(function (args, cli) {
}
try {
engine = engine || require('../lib/store.json.js').create(engineOpts);
engine = engine || require('../lib/store').create(engineOpts);
} catch(e) {
respondWithResults(e);
return;
@ -413,7 +413,7 @@ cli.main(function (args, cli) {
}
if (cli.http) {
try {
engine = engine || require('../lib/store.json.js').create(engineOpts);
engine = engine || require('../lib/store').create(engineOpts);
} catch(e) {
console.error(e);
return;

4
lib/digd.js
View File

@ -184,7 +184,7 @@ function getNs(engine, zs, results, cb) {
// d.vanityNs should only be vanity nameservers (pointing to this same server)
if (z.vanityNs || results.authority.some(function (ns) {
console.log('[debug] ns', ns);
return -1 !== engine.primaryNameservers.indexOf(ns.data.toLowerCase());
return -1 !== engine.primaryNameservers().indexOf(ns.data.toLowerCase());
})) {
results.authority.length = 0;
results.authority.push(engine.zones._toSoa(z));
@ -359,7 +359,7 @@ module.exports.query = function (engine, query, cb) {
// NOTE: I think that the issue here is EXTERNAL vs INTERNAL vanity NS
// We _should_ reply for EXTERNAL vanity NS... but not when it's listed on the SOA internally?
// It's surrounding the problem of what if I do sub domain delegation to the same server.
if (-1 === engine.primaryNameservers.indexOf(r.data.toLowerCase())) {
if (-1 === engine.primaryNameservers().indexOf(r.data.toLowerCase())) {
console.log("It's a vanity NS");
return false;
}

7
lib/httpd.js
View File

@ -230,6 +230,11 @@ module.exports.create = function (cli, engine/*, dnsd*/) {
zone.class = zone.className;
zone.type = zone.typeName;
zone.soa = true;
// TODO: consider sending a predicate object through the engine
// to the actual store in case it is highly inefficient to transfer
// a large number of records from the store that will just be
// thrown away.
engine.records.all(function (err, records) {
records = records.filter(function (r) {
return r.zone === zonename;
@ -239,6 +244,8 @@ module.exports.create = function (cli, engine/*, dnsd*/) {
});
});
});
// I wonder what an API that gets ALL records from all zones is for
app.get('/api/records', function (req, res) {
engine.records.all(function (err, records) {
res.send({ records: records.map(mapRecord) });

268
lib/store.json.js → lib/store/index.js
View File

@ -1,101 +1,80 @@
'use strict';
var crypto = require('crypto');
var dns = require('dns');
var os = require('os');
var path = require('path');
var pathResolvers = {
'.': function fromCwd(relPath) {
return path.join(process.cwd(), relPath);
},
'~': function fromHomedir(relPath) {
if (!os.homedir) {
throw new Error(
'Resolving home directory relative paths is not supported in this version of node.'
);
}
return path.join(os.homedir(), relPath);
},
noop: function (p) { return p; }
}
module.exports.create = function (opts) {
// opts = { filepath };
// `opts.filepath` is a module id or path to a module that contains a store plugin or file
var pathFn = pathResolvers[opts.filepath[0]] || pathResolvers.noop;
var storeId = pathFn(opts.filepath);
var pathToStore = require.resolve(storeId);
var engine = { db: null };
function notDeleted(r) {
return !r.revokedAt && !r.deletedAt;
}
var db = require(opts.filepath);
var stat = require('fs').statSync(opts.filepath);
var crypto = require('crypto');
//
// Manual Migration
//
db.primaryNameservers.forEach(function (ns, i, arr) {
if ('string' === typeof ns) {
ns = { name: ns };
arr[i] = ns;
}
if (!ns.id) {
ns.id = crypto.randomBytes(16).toString('hex');
}
});
db.zones = db.zones || [];
if (db.domains) {
db.zones = db.zones.concat(db.domains);
}
db.zones.forEach(function (zone) {
if (!zone.name) {
zone.name = zone.id;
zone.id = null;
}
if (!zone.id) {
zone.id = crypto.randomBytes(16).toString('hex');
}
if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); }
if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); }
});
db.records.forEach(function (record) {
if (!record.id) {
record.id = crypto.randomBytes(16).toString('hex');
}
});
require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2));
//
// End Migration
//
// instantiate the DB module
var db = (pathToStore.slice(-5) === '.json') ?
// JSON files should be loaded using our built in store.json.js
require('./store.json.js')(pathToStore) :
// everything else should be loaded as a module and passed our opts object
require(storeId)(opts);
db.save = function (cb) {
if (db.save._saving) {
console.log('make pending');
db.save._pending.push(cb);
return;
}
// TODO: examine usage of engine.primaryNameservers to see if we are supporting it right
engine.primaryNameservers = db.primaryNameservers.list;
db.save._saving = true;
require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) {
console.log('done writing');
var pending = db.save._pending.splice(0);
db.save._saving = false;
cb(err);
if (!pending.length) {
return;
}
db.save(function (err) {
console.log('double save');
pending.forEach(function (cb) { cb(err); });
});
});
};
db.save._pending = [];
engine.primaryNameservers = db.primaryNameservers;
engine.peers = {
all: function (cb) {
var dns = require('dns');
var count = db.primaryNameservers.length;
function gotRecord() {
count -= 1;
if (!count) {
cb(null, db.primaryNameservers);
}
}
function getRecord(ns) {
var pNS = db.primaryNameservers.list();
function getRecord(ns, done) {
dns.resolve4(ns.name, function (err, addresses) {
console.log('ns addresses:');
console.log(addresses);
if (err) { console.error(err); gotRecord(); return; }
if (err) { console.error(err); done(); return; }
ns.type = 'A';
ns.address = addresses[0];
gotRecord();
done();
});
}
db.primaryNameservers.forEach(getRecord);
// resolve addreses for all of the primary nameservers in parallel
pNS.forEach(function (ns) {
var status = { pending: true };
function done() {
status.pending = false;
// TODO: determine if the locally stored records should get updated
var incomplete = tasks.filter(function (s) { return s.pending; });
if (incomplete.length < 1) {
cb(null, pNS);
}
}
getRecord(ns, done);
return status;
});
}
};
engine.zones = {
_immutableKeys: [ 'id', 'name', 'primary', 'serial', 'revokedAt', 'changedAt', 'insertedAt', 'updatedAt', 'deletedAt' ]
, _mutableKeys: [ 'admin', 'expiration', 'minimum', 'refresh', 'retry', 'ttl', 'vanity' ]
@ -105,8 +84,9 @@ module.exports.create = function (opts) {
// epoch in seconds will do
return parseInt(Math.round(date/1000).toString().slice(-10), 10);
}
// NOTE/TODO: despite the _, _toSoa is used outside this file (in lib/digd.js and lib/httpd.js)
, _toSoa: function (domain) {
var nameservers = domain.vanityNs || engine.primaryNameservers.map(function (n) { return n.name; });
var nameservers = domain.vanityNs || engine.primaryNameservers().map(function (n) { return n.name; });
var index = Math.floor(Math.random() * nameservers.length) % nameservers.length;
var nameserver = nameservers[index];
@ -122,6 +102,7 @@ module.exports.create = function (opts) {
, name_server: nameserver
// admin -- email address or domain for admin
// default is effectively admin@{domain name}
, admin: domain.admin || ('admin.' + domain.name)
, email_addr: domain.admin || ('admin.' + domain.name)
@ -148,7 +129,7 @@ module.exports.create = function (opts) {
}
, all: function (cb) {
process.nextTick(function () {
cb(null, db.zones.slice(0).filter(notDeleted));
cb(null, db.zones().filter(notDeleted));
});
}
, get: function (queries, cb) {
@ -157,7 +138,7 @@ module.exports.create = function (opts) {
return { name: n };
});
}
var myDomains = db.zones.filter(function (d) {
var myDomains = db.zones().filter(function (d) {
return queries.some(function (q) {
return (d.name.toLowerCase() === q.name) && notDeleted(d);
});
@ -167,19 +148,17 @@ module.exports.create = function (opts) {
});
}
, touch: function (zone, cb) {
var existing;
db.zones.some(function (z) {
if (z.id && zone.id === z.id) { existing = z; return true; }
if (z.name && zone.name === z.name) { existing = z; return true; }
});
if (!existing) {
cb(null, null);
db.zones.get(zone, function (err, existing) {
if (err || !existing) {
cb(err, null);
return;
}
existing.updatedAt = new Date().valueOf(); // toISOString();
console.log('touch saving...');
db.zone.update(existing, function (err) {
cb(err, !err && existing || null);
});
return;
}
existing.updatedAt = new Date().valueOf(); // toISOString();
console.log('touch saving...');
db.save(function (err) {
cb(err, !err && existing || null);
});
}
, save: function (zone, cb) {
@ -191,65 +170,69 @@ module.exports.create = function (opts) {
}
}
, update: function (zone, cb) {
var existing;
var dirty;
db.zones.get({ id: zone.id }, function (err, found) {
var dirty;
db.zones.some(function (z) {
if (z.id === zone.id) {
existing = z;
return true;
if (err) {
console.log('error finding zone');
cb(new Error("Error finding zone for '" + zone.id + "'"), null);
return;
}
});
if (!existing) {
console.log('no existing zone');
cb(new Error("zone for '" + zone.id + "' does not exist"), null);
return;
}
console.log('found existing zone');
console.log(existing);
console.log(zone);
Object.keys(zone).forEach(function (key) {
if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; }
if (existing[key] !== zone[key]) {
dirty = true;
console.log('existing key', key, existing[key], zone[key]);
existing[key] = zone[key];
if (!found) {
console.log('no existing zone');
cb(new Error("zone for '" + zone.id + "' does not exist"), null);
return;
}
});
zone.updatedAt = new Date().valueOf(); // toISOString(); // Math.round(Date.now() / 1000);
if (dirty) {
zone.changedAt = zone.updatedAt;
}
console.log('found existing zone');
console.log(found);
console.log(zone);
Object.keys(zone).forEach(function (key) {
if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; }
if (found[key] !== zone[key]) {
dirty = true;
console.log('existing key', key, found[key], zone[key]);
found[key] = zone[key];
}
});
console.log('saving...');
db.save(function (err) {
cb(err, !err && existing || null);
found.updatedAt = new Date().valueOf(); // toISOString(); // Math.round(Date.now() / 1000);
if (dirty) {
found.changedAt = found.updatedAt;
}
console.log('saving...');
db.zones.update(found, function (err) {
cb(err, !err && found || null);
});
});
}
, create: function (zone, cb) {
var newZone = { id: crypto.randomBytes(16).toString('hex') };
var existing;
var nss = [];
zone.name = (zone.name||'').toLowerCase();
db.zones.some(function (z) {
if (z.name === zone.name) {
existing = z;
return true;
}
});
if (existing) {
cb(new Error("tried to create new zone, but '" + existing.name + "' already exists"));
var zoneName = (zone.name||'').toLowerCase();
db.zones.get({ name: zoneName }, function (err, found) {
if (err) {
console.error(err);
cb(new Error("error attempting to create new zone '" + zoneName + "'"));
return;
}
newZone.name = zone.name;
if (found) {
cb(new Error("tried to create new zone, but '" + found.name + "' already exists"));
return;
}
var newZone = {
id: crypto.randomBytes(16).toString('hex'),
name: zoneName
};
var nss = [];
newZone.createdAt = Date.now();
newZone.updatedAt = newZone.createdAt;
/*
Set only the mutable keys in the new zone from the proposed zone object
*/
Object.keys(zone).forEach(function (key) {
//if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; }
if (-1 === engine.zones._mutableKeys.indexOf(key)) { return; }
@ -262,7 +245,12 @@ module.exports.create = function (opts) {
} else {
newZone.vanity = false;
}
db.primaryNameservers.forEach(function (ns, i) {
// TODO: distinguish between primary and secondary zones
// TODO: determine if we need to do anything special for delegation
// create records for the primary nameservers (or vanity name servers)
db.primaryNameservers.list().forEach(function (ns, i) {
var nsx = 'ns' + (i + 1);
var nsZone;
var ttl = 43200; // 12h // TODO pick a well-reasoned number
@ -302,7 +290,13 @@ module.exports.create = function (opts) {
});
});
db.zones.push(newZone);
db.zones.create(newZone, function (err) {
// WIP: going to need to figure out how to manage this as a transaction
// Significant benefit to having records owned by the zone is we won't have
// records for zones that don't otherwise exist - at least at the engine level.
// every line below this one is not yet modified...
});
nss.forEach(function (ns) {
db.records.push(ns);
});

167
lib/store/store.json.js Normal file
View File

@ -0,0 +1,167 @@
'use strict';
function jsonDeepClone(target) {
return JSON.parse(
JSON.stringify(target)
);
}
/*
init() should return an object with: {
save: function -> undefined - changes to in memory representation should be persisted
This could be considered the equivalent of committing a transaction to the database.
primaryNameservers: {
list: function -> list nameservers
},
zones: {
list: function -> list zones,
find: function -> read zone by ???,
create:
update:
delete:
},
records: {
list: function -> list records,
find: function -> read record by ???,
create:
update:
delete:
}
}
All lists will be a deep copy of the data actually stored.
*/
module.exports = function init (opts) {
// opts = { filepath };
var db = require(opts.filepath);
var stat = require('fs').statSync(opts.filepath);
var crypto = require('crypto');
//
// Manual Migration
//
// Convert the primary nameservers from strings to objects with names and IDs.
db.primaryNameservers.forEach(function (ns, i, arr) {
if ('string' === typeof ns) {
ns = { name: ns };
arr[i] = ns;
}
if (!ns.id) {
ns.id = crypto.randomBytes(16).toString('hex');
}
});
// Convert domains to zones and ensure that they have proper IDs and timestamps
db.zones = db.zones || [];
if (db.domains) {
db.zones = db.zones.concat(db.domains);
}
db.zones.forEach(function (zone) {
if (!zone.name) {
zone.name = zone.id;
zone.id = null;
}
if (!zone.id) {
zone.id = crypto.randomBytes(16).toString('hex');
}
if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); }
if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); }
});
// Records belong to zones, but they (currently) refer to them by a zone property.
// NOTE/TODO: This may pose problems where the whole list of records is not easily
// filtered / kept in memory / indexed and/or retrieved by zone. Traditionally,
// records are stored "within a zone" in a zone file. We may wish to have the
// DB API behave more traditionally, even though some stores (like a SQL database
// table) might actually store the zone as a property of a record as we currently do.
db.records.forEach(function (record) {
if (!record.id) {
record.id = crypto.randomBytes(16).toString('hex');
}
});
// Write the migrated data
require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2));
//
// End Migration
//
var save = function save (cb) {
if (save._saving) {
console.log('make pending');
save._pending.push(cb);
return;
}
save._saving = true;
// TODO: replace with something not destructive to original non-json data
require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) {
console.log('done writing');
var pending = save._pending.splice(0);
save._saving = false;
cb(err);
if (!pending.length) {
return;
}
save(function (err) {
console.log('double save');
pending.forEach(function (cb) { cb(err); });
});
});
};
save._pending = [];
var dbApi = {
save: function () {
// hide _pending and _saving from callers
var args = [].slice.call(arguments);
return save.apply(null, args);
},
// primaryNameservers really isn't editable - it's literally the list of FQDN's
// that this database is replicated to in a multi-master fashion.
//
// However, lib/store/index.js does plenty to update these records in support
// of the SOA records that are built from them (as does this file in the "migration"
// section). I'm toying with the idea of not storing them seperately or creating the
// SOA records somewhat immediately.
primaryNameservers: {
list: function listNameservers() {
return jsonDeepClone(db.primaryNameservers);
}
},
zones: {
list: function listZones() {
return jsonDeepClone(db.zones);
},
find: function getZone(predicate, cb) {
var found;
db.zones.some(function (z) {
if (z.id && predicate.id === z.id) { found = z; return true; }
if (z.name && predicate.name === z.name) { found = z; return true; }
});
if (!found) {
cb(null, null);
return;
}
cb(null, jsonDeepClone(found));
return;
},
create: function() {},
update: function() {},
delete: function() {}
},
records: {
list: function listRecords() {
return jsonDeepClone(db.records);
},
find: function getRecord(predicate, cb) {
},
create: function() {},
update: function() {},
delete: function() {}
}
};
return dbApi;
};