WIP: keep iterating on the store API
This commit is contained in:
Aaron Madsen
parent
8eec24c555
commit
e43c169257
@ -1,10 +1,99 @@
|
||||
'use strict';
|
||||
|
||||
var crypto = require('crypto');
|
||||
|
||||
function jsonDeepClone(target) {
|
||||
return JSON.parse(
|
||||
JSON.stringify(target)
|
||||
);
|
||||
}
|
||||
|
||||
function mergeObjects() {
|
||||
// arguments should be an array of objects. We
|
||||
// reverse it because the last argument to set
|
||||
// a value wins.
|
||||
var args = [].slice.call(arguments).reverse();
|
||||
var len = args.length;
|
||||
if (len === 1) {
|
||||
return args[0];
|
||||
}
|
||||
|
||||
// gather the set of keys from all arguments
|
||||
var keyLists = args.map(function (arg) {
|
||||
return Object.keys(arg);
|
||||
});
|
||||
|
||||
var keys = Object.keys(keyLists.reduce(function (all, list) {
|
||||
list.forEach(function (k) {
|
||||
all[k] = true;
|
||||
});
|
||||
return all;
|
||||
}, {}));
|
||||
|
||||
// for each key
|
||||
return keys.reduce(function (target, k) {
|
||||
// find the first argument (because of the reverse() above) with the key set
|
||||
var values = [];
|
||||
var isObject = false;
|
||||
for (var i = 0; i < len; i++) {
|
||||
var v = args[i];
|
||||
var vType = typeof v;
|
||||
|
||||
if (vType === 'object') {
|
||||
if (!v) {
|
||||
// typeof null is object. null is the only falsey object. null represents
|
||||
// a delete or the end of our argument list;
|
||||
break;
|
||||
}
|
||||
// we need to collect values until we get a non-object, so we can merge them
|
||||
values.push(v);
|
||||
isObject = true;
|
||||
} else if (!isObject) {
|
||||
if (vType === 'undefined') {
|
||||
// if the arg actually has the key set this is effectively a "delete"
|
||||
if (keyList[i].indexOf(k) != -1) {
|
||||
break;
|
||||
}
|
||||
// otherwise we need to check the next argument's value, so we don't break the loop
|
||||
} else {
|
||||
values.push(v);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// a previous value was an object, this one isn't
|
||||
// That means we are done collecting values.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (values.length > 0) {
|
||||
target[k] = mergeObjects.apply(null, values);
|
||||
}
|
||||
|
||||
return target;
|
||||
}, {});
|
||||
}
|
||||
|
||||
function prepareZone(zone, options) {
|
||||
var opts = options || {};
|
||||
var timestamp = opts.timestamp || Date.now();
|
||||
if (!zone.name) {
|
||||
zone.name = zone.id;
|
||||
zone.id = null;
|
||||
}
|
||||
if (!zone.id) {
|
||||
zone.id = crypto.randomBytes(16).toString('hex');
|
||||
}
|
||||
if (!zone.createdAt) { zone.createdAt = timestamp; }
|
||||
if (!zone.updatedAt || opts.isUpdate) { zone.updatedAt = timestamp; }
|
||||
|
||||
// create a names set for the zone, keyed by record name mapped to
|
||||
// an object for the various records with that name, by type (A, MX, TXT, etc.)
|
||||
zone.records = zone.records || {};
|
||||
|
||||
return zone;
|
||||
}
|
||||
|
||||
/*
|
||||
init() should return an object with: {
|
||||
save: function -> undefined - changes to in memory representation should be persisted
|
||||
@ -14,14 +103,12 @@ init() should return an object with: {
|
||||
},
|
||||
zones: {
|
||||
list: function -> list zones,
|
||||
find: function -> read zone by ???,
|
||||
create:
|
||||
update:
|
||||
delete:
|
||||
},
|
||||
records: {
|
||||
list: function -> list records,
|
||||
find: function -> read record by ???,
|
||||
create:
|
||||
update:
|
||||
delete:
|
||||
@ -35,51 +122,71 @@ module.exports = function init (opts) {
|
||||
// opts = { filepath };
|
||||
|
||||
var db = require(opts.filepath);
|
||||
var stat = require('fs').statSync(opts.filepath);
|
||||
var crypto = require('crypto');
|
||||
var mtime = require('fs').statSync(opts.filepath).mtime.valueOf();
|
||||
|
||||
//
|
||||
// Manual Migration
|
||||
// Migration from other formats
|
||||
//
|
||||
|
||||
// Convert the primary nameservers from strings to objects with names and IDs.
|
||||
db.primaryNameservers.forEach(function (ns, i, arr) {
|
||||
if ('string' === typeof ns) {
|
||||
ns = { name: ns };
|
||||
arr[i] = ns;
|
||||
}
|
||||
if (!ns.id) {
|
||||
ns.id = crypto.randomBytes(16).toString('hex');
|
||||
}
|
||||
// Convert the primary nameservers from an array of strings to objects with names and IDs.
|
||||
// also switch to the 'peers' name, since we are really interested in the other FQDNs that
|
||||
// use the same data store and are kept in sync.
|
||||
var peerList = (!db.peers || Array.isArray(db.peers))? db.peers : Object.keys(db.peers).map(function (p) {
|
||||
return db.peers[p];
|
||||
});
|
||||
db.peers = [].concat(db.primaryNameservers, peerList).filter(function (p) {
|
||||
// filer out empty strings, undefined, etc.
|
||||
return !!p;
|
||||
}).map(function (ns) {
|
||||
var peer = ('string' === typeof ns)? ns : { name: ns };
|
||||
if (!ns.id) {
|
||||
peer.id = crypto.randomBytes(16).toString('hex');
|
||||
}
|
||||
return peer;
|
||||
}).reduce(function (peers, p) {
|
||||
peers[p.name] = p;
|
||||
return peers;
|
||||
}, {});
|
||||
delete db.primaryNameservers;
|
||||
|
||||
// Convert domains to zones and ensure that they have proper IDs and timestamps
|
||||
db.zones = db.zones || [];
|
||||
if (db.domains) {
|
||||
db.zones = db.zones.concat(db.domains);
|
||||
}
|
||||
db.zones.forEach(function (zone) {
|
||||
if (!zone.name) {
|
||||
zone.name = zone.id;
|
||||
zone.id = null;
|
||||
}
|
||||
if (!zone.id) {
|
||||
zone.id = crypto.randomBytes(16).toString('hex');
|
||||
}
|
||||
if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); }
|
||||
if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); }
|
||||
// Organize zones as a set of zone names
|
||||
var zoneList = (!db.zones || Array.isArray(db.zones))? db.zones : Object.keys(db.zones).map(function (z) {
|
||||
return db.zones[z];
|
||||
});
|
||||
|
||||
// Records belong to zones, but they (currently) refer to them by a zone property.
|
||||
// NOTE/TODO: This may pose problems where the whole list of records is not easily
|
||||
db.zones = [].concat(db.domains, zoneList).filter(function (z) {
|
||||
// filer out empty strings, undefined, etc.
|
||||
return !!z;
|
||||
}).map(function (zone) {
|
||||
return prepareZone(zone, { timestamp: mtime });
|
||||
}).reduce(function (zones, z) {
|
||||
zones[z.name] = z;
|
||||
return zones;
|
||||
}, {});
|
||||
delete db.domains;
|
||||
|
||||
// NOTE: Records belong to zones, but they previously referred to them only by a
|
||||
// zone property. This may pose problems where the whole list of records is not easily
|
||||
// filtered / kept in memory / indexed and/or retrieved by zone. Traditionally,
|
||||
// records are stored "within a zone" in a zone file. We may wish to have the
|
||||
// DB API behave more traditionally, even though some stores (like a SQL database
|
||||
// records are stored "within a zone" in a zone file. We want to have the store API
|
||||
// behave more traditionally, even though some stores (like a SQL database
|
||||
// table) might actually store the zone as a property of a record as we currently do.
|
||||
db.records.forEach(function (record) {
|
||||
// (This fits with the somewhat unexpected and confusing logic of wildcard records.)
|
||||
(db.records || []).forEach(function (record) {
|
||||
// make sure the record has an ID
|
||||
if (!record.id) {
|
||||
record.id = crypto.randomBytes(16).toString('hex');
|
||||
}
|
||||
|
||||
// put it in it's zone - synthesize one if needed
|
||||
db.zones[record.zone] = db.zones[record.zone] || prepareZone({ name: record.zone });
|
||||
var zone = db.zones[record.zone];
|
||||
zone.records[record.name] = zone.records[record.name] || [];
|
||||
var recordsForName = zone.records[record.name];
|
||||
recordsForName.push(record);
|
||||
});
|
||||
delete db.records;
|
||||
|
||||
// Write the migrated data
|
||||
require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2));
|
||||
@ -95,7 +202,6 @@ module.exports = function init (opts) {
|
||||
}
|
||||
|
||||
save._saving = true;
|
||||
// TODO: replace with something not destructive to original non-json data
|
||||
require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) {
|
||||
console.log('done writing');
|
||||
var pending = save._pending.splice(0);
|
||||
@ -112,54 +218,217 @@ module.exports = function init (opts) {
|
||||
};
|
||||
save._pending = [];
|
||||
|
||||
function matchPredicate(predicate) {
|
||||
return function (toCheck) {
|
||||
// which items match the predicate?
|
||||
if (!toCheck) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// check all the keys in the predicate - only supporting exact match
|
||||
// of at least one listed option for all keys right now
|
||||
if (Object.keys(predicate || {}).some(function (k) {
|
||||
return [].concat(predicate[k]).indexOf(toCheck[k]) === -1;
|
||||
})) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// we have a match
|
||||
return true;
|
||||
};
|
||||
}
|
||||
|
||||
function matchZone(predicate) {
|
||||
var zonenames = !!predicate.name ? [].concat(predicate.name) : Object.keys(db.zones);
|
||||
var check = matchPredicate(predicate);
|
||||
// TODO: swap the filter() for a functional style "loop" recursive function
|
||||
// that lets us return early if we have a limit, etc.
|
||||
var found = zonenames.filter(function (zonename) {
|
||||
/*
|
||||
if (predicate.id && predicate.id !== z.id) { return false; }
|
||||
if (predicate.name && predicate.name !== z.name) { return false; }
|
||||
*/
|
||||
return check(db.zones[zonename]);
|
||||
}).map(function (zonename) {
|
||||
return db.zones[zonename];
|
||||
});
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
var dbApi = {
|
||||
save: function () {
|
||||
// hide _pending and _saving from callers
|
||||
var args = [].slice.call(arguments);
|
||||
return save.apply(null, args);
|
||||
},
|
||||
// primaryNameservers really isn't editable - it's literally the list of FQDN's
|
||||
// peers really isn't editable - it's literally the list of FQDN's
|
||||
// that this database is replicated to in a multi-master fashion.
|
||||
//
|
||||
// However, lib/store/index.js does plenty to update these records in support
|
||||
// of the SOA records that are built from them (as does this file in the "migration"
|
||||
// section). I'm toying with the idea of not storing them seperately or creating the
|
||||
// SOA records somewhat immediately.
|
||||
primaryNameservers: {
|
||||
list: function listNameservers() {
|
||||
return jsonDeepClone(db.primaryNameservers);
|
||||
}
|
||||
peers: function listPeers(cb) {
|
||||
// Most data stores are going to have an asynchronous storage API. If we need
|
||||
// synchronous access to the data it is going to have to be cached. If it is
|
||||
// cached, there is still the issue the cache getting out of sync (a legitimate
|
||||
// issue anyway). If we explicitly make all of these operations async then we
|
||||
// have greater flexibility for store implmentations to address these issues.
|
||||
return setImmediate(cb, null, jsonDeepClone(db.peers));
|
||||
},
|
||||
zones: {
|
||||
list: function listZones() {
|
||||
return jsonDeepClone(db.zones);
|
||||
},
|
||||
find: function getZone(predicate, cb) {
|
||||
var found;
|
||||
db.zones.some(function (z) {
|
||||
if (z.id && predicate.id === z.id) { found = z; return true; }
|
||||
if (z.name && predicate.name === z.name) { found = z; return true; }
|
||||
/*
|
||||
I'm fairly certan that zone names must be unique and therefore are legitimately
|
||||
IDs within the zones namespace. This is similarly true of record names within a zone.
|
||||
I'm not certain that having a distinct ID adds value and it may add confusion / complexity.
|
||||
*/
|
||||
// NOTE: `opts` exists so we can add options - like properties to read - easily in the future
|
||||
// without modifying the function signature
|
||||
list: function listZones(predicate, opts, cb) {
|
||||
// TODO: consider whether we should just return the zone names
|
||||
var found = jsonDeepClone(matchZone(predicate)).map(function (z) {
|
||||
// This is fairly inefficient!! Consider alternative storage
|
||||
// that does not require deleting the records like this.
|
||||
delete z.records;
|
||||
return z;
|
||||
});
|
||||
if (!found) {
|
||||
cb(null, null);
|
||||
return;
|
||||
}
|
||||
cb(null, jsonDeepClone(found));
|
||||
return;
|
||||
return setImmediate(cb, null, found);
|
||||
},
|
||||
create: function() {},
|
||||
update: function() {},
|
||||
delete: function() {}
|
||||
// // NOTE: I'm not sure we need a distinct 'find()' operation in the API
|
||||
// // unless we are going to limit the output of the
|
||||
// // 'list()' operation in some incompatible way.
|
||||
// // NOTE: `opts` exists so we can add options - like properties to read - easily in the future
|
||||
// // without modifying the function signature
|
||||
// find: function getZone(predicate, opts, cb) {
|
||||
// if (!predicate.name || predicate.id) {
|
||||
// return setImmediate(cb, new Error('Finding a zone requires a `name` or `id`'));
|
||||
// }
|
||||
// // TODO: implement a limit / short circuit and possibly offset
|
||||
// // to allow for paging of zone data.
|
||||
// var found = matchZone(predicate);
|
||||
// if (!found[0]) {
|
||||
// // TODO: make error message more specific?
|
||||
// return setImmediate(cb, new Error('Zone not found'));
|
||||
// }
|
||||
|
||||
// var z = jsonDeepClone(found[0]);
|
||||
// delete z.records;
|
||||
// return setImmediate(cb, null, z);
|
||||
// },
|
||||
create: function createZone(zone, cb) {
|
||||
// We'll need a lock mechanism of some sort that works
|
||||
// for simultaneous requests and multiple processes.
|
||||
matchZone({ name: zone.name }, function (err, matched) {
|
||||
if (err) {
|
||||
return setImmediate(cb, err);
|
||||
}
|
||||
|
||||
var found = matched[0];
|
||||
if (found) {
|
||||
return setImmediate(cb, new Error('Zone ' + zone.name + ' already exists'));
|
||||
}
|
||||
|
||||
db.zones[zone.name] = prepareZone(zone);
|
||||
return setImmediate(function () {
|
||||
cb(null, jsonDeepClone(db.zones[zone.name]));
|
||||
// release lock
|
||||
});
|
||||
});
|
||||
},
|
||||
update: function updateZone(zone, cb) {
|
||||
// We'll need a lock mechanism of some sort that works
|
||||
// for simultaneous requests and multiple processes.
|
||||
matchZone({ name: zone.name }, function (err, matched) {
|
||||
if (err) {
|
||||
return setImmediate(cb, err);
|
||||
}
|
||||
var found = matched[0];
|
||||
if (!found) {
|
||||
return setImmediate(cb, new Error('Zone not found'));
|
||||
}
|
||||
// make sure we are not writing records through this interface
|
||||
delete zone.records;
|
||||
|
||||
var combined = mergeObjects(found, zone);
|
||||
db.zones[zone.name] = prepareZone(combined, { isUpdate: true });
|
||||
return setImmediate(function () {
|
||||
cb(null, jsonDeepClone(db.zones[zone.name]));
|
||||
// release lock
|
||||
});
|
||||
});
|
||||
},
|
||||
delete: function(zone, cb) {
|
||||
// We'll need a lock mechanism of some sort that works
|
||||
// for simultaneous requests and multiple processes.
|
||||
matchZone({ name: zone.name }, function (err, matched) {
|
||||
if (err) {
|
||||
return setImmediate(cb, err);
|
||||
}
|
||||
var found = matched[0];
|
||||
if (!found) {
|
||||
return setImmediate(cb, new Error('Zone not found'));
|
||||
}
|
||||
|
||||
delete db.zones[zone.name];
|
||||
return setImmediate(function () {
|
||||
cb();
|
||||
// release lock
|
||||
});
|
||||
});
|
||||
}
|
||||
},
|
||||
records: {
|
||||
list: function listRecords() {
|
||||
return jsonDeepClone(db.records);
|
||||
list: function listRecords(rPredicate, cb) {
|
||||
var recordNames = [].concat(rPredicate.name);
|
||||
var check = matchPredicate(rPredicate);
|
||||
|
||||
var found = matchZone({ name: rPredicate.zone }).reduce(function (records, zone) {
|
||||
// get the records from the zone that match the record predicate
|
||||
var zFound = recordNames.filter(function (name) {
|
||||
return !!zone.records[name];
|
||||
}).map(function (name) {
|
||||
return zone.records[name].filter(check);
|
||||
});
|
||||
return records.concat(zFound);
|
||||
}, []);
|
||||
|
||||
return setImmediate(cb, null, jsonDeepClone(found));
|
||||
},
|
||||
find: function getRecord(predicate, cb) {
|
||||
// find: function getRecord(rPredicate, cb) {
|
||||
// var recordNames = [].concat(rPredicate.name);
|
||||
// var check = matchPredicate(rPredicate);
|
||||
|
||||
// // TODO: swap the `filter()` and `map()` for a functional style "loop"
|
||||
// // recursive function that lets us return early if we have a limit, etc.
|
||||
// var found = matchZone({ name: rPredicate.zone }).reduce(function (records, zone) {
|
||||
// // get the records from the zone that match the record predicate
|
||||
// var zFound = recordNames.filter(function (name) {
|
||||
// return !!zone.records[name];
|
||||
// }).map(function (name) {
|
||||
// return zone.records[name].filter(check);
|
||||
// });
|
||||
// return records.concat(zFound);
|
||||
// }, []);
|
||||
|
||||
// return setImmediate(cb, null, jsonDeepClone(found[0]));
|
||||
// },
|
||||
create: function(record, cb) {
|
||||
var zone = matchZone({ name: record.zone })[0];
|
||||
if (!zone) {
|
||||
return setImmediate(cb, new Error('Unble to find zone ' + record.zone + ' to create record'));
|
||||
}
|
||||
|
||||
var records = zone.records[record.name] = zone.records[record.name] || [];
|
||||
var check = matchPredicate(record);
|
||||
if (records.filter(check)[0]) {
|
||||
return setImmediate(cb, new Error('Exact record already exists in zone ' + record.zone ));
|
||||
}
|
||||
|
||||
return setImmediate(cb, null, jsonDeepClone(found));
|
||||
},
|
||||
create: function() {},
|
||||
update: function() {},
|
||||
delete: function() {}
|
||||
update: function(record, cb) {},
|
||||
delete: function(record, cb) {}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user