v1.3.1: burn stray log

README.md

@@ -81,9 +81,10 @@ API
 
 It's kinda CRUDdy... but don't let that scare you.
 
-* `upsert(id, data)` - creates or updates based on existence in DB (use this)
+* `upsert(id, data[, oldId])` - creates or updates based on existence in DB (use this)
   * modifies `createdAt` and or `updatedAt`
-* `save(data)` - (just don't use this, please) creates or updates based on presence of ID
+* `create(id, obj)` - same as above, but fails if the object exists
+* `save(data[, oldId])` - (just don't use this, please) creates or updates based on presence of ID
 * `destroy(id)` - mark a record as `deletedAt` from DB
 * `get(id)` - grab one by id
 * `find(attrs, opts)` - grab many by indexable attributes

lib/dbwrap.js

@@ -1,48 +1,62 @@
 'use strict';
+/*global Promise*/
+var PromiseA = Promise;
 
-function wrap(db, dir) {
+function lowerFirst(str) {
+  return str.charAt(0).toLowerCase() + str.substr(1);
+}
+function snakeCase(str) {
+  return lowerFirst(str).replace(/([A-Z])/g, function (match) {
+    return "_" + match.toLowerCase();
+  });
+}
+function camelCase(str) {
+  return str.replace(/_([a-z])/g, function (match) {
+    return match[1].toUpperCase();
+  });
+}
+function upperCamelCase(str) {
+  var camel = camelCase(str);
+  return camel.charAt(0).toUpperCase() + camel.substr(1);
+}
+
+var searchConditions = {
+  '=':  true,
+  '==': true,
+  '!=': true,
+  '<>': true,
+  '<':  true,
+  '<=': true,
+  '!<': true,
+  '>':  true,
+  '>=': true,
+  '!>': true,
+  'IS': true,
+  'IS NOT': true,
+
+  'IN':          true,
+  'NOT IN':      true,
+  'LIKE':        true,
+  'NOT LIKE':    true,
+  'GLOB':        true,
+  'NOT GLOB':    true,
+  'BETWEEN':     true,
+  'NOT BETWEEN': true,
+};
+
+function wrap(db, dir, dbsMap) {
   // TODO if I put a failure right here,
   // why doesn't the unhandled promise rejection fire?
-  var PromiseA = require('bluebird');
   var promises = [];
   var earr = [];
-  var dbsMap = {};
   var debug = false;
 
-  db.escape = function (str) {
+  if (!dbsMap) {
-    return (str||'').replace(/'/g, "''");
+    dbsMap = {};
-  };
-
-  function lowerFirst(str) {
-    return str.charAt(0).toLowerCase() + str.slice(1);
-  }
-
-  function snakeCase(str) {
-    return lowerFirst(str).replace(
-      /([A-Z])/g
-    , function ($1) {
-        return "_" + $1.toLowerCase();
-      }
-    );
-  }
-
-  function camelCase(str) {
-    str = str.replace(
-      /_([a-z])/g
-    , function (g) {
-        return g[1].toUpperCase();
-      }
-    );
-    return str;
-  }
-
-  function upperCamelCase(str) {
-    // TODO handle UTF-8 properly (use codePointAt, don't use slice)
-    return str.charAt(0).toUpperCase() + camelCase(str).slice(1);
   }
 
   // PRAGMA schema.table_info(table-name);
-  // 
+  //
   function sqlite3GetColumns(tablename, columns, cb) {
     var sql = "PRAGMA table_info(" + db.escape(tablename) + ")";
 
@@ -89,7 +103,8 @@ function wrap(db, dir) {
         db.all(sql, earr, function (err, results) {
           if (err) {
             console.error("[Error] add column '" + tablename + "'");
-            console.error(err.stack);
+            console.error(sql);
+            console.error(err.stack || new Error('stack').stack);
             cb(err);
             return;
           }
@@ -146,27 +161,11 @@ function wrap(db, dir) {
     var idnameCased = (camelCase(dir.idname || 'id'));
 
     dir.indices.forEach(normalizeColumn);
-
+    DB._indices = dir.indices;
-    db = PromiseA.promisifyAll(db);
+    DB._indicesMap = {};
-
+    DB._indices.forEach(function (col) {
-    if (dir && dir.verbose || db.verbose) {
+      DB._indicesMap[col.name] = col;
-      console.log('Getting Verbose up in here');
+    });
-      db.on('trace', function (str) {
-        console.log('SQL:', str);
-      });
-
-      db.on('profile', function (sql, ms) {
-        console.log('Profile:', ms);
-      });
-    }
-
-    function simpleParse(row) {
-      if (!row) {
-        return null;
-      }
-
-      return simpleMap([row])[0] || null;
-    }
 
     function simpleMap(rows) {
       if (!rows) {
@@ -181,17 +180,24 @@ function wrap(db, dir) {
 
         if (row.json) {
           obj = JSON.parse(row.json);
-          delete row.json;
         } else {
           obj = {};
         }
+        delete row.json;
 
         obj[idnameCased] = row[idname];
         delete row[idname];
 
         Object.keys(row).forEach(function (fieldname) {
-          // TODO warn if overriding proper field? (shouldn't be possible)
+          // Ideally it shouldn't be possible to overriding a former proper column,
-          obj[camelCase(fieldname)] = row[fieldname];
+          // but when a new indexable field is added, the old value is still in json
+          // TODO one-time upgrade of all rows when a new column is added
+          if (null === row[fieldname] || 'undefined' === typeof row[fieldname] || '' === row[fieldname]) {
+            obj[camelCase(fieldname)] = row[fieldname] || obj[camelCase(fieldname)] || row[fieldname];
+          }
+          else {
+            obj[camelCase(fieldname)] = row[fieldname];
+          }
         });
 
         return obj;
@@ -203,6 +209,93 @@ function wrap(db, dir) {
       return results;
     }
 
+    function simpleParse(row) {
+      if (!row) {
+        return null;
+      }
+
+      return simpleMap([row])[0] || null;
+    }
+
+    // pull indices from object
+    function strainUpdate(id, data/*, vals*/, cb, oldId) {
+      var fieldable = [];
+      var sql;
+      var vals = [];
+
+      ['hasOne', 'hasMany', 'hasAndBelongsToMany', 'belongsTo', 'belongsToMany'].forEach(function (relname) {
+        var rels = dir[relname];
+
+        if (!rels) {
+          return;
+        }
+
+        if (!Array.isArray(rels)) {
+          rels = [rels];
+        }
+
+        // don't save relationships
+        rels.forEach(function (colname) {
+          delete data[colname];
+          delete data[camelCase(colname)];
+          // TODO placehold relationships on find / get?
+          // data[camelCase(colname)] = null;
+        });
+      });
+
+      dir.indices.forEach(function (col) {
+        // We prioritze the raw name rather than the camelCase name because it's not in the object
+        // we give for retrieved entries, so if it's present then the user put it there themselves.
+        var val = data[col.name] || data[camelCase(col.name)];
+
+        //if (col.name in data)
+        if ('undefined' !== typeof val) {
+          /*
+          fieldable.push(
+            db.escape(snakeCase(col.name))
+          + " = '" + db.escape(val) + "'"
+          );
+          */
+          fieldable.push(db.escape(snakeCase(col.name)));
+          vals.push(val);
+        }
+
+        delete data[col.name];
+        delete data[camelCase(col.name)];
+      });
+
+      if (!oldId) {
+        delete data[idnameCased];
+      }
+
+      if (!fieldable.length || Object.keys(data).length) {
+        vals.push(JSON.stringify(data));
+      } else {
+        vals.push(null);
+      }
+      fieldable.push('json');
+
+      vals.push(id);
+
+      sql = cb(fieldable);
+
+      if (debug) {
+        console.log('[masterquest-sqlite3] dbwrap.js');
+        console.log(sql);
+        console.log(vals);
+      }
+
+      vals.forEach(function (val) {
+        if (null === val || 'number' === typeof val) {
+          sql = sql.replace('?', String(val));
+        } else {
+          sql = sql.replace('?', "'" + db.escape(val) + "'");
+        }
+      });
+
+      return sql;
+    }
+
     DB.migrate = function (columns) {
       columns.forEach(normalizeColumn);
 
@@ -219,36 +312,144 @@ function wrap(db, dir) {
     };
 
     DB.find = function (obj, params) {
+      var err;
       var sql = 'SELECT * FROM \'' + tablename + '\' ';
       var keys = obj && Object.keys(obj);
 
-      if (obj && keys.length) {
+      if (obj) {
-        sql += 'WHERE ';
+        Object.keys(obj).forEach(function (key) {
-
+          if (undefined === obj[key]) {
-        keys.forEach(function (key, i) {
+            err = new Error("'" + key + "' was `undefined'. For security purposes you must explicitly set the value to null or ''");
-          if (i !== 0) {
-            sql += 'AND ';
-          }
-          if (null === obj[key]) {
-            sql += db.escape(snakeCase(key)) + " IS '" + db.escape(obj[key]) + "'";
-          }
-          else {
-            sql += db.escape(snakeCase(key)) + " = '" + db.escape(obj[key]) + "'";
           }
         });
       }
-      else if (null !== obj || (params && !params.limit)) {
+      if (err) {
+        return PromiseA.reject(err);
+      }
+
+      if (params && params.limit) {
+        params.limit = parseInt(params.limit, 10);
+        // remember to check for the case of NaN
+        if (!params.limit || params.limit <= 0) {
+          return PromiseA.reject(new Error('limit must be a positive integer'));
+        }
+      }
+
+      if (obj && keys.length) {
+        var conditions = keys.map(function (key) {
+          var dbKey = db.escape(snakeCase(key));
+          var value = obj[key];
+          if (null === value) {
+            return dbKey + ' IS NULL';
+          }
+
+          var split, cmd;
+          if (typeof value === 'string') {
+            value = value.trim();
+            if (['IS NULL', 'IS NOT NULL'].indexOf(value.toUpperCase()) !== -1) {
+              return dbKey + ' ' + value.toUpperCase();
+            }
+
+            split = value.split(' ');
+            if (searchConditions[split[0].toUpperCase()]) {
+              cmd = split[0].toUpperCase();
+              value = split.slice(1).join(' ');
+            } else if (searchConditions[split.slice(0, 2).join(' ').toUpperCase()]) {
+              cmd = split.slice(0, 2).join(' ').toUpperCase();
+              value = split.slice(2).join(' ');
+            }
+            // If we were given something like "BEGINS WITH 'something quoted'" we don't want
+            // to include the quotes (we'll quote it again later) so we strip them out here.
+            if (cmd) {
+              value = value.replace(/^(['"])(.*)\1$/, '$2');
+            }
+          }
+          if (typeof value === 'object') {
+            cmd = value.condition || value.relation || value.cmd;
+            value = value.value;
+            if (!cmd || !value) {
+              err = new Error("'"+key+"' was an object, but missing condition and/or value");
+              return;
+            }
+            if (typeof cmd !== 'string' || !searchConditions[cmd.toUpperCase()]) {
+              err = new Error("'"+key+"' tried to use invalid condition '"+cmd+"'");
+              return;
+            } else {
+              cmd = cmd.toUpperCase();
+            }
+          }
+
+          if (!cmd) {
+            cmd = '=';
+          }
+          // The IN condition is special in that we can't quote the value as a single value,
+          // so it requires a little more logic to actually work and still be sanitary.
+          if (cmd === 'IN' || cmd === 'NOT IN') {
+            if (typeof value === 'string') {
+              value = value.split((params || {}).seperator || /[\s,]+/);
+            }
+            if (!Array.isArray(value)) {
+              err = new Error("'"+key+"' has invalid value for use with 'IN'");
+              return;
+            }
+            value = value.map(function (val) {
+              return "'"+db.escape(val)+"'";
+            });
+            return dbKey + ' ' + cmd + ' (' + value.join(',') + ')';
+          }
+          // The BETWEEN condition is also special for the same reason as IN
+          if (cmd === 'BETWEEN' || cmd === 'NOT BETWEEN') {
+            if (typeof value === 'string') {
+              value = value.split((params || {}).seperator || /[\s,]+(AND\s+)?/i);
+            }
+            if (!Array.isArray(value) || value.length !== 2) {
+              err = new Error("'"+key+"' has invalid value for use with 'BETWEEN'");
+              return;
+            }
+            value = value.map(function (val) {
+              return "'"+db.escape(val)+"'";
+            });
+            return dbKey + ' ' + cmd + ' ' + value.join(' AND ');
+          }
+          // If we are supposed to compare to another field then make sure the name is correct,
+          // and that we don't try to quote the name.
+          if (typeof value === 'string' && /^[a-zA-Z0-9_]*$/.test(value)) {
+            var snake = snakeCase(value);
+            if (dir.indices.some(function (col) { return snake === col.name; })) {
+              return dbKey + ' ' + cmd + ' ' + snake;
+            }
+          }
+          return dbKey + ' ' + cmd + " '" + db.escape(value) + "'";
+        });
+        if (err) {
+          return PromiseA.reject(err);
+        }
+
+        sql += 'WHERE ' + conditions.join(' AND ');
+      }
+      else if (null !== obj || !(params && params.limit)) {
         return PromiseA.reject(new Error("to find all you must explicitly specify find(null, { limit: <<int>> })"));
       }
 
       if (params) {
-        if (params.orderBy) {
+        if (typeof params.orderByDesc === 'string' && !params.orderBy) {
-          sql += " ORDER BY \"" + db.escape(snakeCase(params.orderBy) + "\" ");
+          params.orderBy = params.orderByDesc;
-          if (params.orderByDesc) {
+          params.orderByDesc = true;
-            sql += 'DESC ';
-          }
         }
-        if (params.limit) {
+
+        // IMPORTANT: " is not the same to sqlite as '.
+        // // " is exact and necessary
+        if (params.orderBy) {
+          sql += " ORDER BY \"" + db.escape(snakeCase(params.orderBy)) + "\" ";
+          if (params.orderByDesc) {
+            sql += "DESC ";
+          }
+        } else if (DB._indicesMap.updated_at) {
+          sql += " ORDER BY \"updated_at\" DESC ";
+        } else if (DB._indicesMap.created_at) {
+          sql += " ORDER BY \"created_at\" DESC ";
+        }
+        if (isFinite(params.limit)) {
           sql += " LIMIT " + parseInt(params.limit, 10);
         }
       }
@@ -273,13 +474,13 @@ function wrap(db, dir) {
       }).then(simpleParse);
     };
 
-    DB.upsert = function (id, data) {
+    DB.upsert = function (id, data, oldId) {
       if (!data) {
         data = id;
         id = data[idnameCased];
       }
 
-      return DB.set(id, data).then(function (result) {
+      return DB.set(oldId || id, data, oldId).then(function (result) {
         var success = result.changes >= 1;
 
         if (success) {
@@ -290,18 +491,23 @@ function wrap(db, dir) {
       });
     };
 
-    DB.save = function (data) {
+    DB.save = function (data, oldId) {
-      if (!data[idnameCased]) {
+      if (!data[idnameCased] && !oldId) {
         // NOTE saving the id both in the object and the id for now
-        var UUID = require('node-uuid');
+        data[idnameCased] = require('crypto').randomBytes(16).toString('hex').split('');
-        data[idnameCased] = UUID.v4();
+        data[idnameCased].splice(8, 0, '-');
+        data[idnameCased].splice(8 + 1 + 4, 0, '-');
+        data[idnameCased].splice(8 + 1 + 4 + 1 + 4, 0, '-');
+        data[idnameCased].splice(8 + 1 + 4 + 1 + 4 + 1 + 4, 0, '-');
+        data[idnameCased][14] = 4; // TODO look at the mock uuid in the Go code I wrote
+        data[idnameCased] = data[idnameCased].join('');
         return DB.create(data[idnameCased], data).then(function (/*stats*/) {
           //data._rowid = stats.id;
           return data;
         });
       }
 
-      return DB.set(data[idnameCased], data).then(function (result) {
+      return DB.set(oldId || data[idnameCased], data, oldId).then(function (result) {
         var success = result.changes >= 1;
 
         if (success) {
@@ -324,6 +530,9 @@ function wrap(db, dir) {
         return PromiseA.reject(new Error("no id supplied"));
       }
 
+      obj.createdAt = Date.now();
+      obj.updatedAt = Date.now();
+
       return new PromiseA(function (resolve, reject) {
         var json = JSON.stringify(obj);
         var data = JSON.parse(json);
@@ -362,73 +571,9 @@ function wrap(db, dir) {
       });
     };
 
-    // pull indices from object
+    DB.set = function (id, obj, oldId) {
-    function strainUpdate(id, data/*, vals*/, cb) {
+      obj.updatedAt = Date.now();
-      var fieldable = [];
-      var json;
-      var sql;
-      var vals = [];
 
-      ['hasOne', 'hasMany', 'hasAndBelongsToMany', 'belongsTo', 'belongsToMany'].forEach(function (relname) {
-        var rels = dir[relname];
-
-        if (!rels) {
-          return;
-        }
-
-        if (!Array.isArray(rels)) {
-          rels = [rels];
-        }
-
-        // don't save relationships
-        rels.forEach(function (colname) {
-          delete data[colname];
-          delete data[camelCase(colname)];
-          // TODO placehold relationships on find / get?
-          // data[camelCase(colname)] = null;
-        });
-      });
-
-      dir.indices.forEach(function (col) {
-        var val = data[camelCase(col.name)];
-
-        //if (col.name in data)
-        if ('undefined' !== typeof val) {
-          /*
-          fieldable.push(
-            db.escape(snakeCase(col.name))
-          + " = '" + db.escape(val) + "'"
-          );
-          */
-          fieldable.push(db.escape(snakeCase(col.name)));
-          vals.push(val);
-        }
-
-        delete data[col.name];
-        delete data[camelCase(col.name)];
-      });
-
-      delete data[idnameCased];
-
-      if (!fieldable.length || Object.keys(data).length) {
-        json = JSON.stringify(data);
-        fieldable.push("json");
-        //fieldable.push("json = '" + db.escape(json) + "'");
-        vals.push(json);
-      }
-
-      vals.push(id);
-
-      sql = cb(fieldable);
-
-      while (vals.length) {
-        sql = sql.replace(/\?/, "'" + db.escape(vals.shift()) + "'");
-      }
-
-      return sql;
-    }
-
-    DB.set = function (id, obj) {
       var json = JSON.stringify(obj);
       var data = JSON.parse(json);
 
@@ -443,7 +588,8 @@ function wrap(db, dir) {
 
         //var vals = [];
         // removes known fields from data
-        var sql = strainUpdate(id, data/*, vals*/, sqlTpl);
+        data.updated_at = Date.now();
+        var sql = strainUpdate(id, data/*, vals*/, sqlTpl, oldId);
 
         //console.log('[debug] DB.set() sql:', sql);
         db.run(sql, /*vals*/[], function (err) {
@@ -537,7 +683,39 @@ function wrap(db, dir) {
     });
   }
 
+  function promisify(key) {
+    if ('function' !== typeof db[key] || /Async$/.test(key) || db[key + 'Async']) { return; }
+    db[key + 'Async'] = require('util').promisify(db[key]);
+  }
+  if (!db.__masterquest_init) {
+    db.__masterquest_init = true;
+    Object.keys(db).forEach(promisify);
+    ['run', 'all'].forEach(promisify);
+    db.__masterquest_init = true;
+    db.escape = function (str) {
+      // TODO? literals for true,false,null
+      // error on undefined?
+      if (undefined === str) {
+        str = '';
+      }
+      return String(str).replace(/'/g, "''");
+    };
+
+    if (dir && dir.verbose || db.verbose) {
+      console.log('Getting Verbose up in here');
+      db.on('trace', function (str) {
+        console.log('SQL:', str);
+      });
+
+      db.on('profile', function (sql, ms) {
+        console.log('Profile:', ms);
+      });
+    }
+  }
+  dbsMap.sql = db;
+
   dir.forEach(function (dir) {
+    // TODO if directive is the same as existing dbsMap, skip it
     promises.push(createTable(dir).then(function (dbw) {
 
       dbsMap[dir.modelname] = dbw;
@@ -546,8 +724,6 @@ function wrap(db, dir) {
     }));
   });
 
-  dbsMap.sql = db;
-
   return PromiseA.all(promises).then(function (/*dbs*/) {
     return dbsMap;
   });

package-lock.json

@@ -0,0 +1,5 @@
+{
+  "name": "masterquest-sqlite3",
+  "version": "1.3.0",
+  "lockfileVersion": 1
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "masterquest-sqlite3",
-  "version": "1.0.2",
+  "version": "1.3.1",
   "description": "A NoSQL / SQLite3 Hybrid. All your indices are belong to us. Master Quest.",
   "main": "lib/dbwrap",
   "scripts": {
@@ -8,12 +8,10 @@
   },
   "repository": {
     "type": "git",
-    "url": "git@github.com:coolaj86/node-masterquest-sqlite3.git"
+    "url": "https://git.coolaj86.com:coolaj86/masterquest-sqlite3.js"
   },
   "author": "AJ ONeal <coolaj86@gmail.com> (https://coolaj86.com/)",
   "license": "(MIT OR Apache-2.0)",
-  "homepage": "https://github.com/coolaj86/masterquest-sqlite3",
+  "homepage": "https://git.coolaj86.com:coolaj86/masterquest-sqlite3.js",
-  "dependencies": {
+  "dependencies": {}
-    "bluebird": "^3.0.5"
-  }
 }