bandolier-template/lib/ormish.js

/*
 The ORMish object-relational-mapping-ish library for knex. It's just enough ORM added to knex to be useful, and anything else you need can be done with knex.
 */
import config from '../knexfile.cjs';
import knexConfig from 'knex';
import assert from 'assert';
import { attachPaginate } from 'knex-paginate';

/*
 A preconfigured knex.js driver using the config.development configuration
 by default.

 _TODO_: Need to make this configurable, even though I just use one config right now since I run sqlite3 all the time.
 */
export const knex = knexConfig(config.development);

/*
 run the PERF_TRICKS to configure sqlite3 when thing start, really need to make this
 a configuration and only do it with sqlite3, but for now just get this done
 */
if(config.development.client === "sqlite3") {
  const PERF_TRICKS = [
    "pragma journal_mode = WAL", // use a WAL journal to not block writers/readers
    "pragma synchronous = normal", // set concurrency to normal now that we have WAL
    "pragma temp_store = memory", // use RAM to make the temporary indices
    // "pragma mmap_size = 30000000000", // use 30GB of mmap to store DB, not useful in multi-process settings
    // "pragma page_size = 32768", // improve performance only if you store large stuff
     "pragma vacuum",  // give it a vacuum, this could impact startup time significantly
    "pragma optimize", // optimize it, but should be done periodically
  ];

  for(let sql of PERF_TRICKS) {
    await knex.raw(sql);
  }
}

attachPaginate();

/*
 Filled in by `load_schema` to give access to the database scheme in the admin
 tool and generally through the API.
 */
export const SCHEMA = {};

const load_schema = async () => {
  const raw = await knex("sqlite_schema").where({"type": "table"}).select(['type', 'name']);
  for(let table of raw) {
    table._columns = await knex(table.name).columnInfo();
    SCHEMA[table.name] = table;
  }
}

await load_schema();

/*
 In some cases (like the generic admin) you need to get validation
 rules but you don't have a specific class to work with.  This function
 is called by Model.validation and you can call it directly to get rules
 for a database table.

 1. `name string` - the table name.
 2. `rules Object` - default rules with empty "" for the rules you want filled in
 3. `all boolean` - set this to true if you want everything
 4. `no_id boolean` - defaults to true, set false if you also want the id
 5. `return Object` - the resulting rules to use with Validator
 */
export const validation = (name, rules, all=false, no_id=true) => {
  assert(rules, "rules parameter is required and will be modified");
  const schema = SCHEMA[name];
  assert(schema, `There is no schema for table named ${name}. Did you forget to migrate:latest?`);

  for(let [key, opts] of Object.entries(schema._columns)) {
    // most of the time you don't want the id
    if(no_id && key === "id") continue;

    if(all || rules[key] === "") {
      let required = opts.nullable || opts.defaultValue ? "nullable|" : "required|";

      switch(opts.type) {
        case "varchar": // fallthrough
        case "text":
          // some databases have an email type but we'll also look for ones named email
          if(key === "email") {
            rules[key] = required + "email";
          } else {
            rules[key] = required + "string";
          }

          if(opts.maxLength) {
            rules[key] += `|min:0|max:${opts.maxLength}`;
          }
          break;
        case "boolean":
          rules[key] = required + "boolean";
          break;
        case "integer":
        case "float":
          rules[key] = required + "numeric";
          break;
        case "date":
          // BUG: validator doesn't do proper date formatting yet
          // rules[key] = required + "date";
          break;
        case "email":
          // not all databases have this
          rules[key] = required + "email";
          break;
        default:
          rules[key] = required;
      }
    }
  }

  return rules;
}

/*
 The base class for all models found in `lib/models.js`.  You use this by extending it with:

 ```javascript
 class User extends Model.from_table('user') {
 }
 ```

 This will create a `User` class that is automatically configured using the SCHEMA create from the `user` table in your database.  You won't need to define the attributes on this class as it will be correctly populated from the database.

 The database is therefore the "source of truth" for all of the models.  You can then add functions to extend what this class does.
 */
export class Model {

  /*
   Allows you to build a new object of this Model with the given `attr`
   already set, but really you should use the `Model.from(attr)` method instead.
   This does _no_ object sanitization with `Model.clean(attr)` method, and if
   it doesn't match the underlying database it will throw an exception.

   - `attr Object` - the attributes for this model
   */
  constructor(attr) {
    assert(attr, "Must give attributes.");
    Object.assign(this, attr);
  }

  /*
   How to actually create a new instance of this model. This
   will do two things:

   1. Correctly use the schema for the subclass model.
   2. Sanitize the input to remove anything that shouldn't be in the database.

   The `also_remove` parameter is a list of additional keys to also scrub from the object.

   - `attr Object` -- The attributes this should start with.
   - `also_remove Array` -- list of additional attributes to remove.
   */
  static from(attr, also_remove=undefined) {
    return new this(this.clean(attr, also_remove));
  }

  /*
   Returns an object representing the schema for this Model. Remember that this
   will reflect what's in the database schema, which is formatted however
   `knex.js` formats your database Schema.  Might not be portable between
   databases and only tested with SQlite3.

   _This is an attribute accessor, so just do `obj.schema` rather than call it like a function._

   - `return Object` - The schema for this model.
   */
  get schema() {
    return this.constructor.schema;
  }

  /*
   Uses the `this.schema` scrub out any attributes that are not valid for the
   schema.  This is effectively a whitelist for the allowed attributes based on
   the database schema.  You can use the `also_remove` parameter to list
   additional attributes to remove, which you should do to sanitize incoming
   objects for things like password fields.

   - `attr Object` - The attributes to clean.
   - `also_remove Array` - Additional attributes to remove.
   */
  static clean(attr, also_remove=undefined) {
    assert(attr, "Must give attributes to clean.");

    let clean_entries = Object.entries(attr)
      .filter(([k,v]) => k in this.schema);

    if(also_remove) also_remove.forEach(k => delete clean_entries[k]);

    return Object.fromEntries(clean_entries);
  }

  get table_name() {
    return this.constructor.table_name;
  }

  /*
    Returns an object of basic rules meant for lib/api.js:validate
    based on what's in the database.  It's meant to be an easy to
    pass in starter which you can augment.  It expects a set of rules
    with keys you want configured. Any key that's set to an empty string ""
    will be filled in with a minimum rule to match the database schema.

    It's designed to be called once at the top of an api/ handler to get
    a basic set of rules.  You could also run it to print out the rules then
    simply write the rules directly where you need them.

    - `param rules {Object}` - rules specifier
   */
  static validation(rules) {
    return validation(this.table_name, rules);
  }

  async destroy() {
    assert(this.table_name !== undefined, "You must set class variable table_name.");

    await knex(this.table_name).
      where({id: this.id}).
      del();
  }

  async has_one(model, where, columns) {
    assert(where.id !== undefined, `where must at least have id for has_one ${model.table_name} you have ${JSON.stringify(where)}`);
    return await model.first(where, columns);
  }

  async has_many(model, where, columns) {
    return await model.all(where, columns);
  }

  async many_to_many(model, through_table) {
    // SECURITY: doing string interpolation which might allow injecting SQL
    let query = knex(model.table_name).where("id", "in",
      knex(through_table).select(`${model.table_name}_id as id`).where(`${this.table_name}_id`, "=", this.id));

    let rows = await query;

    let results = [];

    for(let r of rows) {
      results.push(new model(await model.clean(r)));
    }

    return results;
  }

  static async count(where, columns) {
    // the knex count api returns a DB specific result, so we need
    // to specify what we want, which is count:
    const spec = { count: columns || ['id']};
    let res = await knex(this.table_name).where(where).count(spec);
    if(res.length == 1) {
      // single result, just give the count
      return res[0].count;
    } else {
      console.warn("Your call to count in", this.table_name, "using where=", where, "columns: ", columns, "returned a weird result:", res);
      return res; // weird result let them deal with it
    }
  }

  static async insert(attr) {
    assert(this.table_name !== undefined, "You must set class variable table_name.");
    assert(attr, `You must give some attr to insert into ${this.table_name}`);

    let res = await knex(this.table_name).insert(attr);
    assert(res, `Failed to get an id from the insert for ${this.table_name}`);

    attr.id = res[0];

    return new this(attr);
  }

  /*
    Implements an upsert (insert but update on conflict) for Postgres, MySQL, and SQLite3 only.

    + attr { Object } - The attributes to insert or update.
    + conflict_key { string } - The key that can cause a conflict then update.
    + merge { boolean } - Defaults to true and will change the record. false will ignore and not update on conflict.
    + return { number } - id or undefined
   */
  static async upsert(attr, conflict_key, merge=true) {
    assert(conflict_key !== undefined, `You forgot to set the conflict_key on upsert to table ${this.table_name}`);
    let result = undefined;

    // TODO: allow specifying returns for databases that support it
    if(merge) {
      result = await knex(this.table_name).insert(attr).onConflict(conflict_key).merge();
    } else {
      result = await knex(this.table_name).insert(attr).onConflict(conflict_key).ignore();
    }

    // returns the id of the row or undefined
    return result !== undefined ? result[0] : undefined;
  }

  static async update(where, what) {
    assert(where, "You must give a where options.");
    return knex(this.table_name).where(where).update(what);
  }

  static async first(where, columns) {
    assert(where, "You must give a where options.");
    let attr = undefined;

    if(columns) {
      attr = await knex(this.table_name).column(columns).first().where(where);
    } else {
      attr = await knex(this.table_name).first().where(where);
    }

    return attr !== undefined ? new this(attr) : attr;
  }

  static async delete(where) {
    assert(where, "You must give a where options.");
    return knex(this.table_name).where(where).del();
  }

  static async all(where, columns) {
    assert(where, "You must give a where options.");
    let results = [];

    if(columns) {
      results = await knex(this.table_name).column(columns).where(where).select();
    } else {
      results = await knex(this.table_name).where(where);
    }

    let final = results.map(r => new this(r));
    return final;
  }

  static from_table(table_name) {
    let m = class extends Model { };
    m.table_name = table_name;

    assert(SCHEMA, "schema is not loaded!");
    assert(SCHEMA[table_name], `table named ${table_name} not in SCHEMA: ${ Object.keys(SCHEMA) }`);

    m.schema = SCHEMA[table_name]._columns;
    return m;
  }

  static async exists(where) {
    let res = await knex(this.table_name).select('id').where(where).limit(1).first();
    return res ? res.id : false;
  }
}

export default { knex, SCHEMA, Model };