""" Autogenerated db CRUD and related utilities """
########################################
#
#       D O   N O T   E D I T
#
# This file was written by makeMySQLCRUD.js.  Any changes should be made there.
#
########################################

#pylint: disable=line-too-long
#pylint: disable=too-many-lines
#pylint: disable=trailing-newlines
#pylint: disable=wrong-import-position
#pylint: disable=wrong-import-order
#pylint: disable=invalid-name
#pylint: disable=missing-function-docstring
#pylint: disable=consider-using-in
#pylint: disable=logging-not-lazy
#pylint: disable=inconsistent-return-statements
#pylint: disable=too-many-return-statements
#pylint: disable=too-many-branches
#pylint: disable=too-many-locals
#pylint: disable=unused-argument
import logging
import flask
import re
import datetime
import pickle
import mysql.connector
import py.mconf as mconf

# Notes:
# (1) In general, all processing that might raise a mysql.connector.Error is
# wrapped to raise a ValueError instead, to support callers working at a
# higher level of CRUD abstraction.  The general processing contruct
#    except mysql.connector.Error as e:
#        raise ValueError from e
# is not used for this purpose because it produces an undecorated ValueError
# without the str(e) text, making it harder to track down what the problem
# actually was.  The source can be found from the Error __context__, but
# that is also set when raising a new Error, so the general use here is
#        raise ValueError(str(e) or "No mysql error text")

# Reserved database fields used for every instance:
#  - dsId: a long int, possibly out of range of a javascript integer,
#    possibly non-sequential, uniquely identifying an entity instance.
#    The entity type + dsId uniquely identifies an object in the system.
#  - created: An ISO timestamp when the instance was first written.
#  - modified: An ISO timestamp followed by ';' followed by mod count.
#  - batchconv: Arbitray string for batch database conversion.
dbflds = {"dsId": {"pt": "dbid", "un": True, "dv": 0},
          "created": {"pt": "string", "un": False, "dv": ""},
          "modified": {"pt": "string", "un": False, "dv": ""},
          "batchconv": {"pt": "string", "un": False, "dv": ""}}

entdefs = {
    "AppUser": {  # PastKey User account
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "email": {"pt": "email", "un": True, "dv": ""},
        "phash": {"pt": "string", "un": False, "dv": ""},
        "status": {"pt": "string", "un": False, "dv": ""},
        "actsends": {"pt": "string", "un": False, "dv": ""},
        "actcode": {"pt": "string", "un": False, "dv": ""},
        "accessed": {"pt": "string", "un": False, "dv": ""},
        "name": {"pt": "string", "un": False, "dv": ""},
        "title": {"pt": "string", "un": False, "dv": ""},
        "web": {"pt": "string", "un": False, "dv": ""},
        "lang": {"pt": "string", "un": False, "dv": ""},
        "settings": {"pt": "string", "un": False, "dv": ""},
        "started": {"pt": "string", "un": False, "dv": ""},
        "completed": {"pt": "string", "un": False, "dv": ""},
        "remtls": {"pt": "string", "un": False, "dv": ""},
        "built": {"pt": "string", "un": False, "dv": ""}
    },
    "Point": {  # A data point for use in timelines
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "editors": {"pt": "string", "un": False, "dv": ""},
        "srctl": {"pt": "dbid", "un": False, "dv": 0},
        "lmuid": {"pt": "dbid", "un": False, "dv": 0},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "source": {"pt": "string", "un": False, "dv": ""},
        "date": {"pt": "string", "un": False, "dv": ""},
        "text": {"pt": "string", "un": False, "dv": ""},
        "refs": {"pt": "string", "un": False, "dv": ""},
        "qtype": {"pt": "string", "un": False, "dv": ""},
        "communities": {"pt": "string", "un": False, "dv": ""},
        "regions": {"pt": "string", "un": False, "dv": ""},
        "categories": {"pt": "string", "un": False, "dv": ""},
        "tags": {"pt": "string", "un": False, "dv": ""},
        "codes": {"pt": "string", "un": False, "dv": ""},
        "srclang": {"pt": "string", "un": False, "dv": ""},
        "translations": {"pt": "string", "un": False, "dv": ""},
        "pic": {"pt": "image", "un": False, "dv": None},
        "stats": {"pt": "string", "un": False, "dv": ""}
    },
    "Timeline": {  # Points + suppviz*, or other timelines
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "editors": {"pt": "string", "un": False, "dv": ""},
        "lmuid": {"pt": "dbid", "un": False, "dv": 0},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "name": {"pt": "string", "un": True, "dv": ""},
        "cname": {"pt": "string", "un": False, "dv": ""},
        "slug": {"pt": "string", "un": True, "dv": ""},
        "title": {"pt": "string", "un": False, "dv": ""},
        "subtitle": {"pt": "string", "un": False, "dv": ""},
        "featured": {"pt": "string", "un": False, "dv": ""},
        "lang": {"pt": "string", "un": False, "dv": ""},
        "comment": {"pt": "string", "un": False, "dv": ""},
        "about": {"pt": "string", "un": False, "dv": ""},
        "kwds": {"pt": "string", "un": False, "dv": ""},
        "ctype": {"pt": "string", "un": False, "dv": ""},
        "cids": {"pt": "string", "un": False, "dv": ""},
        "rempts": {"pt": "string", "un": False, "dv": ""},
        "svs": {"pt": "string", "un": False, "dv": ""},
        "preb": {"pt": "string", "un": False, "dv": ""}
    },
    "TLComp": {  # Timeline completion archive record
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "userid": {"pt": "dbid", "un": False, "dv": 0},
        "tlid": {"pt": "dbid", "un": False, "dv": 0},
        "username": {"pt": "string", "un": False, "dv": ""},
        "tlname": {"pt": "string", "un": False, "dv": ""},
        "data": {"pt": "string", "un": False, "dv": ""}
    },
    "DayCount": {  # Traffic access accumulator
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "tstamp": {"pt": "string", "un": False, "dv": ""},
        "rtype": {"pt": "string", "un": False, "dv": ""},
        "detail": {"pt": "string", "un": False, "dv": ""}
    },
    "AppService": {  # Processing service access
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "name": {"pt": "string", "un": True, "dv": ""},
        "ckey": {"pt": "string", "un": False, "dv": ""},
        "csec": {"pt": "string", "un": False, "dv": ""},
        "data": {"pt": "string", "un": False, "dv": ""}
    }
}


entkeys = {
    "AppUser": ["importid", "email"],
    "Point": ["importid"],
    "Timeline": ["importid", "name", "slug"],
    "TLComp": ["importid"],
    "DayCount": ["importid"],
    "AppService": ["importid", "name"]
}


cachedefs = {
    "AppUser": {"minutes": 120, "manualadd": True},
    "Point": {"minutes": 0, "manualadd": False},
    "Timeline": {"minutes": 0, "manualadd": False},
    "TLComp": {"minutes": 0, "manualadd": False},
    "DayCount": {"minutes": 0, "manualadd": False},
    "AppService": {"minutes": 240, "manualadd": False}
}


def timestamp(offset=0):
    now = datetime.datetime.utcnow().replace(microsecond=0)
    return dt2ISO(now + datetime.timedelta(minutes=offset))


def expiration_for_inst(inst):
    if not inst or not inst["dsId"]:
        raise ValueError("Uncacheable instance: " + str(inst))
    cms = cachedefs[inst["dsType"]]
    if not cms or not cms["minutes"]:
        return ""  # not cached, no time to live
    return timestamp(cms["minutes"])


def make_key(dsType, field, value):
    # The value param will always be a string because after retrieving an
    # instance via query, the resulting fields are converted via db2app.
    return dsType + "_" + field + "_" + value


def entkey_vals(inst):
    # dsId key holds the cached instance.  Need img data so pickle.
    instkey = make_key(inst["dsType"], "dsId", inst["dsId"])
    keyvals = [{"key": instkey, "val": pickle.dumps(inst)}]
    # alternate entity keys point to the dsId key
    for field in entkeys[inst["dsType"]]:
        keyvals.append({"key": make_key(inst["dsType"], field, inst[field]),
                        "val": instkey})
    return keyvals


# Avoids repeated calls to the db for the same instance, especially within
# the same call to the webserver.  Used sparingly to avoid chewing memory.
# Time to live can range from zero to whenever in actual runtime use.
class EntityCache():
    """ Special case runtime cache to avoid pounding the db repeatedly """
    entities = {}
    def cache_put(self, inst):
        expir = expiration_for_inst(inst)
        if expir:  # cacheable
            self.cache_remove(inst)  # clear any outdated entries
            kt = inst["dsType"] + "_" + inst["dsId"] + "_cleanup"
            cachekeys = ["TTL_" + expir]
            for keyval in entkey_vals(inst):
                cachekeys.append(keyval["key"])
                self.entities[keyval["key"]] = keyval["val"]
            self.entities[kt] = ",".join(cachekeys)
            # self.log_cache_entries()
    def cache_get(self, entity, field, value):
        instkey = make_key(entity, field, value)
        if instkey not in self.entities:
            return None
        instval = self.entities[instkey]
        if field != "dsId":
            instval = self.entities[instval]
        return pickle.loads(instval)
    def cache_remove(self, inst):
        if inst:
            kt = inst["dsType"] + "_" + inst["dsId"] + "_cleanup"
            cleankeys = self.entities.pop(kt, None)
            if cleankeys:
                for oldkey in cleankeys.split(","):
                    self.entities.pop(oldkey, None)
    def cache_clean(self):
        now = nowISO()
        for key, value in self.entities.items():
            if key.endswith("_cleanup"):
                ttl = value.split(",")[0][4:]
                if ttl < now:
                    kcs = key.split("_")
                    inst = {"dsType": kcs[0], "dsId": kcs[1]}
                    self.cache_remove(inst)
    def log_cache_entries(self):
        txt = "EntityCache entities:\n"
        for key, _ in self.entities.items():
            txt += "    " + key + ": " + str(self.entities[key])[0:74] + "\n"
        logging.info(txt)
        return txt
entcache = EntityCache()


def reqarg(argname, fieldtype="string", required=False):
    argval = flask.request.args.get(argname)  # None if not found
    if not argval:
        argval = flask.request.form.get(argname)  # Ditto
    if required and not argval:
        raise ValueError("Missing required value for " + argname)
    dotidx = fieldtype.find('.')
    if dotidx > 0:
        entity = fieldtype[0:dotidx]
        fieldname = fieldtype[dotidx + 1:]
        fieldtype = entdefs[entity][fieldname]["pt"]
    if fieldtype == "email":
        emaddr = argval or ""
        emaddr = emaddr.lower()
        emaddr = re.sub('%40', '@', emaddr)
        if required and not re.match(r"[^@]+@[^@]+\.[^@]+", emaddr):
            raise ValueError("Invalid " + argname + " value: " + emaddr)
        return emaddr
    # A dbid is an int in the db and a string everywhere else
    if fieldtype in ["dbid", "string", "isodate", "isomod", "srchidcsv", "text",
                     "json", "jsarr", "idcsv", "isodcsv", "gencsv", "url"]:
        return argval or ""
    if fieldtype == "image":
        return argval or None
    if fieldtype == "int":
        argval = argval or 0
        return int(argval)
    raise ValueError("Unknown type " + fieldtype + " for " + argname)


# "cached fetch by key". Field must be dsId or one of the entkeys.
def cfbk(entity, field, value, required=False):
    if field != 'dsId' and field not in entkeys[entity]:
        raise ValueError(field + " not a unique index for " + entity)
    vstr = str(value)
    ci = entcache.cache_get(entity, field, vstr)
    if ci:
        dblogmsg("CAC", entity, ci)
        return ci
    if entdefs[entity][field]["pt"] not in ["dbid", "int"]:
        vstr = "\"" + vstr + "\""
    objs = query_entity(entity, "WHERE " + field + "=" + vstr + " LIMIT 1")
    if len(objs) > 0:
        inst = objs[0]
        if not cachedefs[inst["dsType"]]["manualadd"]:
            entcache.cache_put(inst)
        return inst
    if required:
        raise ValueError(entity + " " + vstr + " not found.")
    return None


# Get a connection to the database.  May throw mysql.connector.Error
# https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html
def get_mysql_connector():
    cnx = None
    try:
        cnx = mysql.connector.connect(user=mconf.db["u"],
                                      password=mconf.db["p"],
                                      host=mconf.db["h"],
                                      database=mconf.db["d"])
    except Exception as e:
        raise ValueError("Connection failed: " + str(e))
    return cnx


# Given what should be a string value, remove preceding or trailing space.
# If unique is true, then treat values of "null" or "None" as "".
def trim_string_val(val, unique=False):
    val = val or ""
    val = str(val)
    val = val.strip()
    if val and unique:
        lowval = val.lower()
        if lowval in ["null", "none"]:
            val = ""
    return val


# Read the given field from the inst or the default values, then convert it
# from an app value to a db value.  All string values are trimmed since
# preceding or trailing space makes matching horrible and buggy.  The UI can
# add a trailing newline for long text if it wants.
def app2db_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]["pt"]
        unique = entdefs[entity][field]["un"]
        val = entdefs[entity][field]["dv"]
    else:
        pt = dbflds[field]["pt"]
        unique = dbflds[field]["un"]
        val = dbflds[field]["dv"]
    if field in inst:
        val = inst[field]
    # convert value based on type and whether the values are unique
    if pt in ["email", "string"]:
        val = val or ""
        val = trim_string_val(val, unique)  # trim all strings. See comment.
        if not val:
            val = None
    elif pt == "image":
        if not val:  # Empty data gets set to null
            val = None
    elif pt == "int":
        val = val or 0
        val = int(val)  # convert possible "0" value
    elif pt == "dbid":
        try:
            val = int(val)  # will fail on "", "null" or other bad values
        except ValueError:
            val = 0
        if unique and not val:  # null vals don't violate UNIQUE constraint
            val = None          # otherwise use 0 as val may be required
    return val


# Read the given field from the inst or the default values, then convert it
# from a db value to an app value.  "app" means the server side module
# calling this module, not the web client.  Image binary values and json
# field values are not decoded, but get safe defaults if NULL.  dbids are
# converted to strings.
def db2app_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]["pt"]
        val = entdefs[entity][field]["dv"]
    else:
        pt = dbflds[field]["pt"]
        val = dbflds[field]["dv"]
    if field in inst:
        val = inst[field]
    # convert value based on type
    if pt in ["email", "string"]:
        if not val:  # A null value gets set to the empty string
            val = ""
        val = str(val)  # db interface might have autoconverted to int
    elif pt == "image":
        if not val:  # A null value gets set to the empty string
            val = ""
    elif pt == "int":
        if not val:  # Convert null values to 0
            val = 0
    elif pt == "dbid":
        if not val:  # A zero or null value gets set to falsey empty string
            val = ""
        else:
            val = str(val)
    return val


def ISO2dt(isostr):
    isostr = re.sub(r"\.\d*Z", "Z", isostr)  # remove microsecond if any
    dt = datetime.datetime.utcnow()
    dt = dt.strptime(isostr, "%Y-%m-%dT%H:%M:%SZ")
    return dt


def dt2ISO(dt):
    iso = str(dt.year) + "-" + str(dt.month).rjust(2, '0') + "-"
    iso += str(dt.day).rjust(2, '0') + "T" + str(dt.hour).rjust(2, '0')
    iso += ":" + str(dt.minute).rjust(2, '0') + ":"
    iso += str(dt.second).rjust(2, '0') + "Z"
    return iso


def nowISO():
    """ Return the current time as an ISO string """
    return dt2ISO(datetime.datetime.utcnow())


def initialize_timestamp_fields(fields, vck):
    ts = nowISO()
    if "created" not in fields or not fields["created"] or vck != "override":
        fields["created"] = ts
    if "modified" not in fields or not fields["modified"] or vck != "override":
        fields["modified"] = ts + ";1"


def verify_timestamp_fields(entity, dsId, fields, vck):
    if vck == "override" and "created" in fields and "modified" in fields:
        return fields # skip query and use specified values
    if not vck or not vck.strip():
        raise ValueError("Version check required to update " + entity +
                         " " + str(dsId))
    existing = cfbk(entity, "dsId", dsId)
    if not existing:
        raise ValueError("Existing " + entity + " " + str(dsId) + " not found.")
    if vck != "override" and existing["modified"] != vck:
        raise ValueError("Update error. Outdated data given for " + entity +
                         " " + str(dsId) + ".")
    if "created" not in fields or not fields["created"] or vck != "override":
        fields["created"] = existing["created"]
    ver = 1
    mods = existing["modified"].split(";")
    if len(mods) > 1:
        ver = int(mods[1]) + 1
    if "modified" not in fields or not fields["modified"] or vck != "override":
        fields["modified"] = nowISO() + ";" + str(ver)
    return existing


# Convert the given AppUser inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_AppUser(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("AppUser", "importid", inst)
    if fill or "email" in inst:
        cnv["email"] = app2db_fieldval("AppUser", "email", inst)
    if fill or "phash" in inst:
        cnv["phash"] = app2db_fieldval("AppUser", "phash", inst)
    if fill or "status" in inst:
        cnv["status"] = app2db_fieldval("AppUser", "status", inst)
    if fill or "actsends" in inst:
        cnv["actsends"] = app2db_fieldval("AppUser", "actsends", inst)
    if fill or "actcode" in inst:
        cnv["actcode"] = app2db_fieldval("AppUser", "actcode", inst)
    if fill or "accessed" in inst:
        cnv["accessed"] = app2db_fieldval("AppUser", "accessed", inst)
    if fill or "name" in inst:
        cnv["name"] = app2db_fieldval("AppUser", "name", inst)
    if fill or "title" in inst:
        cnv["title"] = app2db_fieldval("AppUser", "title", inst)
    if fill or "web" in inst:
        cnv["web"] = app2db_fieldval("AppUser", "web", inst)
    if fill or "lang" in inst:
        cnv["lang"] = app2db_fieldval("AppUser", "lang", inst)
    if fill or "settings" in inst:
        cnv["settings"] = app2db_fieldval("AppUser", "settings", inst)
    if fill or "started" in inst:
        cnv["started"] = app2db_fieldval("AppUser", "started", inst)
    if fill or "completed" in inst:
        cnv["completed"] = app2db_fieldval("AppUser", "completed", inst)
    if fill or "remtls" in inst:
        cnv["remtls"] = app2db_fieldval("AppUser", "remtls", inst)
    if fill or "built" in inst:
        cnv["built"] = app2db_fieldval("AppUser", "built", inst)
    return cnv


# Convert the given AppUser inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_AppUser(inst):
    cnv = {}
    cnv["dsType"] = "AppUser"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("AppUser", "importid", inst)
    cnv["email"] = db2app_fieldval("AppUser", "email", inst)
    cnv["phash"] = db2app_fieldval("AppUser", "phash", inst)
    cnv["status"] = db2app_fieldval("AppUser", "status", inst)
    cnv["actsends"] = db2app_fieldval("AppUser", "actsends", inst)
    cnv["actcode"] = db2app_fieldval("AppUser", "actcode", inst)
    cnv["accessed"] = db2app_fieldval("AppUser", "accessed", inst)
    cnv["name"] = db2app_fieldval("AppUser", "name", inst)
    cnv["title"] = db2app_fieldval("AppUser", "title", inst)
    cnv["web"] = db2app_fieldval("AppUser", "web", inst)
    cnv["lang"] = db2app_fieldval("AppUser", "lang", inst)
    cnv["settings"] = db2app_fieldval("AppUser", "settings", inst)
    cnv["started"] = db2app_fieldval("AppUser", "started", inst)
    cnv["completed"] = db2app_fieldval("AppUser", "completed", inst)
    cnv["remtls"] = db2app_fieldval("AppUser", "remtls", inst)
    cnv["built"] = db2app_fieldval("AppUser", "built", inst)
    return cnv


# Convert the given Point inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_Point(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "editors" in inst:
        cnv["editors"] = app2db_fieldval("Point", "editors", inst)
    if fill or "srctl" in inst:
        cnv["srctl"] = app2db_fieldval("Point", "srctl", inst)
    if fill or "lmuid" in inst:
        cnv["lmuid"] = app2db_fieldval("Point", "lmuid", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("Point", "importid", inst)
    if fill or "source" in inst:
        cnv["source"] = app2db_fieldval("Point", "source", inst)
    if fill or "date" in inst:
        cnv["date"] = app2db_fieldval("Point", "date", inst)
    if fill or "text" in inst:
        cnv["text"] = app2db_fieldval("Point", "text", inst)
    if fill or "refs" in inst:
        cnv["refs"] = app2db_fieldval("Point", "refs", inst)
    if fill or "qtype" in inst:
        cnv["qtype"] = app2db_fieldval("Point", "qtype", inst)
    if fill or "communities" in inst:
        cnv["communities"] = app2db_fieldval("Point", "communities", inst)
    if fill or "regions" in inst:
        cnv["regions"] = app2db_fieldval("Point", "regions", inst)
    if fill or "categories" in inst:
        cnv["categories"] = app2db_fieldval("Point", "categories", inst)
    if fill or "tags" in inst:
        cnv["tags"] = app2db_fieldval("Point", "tags", inst)
    if fill or "codes" in inst:
        cnv["codes"] = app2db_fieldval("Point", "codes", inst)
    if fill or "srclang" in inst:
        cnv["srclang"] = app2db_fieldval("Point", "srclang", inst)
    if fill or "translations" in inst:
        cnv["translations"] = app2db_fieldval("Point", "translations", inst)
    if fill or "pic" in inst:
        cnv["pic"] = app2db_fieldval("Point", "pic", inst)
    if fill or "stats" in inst:
        cnv["stats"] = app2db_fieldval("Point", "stats", inst)
    return cnv


# Convert the given Point inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_Point(inst):
    cnv = {}
    cnv["dsType"] = "Point"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["editors"] = db2app_fieldval("Point", "editors", inst)
    cnv["srctl"] = db2app_fieldval("Point", "srctl", inst)
    cnv["lmuid"] = db2app_fieldval("Point", "lmuid", inst)
    cnv["importid"] = db2app_fieldval("Point", "importid", inst)
    cnv["source"] = db2app_fieldval("Point", "source", inst)
    cnv["date"] = db2app_fieldval("Point", "date", inst)
    cnv["text"] = db2app_fieldval("Point", "text", inst)
    cnv["refs"] = db2app_fieldval("Point", "refs", inst)
    cnv["qtype"] = db2app_fieldval("Point", "qtype", inst)
    cnv["communities"] = db2app_fieldval("Point", "communities", inst)
    cnv["regions"] = db2app_fieldval("Point", "regions", inst)
    cnv["categories"] = db2app_fieldval("Point", "categories", inst)
    cnv["tags"] = db2app_fieldval("Point", "tags", inst)
    cnv["codes"] = db2app_fieldval("Point", "codes", inst)
    cnv["srclang"] = db2app_fieldval("Point", "srclang", inst)
    cnv["translations"] = db2app_fieldval("Point", "translations", inst)
    cnv["pic"] = db2app_fieldval("Point", "pic", inst)
    cnv["stats"] = db2app_fieldval("Point", "stats", inst)
    return cnv


# Convert the given Timeline inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_Timeline(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "editors" in inst:
        cnv["editors"] = app2db_fieldval("Timeline", "editors", inst)
    if fill or "lmuid" in inst:
        cnv["lmuid"] = app2db_fieldval("Timeline", "lmuid", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("Timeline", "importid", inst)
    if fill or "name" in inst:
        cnv["name"] = app2db_fieldval("Timeline", "name", inst)
    if fill or "cname" in inst:
        cnv["cname"] = app2db_fieldval("Timeline", "cname", inst)
    if fill or "slug" in inst:
        cnv["slug"] = app2db_fieldval("Timeline", "slug", inst)
    if fill or "title" in inst:
        cnv["title"] = app2db_fieldval("Timeline", "title", inst)
    if fill or "subtitle" in inst:
        cnv["subtitle"] = app2db_fieldval("Timeline", "subtitle", inst)
    if fill or "featured" in inst:
        cnv["featured"] = app2db_fieldval("Timeline", "featured", inst)
    if fill or "lang" in inst:
        cnv["lang"] = app2db_fieldval("Timeline", "lang", inst)
    if fill or "comment" in inst:
        cnv["comment"] = app2db_fieldval("Timeline", "comment", inst)
    if fill or "about" in inst:
        cnv["about"] = app2db_fieldval("Timeline", "about", inst)
    if fill or "kwds" in inst:
        cnv["kwds"] = app2db_fieldval("Timeline", "kwds", inst)
    if fill or "ctype" in inst:
        cnv["ctype"] = app2db_fieldval("Timeline", "ctype", inst)
    if fill or "cids" in inst:
        cnv["cids"] = app2db_fieldval("Timeline", "cids", inst)
    if fill or "rempts" in inst:
        cnv["rempts"] = app2db_fieldval("Timeline", "rempts", inst)
    if fill or "svs" in inst:
        cnv["svs"] = app2db_fieldval("Timeline", "svs", inst)
    if fill or "preb" in inst:
        cnv["preb"] = app2db_fieldval("Timeline", "preb", inst)
    return cnv


# Convert the given Timeline inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_Timeline(inst):
    cnv = {}
    cnv["dsType"] = "Timeline"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["editors"] = db2app_fieldval("Timeline", "editors", inst)
    cnv["lmuid"] = db2app_fieldval("Timeline", "lmuid", inst)
    cnv["importid"] = db2app_fieldval("Timeline", "importid", inst)
    cnv["name"] = db2app_fieldval("Timeline", "name", inst)
    cnv["cname"] = db2app_fieldval("Timeline", "cname", inst)
    cnv["slug"] = db2app_fieldval("Timeline", "slug", inst)
    cnv["title"] = db2app_fieldval("Timeline", "title", inst)
    cnv["subtitle"] = db2app_fieldval("Timeline", "subtitle", inst)
    cnv["featured"] = db2app_fieldval("Timeline", "featured", inst)
    cnv["lang"] = db2app_fieldval("Timeline", "lang", inst)
    cnv["comment"] = db2app_fieldval("Timeline", "comment", inst)
    cnv["about"] = db2app_fieldval("Timeline", "about", inst)
    cnv["kwds"] = db2app_fieldval("Timeline", "kwds", inst)
    cnv["ctype"] = db2app_fieldval("Timeline", "ctype", inst)
    cnv["cids"] = db2app_fieldval("Timeline", "cids", inst)
    cnv["rempts"] = db2app_fieldval("Timeline", "rempts", inst)
    cnv["svs"] = db2app_fieldval("Timeline", "svs", inst)
    cnv["preb"] = db2app_fieldval("Timeline", "preb", inst)
    return cnv


# Convert the given TLComp inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_TLComp(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("TLComp", "importid", inst)
    if fill or "userid" in inst:
        cnv["userid"] = app2db_fieldval("TLComp", "userid", inst)
    if fill or "tlid" in inst:
        cnv["tlid"] = app2db_fieldval("TLComp", "tlid", inst)
    if fill or "username" in inst:
        cnv["username"] = app2db_fieldval("TLComp", "username", inst)
    if fill or "tlname" in inst:
        cnv["tlname"] = app2db_fieldval("TLComp", "tlname", inst)
    if fill or "data" in inst:
        cnv["data"] = app2db_fieldval("TLComp", "data", inst)
    return cnv


# Convert the given TLComp inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_TLComp(inst):
    cnv = {}
    cnv["dsType"] = "TLComp"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("TLComp", "importid", inst)
    cnv["userid"] = db2app_fieldval("TLComp", "userid", inst)
    cnv["tlid"] = db2app_fieldval("TLComp", "tlid", inst)
    cnv["username"] = db2app_fieldval("TLComp", "username", inst)
    cnv["tlname"] = db2app_fieldval("TLComp", "tlname", inst)
    cnv["data"] = db2app_fieldval("TLComp", "data", inst)
    return cnv


# Convert the given DayCount inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_DayCount(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("DayCount", "importid", inst)
    if fill or "tstamp" in inst:
        cnv["tstamp"] = app2db_fieldval("DayCount", "tstamp", inst)
    if fill or "rtype" in inst:
        cnv["rtype"] = app2db_fieldval("DayCount", "rtype", inst)
    if fill or "detail" in inst:
        cnv["detail"] = app2db_fieldval("DayCount", "detail", inst)
    return cnv


# Convert the given DayCount inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_DayCount(inst):
    cnv = {}
    cnv["dsType"] = "DayCount"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("DayCount", "importid", inst)
    cnv["tstamp"] = db2app_fieldval("DayCount", "tstamp", inst)
    cnv["rtype"] = db2app_fieldval("DayCount", "rtype", inst)
    cnv["detail"] = db2app_fieldval("DayCount", "detail", inst)
    return cnv


# Convert the given AppService inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_AppService(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("AppService", "importid", inst)
    if fill or "name" in inst:
        cnv["name"] = app2db_fieldval("AppService", "name", inst)
    if fill or "ckey" in inst:
        cnv["ckey"] = app2db_fieldval("AppService", "ckey", inst)
    if fill or "csec" in inst:
        cnv["csec"] = app2db_fieldval("AppService", "csec", inst)
    if fill or "data" in inst:
        cnv["data"] = app2db_fieldval("AppService", "data", inst)
    return cnv


# Convert the given AppService inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_AppService(inst):
    cnv = {}
    cnv["dsType"] = "AppService"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("AppService", "importid", inst)
    cnv["name"] = db2app_fieldval("AppService", "name", inst)
    cnv["ckey"] = db2app_fieldval("AppService", "ckey", inst)
    cnv["csec"] = db2app_fieldval("AppService", "csec", inst)
    cnv["data"] = db2app_fieldval("AppService", "data", inst)
    return cnv


def dblogmsg(op, entity, res):
    log_summary_flds = {
        "AppUser": ["email", "name"],
        "Point": ["editors", "date", "text"],
        "Timeline": ["name"],
        "TLComp": ["userid", "tlid", "username", "tlname"],
        "DayCount": ["tstamp", "rtype"],
        "AppService": ["name"]}
    if res:
        if op != "QRY":  # query is already a list, listify anything else
            res = [res]
        for obj in res:
            msg = "db" + op + " " + entity + " " + obj["dsId"]
            if entity in log_summary_flds:
                for field in log_summary_flds[entity]:
                    msg += " " + str(obj[field])[0:60]
            logging.info(msg)
    else:  # no res, probably a delete
        logging.info("db" + op + " " + entity + " -no obj details-")


# Write a new AppUser row, using the given field values or defaults.
def insert_new_AppUser(cnx, cursor, fields):
    fields = app2db_AppUser(fields)
    stmt = (
        "INSERT INTO AppUser (created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(email)s, %(phash)s, %(status)s, %(actsends)s, %(actcode)s, %(accessed)s, %(name)s, %(title)s, %(web)s, %(lang)s, %(settings)s, %(started)s, %(completed)s, %(remtls)s, %(built)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["AppUser"]["importid"]["dv"]),
        'email': fields.get("email", entdefs["AppUser"]["email"]["dv"]),
        'phash': fields.get("phash", entdefs["AppUser"]["phash"]["dv"]),
        'status': fields.get("status", entdefs["AppUser"]["status"]["dv"]),
        'actsends': fields.get("actsends", entdefs["AppUser"]["actsends"]["dv"]),
        'actcode': fields.get("actcode", entdefs["AppUser"]["actcode"]["dv"]),
        'accessed': fields.get("accessed", entdefs["AppUser"]["accessed"]["dv"]),
        'name': fields.get("name", entdefs["AppUser"]["name"]["dv"]),
        'title': fields.get("title", entdefs["AppUser"]["title"]["dv"]),
        'web': fields.get("web", entdefs["AppUser"]["web"]["dv"]),
        'lang': fields.get("lang", entdefs["AppUser"]["lang"]["dv"]),
        'settings': fields.get("settings", entdefs["AppUser"]["settings"]["dv"]),
        'started': fields.get("started", entdefs["AppUser"]["started"]["dv"]),
        'completed': fields.get("completed", entdefs["AppUser"]["completed"]["dv"]),
        'remtls': fields.get("remtls", entdefs["AppUser"]["remtls"]["dv"]),
        'built': fields.get("built", entdefs["AppUser"]["built"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppUser(fields)
    dblogmsg("ADD", "AppUser", fields)
    return fields


# Update the specified AppUser row with the given field values.
def update_existing_AppUser(context, fields):
    fields = app2db_AppUser(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE AppUser SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("AppUser" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppUser(result)
    dblogmsg("UPD", "AppUser", result)
    entcache.cache_remove(result)
    return result


# Write a new Point row, using the given field values or defaults.
def insert_new_Point(cnx, cursor, fields):
    fields = app2db_Point(fields)
    stmt = (
        "INSERT INTO Point (created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats) "
        "VALUES (%(created)s, %(modified)s, %(editors)s, %(srctl)s, %(lmuid)s, %(importid)s, %(source)s, %(date)s, %(text)s, %(refs)s, %(qtype)s, %(communities)s, %(regions)s, %(categories)s, %(tags)s, %(codes)s, %(srclang)s, %(translations)s, %(pic)s, %(stats)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'editors': fields.get("editors", entdefs["Point"]["editors"]["dv"]),
        'srctl': fields.get("srctl", entdefs["Point"]["srctl"]["dv"]),
        'lmuid': fields.get("lmuid", entdefs["Point"]["lmuid"]["dv"]),
        'importid': fields.get("importid", entdefs["Point"]["importid"]["dv"]),
        'source': fields.get("source", entdefs["Point"]["source"]["dv"]),
        'date': fields.get("date", entdefs["Point"]["date"]["dv"]),
        'text': fields.get("text", entdefs["Point"]["text"]["dv"]),
        'refs': fields.get("refs", entdefs["Point"]["refs"]["dv"]),
        'qtype': fields.get("qtype", entdefs["Point"]["qtype"]["dv"]),
        'communities': fields.get("communities", entdefs["Point"]["communities"]["dv"]),
        'regions': fields.get("regions", entdefs["Point"]["regions"]["dv"]),
        'categories': fields.get("categories", entdefs["Point"]["categories"]["dv"]),
        'tags': fields.get("tags", entdefs["Point"]["tags"]["dv"]),
        'codes': fields.get("codes", entdefs["Point"]["codes"]["dv"]),
        'srclang': fields.get("srclang", entdefs["Point"]["srclang"]["dv"]),
        'translations': fields.get("translations", entdefs["Point"]["translations"]["dv"]),
        'pic': fields.get("pic", entdefs["Point"]["pic"]["dv"]),
        'stats': fields.get("stats", entdefs["Point"]["stats"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Point(fields)
    dblogmsg("ADD", "Point", fields)
    return fields


# Update the specified Point row with the given field values.
def update_existing_Point(context, fields):
    fields = app2db_Point(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE Point SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("Point" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_Point(result)
    dblogmsg("UPD", "Point", result)
    entcache.cache_remove(result)
    return result


# Write a new Timeline row, using the given field values or defaults.
def insert_new_Timeline(cnx, cursor, fields):
    fields = app2db_Timeline(fields)
    stmt = (
        "INSERT INTO Timeline (created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) "
        "VALUES (%(created)s, %(modified)s, %(editors)s, %(lmuid)s, %(importid)s, %(name)s, %(cname)s, %(slug)s, %(title)s, %(subtitle)s, %(featured)s, %(lang)s, %(comment)s, %(about)s, %(kwds)s, %(ctype)s, %(cids)s, %(rempts)s, %(svs)s, %(preb)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'editors': fields.get("editors", entdefs["Timeline"]["editors"]["dv"]),
        'lmuid': fields.get("lmuid", entdefs["Timeline"]["lmuid"]["dv"]),
        'importid': fields.get("importid", entdefs["Timeline"]["importid"]["dv"]),
        'name': fields.get("name", entdefs["Timeline"]["name"]["dv"]),
        'cname': fields.get("cname", entdefs["Timeline"]["cname"]["dv"]),
        'slug': fields.get("slug", entdefs["Timeline"]["slug"]["dv"]),
        'title': fields.get("title", entdefs["Timeline"]["title"]["dv"]),
        'subtitle': fields.get("subtitle", entdefs["Timeline"]["subtitle"]["dv"]),
        'featured': fields.get("featured", entdefs["Timeline"]["featured"]["dv"]),
        'lang': fields.get("lang", entdefs["Timeline"]["lang"]["dv"]),
        'comment': fields.get("comment", entdefs["Timeline"]["comment"]["dv"]),
        'about': fields.get("about", entdefs["Timeline"]["about"]["dv"]),
        'kwds': fields.get("kwds", entdefs["Timeline"]["kwds"]["dv"]),
        'ctype': fields.get("ctype", entdefs["Timeline"]["ctype"]["dv"]),
        'cids': fields.get("cids", entdefs["Timeline"]["cids"]["dv"]),
        'rempts': fields.get("rempts", entdefs["Timeline"]["rempts"]["dv"]),
        'svs': fields.get("svs", entdefs["Timeline"]["svs"]["dv"]),
        'preb': fields.get("preb", entdefs["Timeline"]["preb"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Timeline(fields)
    dblogmsg("ADD", "Timeline", fields)
    return fields


# Update the specified Timeline row with the given field values.
def update_existing_Timeline(context, fields):
    fields = app2db_Timeline(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE Timeline SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("Timeline" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_Timeline(result)
    dblogmsg("UPD", "Timeline", result)
    entcache.cache_remove(result)
    return result


# Write a new TLComp row, using the given field values or defaults.
def insert_new_TLComp(cnx, cursor, fields):
    fields = app2db_TLComp(fields)
    stmt = (
        "INSERT INTO TLComp (created, modified, importid, userid, tlid, username, tlname, data) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(userid)s, %(tlid)s, %(username)s, %(tlname)s, %(data)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["TLComp"]["importid"]["dv"]),
        'userid': fields.get("userid", entdefs["TLComp"]["userid"]["dv"]),
        'tlid': fields.get("tlid", entdefs["TLComp"]["tlid"]["dv"]),
        'username': fields.get("username", entdefs["TLComp"]["username"]["dv"]),
        'tlname': fields.get("tlname", entdefs["TLComp"]["tlname"]["dv"]),
        'data': fields.get("data", entdefs["TLComp"]["data"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_TLComp(fields)
    dblogmsg("ADD", "TLComp", fields)
    return fields


# Update the specified TLComp row with the given field values.
def update_existing_TLComp(context, fields):
    fields = app2db_TLComp(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE TLComp SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("TLComp" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_TLComp(result)
    dblogmsg("UPD", "TLComp", result)
    entcache.cache_remove(result)
    return result


# Write a new DayCount row, using the given field values or defaults.
def insert_new_DayCount(cnx, cursor, fields):
    fields = app2db_DayCount(fields)
    stmt = (
        "INSERT INTO DayCount (created, modified, importid, tstamp, rtype, detail) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(tstamp)s, %(rtype)s, %(detail)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["DayCount"]["importid"]["dv"]),
        'tstamp': fields.get("tstamp", entdefs["DayCount"]["tstamp"]["dv"]),
        'rtype': fields.get("rtype", entdefs["DayCount"]["rtype"]["dv"]),
        'detail': fields.get("detail", entdefs["DayCount"]["detail"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_DayCount(fields)
    dblogmsg("ADD", "DayCount", fields)
    return fields


# Update the specified DayCount row with the given field values.
def update_existing_DayCount(context, fields):
    fields = app2db_DayCount(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE DayCount SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("DayCount" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_DayCount(result)
    dblogmsg("UPD", "DayCount", result)
    entcache.cache_remove(result)
    return result


# Write a new AppService row, using the given field values or defaults.
def insert_new_AppService(cnx, cursor, fields):
    fields = app2db_AppService(fields)
    stmt = (
        "INSERT INTO AppService (created, modified, importid, name, ckey, csec, data) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(name)s, %(ckey)s, %(csec)s, %(data)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["AppService"]["importid"]["dv"]),
        'name': fields.get("name", entdefs["AppService"]["name"]["dv"]),
        'ckey': fields.get("ckey", entdefs["AppService"]["ckey"]["dv"]),
        'csec': fields.get("csec", entdefs["AppService"]["csec"]["dv"]),
        'data': fields.get("data", entdefs["AppService"]["data"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppService(fields)
    dblogmsg("ADD", "AppService", fields)
    return fields


# Update the specified AppService row with the given field values.
def update_existing_AppService(context, fields):
    fields = app2db_AppService(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE AppService SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("AppService" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppService(result)
    dblogmsg("UPD", "AppService", result)
    entcache.cache_put(result)
    return result


# Write the given dict/object based on the dsType.  Binary field values must
# be base64.b64encode.  Unspecified fields are set to default values for a
# new instance, and left alone on update.  For update, the verification
# check value must match the modified value of the existing instance.
def write_entity(inst, vck="1234-12-12T00:00:00Z"):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError("Database connection failed.")
    try:
        cursor = cnx.cursor()
        try:
            entity = inst.get("dsType", None)
            dsId = inst.get("dsId", 0)
            if dsId:
                existing = verify_timestamp_fields(entity, dsId, inst, vck)
                context = {"cnx":cnx, "cursor":cursor, "vck":vck,
                           "existing":existing}
                if entity == "AppUser":
                    return update_existing_AppUser(context, inst)
                if entity == "Point":
                    return update_existing_Point(context, inst)
                if entity == "Timeline":
                    return update_existing_Timeline(context, inst)
                if entity == "TLComp":
                    return update_existing_TLComp(context, inst)
                if entity == "DayCount":
                    return update_existing_DayCount(context, inst)
                if entity == "AppService":
                    return update_existing_AppService(context, inst)
                raise ValueError("Cannot modify unknown entity dsType " +
                                 str(entity))
            # No existing instance to update.  Insert new.
            initialize_timestamp_fields(inst, vck)
            if entity == "AppUser":
                return insert_new_AppUser(cnx, cursor, inst)
            if entity == "Point":
                return insert_new_Point(cnx, cursor, inst)
            if entity == "Timeline":
                return insert_new_Timeline(cnx, cursor, inst)
            if entity == "TLComp":
                return insert_new_TLComp(cnx, cursor, inst)
            if entity == "DayCount":
                return insert_new_DayCount(cnx, cursor, inst)
            if entity == "AppService":
                return insert_new_AppService(cnx, cursor, inst)
            raise ValueError("Cannot create unknown entity dsType " +
                             str(entity))
        except mysql.connector.Error as e:
            raise ValueError(str(e) or "No mysql error text")  # see note 1
        finally:
            cursor.close()
    finally:
        cnx.close()


def delete_entity(entity, dsId):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError("Database connection failed.")
    try:
        cursor = cnx.cursor()
        try:
            stmt = "DELETE FROM " + entity + " WHERE dsId=" + str(dsId)
            cursor.execute(stmt)
            cnx.commit()
            dblogmsg("DEL", entity + " " + str(dsId), None)
            # if cache cleanup is needed that is up to caller
        except mysql.connector.Error as e:
            raise ValueError(str(e) or "No mysql error text")  # see note 1
        finally:
            cursor.close()
    finally:
        cnx.close()


def query_AppUser(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built"
    query += " FROM AppUser " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built) in cursor:
        inst = {"dsType": "AppUser", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "email": email, "phash": phash, "status": status, "actsends": actsends, "actcode": actcode, "accessed": accessed, "name": name, "title": title, "web": web, "lang": lang, "settings": settings, "started": started, "completed": completed, "remtls": remtls, "built": built}
        inst = db2app_AppUser(inst)
        res.append(inst)
    dblogmsg("QRY", "AppUser", res)
    return res


def query_Point(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats"
    query += " FROM Point " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats) in cursor:
        inst = {"dsType": "Point", "dsId": dsId, "created": created, "modified": modified, "editors": editors, "srctl": srctl, "lmuid": lmuid, "importid": importid, "source": source, "date": date, "text": text, "refs": refs, "qtype": qtype, "communities": communities, "regions": regions, "categories": categories, "tags": tags, "codes": codes, "srclang": srclang, "translations": translations, "pic": pic, "stats": stats}
        inst = db2app_Point(inst)
        res.append(inst)
    dblogmsg("QRY", "Point", res)
    return res


def query_Timeline(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb"
    query += " FROM Timeline " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) in cursor:
        inst = {"dsType": "Timeline", "dsId": dsId, "created": created, "modified": modified, "editors": editors, "lmuid": lmuid, "importid": importid, "name": name, "cname": cname, "slug": slug, "title": title, "subtitle": subtitle, "featured": featured, "lang": lang, "comment": comment, "about": about, "kwds": kwds, "ctype": ctype, "cids": cids, "rempts": rempts, "svs": svs, "preb": preb}
        inst = db2app_Timeline(inst)
        res.append(inst)
    dblogmsg("QRY", "Timeline", res)
    return res


def query_TLComp(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, userid, tlid, username, tlname, data"
    query += " FROM TLComp " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, userid, tlid, username, tlname, data) in cursor:
        inst = {"dsType": "TLComp", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "userid": userid, "tlid": tlid, "username": username, "tlname": tlname, "data": data}
        inst = db2app_TLComp(inst)
        res.append(inst)
    dblogmsg("QRY", "TLComp", res)
    return res


def query_DayCount(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, tstamp, rtype, detail"
    query += " FROM DayCount " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, tstamp, rtype, detail) in cursor:
        inst = {"dsType": "DayCount", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "tstamp": tstamp, "rtype": rtype, "detail": detail}
        inst = db2app_DayCount(inst)
        res.append(inst)
    dblogmsg("QRY", "DayCount", res)
    return res


def query_AppService(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, name, ckey, csec, data"
    query += " FROM AppService " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, name, ckey, csec, data) in cursor:
        inst = {"dsType": "AppService", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "name": name, "ckey": ckey, "csec": csec, "data": data}
        inst = db2app_AppService(inst)
        res.append(inst)
    dblogmsg("QRY", "AppService", res)
    return res


# Fetch all instances of the specified entity kind for the given WHERE
# clause.  The WHERE clause should include a LIMIT, and should only match on
# indexed fields and/or declared query indexes.  For speed and general
# compatibility, only one inequality operator should be used in the match.
def query_entity(entity, where):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError("Database connection failed.")
    try:
        cursor = cnx.cursor()
        try:
            if entity == "AppUser":
                return query_AppUser(cnx, cursor, where)
            if entity == "Point":
                return query_Point(cnx, cursor, where)
            if entity == "Timeline":
                return query_Timeline(cnx, cursor, where)
            if entity == "TLComp":
                return query_TLComp(cnx, cursor, where)
            if entity == "DayCount":
                return query_DayCount(cnx, cursor, where)
            if entity == "AppService":
                return query_AppService(cnx, cursor, where)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or "No mysql error text")  # see note 1
        finally:
            cursor.close()
    finally:
        cnx.close()


def visible_AppUser_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        if fld == "email" and audience != "private":
            continue
        if fld == "phash":
            continue
        if fld == "status" and audience != "private":
            continue
        if fld == "actsends":
            continue
        if fld == "actcode":
            continue
        filtobj[fld] = val
    return filtobj


def visible_Point_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        if fld == "pic":
            if obj["pic"]:
                val = obj["dsId"]
            else:
                val = ""
        filtobj[fld] = val
    return filtobj


def visible_Timeline_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


def visible_TLComp_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


def visible_DayCount_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


def visible_AppService_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


# Return a copied object with only the fields appropriate to the audience.
# Specifying audience="private" includes peronal info.  The given obj is
# assumed to already have been through db2app conversion.  Image fields are
# converted to dsId values for separate download.
def visible_fields(obj, audience="public"):
    if obj["dsType"] == "AppUser":
        return visible_AppUser_fields(obj, audience)
    if obj["dsType"] == "Point":
        return visible_Point_fields(obj, audience)
    if obj["dsType"] == "Timeline":
        return visible_Timeline_fields(obj, audience)
    if obj["dsType"] == "TLComp":
        return visible_TLComp_fields(obj, audience)
    if obj["dsType"] == "DayCount":
        return visible_DayCount_fields(obj, audience)
    if obj["dsType"] == "AppService":
        return visible_AppService_fields(obj, audience)
    raise ValueError("Unknown object dsType: " + obj["dsType"])



4a9bb9663a93d6b2a3897913a25613630116be04
<mask token>


def timestamp(offset=0):
    now = datetime.datetime.utcnow().replace(microsecond=0)
    return dt2ISO(now + datetime.timedelta(minutes=offset))


<mask token>


def make_key(dsType, field, value):
    return dsType + '_' + field + '_' + value


def entkey_vals(inst):
    instkey = make_key(inst['dsType'], 'dsId', inst['dsId'])
    keyvals = [{'key': instkey, 'val': pickle.dumps(inst)}]
    for field in entkeys[inst['dsType']]:
        keyvals.append({'key': make_key(inst['dsType'], field, inst[field]),
            'val': instkey})
    return keyvals


class EntityCache:
    """ Special case runtime cache to avoid pounding the db repeatedly """
    entities = {}

    def cache_put(self, inst):
        expir = expiration_for_inst(inst)
        if expir:
            self.cache_remove(inst)
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cachekeys = ['TTL_' + expir]
            for keyval in entkey_vals(inst):
                cachekeys.append(keyval['key'])
                self.entities[keyval['key']] = keyval['val']
            self.entities[kt] = ','.join(cachekeys)

    def cache_get(self, entity, field, value):
        instkey = make_key(entity, field, value)
        if instkey not in self.entities:
            return None
        instval = self.entities[instkey]
        if field != 'dsId':
            instval = self.entities[instval]
        return pickle.loads(instval)

    def cache_remove(self, inst):
        if inst:
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cleankeys = self.entities.pop(kt, None)
            if cleankeys:
                for oldkey in cleankeys.split(','):
                    self.entities.pop(oldkey, None)

    def cache_clean(self):
        now = nowISO()
        for key, value in self.entities.items():
            if key.endswith('_cleanup'):
                ttl = value.split(',')[0][4:]
                if ttl < now:
                    kcs = key.split('_')
                    inst = {'dsType': kcs[0], 'dsId': kcs[1]}
                    self.cache_remove(inst)

    def log_cache_entries(self):
        txt = 'EntityCache entities:\n'
        for key, _ in self.entities.items():
            txt += '    ' + key + ': ' + str(self.entities[key])[0:74] + '\n'
        logging.info(txt)
        return txt


<mask token>


def reqarg(argname, fieldtype='string', required=False):
    argval = flask.request.args.get(argname)
    if not argval:
        argval = flask.request.form.get(argname)
    if required and not argval:
        raise ValueError('Missing required value for ' + argname)
    dotidx = fieldtype.find('.')
    if dotidx > 0:
        entity = fieldtype[0:dotidx]
        fieldname = fieldtype[dotidx + 1:]
        fieldtype = entdefs[entity][fieldname]['pt']
    if fieldtype == 'email':
        emaddr = argval or ''
        emaddr = emaddr.lower()
        emaddr = re.sub('%40', '@', emaddr)
        if required and not re.match('[^@]+@[^@]+\\.[^@]+', emaddr):
            raise ValueError('Invalid ' + argname + ' value: ' + emaddr)
        return emaddr
    if fieldtype in ['dbid', 'string', 'isodate', 'isomod', 'srchidcsv',
        'text', 'json', 'jsarr', 'idcsv', 'isodcsv', 'gencsv', 'url']:
        return argval or ''
    if fieldtype == 'image':
        return argval or None
    if fieldtype == 'int':
        argval = argval or 0
        return int(argval)
    raise ValueError('Unknown type ' + fieldtype + ' for ' + argname)


<mask token>


def trim_string_val(val, unique=False):
    val = val or ''
    val = str(val)
    val = val.strip()
    if val and unique:
        lowval = val.lower()
        if lowval in ['null', 'none']:
            val = ''
    return val


def app2db_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]['pt']
        unique = entdefs[entity][field]['un']
        val = entdefs[entity][field]['dv']
    else:
        pt = dbflds[field]['pt']
        unique = dbflds[field]['un']
        val = dbflds[field]['dv']
    if field in inst:
        val = inst[field]
    if pt in ['email', 'string']:
        val = val or ''
        val = trim_string_val(val, unique)
        if not val:
            val = None
    elif pt == 'image':
        if not val:
            val = None
    elif pt == 'int':
        val = val or 0
        val = int(val)
    elif pt == 'dbid':
        try:
            val = int(val)
        except ValueError:
            val = 0
        if unique and not val:
            val = None
    return val


<mask token>


def ISO2dt(isostr):
    isostr = re.sub('\\.\\d*Z', 'Z', isostr)
    dt = datetime.datetime.utcnow()
    dt = dt.strptime(isostr, '%Y-%m-%dT%H:%M:%SZ')
    return dt


def dt2ISO(dt):
    iso = str(dt.year) + '-' + str(dt.month).rjust(2, '0') + '-'
    iso += str(dt.day).rjust(2, '0') + 'T' + str(dt.hour).rjust(2, '0')
    iso += ':' + str(dt.minute).rjust(2, '0') + ':'
    iso += str(dt.second).rjust(2, '0') + 'Z'
    return iso


<mask token>


def initialize_timestamp_fields(fields, vck):
    ts = nowISO()
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = ts
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = ts + ';1'


def verify_timestamp_fields(entity, dsId, fields, vck):
    if vck == 'override' and 'created' in fields and 'modified' in fields:
        return fields
    if not vck or not vck.strip():
        raise ValueError('Version check required to update ' + entity + ' ' +
            str(dsId))
    existing = cfbk(entity, 'dsId', dsId)
    if not existing:
        raise ValueError('Existing ' + entity + ' ' + str(dsId) + ' not found.'
            )
    if vck != 'override' and existing['modified'] != vck:
        raise ValueError('Update error. Outdated data given for ' + entity +
            ' ' + str(dsId) + '.')
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = existing['created']
    ver = 1
    mods = existing['modified'].split(';')
    if len(mods) > 1:
        ver = int(mods[1]) + 1
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = nowISO() + ';' + str(ver)
    return existing


<mask token>


def db2app_AppUser(inst):
    cnv = {}
    cnv['dsType'] = 'AppUser'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppUser', 'importid', inst)
    cnv['email'] = db2app_fieldval('AppUser', 'email', inst)
    cnv['phash'] = db2app_fieldval('AppUser', 'phash', inst)
    cnv['status'] = db2app_fieldval('AppUser', 'status', inst)
    cnv['actsends'] = db2app_fieldval('AppUser', 'actsends', inst)
    cnv['actcode'] = db2app_fieldval('AppUser', 'actcode', inst)
    cnv['accessed'] = db2app_fieldval('AppUser', 'accessed', inst)
    cnv['name'] = db2app_fieldval('AppUser', 'name', inst)
    cnv['title'] = db2app_fieldval('AppUser', 'title', inst)
    cnv['web'] = db2app_fieldval('AppUser', 'web', inst)
    cnv['lang'] = db2app_fieldval('AppUser', 'lang', inst)
    cnv['settings'] = db2app_fieldval('AppUser', 'settings', inst)
    cnv['started'] = db2app_fieldval('AppUser', 'started', inst)
    cnv['completed'] = db2app_fieldval('AppUser', 'completed', inst)
    cnv['remtls'] = db2app_fieldval('AppUser', 'remtls', inst)
    cnv['built'] = db2app_fieldval('AppUser', 'built', inst)
    return cnv


<mask token>


def db2app_Point(inst):
    cnv = {}
    cnv['dsType'] = 'Point'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['editors'] = db2app_fieldval('Point', 'editors', inst)
    cnv['srctl'] = db2app_fieldval('Point', 'srctl', inst)
    cnv['lmuid'] = db2app_fieldval('Point', 'lmuid', inst)
    cnv['importid'] = db2app_fieldval('Point', 'importid', inst)
    cnv['source'] = db2app_fieldval('Point', 'source', inst)
    cnv['date'] = db2app_fieldval('Point', 'date', inst)
    cnv['text'] = db2app_fieldval('Point', 'text', inst)
    cnv['refs'] = db2app_fieldval('Point', 'refs', inst)
    cnv['qtype'] = db2app_fieldval('Point', 'qtype', inst)
    cnv['communities'] = db2app_fieldval('Point', 'communities', inst)
    cnv['regions'] = db2app_fieldval('Point', 'regions', inst)
    cnv['categories'] = db2app_fieldval('Point', 'categories', inst)
    cnv['tags'] = db2app_fieldval('Point', 'tags', inst)
    cnv['codes'] = db2app_fieldval('Point', 'codes', inst)
    cnv['srclang'] = db2app_fieldval('Point', 'srclang', inst)
    cnv['translations'] = db2app_fieldval('Point', 'translations', inst)
    cnv['pic'] = db2app_fieldval('Point', 'pic', inst)
    cnv['stats'] = db2app_fieldval('Point', 'stats', inst)
    return cnv


def app2db_Timeline(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'editors' in inst:
        cnv['editors'] = app2db_fieldval('Timeline', 'editors', inst)
    if fill or 'lmuid' in inst:
        cnv['lmuid'] = app2db_fieldval('Timeline', 'lmuid', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('Timeline', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('Timeline', 'name', inst)
    if fill or 'cname' in inst:
        cnv['cname'] = app2db_fieldval('Timeline', 'cname', inst)
    if fill or 'slug' in inst:
        cnv['slug'] = app2db_fieldval('Timeline', 'slug', inst)
    if fill or 'title' in inst:
        cnv['title'] = app2db_fieldval('Timeline', 'title', inst)
    if fill or 'subtitle' in inst:
        cnv['subtitle'] = app2db_fieldval('Timeline', 'subtitle', inst)
    if fill or 'featured' in inst:
        cnv['featured'] = app2db_fieldval('Timeline', 'featured', inst)
    if fill or 'lang' in inst:
        cnv['lang'] = app2db_fieldval('Timeline', 'lang', inst)
    if fill or 'comment' in inst:
        cnv['comment'] = app2db_fieldval('Timeline', 'comment', inst)
    if fill or 'about' in inst:
        cnv['about'] = app2db_fieldval('Timeline', 'about', inst)
    if fill or 'kwds' in inst:
        cnv['kwds'] = app2db_fieldval('Timeline', 'kwds', inst)
    if fill or 'ctype' in inst:
        cnv['ctype'] = app2db_fieldval('Timeline', 'ctype', inst)
    if fill or 'cids' in inst:
        cnv['cids'] = app2db_fieldval('Timeline', 'cids', inst)
    if fill or 'rempts' in inst:
        cnv['rempts'] = app2db_fieldval('Timeline', 'rempts', inst)
    if fill or 'svs' in inst:
        cnv['svs'] = app2db_fieldval('Timeline', 'svs', inst)
    if fill or 'preb' in inst:
        cnv['preb'] = app2db_fieldval('Timeline', 'preb', inst)
    return cnv


<mask token>


def app2db_TLComp(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('TLComp', 'importid', inst)
    if fill or 'userid' in inst:
        cnv['userid'] = app2db_fieldval('TLComp', 'userid', inst)
    if fill or 'tlid' in inst:
        cnv['tlid'] = app2db_fieldval('TLComp', 'tlid', inst)
    if fill or 'username' in inst:
        cnv['username'] = app2db_fieldval('TLComp', 'username', inst)
    if fill or 'tlname' in inst:
        cnv['tlname'] = app2db_fieldval('TLComp', 'tlname', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('TLComp', 'data', inst)
    return cnv


def db2app_TLComp(inst):
    cnv = {}
    cnv['dsType'] = 'TLComp'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('TLComp', 'importid', inst)
    cnv['userid'] = db2app_fieldval('TLComp', 'userid', inst)
    cnv['tlid'] = db2app_fieldval('TLComp', 'tlid', inst)
    cnv['username'] = db2app_fieldval('TLComp', 'username', inst)
    cnv['tlname'] = db2app_fieldval('TLComp', 'tlname', inst)
    cnv['data'] = db2app_fieldval('TLComp', 'data', inst)
    return cnv


def app2db_DayCount(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('DayCount', 'importid', inst)
    if fill or 'tstamp' in inst:
        cnv['tstamp'] = app2db_fieldval('DayCount', 'tstamp', inst)
    if fill or 'rtype' in inst:
        cnv['rtype'] = app2db_fieldval('DayCount', 'rtype', inst)
    if fill or 'detail' in inst:
        cnv['detail'] = app2db_fieldval('DayCount', 'detail', inst)
    return cnv


<mask token>


def app2db_AppService(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('AppService', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('AppService', 'name', inst)
    if fill or 'ckey' in inst:
        cnv['ckey'] = app2db_fieldval('AppService', 'ckey', inst)
    if fill or 'csec' in inst:
        cnv['csec'] = app2db_fieldval('AppService', 'csec', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('AppService', 'data', inst)
    return cnv


def db2app_AppService(inst):
    cnv = {}
    cnv['dsType'] = 'AppService'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppService', 'importid', inst)
    cnv['name'] = db2app_fieldval('AppService', 'name', inst)
    cnv['ckey'] = db2app_fieldval('AppService', 'ckey', inst)
    cnv['csec'] = db2app_fieldval('AppService', 'csec', inst)
    cnv['data'] = db2app_fieldval('AppService', 'data', inst)
    return cnv


def dblogmsg(op, entity, res):
    log_summary_flds = {'AppUser': ['email', 'name'], 'Point': ['editors',
        'date', 'text'], 'Timeline': ['name'], 'TLComp': ['userid', 'tlid',
        'username', 'tlname'], 'DayCount': ['tstamp', 'rtype'],
        'AppService': ['name']}
    if res:
        if op != 'QRY':
            res = [res]
        for obj in res:
            msg = 'db' + op + ' ' + entity + ' ' + obj['dsId']
            if entity in log_summary_flds:
                for field in log_summary_flds[entity]:
                    msg += ' ' + str(obj[field])[0:60]
            logging.info(msg)
    else:
        logging.info('db' + op + ' ' + entity + ' -no obj details-')


<mask token>


def update_existing_AppUser(context, fields):
    fields = app2db_AppUser(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppUser SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppUser' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppUser(result)
    dblogmsg('UPD', 'AppUser', result)
    entcache.cache_remove(result)
    return result


<mask token>


def insert_new_Timeline(cnx, cursor, fields):
    fields = app2db_Timeline(fields)
    stmt = (
        'INSERT INTO Timeline (created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) VALUES (%(created)s, %(modified)s, %(editors)s, %(lmuid)s, %(importid)s, %(name)s, %(cname)s, %(slug)s, %(title)s, %(subtitle)s, %(featured)s, %(lang)s, %(comment)s, %(about)s, %(kwds)s, %(ctype)s, %(cids)s, %(rempts)s, %(svs)s, %(preb)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'editors': fields.get('editors', entdefs['Timeline'][
        'editors']['dv']), 'lmuid': fields.get('lmuid', entdefs['Timeline']
        ['lmuid']['dv']), 'importid': fields.get('importid', entdefs[
        'Timeline']['importid']['dv']), 'name': fields.get('name', entdefs[
        'Timeline']['name']['dv']), 'cname': fields.get('cname', entdefs[
        'Timeline']['cname']['dv']), 'slug': fields.get('slug', entdefs[
        'Timeline']['slug']['dv']), 'title': fields.get('title', entdefs[
        'Timeline']['title']['dv']), 'subtitle': fields.get('subtitle',
        entdefs['Timeline']['subtitle']['dv']), 'featured': fields.get(
        'featured', entdefs['Timeline']['featured']['dv']), 'lang': fields.
        get('lang', entdefs['Timeline']['lang']['dv']), 'comment': fields.
        get('comment', entdefs['Timeline']['comment']['dv']), 'about':
        fields.get('about', entdefs['Timeline']['about']['dv']), 'kwds':
        fields.get('kwds', entdefs['Timeline']['kwds']['dv']), 'ctype':
        fields.get('ctype', entdefs['Timeline']['ctype']['dv']), 'cids':
        fields.get('cids', entdefs['Timeline']['cids']['dv']), 'rempts':
        fields.get('rempts', entdefs['Timeline']['rempts']['dv']), 'svs':
        fields.get('svs', entdefs['Timeline']['svs']['dv']), 'preb': fields
        .get('preb', entdefs['Timeline']['preb']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Timeline(fields)
    dblogmsg('ADD', 'Timeline', fields)
    return fields


def update_existing_Timeline(context, fields):
    fields = app2db_Timeline(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE Timeline SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('Timeline' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_Timeline(result)
    dblogmsg('UPD', 'Timeline', result)
    entcache.cache_remove(result)
    return result


def insert_new_TLComp(cnx, cursor, fields):
    fields = app2db_TLComp(fields)
    stmt = (
        'INSERT INTO TLComp (created, modified, importid, userid, tlid, username, tlname, data) VALUES (%(created)s, %(modified)s, %(importid)s, %(userid)s, %(tlid)s, %(username)s, %(tlname)s, %(data)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs['TLComp'][
        'importid']['dv']), 'userid': fields.get('userid', entdefs['TLComp'
        ]['userid']['dv']), 'tlid': fields.get('tlid', entdefs['TLComp'][
        'tlid']['dv']), 'username': fields.get('username', entdefs['TLComp'
        ]['username']['dv']), 'tlname': fields.get('tlname', entdefs[
        'TLComp']['tlname']['dv']), 'data': fields.get('data', entdefs[
        'TLComp']['data']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_TLComp(fields)
    dblogmsg('ADD', 'TLComp', fields)
    return fields


<mask token>


def update_existing_DayCount(context, fields):
    fields = app2db_DayCount(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE DayCount SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('DayCount' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_DayCount(result)
    dblogmsg('UPD', 'DayCount', result)
    entcache.cache_remove(result)
    return result


<mask token>


def update_existing_AppService(context, fields):
    fields = app2db_AppService(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppService SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppService' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppService(result)
    dblogmsg('UPD', 'AppService', result)
    entcache.cache_put(result)
    return result


def write_entity(inst, vck='1234-12-12T00:00:00Z'):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            entity = inst.get('dsType', None)
            dsId = inst.get('dsId', 0)
            if dsId:
                existing = verify_timestamp_fields(entity, dsId, inst, vck)
                context = {'cnx': cnx, 'cursor': cursor, 'vck': vck,
                    'existing': existing}
                if entity == 'AppUser':
                    return update_existing_AppUser(context, inst)
                if entity == 'Point':
                    return update_existing_Point(context, inst)
                if entity == 'Timeline':
                    return update_existing_Timeline(context, inst)
                if entity == 'TLComp':
                    return update_existing_TLComp(context, inst)
                if entity == 'DayCount':
                    return update_existing_DayCount(context, inst)
                if entity == 'AppService':
                    return update_existing_AppService(context, inst)
                raise ValueError('Cannot modify unknown entity dsType ' +
                    str(entity))
            initialize_timestamp_fields(inst, vck)
            if entity == 'AppUser':
                return insert_new_AppUser(cnx, cursor, inst)
            if entity == 'Point':
                return insert_new_Point(cnx, cursor, inst)
            if entity == 'Timeline':
                return insert_new_Timeline(cnx, cursor, inst)
            if entity == 'TLComp':
                return insert_new_TLComp(cnx, cursor, inst)
            if entity == 'DayCount':
                return insert_new_DayCount(cnx, cursor, inst)
            if entity == 'AppService':
                return insert_new_AppService(cnx, cursor, inst)
            raise ValueError('Cannot create unknown entity dsType ' + str(
                entity))
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


<mask token>


def query_AppUser(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built'
        )
    query += ' FROM AppUser ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built in cursor:
        inst = {'dsType': 'AppUser', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'email': email,
            'phash': phash, 'status': status, 'actsends': actsends,
            'actcode': actcode, 'accessed': accessed, 'name': name, 'title':
            title, 'web': web, 'lang': lang, 'settings': settings,
            'started': started, 'completed': completed, 'remtls': remtls,
            'built': built}
        inst = db2app_AppUser(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppUser', res)
    return res


def query_Point(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats'
        )
    query += ' FROM Point ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats in cursor:
        inst = {'dsType': 'Point', 'dsId': dsId, 'created': created,
            'modified': modified, 'editors': editors, 'srctl': srctl,
            'lmuid': lmuid, 'importid': importid, 'source': source, 'date':
            date, 'text': text, 'refs': refs, 'qtype': qtype, 'communities':
            communities, 'regions': regions, 'categories': categories,
            'tags': tags, 'codes': codes, 'srclang': srclang,
            'translations': translations, 'pic': pic, 'stats': stats}
        inst = db2app_Point(inst)
        res.append(inst)
    dblogmsg('QRY', 'Point', res)
    return res


<mask token>


def query_DayCount(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, tstamp, rtype, detail'
    query += ' FROM DayCount ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, tstamp, rtype, detail in cursor:
        inst = {'dsType': 'DayCount', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'tstamp': tstamp,
            'rtype': rtype, 'detail': detail}
        inst = db2app_DayCount(inst)
        res.append(inst)
    dblogmsg('QRY', 'DayCount', res)
    return res


def query_AppService(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, name, ckey, csec, data'
    query += ' FROM AppService ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, name, ckey, csec, data in cursor:
        inst = {'dsType': 'AppService', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'name': name,
            'ckey': ckey, 'csec': csec, 'data': data}
        inst = db2app_AppService(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppService', res)
    return res


def query_entity(entity, where):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            if entity == 'AppUser':
                return query_AppUser(cnx, cursor, where)
            if entity == 'Point':
                return query_Point(cnx, cursor, where)
            if entity == 'Timeline':
                return query_Timeline(cnx, cursor, where)
            if entity == 'TLComp':
                return query_TLComp(cnx, cursor, where)
            if entity == 'DayCount':
                return query_DayCount(cnx, cursor, where)
            if entity == 'AppService':
                return query_AppService(cnx, cursor, where)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def visible_AppUser_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        if fld == 'email' and audience != 'private':
            continue
        if fld == 'phash':
            continue
        if fld == 'status' and audience != 'private':
            continue
        if fld == 'actsends':
            continue
        if fld == 'actcode':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>


def visible_Timeline_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>


def visible_DayCount_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>

<mask token>


def timestamp(offset=0):
    now = datetime.datetime.utcnow().replace(microsecond=0)
    return dt2ISO(now + datetime.timedelta(minutes=offset))


def expiration_for_inst(inst):
    if not inst or not inst['dsId']:
        raise ValueError('Uncacheable instance: ' + str(inst))
    cms = cachedefs[inst['dsType']]
    if not cms or not cms['minutes']:
        return ''
    return timestamp(cms['minutes'])


def make_key(dsType, field, value):
    return dsType + '_' + field + '_' + value


def entkey_vals(inst):
    instkey = make_key(inst['dsType'], 'dsId', inst['dsId'])
    keyvals = [{'key': instkey, 'val': pickle.dumps(inst)}]
    for field in entkeys[inst['dsType']]:
        keyvals.append({'key': make_key(inst['dsType'], field, inst[field]),
            'val': instkey})
    return keyvals


class EntityCache:
    """ Special case runtime cache to avoid pounding the db repeatedly """
    entities = {}

    def cache_put(self, inst):
        expir = expiration_for_inst(inst)
        if expir:
            self.cache_remove(inst)
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cachekeys = ['TTL_' + expir]
            for keyval in entkey_vals(inst):
                cachekeys.append(keyval['key'])
                self.entities[keyval['key']] = keyval['val']
            self.entities[kt] = ','.join(cachekeys)

    def cache_get(self, entity, field, value):
        instkey = make_key(entity, field, value)
        if instkey not in self.entities:
            return None
        instval = self.entities[instkey]
        if field != 'dsId':
            instval = self.entities[instval]
        return pickle.loads(instval)

    def cache_remove(self, inst):
        if inst:
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cleankeys = self.entities.pop(kt, None)
            if cleankeys:
                for oldkey in cleankeys.split(','):
                    self.entities.pop(oldkey, None)

    def cache_clean(self):
        now = nowISO()
        for key, value in self.entities.items():
            if key.endswith('_cleanup'):
                ttl = value.split(',')[0][4:]
                if ttl < now:
                    kcs = key.split('_')
                    inst = {'dsType': kcs[0], 'dsId': kcs[1]}
                    self.cache_remove(inst)

    def log_cache_entries(self):
        txt = 'EntityCache entities:\n'
        for key, _ in self.entities.items():
            txt += '    ' + key + ': ' + str(self.entities[key])[0:74] + '\n'
        logging.info(txt)
        return txt


<mask token>


def reqarg(argname, fieldtype='string', required=False):
    argval = flask.request.args.get(argname)
    if not argval:
        argval = flask.request.form.get(argname)
    if required and not argval:
        raise ValueError('Missing required value for ' + argname)
    dotidx = fieldtype.find('.')
    if dotidx > 0:
        entity = fieldtype[0:dotidx]
        fieldname = fieldtype[dotidx + 1:]
        fieldtype = entdefs[entity][fieldname]['pt']
    if fieldtype == 'email':
        emaddr = argval or ''
        emaddr = emaddr.lower()
        emaddr = re.sub('%40', '@', emaddr)
        if required and not re.match('[^@]+@[^@]+\\.[^@]+', emaddr):
            raise ValueError('Invalid ' + argname + ' value: ' + emaddr)
        return emaddr
    if fieldtype in ['dbid', 'string', 'isodate', 'isomod', 'srchidcsv',
        'text', 'json', 'jsarr', 'idcsv', 'isodcsv', 'gencsv', 'url']:
        return argval or ''
    if fieldtype == 'image':
        return argval or None
    if fieldtype == 'int':
        argval = argval or 0
        return int(argval)
    raise ValueError('Unknown type ' + fieldtype + ' for ' + argname)


<mask token>


def trim_string_val(val, unique=False):
    val = val or ''
    val = str(val)
    val = val.strip()
    if val and unique:
        lowval = val.lower()
        if lowval in ['null', 'none']:
            val = ''
    return val


def app2db_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]['pt']
        unique = entdefs[entity][field]['un']
        val = entdefs[entity][field]['dv']
    else:
        pt = dbflds[field]['pt']
        unique = dbflds[field]['un']
        val = dbflds[field]['dv']
    if field in inst:
        val = inst[field]
    if pt in ['email', 'string']:
        val = val or ''
        val = trim_string_val(val, unique)
        if not val:
            val = None
    elif pt == 'image':
        if not val:
            val = None
    elif pt == 'int':
        val = val or 0
        val = int(val)
    elif pt == 'dbid':
        try:
            val = int(val)
        except ValueError:
            val = 0
        if unique and not val:
            val = None
    return val


def db2app_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]['pt']
        val = entdefs[entity][field]['dv']
    else:
        pt = dbflds[field]['pt']
        val = dbflds[field]['dv']
    if field in inst:
        val = inst[field]
    if pt in ['email', 'string']:
        if not val:
            val = ''
        val = str(val)
    elif pt == 'image':
        if not val:
            val = ''
    elif pt == 'int':
        if not val:
            val = 0
    elif pt == 'dbid':
        if not val:
            val = ''
        else:
            val = str(val)
    return val


def ISO2dt(isostr):
    isostr = re.sub('\\.\\d*Z', 'Z', isostr)
    dt = datetime.datetime.utcnow()
    dt = dt.strptime(isostr, '%Y-%m-%dT%H:%M:%SZ')
    return dt


def dt2ISO(dt):
    iso = str(dt.year) + '-' + str(dt.month).rjust(2, '0') + '-'
    iso += str(dt.day).rjust(2, '0') + 'T' + str(dt.hour).rjust(2, '0')
    iso += ':' + str(dt.minute).rjust(2, '0') + ':'
    iso += str(dt.second).rjust(2, '0') + 'Z'
    return iso


def nowISO():
    """ Return the current time as an ISO string """
    return dt2ISO(datetime.datetime.utcnow())


def initialize_timestamp_fields(fields, vck):
    ts = nowISO()
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = ts
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = ts + ';1'


def verify_timestamp_fields(entity, dsId, fields, vck):
    if vck == 'override' and 'created' in fields and 'modified' in fields:
        return fields
    if not vck or not vck.strip():
        raise ValueError('Version check required to update ' + entity + ' ' +
            str(dsId))
    existing = cfbk(entity, 'dsId', dsId)
    if not existing:
        raise ValueError('Existing ' + entity + ' ' + str(dsId) + ' not found.'
            )
    if vck != 'override' and existing['modified'] != vck:
        raise ValueError('Update error. Outdated data given for ' + entity +
            ' ' + str(dsId) + '.')
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = existing['created']
    ver = 1
    mods = existing['modified'].split(';')
    if len(mods) > 1:
        ver = int(mods[1]) + 1
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = nowISO() + ';' + str(ver)
    return existing


<mask token>


def db2app_AppUser(inst):
    cnv = {}
    cnv['dsType'] = 'AppUser'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppUser', 'importid', inst)
    cnv['email'] = db2app_fieldval('AppUser', 'email', inst)
    cnv['phash'] = db2app_fieldval('AppUser', 'phash', inst)
    cnv['status'] = db2app_fieldval('AppUser', 'status', inst)
    cnv['actsends'] = db2app_fieldval('AppUser', 'actsends', inst)
    cnv['actcode'] = db2app_fieldval('AppUser', 'actcode', inst)
    cnv['accessed'] = db2app_fieldval('AppUser', 'accessed', inst)
    cnv['name'] = db2app_fieldval('AppUser', 'name', inst)
    cnv['title'] = db2app_fieldval('AppUser', 'title', inst)
    cnv['web'] = db2app_fieldval('AppUser', 'web', inst)
    cnv['lang'] = db2app_fieldval('AppUser', 'lang', inst)
    cnv['settings'] = db2app_fieldval('AppUser', 'settings', inst)
    cnv['started'] = db2app_fieldval('AppUser', 'started', inst)
    cnv['completed'] = db2app_fieldval('AppUser', 'completed', inst)
    cnv['remtls'] = db2app_fieldval('AppUser', 'remtls', inst)
    cnv['built'] = db2app_fieldval('AppUser', 'built', inst)
    return cnv


def app2db_Point(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'editors' in inst:
        cnv['editors'] = app2db_fieldval('Point', 'editors', inst)
    if fill or 'srctl' in inst:
        cnv['srctl'] = app2db_fieldval('Point', 'srctl', inst)
    if fill or 'lmuid' in inst:
        cnv['lmuid'] = app2db_fieldval('Point', 'lmuid', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('Point', 'importid', inst)
    if fill or 'source' in inst:
        cnv['source'] = app2db_fieldval('Point', 'source', inst)
    if fill or 'date' in inst:
        cnv['date'] = app2db_fieldval('Point', 'date', inst)
    if fill or 'text' in inst:
        cnv['text'] = app2db_fieldval('Point', 'text', inst)
    if fill or 'refs' in inst:
        cnv['refs'] = app2db_fieldval('Point', 'refs', inst)
    if fill or 'qtype' in inst:
        cnv['qtype'] = app2db_fieldval('Point', 'qtype', inst)
    if fill or 'communities' in inst:
        cnv['communities'] = app2db_fieldval('Point', 'communities', inst)
    if fill or 'regions' in inst:
        cnv['regions'] = app2db_fieldval('Point', 'regions', inst)
    if fill or 'categories' in inst:
        cnv['categories'] = app2db_fieldval('Point', 'categories', inst)
    if fill or 'tags' in inst:
        cnv['tags'] = app2db_fieldval('Point', 'tags', inst)
    if fill or 'codes' in inst:
        cnv['codes'] = app2db_fieldval('Point', 'codes', inst)
    if fill or 'srclang' in inst:
        cnv['srclang'] = app2db_fieldval('Point', 'srclang', inst)
    if fill or 'translations' in inst:
        cnv['translations'] = app2db_fieldval('Point', 'translations', inst)
    if fill or 'pic' in inst:
        cnv['pic'] = app2db_fieldval('Point', 'pic', inst)
    if fill or 'stats' in inst:
        cnv['stats'] = app2db_fieldval('Point', 'stats', inst)
    return cnv


def db2app_Point(inst):
    cnv = {}
    cnv['dsType'] = 'Point'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['editors'] = db2app_fieldval('Point', 'editors', inst)
    cnv['srctl'] = db2app_fieldval('Point', 'srctl', inst)
    cnv['lmuid'] = db2app_fieldval('Point', 'lmuid', inst)
    cnv['importid'] = db2app_fieldval('Point', 'importid', inst)
    cnv['source'] = db2app_fieldval('Point', 'source', inst)
    cnv['date'] = db2app_fieldval('Point', 'date', inst)
    cnv['text'] = db2app_fieldval('Point', 'text', inst)
    cnv['refs'] = db2app_fieldval('Point', 'refs', inst)
    cnv['qtype'] = db2app_fieldval('Point', 'qtype', inst)
    cnv['communities'] = db2app_fieldval('Point', 'communities', inst)
    cnv['regions'] = db2app_fieldval('Point', 'regions', inst)
    cnv['categories'] = db2app_fieldval('Point', 'categories', inst)
    cnv['tags'] = db2app_fieldval('Point', 'tags', inst)
    cnv['codes'] = db2app_fieldval('Point', 'codes', inst)
    cnv['srclang'] = db2app_fieldval('Point', 'srclang', inst)
    cnv['translations'] = db2app_fieldval('Point', 'translations', inst)
    cnv['pic'] = db2app_fieldval('Point', 'pic', inst)
    cnv['stats'] = db2app_fieldval('Point', 'stats', inst)
    return cnv


def app2db_Timeline(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'editors' in inst:
        cnv['editors'] = app2db_fieldval('Timeline', 'editors', inst)
    if fill or 'lmuid' in inst:
        cnv['lmuid'] = app2db_fieldval('Timeline', 'lmuid', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('Timeline', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('Timeline', 'name', inst)
    if fill or 'cname' in inst:
        cnv['cname'] = app2db_fieldval('Timeline', 'cname', inst)
    if fill or 'slug' in inst:
        cnv['slug'] = app2db_fieldval('Timeline', 'slug', inst)
    if fill or 'title' in inst:
        cnv['title'] = app2db_fieldval('Timeline', 'title', inst)
    if fill or 'subtitle' in inst:
        cnv['subtitle'] = app2db_fieldval('Timeline', 'subtitle', inst)
    if fill or 'featured' in inst:
        cnv['featured'] = app2db_fieldval('Timeline', 'featured', inst)
    if fill or 'lang' in inst:
        cnv['lang'] = app2db_fieldval('Timeline', 'lang', inst)
    if fill or 'comment' in inst:
        cnv['comment'] = app2db_fieldval('Timeline', 'comment', inst)
    if fill or 'about' in inst:
        cnv['about'] = app2db_fieldval('Timeline', 'about', inst)
    if fill or 'kwds' in inst:
        cnv['kwds'] = app2db_fieldval('Timeline', 'kwds', inst)
    if fill or 'ctype' in inst:
        cnv['ctype'] = app2db_fieldval('Timeline', 'ctype', inst)
    if fill or 'cids' in inst:
        cnv['cids'] = app2db_fieldval('Timeline', 'cids', inst)
    if fill or 'rempts' in inst:
        cnv['rempts'] = app2db_fieldval('Timeline', 'rempts', inst)
    if fill or 'svs' in inst:
        cnv['svs'] = app2db_fieldval('Timeline', 'svs', inst)
    if fill or 'preb' in inst:
        cnv['preb'] = app2db_fieldval('Timeline', 'preb', inst)
    return cnv


<mask token>


def app2db_TLComp(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('TLComp', 'importid', inst)
    if fill or 'userid' in inst:
        cnv['userid'] = app2db_fieldval('TLComp', 'userid', inst)
    if fill or 'tlid' in inst:
        cnv['tlid'] = app2db_fieldval('TLComp', 'tlid', inst)
    if fill or 'username' in inst:
        cnv['username'] = app2db_fieldval('TLComp', 'username', inst)
    if fill or 'tlname' in inst:
        cnv['tlname'] = app2db_fieldval('TLComp', 'tlname', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('TLComp', 'data', inst)
    return cnv


def db2app_TLComp(inst):
    cnv = {}
    cnv['dsType'] = 'TLComp'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('TLComp', 'importid', inst)
    cnv['userid'] = db2app_fieldval('TLComp', 'userid', inst)
    cnv['tlid'] = db2app_fieldval('TLComp', 'tlid', inst)
    cnv['username'] = db2app_fieldval('TLComp', 'username', inst)
    cnv['tlname'] = db2app_fieldval('TLComp', 'tlname', inst)
    cnv['data'] = db2app_fieldval('TLComp', 'data', inst)
    return cnv


def app2db_DayCount(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('DayCount', 'importid', inst)
    if fill or 'tstamp' in inst:
        cnv['tstamp'] = app2db_fieldval('DayCount', 'tstamp', inst)
    if fill or 'rtype' in inst:
        cnv['rtype'] = app2db_fieldval('DayCount', 'rtype', inst)
    if fill or 'detail' in inst:
        cnv['detail'] = app2db_fieldval('DayCount', 'detail', inst)
    return cnv


<mask token>


def app2db_AppService(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('AppService', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('AppService', 'name', inst)
    if fill or 'ckey' in inst:
        cnv['ckey'] = app2db_fieldval('AppService', 'ckey', inst)
    if fill or 'csec' in inst:
        cnv['csec'] = app2db_fieldval('AppService', 'csec', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('AppService', 'data', inst)
    return cnv


def db2app_AppService(inst):
    cnv = {}
    cnv['dsType'] = 'AppService'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppService', 'importid', inst)
    cnv['name'] = db2app_fieldval('AppService', 'name', inst)
    cnv['ckey'] = db2app_fieldval('AppService', 'ckey', inst)
    cnv['csec'] = db2app_fieldval('AppService', 'csec', inst)
    cnv['data'] = db2app_fieldval('AppService', 'data', inst)
    return cnv


def dblogmsg(op, entity, res):
    log_summary_flds = {'AppUser': ['email', 'name'], 'Point': ['editors',
        'date', 'text'], 'Timeline': ['name'], 'TLComp': ['userid', 'tlid',
        'username', 'tlname'], 'DayCount': ['tstamp', 'rtype'],
        'AppService': ['name']}
    if res:
        if op != 'QRY':
            res = [res]
        for obj in res:
            msg = 'db' + op + ' ' + entity + ' ' + obj['dsId']
            if entity in log_summary_flds:
                for field in log_summary_flds[entity]:
                    msg += ' ' + str(obj[field])[0:60]
            logging.info(msg)
    else:
        logging.info('db' + op + ' ' + entity + ' -no obj details-')


<mask token>


def update_existing_AppUser(context, fields):
    fields = app2db_AppUser(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppUser SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppUser' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppUser(result)
    dblogmsg('UPD', 'AppUser', result)
    entcache.cache_remove(result)
    return result


<mask token>


def insert_new_Timeline(cnx, cursor, fields):
    fields = app2db_Timeline(fields)
    stmt = (
        'INSERT INTO Timeline (created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) VALUES (%(created)s, %(modified)s, %(editors)s, %(lmuid)s, %(importid)s, %(name)s, %(cname)s, %(slug)s, %(title)s, %(subtitle)s, %(featured)s, %(lang)s, %(comment)s, %(about)s, %(kwds)s, %(ctype)s, %(cids)s, %(rempts)s, %(svs)s, %(preb)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'editors': fields.get('editors', entdefs['Timeline'][
        'editors']['dv']), 'lmuid': fields.get('lmuid', entdefs['Timeline']
        ['lmuid']['dv']), 'importid': fields.get('importid', entdefs[
        'Timeline']['importid']['dv']), 'name': fields.get('name', entdefs[
        'Timeline']['name']['dv']), 'cname': fields.get('cname', entdefs[
        'Timeline']['cname']['dv']), 'slug': fields.get('slug', entdefs[
        'Timeline']['slug']['dv']), 'title': fields.get('title', entdefs[
        'Timeline']['title']['dv']), 'subtitle': fields.get('subtitle',
        entdefs['Timeline']['subtitle']['dv']), 'featured': fields.get(
        'featured', entdefs['Timeline']['featured']['dv']), 'lang': fields.
        get('lang', entdefs['Timeline']['lang']['dv']), 'comment': fields.
        get('comment', entdefs['Timeline']['comment']['dv']), 'about':
        fields.get('about', entdefs['Timeline']['about']['dv']), 'kwds':
        fields.get('kwds', entdefs['Timeline']['kwds']['dv']), 'ctype':
        fields.get('ctype', entdefs['Timeline']['ctype']['dv']), 'cids':
        fields.get('cids', entdefs['Timeline']['cids']['dv']), 'rempts':
        fields.get('rempts', entdefs['Timeline']['rempts']['dv']), 'svs':
        fields.get('svs', entdefs['Timeline']['svs']['dv']), 'preb': fields
        .get('preb', entdefs['Timeline']['preb']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Timeline(fields)
    dblogmsg('ADD', 'Timeline', fields)
    return fields


def update_existing_Timeline(context, fields):
    fields = app2db_Timeline(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE Timeline SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('Timeline' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_Timeline(result)
    dblogmsg('UPD', 'Timeline', result)
    entcache.cache_remove(result)
    return result


def insert_new_TLComp(cnx, cursor, fields):
    fields = app2db_TLComp(fields)
    stmt = (
        'INSERT INTO TLComp (created, modified, importid, userid, tlid, username, tlname, data) VALUES (%(created)s, %(modified)s, %(importid)s, %(userid)s, %(tlid)s, %(username)s, %(tlname)s, %(data)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs['TLComp'][
        'importid']['dv']), 'userid': fields.get('userid', entdefs['TLComp'
        ]['userid']['dv']), 'tlid': fields.get('tlid', entdefs['TLComp'][
        'tlid']['dv']), 'username': fields.get('username', entdefs['TLComp'
        ]['username']['dv']), 'tlname': fields.get('tlname', entdefs[
        'TLComp']['tlname']['dv']), 'data': fields.get('data', entdefs[
        'TLComp']['data']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_TLComp(fields)
    dblogmsg('ADD', 'TLComp', fields)
    return fields


<mask token>


def update_existing_DayCount(context, fields):
    fields = app2db_DayCount(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE DayCount SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('DayCount' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_DayCount(result)
    dblogmsg('UPD', 'DayCount', result)
    entcache.cache_remove(result)
    return result


<mask token>


def update_existing_AppService(context, fields):
    fields = app2db_AppService(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppService SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppService' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppService(result)
    dblogmsg('UPD', 'AppService', result)
    entcache.cache_put(result)
    return result


def write_entity(inst, vck='1234-12-12T00:00:00Z'):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            entity = inst.get('dsType', None)
            dsId = inst.get('dsId', 0)
            if dsId:
                existing = verify_timestamp_fields(entity, dsId, inst, vck)
                context = {'cnx': cnx, 'cursor': cursor, 'vck': vck,
                    'existing': existing}
                if entity == 'AppUser':
                    return update_existing_AppUser(context, inst)
                if entity == 'Point':
                    return update_existing_Point(context, inst)
                if entity == 'Timeline':
                    return update_existing_Timeline(context, inst)
                if entity == 'TLComp':
                    return update_existing_TLComp(context, inst)
                if entity == 'DayCount':
                    return update_existing_DayCount(context, inst)
                if entity == 'AppService':
                    return update_existing_AppService(context, inst)
                raise ValueError('Cannot modify unknown entity dsType ' +
                    str(entity))
            initialize_timestamp_fields(inst, vck)
            if entity == 'AppUser':
                return insert_new_AppUser(cnx, cursor, inst)
            if entity == 'Point':
                return insert_new_Point(cnx, cursor, inst)
            if entity == 'Timeline':
                return insert_new_Timeline(cnx, cursor, inst)
            if entity == 'TLComp':
                return insert_new_TLComp(cnx, cursor, inst)
            if entity == 'DayCount':
                return insert_new_DayCount(cnx, cursor, inst)
            if entity == 'AppService':
                return insert_new_AppService(cnx, cursor, inst)
            raise ValueError('Cannot create unknown entity dsType ' + str(
                entity))
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def delete_entity(entity, dsId):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            stmt = 'DELETE FROM ' + entity + ' WHERE dsId=' + str(dsId)
            cursor.execute(stmt)
            cnx.commit()
            dblogmsg('DEL', entity + ' ' + str(dsId), None)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def query_AppUser(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built'
        )
    query += ' FROM AppUser ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built in cursor:
        inst = {'dsType': 'AppUser', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'email': email,
            'phash': phash, 'status': status, 'actsends': actsends,
            'actcode': actcode, 'accessed': accessed, 'name': name, 'title':
            title, 'web': web, 'lang': lang, 'settings': settings,
            'started': started, 'completed': completed, 'remtls': remtls,
            'built': built}
        inst = db2app_AppUser(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppUser', res)
    return res


def query_Point(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats'
        )
    query += ' FROM Point ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats in cursor:
        inst = {'dsType': 'Point', 'dsId': dsId, 'created': created,
            'modified': modified, 'editors': editors, 'srctl': srctl,
            'lmuid': lmuid, 'importid': importid, 'source': source, 'date':
            date, 'text': text, 'refs': refs, 'qtype': qtype, 'communities':
            communities, 'regions': regions, 'categories': categories,
            'tags': tags, 'codes': codes, 'srclang': srclang,
            'translations': translations, 'pic': pic, 'stats': stats}
        inst = db2app_Point(inst)
        res.append(inst)
    dblogmsg('QRY', 'Point', res)
    return res


<mask token>


def query_TLComp(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, userid, tlid, username, tlname, data'
    query += ' FROM TLComp ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, userid, tlid, username, tlname, data in cursor:
        inst = {'dsType': 'TLComp', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'userid': userid,
            'tlid': tlid, 'username': username, 'tlname': tlname, 'data': data}
        inst = db2app_TLComp(inst)
        res.append(inst)
    dblogmsg('QRY', 'TLComp', res)
    return res


def query_DayCount(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, tstamp, rtype, detail'
    query += ' FROM DayCount ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, tstamp, rtype, detail in cursor:
        inst = {'dsType': 'DayCount', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'tstamp': tstamp,
            'rtype': rtype, 'detail': detail}
        inst = db2app_DayCount(inst)
        res.append(inst)
    dblogmsg('QRY', 'DayCount', res)
    return res


def query_AppService(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, name, ckey, csec, data'
    query += ' FROM AppService ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, name, ckey, csec, data in cursor:
        inst = {'dsType': 'AppService', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'name': name,
            'ckey': ckey, 'csec': csec, 'data': data}
        inst = db2app_AppService(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppService', res)
    return res


def query_entity(entity, where):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            if entity == 'AppUser':
                return query_AppUser(cnx, cursor, where)
            if entity == 'Point':
                return query_Point(cnx, cursor, where)
            if entity == 'Timeline':
                return query_Timeline(cnx, cursor, where)
            if entity == 'TLComp':
                return query_TLComp(cnx, cursor, where)
            if entity == 'DayCount':
                return query_DayCount(cnx, cursor, where)
            if entity == 'AppService':
                return query_AppService(cnx, cursor, where)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def visible_AppUser_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        if fld == 'email' and audience != 'private':
            continue
        if fld == 'phash':
            continue
        if fld == 'status' and audience != 'private':
            continue
        if fld == 'actsends':
            continue
        if fld == 'actcode':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>


def visible_Timeline_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>


def visible_DayCount_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>

<mask token>


def timestamp(offset=0):
    now = datetime.datetime.utcnow().replace(microsecond=0)
    return dt2ISO(now + datetime.timedelta(minutes=offset))


def expiration_for_inst(inst):
    if not inst or not inst['dsId']:
        raise ValueError('Uncacheable instance: ' + str(inst))
    cms = cachedefs[inst['dsType']]
    if not cms or not cms['minutes']:
        return ''
    return timestamp(cms['minutes'])


def make_key(dsType, field, value):
    return dsType + '_' + field + '_' + value


def entkey_vals(inst):
    instkey = make_key(inst['dsType'], 'dsId', inst['dsId'])
    keyvals = [{'key': instkey, 'val': pickle.dumps(inst)}]
    for field in entkeys[inst['dsType']]:
        keyvals.append({'key': make_key(inst['dsType'], field, inst[field]),
            'val': instkey})
    return keyvals


class EntityCache:
    """ Special case runtime cache to avoid pounding the db repeatedly """
    entities = {}

    def cache_put(self, inst):
        expir = expiration_for_inst(inst)
        if expir:
            self.cache_remove(inst)
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cachekeys = ['TTL_' + expir]
            for keyval in entkey_vals(inst):
                cachekeys.append(keyval['key'])
                self.entities[keyval['key']] = keyval['val']
            self.entities[kt] = ','.join(cachekeys)

    def cache_get(self, entity, field, value):
        instkey = make_key(entity, field, value)
        if instkey not in self.entities:
            return None
        instval = self.entities[instkey]
        if field != 'dsId':
            instval = self.entities[instval]
        return pickle.loads(instval)

    def cache_remove(self, inst):
        if inst:
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cleankeys = self.entities.pop(kt, None)
            if cleankeys:
                for oldkey in cleankeys.split(','):
                    self.entities.pop(oldkey, None)

    def cache_clean(self):
        now = nowISO()
        for key, value in self.entities.items():
            if key.endswith('_cleanup'):
                ttl = value.split(',')[0][4:]
                if ttl < now:
                    kcs = key.split('_')
                    inst = {'dsType': kcs[0], 'dsId': kcs[1]}
                    self.cache_remove(inst)

    def log_cache_entries(self):
        txt = 'EntityCache entities:\n'
        for key, _ in self.entities.items():
            txt += '    ' + key + ': ' + str(self.entities[key])[0:74] + '\n'
        logging.info(txt)
        return txt


<mask token>


def reqarg(argname, fieldtype='string', required=False):
    argval = flask.request.args.get(argname)
    if not argval:
        argval = flask.request.form.get(argname)
    if required and not argval:
        raise ValueError('Missing required value for ' + argname)
    dotidx = fieldtype.find('.')
    if dotidx > 0:
        entity = fieldtype[0:dotidx]
        fieldname = fieldtype[dotidx + 1:]
        fieldtype = entdefs[entity][fieldname]['pt']
    if fieldtype == 'email':
        emaddr = argval or ''
        emaddr = emaddr.lower()
        emaddr = re.sub('%40', '@', emaddr)
        if required and not re.match('[^@]+@[^@]+\\.[^@]+', emaddr):
            raise ValueError('Invalid ' + argname + ' value: ' + emaddr)
        return emaddr
    if fieldtype in ['dbid', 'string', 'isodate', 'isomod', 'srchidcsv',
        'text', 'json', 'jsarr', 'idcsv', 'isodcsv', 'gencsv', 'url']:
        return argval or ''
    if fieldtype == 'image':
        return argval or None
    if fieldtype == 'int':
        argval = argval or 0
        return int(argval)
    raise ValueError('Unknown type ' + fieldtype + ' for ' + argname)


def cfbk(entity, field, value, required=False):
    if field != 'dsId' and field not in entkeys[entity]:
        raise ValueError(field + ' not a unique index for ' + entity)
    vstr = str(value)
    ci = entcache.cache_get(entity, field, vstr)
    if ci:
        dblogmsg('CAC', entity, ci)
        return ci
    if entdefs[entity][field]['pt'] not in ['dbid', 'int']:
        vstr = '"' + vstr + '"'
    objs = query_entity(entity, 'WHERE ' + field + '=' + vstr + ' LIMIT 1')
    if len(objs) > 0:
        inst = objs[0]
        if not cachedefs[inst['dsType']]['manualadd']:
            entcache.cache_put(inst)
        return inst
    if required:
        raise ValueError(entity + ' ' + vstr + ' not found.')
    return None


<mask token>


def trim_string_val(val, unique=False):
    val = val or ''
    val = str(val)
    val = val.strip()
    if val and unique:
        lowval = val.lower()
        if lowval in ['null', 'none']:
            val = ''
    return val


def app2db_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]['pt']
        unique = entdefs[entity][field]['un']
        val = entdefs[entity][field]['dv']
    else:
        pt = dbflds[field]['pt']
        unique = dbflds[field]['un']
        val = dbflds[field]['dv']
    if field in inst:
        val = inst[field]
    if pt in ['email', 'string']:
        val = val or ''
        val = trim_string_val(val, unique)
        if not val:
            val = None
    elif pt == 'image':
        if not val:
            val = None
    elif pt == 'int':
        val = val or 0
        val = int(val)
    elif pt == 'dbid':
        try:
            val = int(val)
        except ValueError:
            val = 0
        if unique and not val:
            val = None
    return val


def db2app_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]['pt']
        val = entdefs[entity][field]['dv']
    else:
        pt = dbflds[field]['pt']
        val = dbflds[field]['dv']
    if field in inst:
        val = inst[field]
    if pt in ['email', 'string']:
        if not val:
            val = ''
        val = str(val)
    elif pt == 'image':
        if not val:
            val = ''
    elif pt == 'int':
        if not val:
            val = 0
    elif pt == 'dbid':
        if not val:
            val = ''
        else:
            val = str(val)
    return val


def ISO2dt(isostr):
    isostr = re.sub('\\.\\d*Z', 'Z', isostr)
    dt = datetime.datetime.utcnow()
    dt = dt.strptime(isostr, '%Y-%m-%dT%H:%M:%SZ')
    return dt


def dt2ISO(dt):
    iso = str(dt.year) + '-' + str(dt.month).rjust(2, '0') + '-'
    iso += str(dt.day).rjust(2, '0') + 'T' + str(dt.hour).rjust(2, '0')
    iso += ':' + str(dt.minute).rjust(2, '0') + ':'
    iso += str(dt.second).rjust(2, '0') + 'Z'
    return iso


def nowISO():
    """ Return the current time as an ISO string """
    return dt2ISO(datetime.datetime.utcnow())


def initialize_timestamp_fields(fields, vck):
    ts = nowISO()
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = ts
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = ts + ';1'


def verify_timestamp_fields(entity, dsId, fields, vck):
    if vck == 'override' and 'created' in fields and 'modified' in fields:
        return fields
    if not vck or not vck.strip():
        raise ValueError('Version check required to update ' + entity + ' ' +
            str(dsId))
    existing = cfbk(entity, 'dsId', dsId)
    if not existing:
        raise ValueError('Existing ' + entity + ' ' + str(dsId) + ' not found.'
            )
    if vck != 'override' and existing['modified'] != vck:
        raise ValueError('Update error. Outdated data given for ' + entity +
            ' ' + str(dsId) + '.')
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = existing['created']
    ver = 1
    mods = existing['modified'].split(';')
    if len(mods) > 1:
        ver = int(mods[1]) + 1
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = nowISO() + ';' + str(ver)
    return existing


def app2db_AppUser(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('AppUser', 'importid', inst)
    if fill or 'email' in inst:
        cnv['email'] = app2db_fieldval('AppUser', 'email', inst)
    if fill or 'phash' in inst:
        cnv['phash'] = app2db_fieldval('AppUser', 'phash', inst)
    if fill or 'status' in inst:
        cnv['status'] = app2db_fieldval('AppUser', 'status', inst)
    if fill or 'actsends' in inst:
        cnv['actsends'] = app2db_fieldval('AppUser', 'actsends', inst)
    if fill or 'actcode' in inst:
        cnv['actcode'] = app2db_fieldval('AppUser', 'actcode', inst)
    if fill or 'accessed' in inst:
        cnv['accessed'] = app2db_fieldval('AppUser', 'accessed', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('AppUser', 'name', inst)
    if fill or 'title' in inst:
        cnv['title'] = app2db_fieldval('AppUser', 'title', inst)
    if fill or 'web' in inst:
        cnv['web'] = app2db_fieldval('AppUser', 'web', inst)
    if fill or 'lang' in inst:
        cnv['lang'] = app2db_fieldval('AppUser', 'lang', inst)
    if fill or 'settings' in inst:
        cnv['settings'] = app2db_fieldval('AppUser', 'settings', inst)
    if fill or 'started' in inst:
        cnv['started'] = app2db_fieldval('AppUser', 'started', inst)
    if fill or 'completed' in inst:
        cnv['completed'] = app2db_fieldval('AppUser', 'completed', inst)
    if fill or 'remtls' in inst:
        cnv['remtls'] = app2db_fieldval('AppUser', 'remtls', inst)
    if fill or 'built' in inst:
        cnv['built'] = app2db_fieldval('AppUser', 'built', inst)
    return cnv


def db2app_AppUser(inst):
    cnv = {}
    cnv['dsType'] = 'AppUser'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppUser', 'importid', inst)
    cnv['email'] = db2app_fieldval('AppUser', 'email', inst)
    cnv['phash'] = db2app_fieldval('AppUser', 'phash', inst)
    cnv['status'] = db2app_fieldval('AppUser', 'status', inst)
    cnv['actsends'] = db2app_fieldval('AppUser', 'actsends', inst)
    cnv['actcode'] = db2app_fieldval('AppUser', 'actcode', inst)
    cnv['accessed'] = db2app_fieldval('AppUser', 'accessed', inst)
    cnv['name'] = db2app_fieldval('AppUser', 'name', inst)
    cnv['title'] = db2app_fieldval('AppUser', 'title', inst)
    cnv['web'] = db2app_fieldval('AppUser', 'web', inst)
    cnv['lang'] = db2app_fieldval('AppUser', 'lang', inst)
    cnv['settings'] = db2app_fieldval('AppUser', 'settings', inst)
    cnv['started'] = db2app_fieldval('AppUser', 'started', inst)
    cnv['completed'] = db2app_fieldval('AppUser', 'completed', inst)
    cnv['remtls'] = db2app_fieldval('AppUser', 'remtls', inst)
    cnv['built'] = db2app_fieldval('AppUser', 'built', inst)
    return cnv


def app2db_Point(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'editors' in inst:
        cnv['editors'] = app2db_fieldval('Point', 'editors', inst)
    if fill or 'srctl' in inst:
        cnv['srctl'] = app2db_fieldval('Point', 'srctl', inst)
    if fill or 'lmuid' in inst:
        cnv['lmuid'] = app2db_fieldval('Point', 'lmuid', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('Point', 'importid', inst)
    if fill or 'source' in inst:
        cnv['source'] = app2db_fieldval('Point', 'source', inst)
    if fill or 'date' in inst:
        cnv['date'] = app2db_fieldval('Point', 'date', inst)
    if fill or 'text' in inst:
        cnv['text'] = app2db_fieldval('Point', 'text', inst)
    if fill or 'refs' in inst:
        cnv['refs'] = app2db_fieldval('Point', 'refs', inst)
    if fill or 'qtype' in inst:
        cnv['qtype'] = app2db_fieldval('Point', 'qtype', inst)
    if fill or 'communities' in inst:
        cnv['communities'] = app2db_fieldval('Point', 'communities', inst)
    if fill or 'regions' in inst:
        cnv['regions'] = app2db_fieldval('Point', 'regions', inst)
    if fill or 'categories' in inst:
        cnv['categories'] = app2db_fieldval('Point', 'categories', inst)
    if fill or 'tags' in inst:
        cnv['tags'] = app2db_fieldval('Point', 'tags', inst)
    if fill or 'codes' in inst:
        cnv['codes'] = app2db_fieldval('Point', 'codes', inst)
    if fill or 'srclang' in inst:
        cnv['srclang'] = app2db_fieldval('Point', 'srclang', inst)
    if fill or 'translations' in inst:
        cnv['translations'] = app2db_fieldval('Point', 'translations', inst)
    if fill or 'pic' in inst:
        cnv['pic'] = app2db_fieldval('Point', 'pic', inst)
    if fill or 'stats' in inst:
        cnv['stats'] = app2db_fieldval('Point', 'stats', inst)
    return cnv


def db2app_Point(inst):
    cnv = {}
    cnv['dsType'] = 'Point'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['editors'] = db2app_fieldval('Point', 'editors', inst)
    cnv['srctl'] = db2app_fieldval('Point', 'srctl', inst)
    cnv['lmuid'] = db2app_fieldval('Point', 'lmuid', inst)
    cnv['importid'] = db2app_fieldval('Point', 'importid', inst)
    cnv['source'] = db2app_fieldval('Point', 'source', inst)
    cnv['date'] = db2app_fieldval('Point', 'date', inst)
    cnv['text'] = db2app_fieldval('Point', 'text', inst)
    cnv['refs'] = db2app_fieldval('Point', 'refs', inst)
    cnv['qtype'] = db2app_fieldval('Point', 'qtype', inst)
    cnv['communities'] = db2app_fieldval('Point', 'communities', inst)
    cnv['regions'] = db2app_fieldval('Point', 'regions', inst)
    cnv['categories'] = db2app_fieldval('Point', 'categories', inst)
    cnv['tags'] = db2app_fieldval('Point', 'tags', inst)
    cnv['codes'] = db2app_fieldval('Point', 'codes', inst)
    cnv['srclang'] = db2app_fieldval('Point', 'srclang', inst)
    cnv['translations'] = db2app_fieldval('Point', 'translations', inst)
    cnv['pic'] = db2app_fieldval('Point', 'pic', inst)
    cnv['stats'] = db2app_fieldval('Point', 'stats', inst)
    return cnv


def app2db_Timeline(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'editors' in inst:
        cnv['editors'] = app2db_fieldval('Timeline', 'editors', inst)
    if fill or 'lmuid' in inst:
        cnv['lmuid'] = app2db_fieldval('Timeline', 'lmuid', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('Timeline', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('Timeline', 'name', inst)
    if fill or 'cname' in inst:
        cnv['cname'] = app2db_fieldval('Timeline', 'cname', inst)
    if fill or 'slug' in inst:
        cnv['slug'] = app2db_fieldval('Timeline', 'slug', inst)
    if fill or 'title' in inst:
        cnv['title'] = app2db_fieldval('Timeline', 'title', inst)
    if fill or 'subtitle' in inst:
        cnv['subtitle'] = app2db_fieldval('Timeline', 'subtitle', inst)
    if fill or 'featured' in inst:
        cnv['featured'] = app2db_fieldval('Timeline', 'featured', inst)
    if fill or 'lang' in inst:
        cnv['lang'] = app2db_fieldval('Timeline', 'lang', inst)
    if fill or 'comment' in inst:
        cnv['comment'] = app2db_fieldval('Timeline', 'comment', inst)
    if fill or 'about' in inst:
        cnv['about'] = app2db_fieldval('Timeline', 'about', inst)
    if fill or 'kwds' in inst:
        cnv['kwds'] = app2db_fieldval('Timeline', 'kwds', inst)
    if fill or 'ctype' in inst:
        cnv['ctype'] = app2db_fieldval('Timeline', 'ctype', inst)
    if fill or 'cids' in inst:
        cnv['cids'] = app2db_fieldval('Timeline', 'cids', inst)
    if fill or 'rempts' in inst:
        cnv['rempts'] = app2db_fieldval('Timeline', 'rempts', inst)
    if fill or 'svs' in inst:
        cnv['svs'] = app2db_fieldval('Timeline', 'svs', inst)
    if fill or 'preb' in inst:
        cnv['preb'] = app2db_fieldval('Timeline', 'preb', inst)
    return cnv


<mask token>


def app2db_TLComp(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('TLComp', 'importid', inst)
    if fill or 'userid' in inst:
        cnv['userid'] = app2db_fieldval('TLComp', 'userid', inst)
    if fill or 'tlid' in inst:
        cnv['tlid'] = app2db_fieldval('TLComp', 'tlid', inst)
    if fill or 'username' in inst:
        cnv['username'] = app2db_fieldval('TLComp', 'username', inst)
    if fill or 'tlname' in inst:
        cnv['tlname'] = app2db_fieldval('TLComp', 'tlname', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('TLComp', 'data', inst)
    return cnv


def db2app_TLComp(inst):
    cnv = {}
    cnv['dsType'] = 'TLComp'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('TLComp', 'importid', inst)
    cnv['userid'] = db2app_fieldval('TLComp', 'userid', inst)
    cnv['tlid'] = db2app_fieldval('TLComp', 'tlid', inst)
    cnv['username'] = db2app_fieldval('TLComp', 'username', inst)
    cnv['tlname'] = db2app_fieldval('TLComp', 'tlname', inst)
    cnv['data'] = db2app_fieldval('TLComp', 'data', inst)
    return cnv


def app2db_DayCount(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('DayCount', 'importid', inst)
    if fill or 'tstamp' in inst:
        cnv['tstamp'] = app2db_fieldval('DayCount', 'tstamp', inst)
    if fill or 'rtype' in inst:
        cnv['rtype'] = app2db_fieldval('DayCount', 'rtype', inst)
    if fill or 'detail' in inst:
        cnv['detail'] = app2db_fieldval('DayCount', 'detail', inst)
    return cnv


def db2app_DayCount(inst):
    cnv = {}
    cnv['dsType'] = 'DayCount'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('DayCount', 'importid', inst)
    cnv['tstamp'] = db2app_fieldval('DayCount', 'tstamp', inst)
    cnv['rtype'] = db2app_fieldval('DayCount', 'rtype', inst)
    cnv['detail'] = db2app_fieldval('DayCount', 'detail', inst)
    return cnv


def app2db_AppService(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('AppService', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('AppService', 'name', inst)
    if fill or 'ckey' in inst:
        cnv['ckey'] = app2db_fieldval('AppService', 'ckey', inst)
    if fill or 'csec' in inst:
        cnv['csec'] = app2db_fieldval('AppService', 'csec', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('AppService', 'data', inst)
    return cnv


def db2app_AppService(inst):
    cnv = {}
    cnv['dsType'] = 'AppService'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppService', 'importid', inst)
    cnv['name'] = db2app_fieldval('AppService', 'name', inst)
    cnv['ckey'] = db2app_fieldval('AppService', 'ckey', inst)
    cnv['csec'] = db2app_fieldval('AppService', 'csec', inst)
    cnv['data'] = db2app_fieldval('AppService', 'data', inst)
    return cnv


def dblogmsg(op, entity, res):
    log_summary_flds = {'AppUser': ['email', 'name'], 'Point': ['editors',
        'date', 'text'], 'Timeline': ['name'], 'TLComp': ['userid', 'tlid',
        'username', 'tlname'], 'DayCount': ['tstamp', 'rtype'],
        'AppService': ['name']}
    if res:
        if op != 'QRY':
            res = [res]
        for obj in res:
            msg = 'db' + op + ' ' + entity + ' ' + obj['dsId']
            if entity in log_summary_flds:
                for field in log_summary_flds[entity]:
                    msg += ' ' + str(obj[field])[0:60]
            logging.info(msg)
    else:
        logging.info('db' + op + ' ' + entity + ' -no obj details-')


def insert_new_AppUser(cnx, cursor, fields):
    fields = app2db_AppUser(fields)
    stmt = (
        'INSERT INTO AppUser (created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built) VALUES (%(created)s, %(modified)s, %(importid)s, %(email)s, %(phash)s, %(status)s, %(actsends)s, %(actcode)s, %(accessed)s, %(name)s, %(title)s, %(web)s, %(lang)s, %(settings)s, %(started)s, %(completed)s, %(remtls)s, %(built)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs['AppUser'][
        'importid']['dv']), 'email': fields.get('email', entdefs['AppUser']
        ['email']['dv']), 'phash': fields.get('phash', entdefs['AppUser'][
        'phash']['dv']), 'status': fields.get('status', entdefs['AppUser'][
        'status']['dv']), 'actsends': fields.get('actsends', entdefs[
        'AppUser']['actsends']['dv']), 'actcode': fields.get('actcode',
        entdefs['AppUser']['actcode']['dv']), 'accessed': fields.get(
        'accessed', entdefs['AppUser']['accessed']['dv']), 'name': fields.
        get('name', entdefs['AppUser']['name']['dv']), 'title': fields.get(
        'title', entdefs['AppUser']['title']['dv']), 'web': fields.get(
        'web', entdefs['AppUser']['web']['dv']), 'lang': fields.get('lang',
        entdefs['AppUser']['lang']['dv']), 'settings': fields.get(
        'settings', entdefs['AppUser']['settings']['dv']), 'started':
        fields.get('started', entdefs['AppUser']['started']['dv']),
        'completed': fields.get('completed', entdefs['AppUser']['completed'
        ]['dv']), 'remtls': fields.get('remtls', entdefs['AppUser'][
        'remtls']['dv']), 'built': fields.get('built', entdefs['AppUser'][
        'built']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppUser(fields)
    dblogmsg('ADD', 'AppUser', fields)
    return fields


def update_existing_AppUser(context, fields):
    fields = app2db_AppUser(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppUser SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppUser' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppUser(result)
    dblogmsg('UPD', 'AppUser', result)
    entcache.cache_remove(result)
    return result


<mask token>


def insert_new_Timeline(cnx, cursor, fields):
    fields = app2db_Timeline(fields)
    stmt = (
        'INSERT INTO Timeline (created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) VALUES (%(created)s, %(modified)s, %(editors)s, %(lmuid)s, %(importid)s, %(name)s, %(cname)s, %(slug)s, %(title)s, %(subtitle)s, %(featured)s, %(lang)s, %(comment)s, %(about)s, %(kwds)s, %(ctype)s, %(cids)s, %(rempts)s, %(svs)s, %(preb)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'editors': fields.get('editors', entdefs['Timeline'][
        'editors']['dv']), 'lmuid': fields.get('lmuid', entdefs['Timeline']
        ['lmuid']['dv']), 'importid': fields.get('importid', entdefs[
        'Timeline']['importid']['dv']), 'name': fields.get('name', entdefs[
        'Timeline']['name']['dv']), 'cname': fields.get('cname', entdefs[
        'Timeline']['cname']['dv']), 'slug': fields.get('slug', entdefs[
        'Timeline']['slug']['dv']), 'title': fields.get('title', entdefs[
        'Timeline']['title']['dv']), 'subtitle': fields.get('subtitle',
        entdefs['Timeline']['subtitle']['dv']), 'featured': fields.get(
        'featured', entdefs['Timeline']['featured']['dv']), 'lang': fields.
        get('lang', entdefs['Timeline']['lang']['dv']), 'comment': fields.
        get('comment', entdefs['Timeline']['comment']['dv']), 'about':
        fields.get('about', entdefs['Timeline']['about']['dv']), 'kwds':
        fields.get('kwds', entdefs['Timeline']['kwds']['dv']), 'ctype':
        fields.get('ctype', entdefs['Timeline']['ctype']['dv']), 'cids':
        fields.get('cids', entdefs['Timeline']['cids']['dv']), 'rempts':
        fields.get('rempts', entdefs['Timeline']['rempts']['dv']), 'svs':
        fields.get('svs', entdefs['Timeline']['svs']['dv']), 'preb': fields
        .get('preb', entdefs['Timeline']['preb']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Timeline(fields)
    dblogmsg('ADD', 'Timeline', fields)
    return fields


def update_existing_Timeline(context, fields):
    fields = app2db_Timeline(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE Timeline SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('Timeline' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_Timeline(result)
    dblogmsg('UPD', 'Timeline', result)
    entcache.cache_remove(result)
    return result


def insert_new_TLComp(cnx, cursor, fields):
    fields = app2db_TLComp(fields)
    stmt = (
        'INSERT INTO TLComp (created, modified, importid, userid, tlid, username, tlname, data) VALUES (%(created)s, %(modified)s, %(importid)s, %(userid)s, %(tlid)s, %(username)s, %(tlname)s, %(data)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs['TLComp'][
        'importid']['dv']), 'userid': fields.get('userid', entdefs['TLComp'
        ]['userid']['dv']), 'tlid': fields.get('tlid', entdefs['TLComp'][
        'tlid']['dv']), 'username': fields.get('username', entdefs['TLComp'
        ]['username']['dv']), 'tlname': fields.get('tlname', entdefs[
        'TLComp']['tlname']['dv']), 'data': fields.get('data', entdefs[
        'TLComp']['data']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_TLComp(fields)
    dblogmsg('ADD', 'TLComp', fields)
    return fields


<mask token>


def update_existing_DayCount(context, fields):
    fields = app2db_DayCount(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE DayCount SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('DayCount' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_DayCount(result)
    dblogmsg('UPD', 'DayCount', result)
    entcache.cache_remove(result)
    return result


def insert_new_AppService(cnx, cursor, fields):
    fields = app2db_AppService(fields)
    stmt = (
        'INSERT INTO AppService (created, modified, importid, name, ckey, csec, data) VALUES (%(created)s, %(modified)s, %(importid)s, %(name)s, %(ckey)s, %(csec)s, %(data)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs[
        'AppService']['importid']['dv']), 'name': fields.get('name',
        entdefs['AppService']['name']['dv']), 'ckey': fields.get('ckey',
        entdefs['AppService']['ckey']['dv']), 'csec': fields.get('csec',
        entdefs['AppService']['csec']['dv']), 'data': fields.get('data',
        entdefs['AppService']['data']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppService(fields)
    dblogmsg('ADD', 'AppService', fields)
    return fields


def update_existing_AppService(context, fields):
    fields = app2db_AppService(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppService SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppService' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppService(result)
    dblogmsg('UPD', 'AppService', result)
    entcache.cache_put(result)
    return result


def write_entity(inst, vck='1234-12-12T00:00:00Z'):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            entity = inst.get('dsType', None)
            dsId = inst.get('dsId', 0)
            if dsId:
                existing = verify_timestamp_fields(entity, dsId, inst, vck)
                context = {'cnx': cnx, 'cursor': cursor, 'vck': vck,
                    'existing': existing}
                if entity == 'AppUser':
                    return update_existing_AppUser(context, inst)
                if entity == 'Point':
                    return update_existing_Point(context, inst)
                if entity == 'Timeline':
                    return update_existing_Timeline(context, inst)
                if entity == 'TLComp':
                    return update_existing_TLComp(context, inst)
                if entity == 'DayCount':
                    return update_existing_DayCount(context, inst)
                if entity == 'AppService':
                    return update_existing_AppService(context, inst)
                raise ValueError('Cannot modify unknown entity dsType ' +
                    str(entity))
            initialize_timestamp_fields(inst, vck)
            if entity == 'AppUser':
                return insert_new_AppUser(cnx, cursor, inst)
            if entity == 'Point':
                return insert_new_Point(cnx, cursor, inst)
            if entity == 'Timeline':
                return insert_new_Timeline(cnx, cursor, inst)
            if entity == 'TLComp':
                return insert_new_TLComp(cnx, cursor, inst)
            if entity == 'DayCount':
                return insert_new_DayCount(cnx, cursor, inst)
            if entity == 'AppService':
                return insert_new_AppService(cnx, cursor, inst)
            raise ValueError('Cannot create unknown entity dsType ' + str(
                entity))
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def delete_entity(entity, dsId):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            stmt = 'DELETE FROM ' + entity + ' WHERE dsId=' + str(dsId)
            cursor.execute(stmt)
            cnx.commit()
            dblogmsg('DEL', entity + ' ' + str(dsId), None)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def query_AppUser(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built'
        )
    query += ' FROM AppUser ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built in cursor:
        inst = {'dsType': 'AppUser', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'email': email,
            'phash': phash, 'status': status, 'actsends': actsends,
            'actcode': actcode, 'accessed': accessed, 'name': name, 'title':
            title, 'web': web, 'lang': lang, 'settings': settings,
            'started': started, 'completed': completed, 'remtls': remtls,
            'built': built}
        inst = db2app_AppUser(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppUser', res)
    return res


def query_Point(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats'
        )
    query += ' FROM Point ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats in cursor:
        inst = {'dsType': 'Point', 'dsId': dsId, 'created': created,
            'modified': modified, 'editors': editors, 'srctl': srctl,
            'lmuid': lmuid, 'importid': importid, 'source': source, 'date':
            date, 'text': text, 'refs': refs, 'qtype': qtype, 'communities':
            communities, 'regions': regions, 'categories': categories,
            'tags': tags, 'codes': codes, 'srclang': srclang,
            'translations': translations, 'pic': pic, 'stats': stats}
        inst = db2app_Point(inst)
        res.append(inst)
    dblogmsg('QRY', 'Point', res)
    return res


<mask token>


def query_TLComp(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, userid, tlid, username, tlname, data'
    query += ' FROM TLComp ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, userid, tlid, username, tlname, data in cursor:
        inst = {'dsType': 'TLComp', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'userid': userid,
            'tlid': tlid, 'username': username, 'tlname': tlname, 'data': data}
        inst = db2app_TLComp(inst)
        res.append(inst)
    dblogmsg('QRY', 'TLComp', res)
    return res


def query_DayCount(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, tstamp, rtype, detail'
    query += ' FROM DayCount ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, tstamp, rtype, detail in cursor:
        inst = {'dsType': 'DayCount', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'tstamp': tstamp,
            'rtype': rtype, 'detail': detail}
        inst = db2app_DayCount(inst)
        res.append(inst)
    dblogmsg('QRY', 'DayCount', res)
    return res


def query_AppService(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, name, ckey, csec, data'
    query += ' FROM AppService ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, name, ckey, csec, data in cursor:
        inst = {'dsType': 'AppService', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'name': name,
            'ckey': ckey, 'csec': csec, 'data': data}
        inst = db2app_AppService(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppService', res)
    return res


def query_entity(entity, where):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            if entity == 'AppUser':
                return query_AppUser(cnx, cursor, where)
            if entity == 'Point':
                return query_Point(cnx, cursor, where)
            if entity == 'Timeline':
                return query_Timeline(cnx, cursor, where)
            if entity == 'TLComp':
                return query_TLComp(cnx, cursor, where)
            if entity == 'DayCount':
                return query_DayCount(cnx, cursor, where)
            if entity == 'AppService':
                return query_AppService(cnx, cursor, where)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def visible_AppUser_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        if fld == 'email' and audience != 'private':
            continue
        if fld == 'phash':
            continue
        if fld == 'status' and audience != 'private':
            continue
        if fld == 'actsends':
            continue
        if fld == 'actcode':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>


def visible_Timeline_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>


def visible_DayCount_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


def visible_AppService_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


<mask token>

<mask token>
import logging
import flask
import re
import datetime
import pickle
import mysql.connector
import py.mconf as mconf
dbflds = {'dsId': {'pt': 'dbid', 'un': True, 'dv': 0}, 'created': {'pt':
    'string', 'un': False, 'dv': ''}, 'modified': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'batchconv': {'pt': 'string', 'un': False, 'dv': ''}}
entdefs = {'AppUser': {'dsId': {'pt': 'dbid', 'un': True, 'dv': 0},
    'created': {'pt': 'string', 'un': False, 'dv': ''}, 'modified': {'pt':
    'string', 'un': False, 'dv': ''}, 'batchconv': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'importid': {'pt': 'dbid', 'un': True, 'dv': 0},
    'email': {'pt': 'email', 'un': True, 'dv': ''}, 'phash': {'pt':
    'string', 'un': False, 'dv': ''}, 'status': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'actsends': {'pt': 'string', 'un': False, 'dv': ''},
    'actcode': {'pt': 'string', 'un': False, 'dv': ''}, 'accessed': {'pt':
    'string', 'un': False, 'dv': ''}, 'name': {'pt': 'string', 'un': False,
    'dv': ''}, 'title': {'pt': 'string', 'un': False, 'dv': ''}, 'web': {
    'pt': 'string', 'un': False, 'dv': ''}, 'lang': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'settings': {'pt': 'string', 'un': False, 'dv': ''},
    'started': {'pt': 'string', 'un': False, 'dv': ''}, 'completed': {'pt':
    'string', 'un': False, 'dv': ''}, 'remtls': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'built': {'pt': 'string', 'un': False, 'dv': ''}},
    'Point': {'dsId': {'pt': 'dbid', 'un': True, 'dv': 0}, 'created': {'pt':
    'string', 'un': False, 'dv': ''}, 'modified': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'batchconv': {'pt': 'string', 'un': False, 'dv': ''},
    'editors': {'pt': 'string', 'un': False, 'dv': ''}, 'srctl': {'pt':
    'dbid', 'un': False, 'dv': 0}, 'lmuid': {'pt': 'dbid', 'un': False,
    'dv': 0}, 'importid': {'pt': 'dbid', 'un': True, 'dv': 0}, 'source': {
    'pt': 'string', 'un': False, 'dv': ''}, 'date': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'text': {'pt': 'string', 'un': False, 'dv': ''},
    'refs': {'pt': 'string', 'un': False, 'dv': ''}, 'qtype': {'pt':
    'string', 'un': False, 'dv': ''}, 'communities': {'pt': 'string', 'un':
    False, 'dv': ''}, 'regions': {'pt': 'string', 'un': False, 'dv': ''},
    'categories': {'pt': 'string', 'un': False, 'dv': ''}, 'tags': {'pt':
    'string', 'un': False, 'dv': ''}, 'codes': {'pt': 'string', 'un': False,
    'dv': ''}, 'srclang': {'pt': 'string', 'un': False, 'dv': ''},
    'translations': {'pt': 'string', 'un': False, 'dv': ''}, 'pic': {'pt':
    'image', 'un': False, 'dv': None}, 'stats': {'pt': 'string', 'un': 
    False, 'dv': ''}}, 'Timeline': {'dsId': {'pt': 'dbid', 'un': True, 'dv':
    0}, 'created': {'pt': 'string', 'un': False, 'dv': ''}, 'modified': {
    'pt': 'string', 'un': False, 'dv': ''}, 'batchconv': {'pt': 'string',
    'un': False, 'dv': ''}, 'editors': {'pt': 'string', 'un': False, 'dv':
    ''}, 'lmuid': {'pt': 'dbid', 'un': False, 'dv': 0}, 'importid': {'pt':
    'dbid', 'un': True, 'dv': 0}, 'name': {'pt': 'string', 'un': True, 'dv':
    ''}, 'cname': {'pt': 'string', 'un': False, 'dv': ''}, 'slug': {'pt':
    'string', 'un': True, 'dv': ''}, 'title': {'pt': 'string', 'un': False,
    'dv': ''}, 'subtitle': {'pt': 'string', 'un': False, 'dv': ''},
    'featured': {'pt': 'string', 'un': False, 'dv': ''}, 'lang': {'pt':
    'string', 'un': False, 'dv': ''}, 'comment': {'pt': 'string', 'un': 
    False, 'dv': ''}, 'about': {'pt': 'string', 'un': False, 'dv': ''},
    'kwds': {'pt': 'string', 'un': False, 'dv': ''}, 'ctype': {'pt':
    'string', 'un': False, 'dv': ''}, 'cids': {'pt': 'string', 'un': False,
    'dv': ''}, 'rempts': {'pt': 'string', 'un': False, 'dv': ''}, 'svs': {
    'pt': 'string', 'un': False, 'dv': ''}, 'preb': {'pt': 'string', 'un': 
    False, 'dv': ''}}, 'TLComp': {'dsId': {'pt': 'dbid', 'un': True, 'dv': 
    0}, 'created': {'pt': 'string', 'un': False, 'dv': ''}, 'modified': {
    'pt': 'string', 'un': False, 'dv': ''}, 'batchconv': {'pt': 'string',
    'un': False, 'dv': ''}, 'importid': {'pt': 'dbid', 'un': True, 'dv': 0},
    'userid': {'pt': 'dbid', 'un': False, 'dv': 0}, 'tlid': {'pt': 'dbid',
    'un': False, 'dv': 0}, 'username': {'pt': 'string', 'un': False, 'dv':
    ''}, 'tlname': {'pt': 'string', 'un': False, 'dv': ''}, 'data': {'pt':
    'string', 'un': False, 'dv': ''}}, 'DayCount': {'dsId': {'pt': 'dbid',
    'un': True, 'dv': 0}, 'created': {'pt': 'string', 'un': False, 'dv': ''
    }, 'modified': {'pt': 'string', 'un': False, 'dv': ''}, 'batchconv': {
    'pt': 'string', 'un': False, 'dv': ''}, 'importid': {'pt': 'dbid', 'un':
    True, 'dv': 0}, 'tstamp': {'pt': 'string', 'un': False, 'dv': ''},
    'rtype': {'pt': 'string', 'un': False, 'dv': ''}, 'detail': {'pt':
    'string', 'un': False, 'dv': ''}}, 'AppService': {'dsId': {'pt': 'dbid',
    'un': True, 'dv': 0}, 'created': {'pt': 'string', 'un': False, 'dv': ''
    }, 'modified': {'pt': 'string', 'un': False, 'dv': ''}, 'batchconv': {
    'pt': 'string', 'un': False, 'dv': ''}, 'importid': {'pt': 'dbid', 'un':
    True, 'dv': 0}, 'name': {'pt': 'string', 'un': True, 'dv': ''}, 'ckey':
    {'pt': 'string', 'un': False, 'dv': ''}, 'csec': {'pt': 'string', 'un':
    False, 'dv': ''}, 'data': {'pt': 'string', 'un': False, 'dv': ''}}}
entkeys = {'AppUser': ['importid', 'email'], 'Point': ['importid'],
    'Timeline': ['importid', 'name', 'slug'], 'TLComp': ['importid'],
    'DayCount': ['importid'], 'AppService': ['importid', 'name']}
cachedefs = {'AppUser': {'minutes': 120, 'manualadd': True}, 'Point': {
    'minutes': 0, 'manualadd': False}, 'Timeline': {'minutes': 0,
    'manualadd': False}, 'TLComp': {'minutes': 0, 'manualadd': False},
    'DayCount': {'minutes': 0, 'manualadd': False}, 'AppService': {
    'minutes': 240, 'manualadd': False}}


def timestamp(offset=0):
    now = datetime.datetime.utcnow().replace(microsecond=0)
    return dt2ISO(now + datetime.timedelta(minutes=offset))


def expiration_for_inst(inst):
    if not inst or not inst['dsId']:
        raise ValueError('Uncacheable instance: ' + str(inst))
    cms = cachedefs[inst['dsType']]
    if not cms or not cms['minutes']:
        return ''
    return timestamp(cms['minutes'])


def make_key(dsType, field, value):
    return dsType + '_' + field + '_' + value


def entkey_vals(inst):
    instkey = make_key(inst['dsType'], 'dsId', inst['dsId'])
    keyvals = [{'key': instkey, 'val': pickle.dumps(inst)}]
    for field in entkeys[inst['dsType']]:
        keyvals.append({'key': make_key(inst['dsType'], field, inst[field]),
            'val': instkey})
    return keyvals


class EntityCache:
    """ Special case runtime cache to avoid pounding the db repeatedly """
    entities = {}

    def cache_put(self, inst):
        expir = expiration_for_inst(inst)
        if expir:
            self.cache_remove(inst)
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cachekeys = ['TTL_' + expir]
            for keyval in entkey_vals(inst):
                cachekeys.append(keyval['key'])
                self.entities[keyval['key']] = keyval['val']
            self.entities[kt] = ','.join(cachekeys)

    def cache_get(self, entity, field, value):
        instkey = make_key(entity, field, value)
        if instkey not in self.entities:
            return None
        instval = self.entities[instkey]
        if field != 'dsId':
            instval = self.entities[instval]
        return pickle.loads(instval)

    def cache_remove(self, inst):
        if inst:
            kt = inst['dsType'] + '_' + inst['dsId'] + '_cleanup'
            cleankeys = self.entities.pop(kt, None)
            if cleankeys:
                for oldkey in cleankeys.split(','):
                    self.entities.pop(oldkey, None)

    def cache_clean(self):
        now = nowISO()
        for key, value in self.entities.items():
            if key.endswith('_cleanup'):
                ttl = value.split(',')[0][4:]
                if ttl < now:
                    kcs = key.split('_')
                    inst = {'dsType': kcs[0], 'dsId': kcs[1]}
                    self.cache_remove(inst)

    def log_cache_entries(self):
        txt = 'EntityCache entities:\n'
        for key, _ in self.entities.items():
            txt += '    ' + key + ': ' + str(self.entities[key])[0:74] + '\n'
        logging.info(txt)
        return txt


entcache = EntityCache()


def reqarg(argname, fieldtype='string', required=False):
    argval = flask.request.args.get(argname)
    if not argval:
        argval = flask.request.form.get(argname)
    if required and not argval:
        raise ValueError('Missing required value for ' + argname)
    dotidx = fieldtype.find('.')
    if dotidx > 0:
        entity = fieldtype[0:dotidx]
        fieldname = fieldtype[dotidx + 1:]
        fieldtype = entdefs[entity][fieldname]['pt']
    if fieldtype == 'email':
        emaddr = argval or ''
        emaddr = emaddr.lower()
        emaddr = re.sub('%40', '@', emaddr)
        if required and not re.match('[^@]+@[^@]+\\.[^@]+', emaddr):
            raise ValueError('Invalid ' + argname + ' value: ' + emaddr)
        return emaddr
    if fieldtype in ['dbid', 'string', 'isodate', 'isomod', 'srchidcsv',
        'text', 'json', 'jsarr', 'idcsv', 'isodcsv', 'gencsv', 'url']:
        return argval or ''
    if fieldtype == 'image':
        return argval or None
    if fieldtype == 'int':
        argval = argval or 0
        return int(argval)
    raise ValueError('Unknown type ' + fieldtype + ' for ' + argname)


def cfbk(entity, field, value, required=False):
    if field != 'dsId' and field not in entkeys[entity]:
        raise ValueError(field + ' not a unique index for ' + entity)
    vstr = str(value)
    ci = entcache.cache_get(entity, field, vstr)
    if ci:
        dblogmsg('CAC', entity, ci)
        return ci
    if entdefs[entity][field]['pt'] not in ['dbid', 'int']:
        vstr = '"' + vstr + '"'
    objs = query_entity(entity, 'WHERE ' + field + '=' + vstr + ' LIMIT 1')
    if len(objs) > 0:
        inst = objs[0]
        if not cachedefs[inst['dsType']]['manualadd']:
            entcache.cache_put(inst)
        return inst
    if required:
        raise ValueError(entity + ' ' + vstr + ' not found.')
    return None


def get_mysql_connector():
    cnx = None
    try:
        cnx = mysql.connector.connect(user=mconf.db['u'], password=mconf.db
            ['p'], host=mconf.db['h'], database=mconf.db['d'])
    except Exception as e:
        raise ValueError('Connection failed: ' + str(e))
    return cnx


def trim_string_val(val, unique=False):
    val = val or ''
    val = str(val)
    val = val.strip()
    if val and unique:
        lowval = val.lower()
        if lowval in ['null', 'none']:
            val = ''
    return val


def app2db_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]['pt']
        unique = entdefs[entity][field]['un']
        val = entdefs[entity][field]['dv']
    else:
        pt = dbflds[field]['pt']
        unique = dbflds[field]['un']
        val = dbflds[field]['dv']
    if field in inst:
        val = inst[field]
    if pt in ['email', 'string']:
        val = val or ''
        val = trim_string_val(val, unique)
        if not val:
            val = None
    elif pt == 'image':
        if not val:
            val = None
    elif pt == 'int':
        val = val or 0
        val = int(val)
    elif pt == 'dbid':
        try:
            val = int(val)
        except ValueError:
            val = 0
        if unique and not val:
            val = None
    return val


def db2app_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]['pt']
        val = entdefs[entity][field]['dv']
    else:
        pt = dbflds[field]['pt']
        val = dbflds[field]['dv']
    if field in inst:
        val = inst[field]
    if pt in ['email', 'string']:
        if not val:
            val = ''
        val = str(val)
    elif pt == 'image':
        if not val:
            val = ''
    elif pt == 'int':
        if not val:
            val = 0
    elif pt == 'dbid':
        if not val:
            val = ''
        else:
            val = str(val)
    return val


def ISO2dt(isostr):
    isostr = re.sub('\\.\\d*Z', 'Z', isostr)
    dt = datetime.datetime.utcnow()
    dt = dt.strptime(isostr, '%Y-%m-%dT%H:%M:%SZ')
    return dt


def dt2ISO(dt):
    iso = str(dt.year) + '-' + str(dt.month).rjust(2, '0') + '-'
    iso += str(dt.day).rjust(2, '0') + 'T' + str(dt.hour).rjust(2, '0')
    iso += ':' + str(dt.minute).rjust(2, '0') + ':'
    iso += str(dt.second).rjust(2, '0') + 'Z'
    return iso


def nowISO():
    """ Return the current time as an ISO string """
    return dt2ISO(datetime.datetime.utcnow())


def initialize_timestamp_fields(fields, vck):
    ts = nowISO()
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = ts
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = ts + ';1'


def verify_timestamp_fields(entity, dsId, fields, vck):
    if vck == 'override' and 'created' in fields and 'modified' in fields:
        return fields
    if not vck or not vck.strip():
        raise ValueError('Version check required to update ' + entity + ' ' +
            str(dsId))
    existing = cfbk(entity, 'dsId', dsId)
    if not existing:
        raise ValueError('Existing ' + entity + ' ' + str(dsId) + ' not found.'
            )
    if vck != 'override' and existing['modified'] != vck:
        raise ValueError('Update error. Outdated data given for ' + entity +
            ' ' + str(dsId) + '.')
    if 'created' not in fields or not fields['created'] or vck != 'override':
        fields['created'] = existing['created']
    ver = 1
    mods = existing['modified'].split(';')
    if len(mods) > 1:
        ver = int(mods[1]) + 1
    if 'modified' not in fields or not fields['modified'] or vck != 'override':
        fields['modified'] = nowISO() + ';' + str(ver)
    return existing


def app2db_AppUser(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('AppUser', 'importid', inst)
    if fill or 'email' in inst:
        cnv['email'] = app2db_fieldval('AppUser', 'email', inst)
    if fill or 'phash' in inst:
        cnv['phash'] = app2db_fieldval('AppUser', 'phash', inst)
    if fill or 'status' in inst:
        cnv['status'] = app2db_fieldval('AppUser', 'status', inst)
    if fill or 'actsends' in inst:
        cnv['actsends'] = app2db_fieldval('AppUser', 'actsends', inst)
    if fill or 'actcode' in inst:
        cnv['actcode'] = app2db_fieldval('AppUser', 'actcode', inst)
    if fill or 'accessed' in inst:
        cnv['accessed'] = app2db_fieldval('AppUser', 'accessed', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('AppUser', 'name', inst)
    if fill or 'title' in inst:
        cnv['title'] = app2db_fieldval('AppUser', 'title', inst)
    if fill or 'web' in inst:
        cnv['web'] = app2db_fieldval('AppUser', 'web', inst)
    if fill or 'lang' in inst:
        cnv['lang'] = app2db_fieldval('AppUser', 'lang', inst)
    if fill or 'settings' in inst:
        cnv['settings'] = app2db_fieldval('AppUser', 'settings', inst)
    if fill or 'started' in inst:
        cnv['started'] = app2db_fieldval('AppUser', 'started', inst)
    if fill or 'completed' in inst:
        cnv['completed'] = app2db_fieldval('AppUser', 'completed', inst)
    if fill or 'remtls' in inst:
        cnv['remtls'] = app2db_fieldval('AppUser', 'remtls', inst)
    if fill or 'built' in inst:
        cnv['built'] = app2db_fieldval('AppUser', 'built', inst)
    return cnv


def db2app_AppUser(inst):
    cnv = {}
    cnv['dsType'] = 'AppUser'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppUser', 'importid', inst)
    cnv['email'] = db2app_fieldval('AppUser', 'email', inst)
    cnv['phash'] = db2app_fieldval('AppUser', 'phash', inst)
    cnv['status'] = db2app_fieldval('AppUser', 'status', inst)
    cnv['actsends'] = db2app_fieldval('AppUser', 'actsends', inst)
    cnv['actcode'] = db2app_fieldval('AppUser', 'actcode', inst)
    cnv['accessed'] = db2app_fieldval('AppUser', 'accessed', inst)
    cnv['name'] = db2app_fieldval('AppUser', 'name', inst)
    cnv['title'] = db2app_fieldval('AppUser', 'title', inst)
    cnv['web'] = db2app_fieldval('AppUser', 'web', inst)
    cnv['lang'] = db2app_fieldval('AppUser', 'lang', inst)
    cnv['settings'] = db2app_fieldval('AppUser', 'settings', inst)
    cnv['started'] = db2app_fieldval('AppUser', 'started', inst)
    cnv['completed'] = db2app_fieldval('AppUser', 'completed', inst)
    cnv['remtls'] = db2app_fieldval('AppUser', 'remtls', inst)
    cnv['built'] = db2app_fieldval('AppUser', 'built', inst)
    return cnv


def app2db_Point(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'editors' in inst:
        cnv['editors'] = app2db_fieldval('Point', 'editors', inst)
    if fill or 'srctl' in inst:
        cnv['srctl'] = app2db_fieldval('Point', 'srctl', inst)
    if fill or 'lmuid' in inst:
        cnv['lmuid'] = app2db_fieldval('Point', 'lmuid', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('Point', 'importid', inst)
    if fill or 'source' in inst:
        cnv['source'] = app2db_fieldval('Point', 'source', inst)
    if fill or 'date' in inst:
        cnv['date'] = app2db_fieldval('Point', 'date', inst)
    if fill or 'text' in inst:
        cnv['text'] = app2db_fieldval('Point', 'text', inst)
    if fill or 'refs' in inst:
        cnv['refs'] = app2db_fieldval('Point', 'refs', inst)
    if fill or 'qtype' in inst:
        cnv['qtype'] = app2db_fieldval('Point', 'qtype', inst)
    if fill or 'communities' in inst:
        cnv['communities'] = app2db_fieldval('Point', 'communities', inst)
    if fill or 'regions' in inst:
        cnv['regions'] = app2db_fieldval('Point', 'regions', inst)
    if fill or 'categories' in inst:
        cnv['categories'] = app2db_fieldval('Point', 'categories', inst)
    if fill or 'tags' in inst:
        cnv['tags'] = app2db_fieldval('Point', 'tags', inst)
    if fill or 'codes' in inst:
        cnv['codes'] = app2db_fieldval('Point', 'codes', inst)
    if fill or 'srclang' in inst:
        cnv['srclang'] = app2db_fieldval('Point', 'srclang', inst)
    if fill or 'translations' in inst:
        cnv['translations'] = app2db_fieldval('Point', 'translations', inst)
    if fill or 'pic' in inst:
        cnv['pic'] = app2db_fieldval('Point', 'pic', inst)
    if fill or 'stats' in inst:
        cnv['stats'] = app2db_fieldval('Point', 'stats', inst)
    return cnv


def db2app_Point(inst):
    cnv = {}
    cnv['dsType'] = 'Point'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['editors'] = db2app_fieldval('Point', 'editors', inst)
    cnv['srctl'] = db2app_fieldval('Point', 'srctl', inst)
    cnv['lmuid'] = db2app_fieldval('Point', 'lmuid', inst)
    cnv['importid'] = db2app_fieldval('Point', 'importid', inst)
    cnv['source'] = db2app_fieldval('Point', 'source', inst)
    cnv['date'] = db2app_fieldval('Point', 'date', inst)
    cnv['text'] = db2app_fieldval('Point', 'text', inst)
    cnv['refs'] = db2app_fieldval('Point', 'refs', inst)
    cnv['qtype'] = db2app_fieldval('Point', 'qtype', inst)
    cnv['communities'] = db2app_fieldval('Point', 'communities', inst)
    cnv['regions'] = db2app_fieldval('Point', 'regions', inst)
    cnv['categories'] = db2app_fieldval('Point', 'categories', inst)
    cnv['tags'] = db2app_fieldval('Point', 'tags', inst)
    cnv['codes'] = db2app_fieldval('Point', 'codes', inst)
    cnv['srclang'] = db2app_fieldval('Point', 'srclang', inst)
    cnv['translations'] = db2app_fieldval('Point', 'translations', inst)
    cnv['pic'] = db2app_fieldval('Point', 'pic', inst)
    cnv['stats'] = db2app_fieldval('Point', 'stats', inst)
    return cnv


def app2db_Timeline(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'editors' in inst:
        cnv['editors'] = app2db_fieldval('Timeline', 'editors', inst)
    if fill or 'lmuid' in inst:
        cnv['lmuid'] = app2db_fieldval('Timeline', 'lmuid', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('Timeline', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('Timeline', 'name', inst)
    if fill or 'cname' in inst:
        cnv['cname'] = app2db_fieldval('Timeline', 'cname', inst)
    if fill or 'slug' in inst:
        cnv['slug'] = app2db_fieldval('Timeline', 'slug', inst)
    if fill or 'title' in inst:
        cnv['title'] = app2db_fieldval('Timeline', 'title', inst)
    if fill or 'subtitle' in inst:
        cnv['subtitle'] = app2db_fieldval('Timeline', 'subtitle', inst)
    if fill or 'featured' in inst:
        cnv['featured'] = app2db_fieldval('Timeline', 'featured', inst)
    if fill or 'lang' in inst:
        cnv['lang'] = app2db_fieldval('Timeline', 'lang', inst)
    if fill or 'comment' in inst:
        cnv['comment'] = app2db_fieldval('Timeline', 'comment', inst)
    if fill or 'about' in inst:
        cnv['about'] = app2db_fieldval('Timeline', 'about', inst)
    if fill or 'kwds' in inst:
        cnv['kwds'] = app2db_fieldval('Timeline', 'kwds', inst)
    if fill or 'ctype' in inst:
        cnv['ctype'] = app2db_fieldval('Timeline', 'ctype', inst)
    if fill or 'cids' in inst:
        cnv['cids'] = app2db_fieldval('Timeline', 'cids', inst)
    if fill or 'rempts' in inst:
        cnv['rempts'] = app2db_fieldval('Timeline', 'rempts', inst)
    if fill or 'svs' in inst:
        cnv['svs'] = app2db_fieldval('Timeline', 'svs', inst)
    if fill or 'preb' in inst:
        cnv['preb'] = app2db_fieldval('Timeline', 'preb', inst)
    return cnv


def db2app_Timeline(inst):
    cnv = {}
    cnv['dsType'] = 'Timeline'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['editors'] = db2app_fieldval('Timeline', 'editors', inst)
    cnv['lmuid'] = db2app_fieldval('Timeline', 'lmuid', inst)
    cnv['importid'] = db2app_fieldval('Timeline', 'importid', inst)
    cnv['name'] = db2app_fieldval('Timeline', 'name', inst)
    cnv['cname'] = db2app_fieldval('Timeline', 'cname', inst)
    cnv['slug'] = db2app_fieldval('Timeline', 'slug', inst)
    cnv['title'] = db2app_fieldval('Timeline', 'title', inst)
    cnv['subtitle'] = db2app_fieldval('Timeline', 'subtitle', inst)
    cnv['featured'] = db2app_fieldval('Timeline', 'featured', inst)
    cnv['lang'] = db2app_fieldval('Timeline', 'lang', inst)
    cnv['comment'] = db2app_fieldval('Timeline', 'comment', inst)
    cnv['about'] = db2app_fieldval('Timeline', 'about', inst)
    cnv['kwds'] = db2app_fieldval('Timeline', 'kwds', inst)
    cnv['ctype'] = db2app_fieldval('Timeline', 'ctype', inst)
    cnv['cids'] = db2app_fieldval('Timeline', 'cids', inst)
    cnv['rempts'] = db2app_fieldval('Timeline', 'rempts', inst)
    cnv['svs'] = db2app_fieldval('Timeline', 'svs', inst)
    cnv['preb'] = db2app_fieldval('Timeline', 'preb', inst)
    return cnv


def app2db_TLComp(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('TLComp', 'importid', inst)
    if fill or 'userid' in inst:
        cnv['userid'] = app2db_fieldval('TLComp', 'userid', inst)
    if fill or 'tlid' in inst:
        cnv['tlid'] = app2db_fieldval('TLComp', 'tlid', inst)
    if fill or 'username' in inst:
        cnv['username'] = app2db_fieldval('TLComp', 'username', inst)
    if fill or 'tlname' in inst:
        cnv['tlname'] = app2db_fieldval('TLComp', 'tlname', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('TLComp', 'data', inst)
    return cnv


def db2app_TLComp(inst):
    cnv = {}
    cnv['dsType'] = 'TLComp'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('TLComp', 'importid', inst)
    cnv['userid'] = db2app_fieldval('TLComp', 'userid', inst)
    cnv['tlid'] = db2app_fieldval('TLComp', 'tlid', inst)
    cnv['username'] = db2app_fieldval('TLComp', 'username', inst)
    cnv['tlname'] = db2app_fieldval('TLComp', 'tlname', inst)
    cnv['data'] = db2app_fieldval('TLComp', 'data', inst)
    return cnv


def app2db_DayCount(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('DayCount', 'importid', inst)
    if fill or 'tstamp' in inst:
        cnv['tstamp'] = app2db_fieldval('DayCount', 'tstamp', inst)
    if fill or 'rtype' in inst:
        cnv['rtype'] = app2db_fieldval('DayCount', 'rtype', inst)
    if fill or 'detail' in inst:
        cnv['detail'] = app2db_fieldval('DayCount', 'detail', inst)
    return cnv


def db2app_DayCount(inst):
    cnv = {}
    cnv['dsType'] = 'DayCount'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('DayCount', 'importid', inst)
    cnv['tstamp'] = db2app_fieldval('DayCount', 'tstamp', inst)
    cnv['rtype'] = db2app_fieldval('DayCount', 'rtype', inst)
    cnv['detail'] = db2app_fieldval('DayCount', 'detail', inst)
    return cnv


def app2db_AppService(inst, fill=True):
    cnv = {}
    cnv['dsId'] = None
    if 'dsId' in inst:
        cnv['dsId'] = app2db_fieldval(None, 'dsId', inst)
    if fill or 'created' in inst:
        cnv['created'] = app2db_fieldval(None, 'created', inst)
    if fill or 'modified' in inst:
        cnv['modified'] = app2db_fieldval(None, 'modified', inst)
    if fill or 'batchconv' in inst:
        cnv['batchconv'] = app2db_fieldval(None, 'batchconv', inst)
    if fill or 'importid' in inst:
        cnv['importid'] = app2db_fieldval('AppService', 'importid', inst)
    if fill or 'name' in inst:
        cnv['name'] = app2db_fieldval('AppService', 'name', inst)
    if fill or 'ckey' in inst:
        cnv['ckey'] = app2db_fieldval('AppService', 'ckey', inst)
    if fill or 'csec' in inst:
        cnv['csec'] = app2db_fieldval('AppService', 'csec', inst)
    if fill or 'data' in inst:
        cnv['data'] = app2db_fieldval('AppService', 'data', inst)
    return cnv


def db2app_AppService(inst):
    cnv = {}
    cnv['dsType'] = 'AppService'
    cnv['dsId'] = db2app_fieldval(None, 'dsId', inst)
    cnv['created'] = db2app_fieldval(None, 'created', inst)
    cnv['modified'] = db2app_fieldval(None, 'modified', inst)
    cnv['batchconv'] = db2app_fieldval(None, 'batchconv', inst)
    cnv['importid'] = db2app_fieldval('AppService', 'importid', inst)
    cnv['name'] = db2app_fieldval('AppService', 'name', inst)
    cnv['ckey'] = db2app_fieldval('AppService', 'ckey', inst)
    cnv['csec'] = db2app_fieldval('AppService', 'csec', inst)
    cnv['data'] = db2app_fieldval('AppService', 'data', inst)
    return cnv


def dblogmsg(op, entity, res):
    log_summary_flds = {'AppUser': ['email', 'name'], 'Point': ['editors',
        'date', 'text'], 'Timeline': ['name'], 'TLComp': ['userid', 'tlid',
        'username', 'tlname'], 'DayCount': ['tstamp', 'rtype'],
        'AppService': ['name']}
    if res:
        if op != 'QRY':
            res = [res]
        for obj in res:
            msg = 'db' + op + ' ' + entity + ' ' + obj['dsId']
            if entity in log_summary_flds:
                for field in log_summary_flds[entity]:
                    msg += ' ' + str(obj[field])[0:60]
            logging.info(msg)
    else:
        logging.info('db' + op + ' ' + entity + ' -no obj details-')


def insert_new_AppUser(cnx, cursor, fields):
    fields = app2db_AppUser(fields)
    stmt = (
        'INSERT INTO AppUser (created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built) VALUES (%(created)s, %(modified)s, %(importid)s, %(email)s, %(phash)s, %(status)s, %(actsends)s, %(actcode)s, %(accessed)s, %(name)s, %(title)s, %(web)s, %(lang)s, %(settings)s, %(started)s, %(completed)s, %(remtls)s, %(built)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs['AppUser'][
        'importid']['dv']), 'email': fields.get('email', entdefs['AppUser']
        ['email']['dv']), 'phash': fields.get('phash', entdefs['AppUser'][
        'phash']['dv']), 'status': fields.get('status', entdefs['AppUser'][
        'status']['dv']), 'actsends': fields.get('actsends', entdefs[
        'AppUser']['actsends']['dv']), 'actcode': fields.get('actcode',
        entdefs['AppUser']['actcode']['dv']), 'accessed': fields.get(
        'accessed', entdefs['AppUser']['accessed']['dv']), 'name': fields.
        get('name', entdefs['AppUser']['name']['dv']), 'title': fields.get(
        'title', entdefs['AppUser']['title']['dv']), 'web': fields.get(
        'web', entdefs['AppUser']['web']['dv']), 'lang': fields.get('lang',
        entdefs['AppUser']['lang']['dv']), 'settings': fields.get(
        'settings', entdefs['AppUser']['settings']['dv']), 'started':
        fields.get('started', entdefs['AppUser']['started']['dv']),
        'completed': fields.get('completed', entdefs['AppUser']['completed'
        ]['dv']), 'remtls': fields.get('remtls', entdefs['AppUser'][
        'remtls']['dv']), 'built': fields.get('built', entdefs['AppUser'][
        'built']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppUser(fields)
    dblogmsg('ADD', 'AppUser', fields)
    return fields


def update_existing_AppUser(context, fields):
    fields = app2db_AppUser(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppUser SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppUser' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppUser(result)
    dblogmsg('UPD', 'AppUser', result)
    entcache.cache_remove(result)
    return result


def insert_new_Point(cnx, cursor, fields):
    fields = app2db_Point(fields)
    stmt = (
        'INSERT INTO Point (created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats) VALUES (%(created)s, %(modified)s, %(editors)s, %(srctl)s, %(lmuid)s, %(importid)s, %(source)s, %(date)s, %(text)s, %(refs)s, %(qtype)s, %(communities)s, %(regions)s, %(categories)s, %(tags)s, %(codes)s, %(srclang)s, %(translations)s, %(pic)s, %(stats)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'editors': fields.get('editors', entdefs['Point'][
        'editors']['dv']), 'srctl': fields.get('srctl', entdefs['Point'][
        'srctl']['dv']), 'lmuid': fields.get('lmuid', entdefs['Point'][
        'lmuid']['dv']), 'importid': fields.get('importid', entdefs['Point'
        ]['importid']['dv']), 'source': fields.get('source', entdefs[
        'Point']['source']['dv']), 'date': fields.get('date', entdefs[
        'Point']['date']['dv']), 'text': fields.get('text', entdefs['Point'
        ]['text']['dv']), 'refs': fields.get('refs', entdefs['Point'][
        'refs']['dv']), 'qtype': fields.get('qtype', entdefs['Point'][
        'qtype']['dv']), 'communities': fields.get('communities', entdefs[
        'Point']['communities']['dv']), 'regions': fields.get('regions',
        entdefs['Point']['regions']['dv']), 'categories': fields.get(
        'categories', entdefs['Point']['categories']['dv']), 'tags': fields
        .get('tags', entdefs['Point']['tags']['dv']), 'codes': fields.get(
        'codes', entdefs['Point']['codes']['dv']), 'srclang': fields.get(
        'srclang', entdefs['Point']['srclang']['dv']), 'translations':
        fields.get('translations', entdefs['Point']['translations']['dv']),
        'pic': fields.get('pic', entdefs['Point']['pic']['dv']), 'stats':
        fields.get('stats', entdefs['Point']['stats']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Point(fields)
    dblogmsg('ADD', 'Point', fields)
    return fields


def update_existing_Point(context, fields):
    fields = app2db_Point(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE Point SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('Point' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_Point(result)
    dblogmsg('UPD', 'Point', result)
    entcache.cache_remove(result)
    return result


def insert_new_Timeline(cnx, cursor, fields):
    fields = app2db_Timeline(fields)
    stmt = (
        'INSERT INTO Timeline (created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) VALUES (%(created)s, %(modified)s, %(editors)s, %(lmuid)s, %(importid)s, %(name)s, %(cname)s, %(slug)s, %(title)s, %(subtitle)s, %(featured)s, %(lang)s, %(comment)s, %(about)s, %(kwds)s, %(ctype)s, %(cids)s, %(rempts)s, %(svs)s, %(preb)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'editors': fields.get('editors', entdefs['Timeline'][
        'editors']['dv']), 'lmuid': fields.get('lmuid', entdefs['Timeline']
        ['lmuid']['dv']), 'importid': fields.get('importid', entdefs[
        'Timeline']['importid']['dv']), 'name': fields.get('name', entdefs[
        'Timeline']['name']['dv']), 'cname': fields.get('cname', entdefs[
        'Timeline']['cname']['dv']), 'slug': fields.get('slug', entdefs[
        'Timeline']['slug']['dv']), 'title': fields.get('title', entdefs[
        'Timeline']['title']['dv']), 'subtitle': fields.get('subtitle',
        entdefs['Timeline']['subtitle']['dv']), 'featured': fields.get(
        'featured', entdefs['Timeline']['featured']['dv']), 'lang': fields.
        get('lang', entdefs['Timeline']['lang']['dv']), 'comment': fields.
        get('comment', entdefs['Timeline']['comment']['dv']), 'about':
        fields.get('about', entdefs['Timeline']['about']['dv']), 'kwds':
        fields.get('kwds', entdefs['Timeline']['kwds']['dv']), 'ctype':
        fields.get('ctype', entdefs['Timeline']['ctype']['dv']), 'cids':
        fields.get('cids', entdefs['Timeline']['cids']['dv']), 'rempts':
        fields.get('rempts', entdefs['Timeline']['rempts']['dv']), 'svs':
        fields.get('svs', entdefs['Timeline']['svs']['dv']), 'preb': fields
        .get('preb', entdefs['Timeline']['preb']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Timeline(fields)
    dblogmsg('ADD', 'Timeline', fields)
    return fields


def update_existing_Timeline(context, fields):
    fields = app2db_Timeline(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE Timeline SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('Timeline' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_Timeline(result)
    dblogmsg('UPD', 'Timeline', result)
    entcache.cache_remove(result)
    return result


def insert_new_TLComp(cnx, cursor, fields):
    fields = app2db_TLComp(fields)
    stmt = (
        'INSERT INTO TLComp (created, modified, importid, userid, tlid, username, tlname, data) VALUES (%(created)s, %(modified)s, %(importid)s, %(userid)s, %(tlid)s, %(username)s, %(tlname)s, %(data)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs['TLComp'][
        'importid']['dv']), 'userid': fields.get('userid', entdefs['TLComp'
        ]['userid']['dv']), 'tlid': fields.get('tlid', entdefs['TLComp'][
        'tlid']['dv']), 'username': fields.get('username', entdefs['TLComp'
        ]['username']['dv']), 'tlname': fields.get('tlname', entdefs[
        'TLComp']['tlname']['dv']), 'data': fields.get('data', entdefs[
        'TLComp']['data']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_TLComp(fields)
    dblogmsg('ADD', 'TLComp', fields)
    return fields


def update_existing_TLComp(context, fields):
    fields = app2db_TLComp(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE TLComp SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('TLComp' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_TLComp(result)
    dblogmsg('UPD', 'TLComp', result)
    entcache.cache_remove(result)
    return result


def insert_new_DayCount(cnx, cursor, fields):
    fields = app2db_DayCount(fields)
    stmt = (
        'INSERT INTO DayCount (created, modified, importid, tstamp, rtype, detail) VALUES (%(created)s, %(modified)s, %(importid)s, %(tstamp)s, %(rtype)s, %(detail)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs['DayCount']
        ['importid']['dv']), 'tstamp': fields.get('tstamp', entdefs[
        'DayCount']['tstamp']['dv']), 'rtype': fields.get('rtype', entdefs[
        'DayCount']['rtype']['dv']), 'detail': fields.get('detail', entdefs
        ['DayCount']['detail']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_DayCount(fields)
    dblogmsg('ADD', 'DayCount', fields)
    return fields


def update_existing_DayCount(context, fields):
    fields = app2db_DayCount(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE DayCount SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('DayCount' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_DayCount(result)
    dblogmsg('UPD', 'DayCount', result)
    entcache.cache_remove(result)
    return result


def insert_new_AppService(cnx, cursor, fields):
    fields = app2db_AppService(fields)
    stmt = (
        'INSERT INTO AppService (created, modified, importid, name, ckey, csec, data) VALUES (%(created)s, %(modified)s, %(importid)s, %(name)s, %(ckey)s, %(csec)s, %(data)s)'
        )
    data = {'created': fields.get('created'), 'modified': fields.get(
        'modified'), 'importid': fields.get('importid', entdefs[
        'AppService']['importid']['dv']), 'name': fields.get('name',
        entdefs['AppService']['name']['dv']), 'ckey': fields.get('ckey',
        entdefs['AppService']['ckey']['dv']), 'csec': fields.get('csec',
        entdefs['AppService']['csec']['dv']), 'data': fields.get('data',
        entdefs['AppService']['data']['dv'])}
    cursor.execute(stmt, data)
    fields['dsId'] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppService(fields)
    dblogmsg('ADD', 'AppService', fields)
    return fields


def update_existing_AppService(context, fields):
    fields = app2db_AppService(fields, fill=False)
    dsId = int(fields['dsId'])
    stmt = ''
    for field in fields:
        if stmt:
            stmt += ', '
        stmt += field + '=(%(' + field + ')s)'
    stmt = 'UPDATE AppService SET ' + stmt + ' WHERE dsId=' + str(dsId)
    if context['vck'] != 'override':
        stmt += ' AND modified="' + context['vck'] + '"'
    data = {}
    for field in fields:
        data[field] = fields[field]
    context['cursor'].execute(stmt, data)
    if context['cursor'].rowcount < 1 and context['vck'] != 'override':
        raise ValueError('AppService' + str(dsId) +
            ' update received outdated version check value ' + context[
            'vck'] + '.')
    context['cnx'].commit()
    result = context['existing']
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppService(result)
    dblogmsg('UPD', 'AppService', result)
    entcache.cache_put(result)
    return result


def write_entity(inst, vck='1234-12-12T00:00:00Z'):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            entity = inst.get('dsType', None)
            dsId = inst.get('dsId', 0)
            if dsId:
                existing = verify_timestamp_fields(entity, dsId, inst, vck)
                context = {'cnx': cnx, 'cursor': cursor, 'vck': vck,
                    'existing': existing}
                if entity == 'AppUser':
                    return update_existing_AppUser(context, inst)
                if entity == 'Point':
                    return update_existing_Point(context, inst)
                if entity == 'Timeline':
                    return update_existing_Timeline(context, inst)
                if entity == 'TLComp':
                    return update_existing_TLComp(context, inst)
                if entity == 'DayCount':
                    return update_existing_DayCount(context, inst)
                if entity == 'AppService':
                    return update_existing_AppService(context, inst)
                raise ValueError('Cannot modify unknown entity dsType ' +
                    str(entity))
            initialize_timestamp_fields(inst, vck)
            if entity == 'AppUser':
                return insert_new_AppUser(cnx, cursor, inst)
            if entity == 'Point':
                return insert_new_Point(cnx, cursor, inst)
            if entity == 'Timeline':
                return insert_new_Timeline(cnx, cursor, inst)
            if entity == 'TLComp':
                return insert_new_TLComp(cnx, cursor, inst)
            if entity == 'DayCount':
                return insert_new_DayCount(cnx, cursor, inst)
            if entity == 'AppService':
                return insert_new_AppService(cnx, cursor, inst)
            raise ValueError('Cannot create unknown entity dsType ' + str(
                entity))
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def delete_entity(entity, dsId):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            stmt = 'DELETE FROM ' + entity + ' WHERE dsId=' + str(dsId)
            cursor.execute(stmt)
            cnx.commit()
            dblogmsg('DEL', entity + ' ' + str(dsId), None)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def query_AppUser(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built'
        )
    query += ' FROM AppUser ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built in cursor:
        inst = {'dsType': 'AppUser', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'email': email,
            'phash': phash, 'status': status, 'actsends': actsends,
            'actcode': actcode, 'accessed': accessed, 'name': name, 'title':
            title, 'web': web, 'lang': lang, 'settings': settings,
            'started': started, 'completed': completed, 'remtls': remtls,
            'built': built}
        inst = db2app_AppUser(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppUser', res)
    return res


def query_Point(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats'
        )
    query += ' FROM Point ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats in cursor:
        inst = {'dsType': 'Point', 'dsId': dsId, 'created': created,
            'modified': modified, 'editors': editors, 'srctl': srctl,
            'lmuid': lmuid, 'importid': importid, 'source': source, 'date':
            date, 'text': text, 'refs': refs, 'qtype': qtype, 'communities':
            communities, 'regions': regions, 'categories': categories,
            'tags': tags, 'codes': codes, 'srclang': srclang,
            'translations': translations, 'pic': pic, 'stats': stats}
        inst = db2app_Point(inst)
        res.append(inst)
    dblogmsg('QRY', 'Point', res)
    return res


def query_Timeline(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += (
        'editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb'
        )
    query += ' FROM Timeline ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb in cursor:
        inst = {'dsType': 'Timeline', 'dsId': dsId, 'created': created,
            'modified': modified, 'editors': editors, 'lmuid': lmuid,
            'importid': importid, 'name': name, 'cname': cname, 'slug':
            slug, 'title': title, 'subtitle': subtitle, 'featured':
            featured, 'lang': lang, 'comment': comment, 'about': about,
            'kwds': kwds, 'ctype': ctype, 'cids': cids, 'rempts': rempts,
            'svs': svs, 'preb': preb}
        inst = db2app_Timeline(inst)
        res.append(inst)
    dblogmsg('QRY', 'Timeline', res)
    return res


def query_TLComp(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, userid, tlid, username, tlname, data'
    query += ' FROM TLComp ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, userid, tlid, username, tlname, data in cursor:
        inst = {'dsType': 'TLComp', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'userid': userid,
            'tlid': tlid, 'username': username, 'tlname': tlname, 'data': data}
        inst = db2app_TLComp(inst)
        res.append(inst)
    dblogmsg('QRY', 'TLComp', res)
    return res


def query_DayCount(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, tstamp, rtype, detail'
    query += ' FROM DayCount ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, tstamp, rtype, detail in cursor:
        inst = {'dsType': 'DayCount', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'tstamp': tstamp,
            'rtype': rtype, 'detail': detail}
        inst = db2app_DayCount(inst)
        res.append(inst)
    dblogmsg('QRY', 'DayCount', res)
    return res


def query_AppService(cnx, cursor, where):
    query = 'SELECT dsId, created, modified, '
    query += 'importid, name, ckey, csec, data'
    query += ' FROM AppService ' + where
    cursor.execute(query)
    res = []
    for dsId, created, modified, importid, name, ckey, csec, data in cursor:
        inst = {'dsType': 'AppService', 'dsId': dsId, 'created': created,
            'modified': modified, 'importid': importid, 'name': name,
            'ckey': ckey, 'csec': csec, 'data': data}
        inst = db2app_AppService(inst)
        res.append(inst)
    dblogmsg('QRY', 'AppService', res)
    return res


def query_entity(entity, where):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError('Database connection failed.')
    try:
        cursor = cnx.cursor()
        try:
            if entity == 'AppUser':
                return query_AppUser(cnx, cursor, where)
            if entity == 'Point':
                return query_Point(cnx, cursor, where)
            if entity == 'Timeline':
                return query_Timeline(cnx, cursor, where)
            if entity == 'TLComp':
                return query_TLComp(cnx, cursor, where)
            if entity == 'DayCount':
                return query_DayCount(cnx, cursor, where)
            if entity == 'AppService':
                return query_AppService(cnx, cursor, where)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or 'No mysql error text')
        finally:
            cursor.close()
    finally:
        cnx.close()


def visible_AppUser_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        if fld == 'email' and audience != 'private':
            continue
        if fld == 'phash':
            continue
        if fld == 'status' and audience != 'private':
            continue
        if fld == 'actsends':
            continue
        if fld == 'actcode':
            continue
        filtobj[fld] = val
    return filtobj


def visible_Point_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        if fld == 'pic':
            if obj['pic']:
                val = obj['dsId']
            else:
                val = ''
        filtobj[fld] = val
    return filtobj


def visible_Timeline_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


def visible_TLComp_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


def visible_DayCount_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


def visible_AppService_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == 'importid':
            continue
        filtobj[fld] = val
    return filtobj


def visible_fields(obj, audience='public'):
    if obj['dsType'] == 'AppUser':
        return visible_AppUser_fields(obj, audience)
    if obj['dsType'] == 'Point':
        return visible_Point_fields(obj, audience)
    if obj['dsType'] == 'Timeline':
        return visible_Timeline_fields(obj, audience)
    if obj['dsType'] == 'TLComp':
        return visible_TLComp_fields(obj, audience)
    if obj['dsType'] == 'DayCount':
        return visible_DayCount_fields(obj, audience)
    if obj['dsType'] == 'AppService':
        return visible_AppService_fields(obj, audience)
    raise ValueError('Unknown object dsType: ' + obj['dsType'])

""" Autogenerated db CRUD and related utilities """
########################################
#
#       D O   N O T   E D I T
#
# This file was written by makeMySQLCRUD.js.  Any changes should be made there.
#
########################################

#pylint: disable=line-too-long
#pylint: disable=too-many-lines
#pylint: disable=trailing-newlines
#pylint: disable=wrong-import-position
#pylint: disable=wrong-import-order
#pylint: disable=invalid-name
#pylint: disable=missing-function-docstring
#pylint: disable=consider-using-in
#pylint: disable=logging-not-lazy
#pylint: disable=inconsistent-return-statements
#pylint: disable=too-many-return-statements
#pylint: disable=too-many-branches
#pylint: disable=too-many-locals
#pylint: disable=unused-argument
import logging
import flask
import re
import datetime
import pickle
import mysql.connector
import py.mconf as mconf

# Notes:
# (1) In general, all processing that might raise a mysql.connector.Error is
# wrapped to raise a ValueError instead, to support callers working at a
# higher level of CRUD abstraction.  The general processing contruct
#    except mysql.connector.Error as e:
#        raise ValueError from e
# is not used for this purpose because it produces an undecorated ValueError
# without the str(e) text, making it harder to track down what the problem
# actually was.  The source can be found from the Error __context__, but
# that is also set when raising a new Error, so the general use here is
#        raise ValueError(str(e) or "No mysql error text")

# Reserved database fields used for every instance:
#  - dsId: a long int, possibly out of range of a javascript integer,
#    possibly non-sequential, uniquely identifying an entity instance.
#    The entity type + dsId uniquely identifies an object in the system.
#  - created: An ISO timestamp when the instance was first written.
#  - modified: An ISO timestamp followed by ';' followed by mod count.
#  - batchconv: Arbitray string for batch database conversion.
dbflds = {"dsId": {"pt": "dbid", "un": True, "dv": 0},
          "created": {"pt": "string", "un": False, "dv": ""},
          "modified": {"pt": "string", "un": False, "dv": ""},
          "batchconv": {"pt": "string", "un": False, "dv": ""}}

entdefs = {
    "AppUser": {  # PastKey User account
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "email": {"pt": "email", "un": True, "dv": ""},
        "phash": {"pt": "string", "un": False, "dv": ""},
        "status": {"pt": "string", "un": False, "dv": ""},
        "actsends": {"pt": "string", "un": False, "dv": ""},
        "actcode": {"pt": "string", "un": False, "dv": ""},
        "accessed": {"pt": "string", "un": False, "dv": ""},
        "name": {"pt": "string", "un": False, "dv": ""},
        "title": {"pt": "string", "un": False, "dv": ""},
        "web": {"pt": "string", "un": False, "dv": ""},
        "lang": {"pt": "string", "un": False, "dv": ""},
        "settings": {"pt": "string", "un": False, "dv": ""},
        "started": {"pt": "string", "un": False, "dv": ""},
        "completed": {"pt": "string", "un": False, "dv": ""},
        "remtls": {"pt": "string", "un": False, "dv": ""},
        "built": {"pt": "string", "un": False, "dv": ""}
    },
    "Point": {  # A data point for use in timelines
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "editors": {"pt": "string", "un": False, "dv": ""},
        "srctl": {"pt": "dbid", "un": False, "dv": 0},
        "lmuid": {"pt": "dbid", "un": False, "dv": 0},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "source": {"pt": "string", "un": False, "dv": ""},
        "date": {"pt": "string", "un": False, "dv": ""},
        "text": {"pt": "string", "un": False, "dv": ""},
        "refs": {"pt": "string", "un": False, "dv": ""},
        "qtype": {"pt": "string", "un": False, "dv": ""},
        "communities": {"pt": "string", "un": False, "dv": ""},
        "regions": {"pt": "string", "un": False, "dv": ""},
        "categories": {"pt": "string", "un": False, "dv": ""},
        "tags": {"pt": "string", "un": False, "dv": ""},
        "codes": {"pt": "string", "un": False, "dv": ""},
        "srclang": {"pt": "string", "un": False, "dv": ""},
        "translations": {"pt": "string", "un": False, "dv": ""},
        "pic": {"pt": "image", "un": False, "dv": None},
        "stats": {"pt": "string", "un": False, "dv": ""}
    },
    "Timeline": {  # Points + suppviz*, or other timelines
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "editors": {"pt": "string", "un": False, "dv": ""},
        "lmuid": {"pt": "dbid", "un": False, "dv": 0},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "name": {"pt": "string", "un": True, "dv": ""},
        "cname": {"pt": "string", "un": False, "dv": ""},
        "slug": {"pt": "string", "un": True, "dv": ""},
        "title": {"pt": "string", "un": False, "dv": ""},
        "subtitle": {"pt": "string", "un": False, "dv": ""},
        "featured": {"pt": "string", "un": False, "dv": ""},
        "lang": {"pt": "string", "un": False, "dv": ""},
        "comment": {"pt": "string", "un": False, "dv": ""},
        "about": {"pt": "string", "un": False, "dv": ""},
        "kwds": {"pt": "string", "un": False, "dv": ""},
        "ctype": {"pt": "string", "un": False, "dv": ""},
        "cids": {"pt": "string", "un": False, "dv": ""},
        "rempts": {"pt": "string", "un": False, "dv": ""},
        "svs": {"pt": "string", "un": False, "dv": ""},
        "preb": {"pt": "string", "un": False, "dv": ""}
    },
    "TLComp": {  # Timeline completion archive record
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "userid": {"pt": "dbid", "un": False, "dv": 0},
        "tlid": {"pt": "dbid", "un": False, "dv": 0},
        "username": {"pt": "string", "un": False, "dv": ""},
        "tlname": {"pt": "string", "un": False, "dv": ""},
        "data": {"pt": "string", "un": False, "dv": ""}
    },
    "DayCount": {  # Traffic access accumulator
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "tstamp": {"pt": "string", "un": False, "dv": ""},
        "rtype": {"pt": "string", "un": False, "dv": ""},
        "detail": {"pt": "string", "un": False, "dv": ""}
    },
    "AppService": {  # Processing service access
        "dsId": {"pt": "dbid", "un": True, "dv": 0},
        "created": {"pt": "string", "un": False, "dv": ""},
        "modified": {"pt": "string", "un": False, "dv": ""},
        "batchconv": {"pt": "string", "un": False, "dv": ""},
        "importid": {"pt": "dbid", "un": True, "dv": 0},
        "name": {"pt": "string", "un": True, "dv": ""},
        "ckey": {"pt": "string", "un": False, "dv": ""},
        "csec": {"pt": "string", "un": False, "dv": ""},
        "data": {"pt": "string", "un": False, "dv": ""}
    }
}


entkeys = {
    "AppUser": ["importid", "email"],
    "Point": ["importid"],
    "Timeline": ["importid", "name", "slug"],
    "TLComp": ["importid"],
    "DayCount": ["importid"],
    "AppService": ["importid", "name"]
}


cachedefs = {
    "AppUser": {"minutes": 120, "manualadd": True},
    "Point": {"minutes": 0, "manualadd": False},
    "Timeline": {"minutes": 0, "manualadd": False},
    "TLComp": {"minutes": 0, "manualadd": False},
    "DayCount": {"minutes": 0, "manualadd": False},
    "AppService": {"minutes": 240, "manualadd": False}
}


def timestamp(offset=0):
    now = datetime.datetime.utcnow().replace(microsecond=0)
    return dt2ISO(now + datetime.timedelta(minutes=offset))


def expiration_for_inst(inst):
    if not inst or not inst["dsId"]:
        raise ValueError("Uncacheable instance: " + str(inst))
    cms = cachedefs[inst["dsType"]]
    if not cms or not cms["minutes"]:
        return ""  # not cached, no time to live
    return timestamp(cms["minutes"])


def make_key(dsType, field, value):
    # The value param will always be a string because after retrieving an
    # instance via query, the resulting fields are converted via db2app.
    return dsType + "_" + field + "_" + value


def entkey_vals(inst):
    # dsId key holds the cached instance.  Need img data so pickle.
    instkey = make_key(inst["dsType"], "dsId", inst["dsId"])
    keyvals = [{"key": instkey, "val": pickle.dumps(inst)}]
    # alternate entity keys point to the dsId key
    for field in entkeys[inst["dsType"]]:
        keyvals.append({"key": make_key(inst["dsType"], field, inst[field]),
                        "val": instkey})
    return keyvals


# Avoids repeated calls to the db for the same instance, especially within
# the same call to the webserver.  Used sparingly to avoid chewing memory.
# Time to live can range from zero to whenever in actual runtime use.
class EntityCache():
    """ Special case runtime cache to avoid pounding the db repeatedly """
    entities = {}
    def cache_put(self, inst):
        expir = expiration_for_inst(inst)
        if expir:  # cacheable
            self.cache_remove(inst)  # clear any outdated entries
            kt = inst["dsType"] + "_" + inst["dsId"] + "_cleanup"
            cachekeys = ["TTL_" + expir]
            for keyval in entkey_vals(inst):
                cachekeys.append(keyval["key"])
                self.entities[keyval["key"]] = keyval["val"]
            self.entities[kt] = ",".join(cachekeys)
            # self.log_cache_entries()
    def cache_get(self, entity, field, value):
        instkey = make_key(entity, field, value)
        if instkey not in self.entities:
            return None
        instval = self.entities[instkey]
        if field != "dsId":
            instval = self.entities[instval]
        return pickle.loads(instval)
    def cache_remove(self, inst):
        if inst:
            kt = inst["dsType"] + "_" + inst["dsId"] + "_cleanup"
            cleankeys = self.entities.pop(kt, None)
            if cleankeys:
                for oldkey in cleankeys.split(","):
                    self.entities.pop(oldkey, None)
    def cache_clean(self):
        now = nowISO()
        for key, value in self.entities.items():
            if key.endswith("_cleanup"):
                ttl = value.split(",")[0][4:]
                if ttl < now:
                    kcs = key.split("_")
                    inst = {"dsType": kcs[0], "dsId": kcs[1]}
                    self.cache_remove(inst)
    def log_cache_entries(self):
        txt = "EntityCache entities:\n"
        for key, _ in self.entities.items():
            txt += "    " + key + ": " + str(self.entities[key])[0:74] + "\n"
        logging.info(txt)
        return txt
entcache = EntityCache()


def reqarg(argname, fieldtype="string", required=False):
    argval = flask.request.args.get(argname)  # None if not found
    if not argval:
        argval = flask.request.form.get(argname)  # Ditto
    if required and not argval:
        raise ValueError("Missing required value for " + argname)
    dotidx = fieldtype.find('.')
    if dotidx > 0:
        entity = fieldtype[0:dotidx]
        fieldname = fieldtype[dotidx + 1:]
        fieldtype = entdefs[entity][fieldname]["pt"]
    if fieldtype == "email":
        emaddr = argval or ""
        emaddr = emaddr.lower()
        emaddr = re.sub('%40', '@', emaddr)
        if required and not re.match(r"[^@]+@[^@]+\.[^@]+", emaddr):
            raise ValueError("Invalid " + argname + " value: " + emaddr)
        return emaddr
    # A dbid is an int in the db and a string everywhere else
    if fieldtype in ["dbid", "string", "isodate", "isomod", "srchidcsv", "text",
                     "json", "jsarr", "idcsv", "isodcsv", "gencsv", "url"]:
        return argval or ""
    if fieldtype == "image":
        return argval or None
    if fieldtype == "int":
        argval = argval or 0
        return int(argval)
    raise ValueError("Unknown type " + fieldtype + " for " + argname)


# "cached fetch by key". Field must be dsId or one of the entkeys.
def cfbk(entity, field, value, required=False):
    if field != 'dsId' and field not in entkeys[entity]:
        raise ValueError(field + " not a unique index for " + entity)
    vstr = str(value)
    ci = entcache.cache_get(entity, field, vstr)
    if ci:
        dblogmsg("CAC", entity, ci)
        return ci
    if entdefs[entity][field]["pt"] not in ["dbid", "int"]:
        vstr = "\"" + vstr + "\""
    objs = query_entity(entity, "WHERE " + field + "=" + vstr + " LIMIT 1")
    if len(objs) > 0:
        inst = objs[0]
        if not cachedefs[inst["dsType"]]["manualadd"]:
            entcache.cache_put(inst)
        return inst
    if required:
        raise ValueError(entity + " " + vstr + " not found.")
    return None


# Get a connection to the database.  May throw mysql.connector.Error
# https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html
def get_mysql_connector():
    cnx = None
    try:
        cnx = mysql.connector.connect(user=mconf.db["u"],
                                      password=mconf.db["p"],
                                      host=mconf.db["h"],
                                      database=mconf.db["d"])
    except Exception as e:
        raise ValueError("Connection failed: " + str(e))
    return cnx


# Given what should be a string value, remove preceding or trailing space.
# If unique is true, then treat values of "null" or "None" as "".
def trim_string_val(val, unique=False):
    val = val or ""
    val = str(val)
    val = val.strip()
    if val and unique:
        lowval = val.lower()
        if lowval in ["null", "none"]:
            val = ""
    return val


# Read the given field from the inst or the default values, then convert it
# from an app value to a db value.  All string values are trimmed since
# preceding or trailing space makes matching horrible and buggy.  The UI can
# add a trailing newline for long text if it wants.
def app2db_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]["pt"]
        unique = entdefs[entity][field]["un"]
        val = entdefs[entity][field]["dv"]
    else:
        pt = dbflds[field]["pt"]
        unique = dbflds[field]["un"]
        val = dbflds[field]["dv"]
    if field in inst:
        val = inst[field]
    # convert value based on type and whether the values are unique
    if pt in ["email", "string"]:
        val = val or ""
        val = trim_string_val(val, unique)  # trim all strings. See comment.
        if not val:
            val = None
    elif pt == "image":
        if not val:  # Empty data gets set to null
            val = None
    elif pt == "int":
        val = val or 0
        val = int(val)  # convert possible "0" value
    elif pt == "dbid":
        try:
            val = int(val)  # will fail on "", "null" or other bad values
        except ValueError:
            val = 0
        if unique and not val:  # null vals don't violate UNIQUE constraint
            val = None          # otherwise use 0 as val may be required
    return val


# Read the given field from the inst or the default values, then convert it
# from a db value to an app value.  "app" means the server side module
# calling this module, not the web client.  Image binary values and json
# field values are not decoded, but get safe defaults if NULL.  dbids are
# converted to strings.
def db2app_fieldval(entity, field, inst):
    if entity:
        pt = entdefs[entity][field]["pt"]
        val = entdefs[entity][field]["dv"]
    else:
        pt = dbflds[field]["pt"]
        val = dbflds[field]["dv"]
    if field in inst:
        val = inst[field]
    # convert value based on type
    if pt in ["email", "string"]:
        if not val:  # A null value gets set to the empty string
            val = ""
        val = str(val)  # db interface might have autoconverted to int
    elif pt == "image":
        if not val:  # A null value gets set to the empty string
            val = ""
    elif pt == "int":
        if not val:  # Convert null values to 0
            val = 0
    elif pt == "dbid":
        if not val:  # A zero or null value gets set to falsey empty string
            val = ""
        else:
            val = str(val)
    return val


def ISO2dt(isostr):
    isostr = re.sub(r"\.\d*Z", "Z", isostr)  # remove microsecond if any
    dt = datetime.datetime.utcnow()
    dt = dt.strptime(isostr, "%Y-%m-%dT%H:%M:%SZ")
    return dt


def dt2ISO(dt):
    iso = str(dt.year) + "-" + str(dt.month).rjust(2, '0') + "-"
    iso += str(dt.day).rjust(2, '0') + "T" + str(dt.hour).rjust(2, '0')
    iso += ":" + str(dt.minute).rjust(2, '0') + ":"
    iso += str(dt.second).rjust(2, '0') + "Z"
    return iso


def nowISO():
    """ Return the current time as an ISO string """
    return dt2ISO(datetime.datetime.utcnow())


def initialize_timestamp_fields(fields, vck):
    ts = nowISO()
    if "created" not in fields or not fields["created"] or vck != "override":
        fields["created"] = ts
    if "modified" not in fields or not fields["modified"] or vck != "override":
        fields["modified"] = ts + ";1"


def verify_timestamp_fields(entity, dsId, fields, vck):
    if vck == "override" and "created" in fields and "modified" in fields:
        return fields # skip query and use specified values
    if not vck or not vck.strip():
        raise ValueError("Version check required to update " + entity +
                         " " + str(dsId))
    existing = cfbk(entity, "dsId", dsId)
    if not existing:
        raise ValueError("Existing " + entity + " " + str(dsId) + " not found.")
    if vck != "override" and existing["modified"] != vck:
        raise ValueError("Update error. Outdated data given for " + entity +
                         " " + str(dsId) + ".")
    if "created" not in fields or not fields["created"] or vck != "override":
        fields["created"] = existing["created"]
    ver = 1
    mods = existing["modified"].split(";")
    if len(mods) > 1:
        ver = int(mods[1]) + 1
    if "modified" not in fields or not fields["modified"] or vck != "override":
        fields["modified"] = nowISO() + ";" + str(ver)
    return existing


# Convert the given AppUser inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_AppUser(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("AppUser", "importid", inst)
    if fill or "email" in inst:
        cnv["email"] = app2db_fieldval("AppUser", "email", inst)
    if fill or "phash" in inst:
        cnv["phash"] = app2db_fieldval("AppUser", "phash", inst)
    if fill or "status" in inst:
        cnv["status"] = app2db_fieldval("AppUser", "status", inst)
    if fill or "actsends" in inst:
        cnv["actsends"] = app2db_fieldval("AppUser", "actsends", inst)
    if fill or "actcode" in inst:
        cnv["actcode"] = app2db_fieldval("AppUser", "actcode", inst)
    if fill or "accessed" in inst:
        cnv["accessed"] = app2db_fieldval("AppUser", "accessed", inst)
    if fill or "name" in inst:
        cnv["name"] = app2db_fieldval("AppUser", "name", inst)
    if fill or "title" in inst:
        cnv["title"] = app2db_fieldval("AppUser", "title", inst)
    if fill or "web" in inst:
        cnv["web"] = app2db_fieldval("AppUser", "web", inst)
    if fill or "lang" in inst:
        cnv["lang"] = app2db_fieldval("AppUser", "lang", inst)
    if fill or "settings" in inst:
        cnv["settings"] = app2db_fieldval("AppUser", "settings", inst)
    if fill or "started" in inst:
        cnv["started"] = app2db_fieldval("AppUser", "started", inst)
    if fill or "completed" in inst:
        cnv["completed"] = app2db_fieldval("AppUser", "completed", inst)
    if fill or "remtls" in inst:
        cnv["remtls"] = app2db_fieldval("AppUser", "remtls", inst)
    if fill or "built" in inst:
        cnv["built"] = app2db_fieldval("AppUser", "built", inst)
    return cnv


# Convert the given AppUser inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_AppUser(inst):
    cnv = {}
    cnv["dsType"] = "AppUser"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("AppUser", "importid", inst)
    cnv["email"] = db2app_fieldval("AppUser", "email", inst)
    cnv["phash"] = db2app_fieldval("AppUser", "phash", inst)
    cnv["status"] = db2app_fieldval("AppUser", "status", inst)
    cnv["actsends"] = db2app_fieldval("AppUser", "actsends", inst)
    cnv["actcode"] = db2app_fieldval("AppUser", "actcode", inst)
    cnv["accessed"] = db2app_fieldval("AppUser", "accessed", inst)
    cnv["name"] = db2app_fieldval("AppUser", "name", inst)
    cnv["title"] = db2app_fieldval("AppUser", "title", inst)
    cnv["web"] = db2app_fieldval("AppUser", "web", inst)
    cnv["lang"] = db2app_fieldval("AppUser", "lang", inst)
    cnv["settings"] = db2app_fieldval("AppUser", "settings", inst)
    cnv["started"] = db2app_fieldval("AppUser", "started", inst)
    cnv["completed"] = db2app_fieldval("AppUser", "completed", inst)
    cnv["remtls"] = db2app_fieldval("AppUser", "remtls", inst)
    cnv["built"] = db2app_fieldval("AppUser", "built", inst)
    return cnv


# Convert the given Point inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_Point(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "editors" in inst:
        cnv["editors"] = app2db_fieldval("Point", "editors", inst)
    if fill or "srctl" in inst:
        cnv["srctl"] = app2db_fieldval("Point", "srctl", inst)
    if fill or "lmuid" in inst:
        cnv["lmuid"] = app2db_fieldval("Point", "lmuid", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("Point", "importid", inst)
    if fill or "source" in inst:
        cnv["source"] = app2db_fieldval("Point", "source", inst)
    if fill or "date" in inst:
        cnv["date"] = app2db_fieldval("Point", "date", inst)
    if fill or "text" in inst:
        cnv["text"] = app2db_fieldval("Point", "text", inst)
    if fill or "refs" in inst:
        cnv["refs"] = app2db_fieldval("Point", "refs", inst)
    if fill or "qtype" in inst:
        cnv["qtype"] = app2db_fieldval("Point", "qtype", inst)
    if fill or "communities" in inst:
        cnv["communities"] = app2db_fieldval("Point", "communities", inst)
    if fill or "regions" in inst:
        cnv["regions"] = app2db_fieldval("Point", "regions", inst)
    if fill or "categories" in inst:
        cnv["categories"] = app2db_fieldval("Point", "categories", inst)
    if fill or "tags" in inst:
        cnv["tags"] = app2db_fieldval("Point", "tags", inst)
    if fill or "codes" in inst:
        cnv["codes"] = app2db_fieldval("Point", "codes", inst)
    if fill or "srclang" in inst:
        cnv["srclang"] = app2db_fieldval("Point", "srclang", inst)
    if fill or "translations" in inst:
        cnv["translations"] = app2db_fieldval("Point", "translations", inst)
    if fill or "pic" in inst:
        cnv["pic"] = app2db_fieldval("Point", "pic", inst)
    if fill or "stats" in inst:
        cnv["stats"] = app2db_fieldval("Point", "stats", inst)
    return cnv


# Convert the given Point inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_Point(inst):
    cnv = {}
    cnv["dsType"] = "Point"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["editors"] = db2app_fieldval("Point", "editors", inst)
    cnv["srctl"] = db2app_fieldval("Point", "srctl", inst)
    cnv["lmuid"] = db2app_fieldval("Point", "lmuid", inst)
    cnv["importid"] = db2app_fieldval("Point", "importid", inst)
    cnv["source"] = db2app_fieldval("Point", "source", inst)
    cnv["date"] = db2app_fieldval("Point", "date", inst)
    cnv["text"] = db2app_fieldval("Point", "text", inst)
    cnv["refs"] = db2app_fieldval("Point", "refs", inst)
    cnv["qtype"] = db2app_fieldval("Point", "qtype", inst)
    cnv["communities"] = db2app_fieldval("Point", "communities", inst)
    cnv["regions"] = db2app_fieldval("Point", "regions", inst)
    cnv["categories"] = db2app_fieldval("Point", "categories", inst)
    cnv["tags"] = db2app_fieldval("Point", "tags", inst)
    cnv["codes"] = db2app_fieldval("Point", "codes", inst)
    cnv["srclang"] = db2app_fieldval("Point", "srclang", inst)
    cnv["translations"] = db2app_fieldval("Point", "translations", inst)
    cnv["pic"] = db2app_fieldval("Point", "pic", inst)
    cnv["stats"] = db2app_fieldval("Point", "stats", inst)
    return cnv


# Convert the given Timeline inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_Timeline(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "editors" in inst:
        cnv["editors"] = app2db_fieldval("Timeline", "editors", inst)
    if fill or "lmuid" in inst:
        cnv["lmuid"] = app2db_fieldval("Timeline", "lmuid", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("Timeline", "importid", inst)
    if fill or "name" in inst:
        cnv["name"] = app2db_fieldval("Timeline", "name", inst)
    if fill or "cname" in inst:
        cnv["cname"] = app2db_fieldval("Timeline", "cname", inst)
    if fill or "slug" in inst:
        cnv["slug"] = app2db_fieldval("Timeline", "slug", inst)
    if fill or "title" in inst:
        cnv["title"] = app2db_fieldval("Timeline", "title", inst)
    if fill or "subtitle" in inst:
        cnv["subtitle"] = app2db_fieldval("Timeline", "subtitle", inst)
    if fill or "featured" in inst:
        cnv["featured"] = app2db_fieldval("Timeline", "featured", inst)
    if fill or "lang" in inst:
        cnv["lang"] = app2db_fieldval("Timeline", "lang", inst)
    if fill or "comment" in inst:
        cnv["comment"] = app2db_fieldval("Timeline", "comment", inst)
    if fill or "about" in inst:
        cnv["about"] = app2db_fieldval("Timeline", "about", inst)
    if fill or "kwds" in inst:
        cnv["kwds"] = app2db_fieldval("Timeline", "kwds", inst)
    if fill or "ctype" in inst:
        cnv["ctype"] = app2db_fieldval("Timeline", "ctype", inst)
    if fill or "cids" in inst:
        cnv["cids"] = app2db_fieldval("Timeline", "cids", inst)
    if fill or "rempts" in inst:
        cnv["rempts"] = app2db_fieldval("Timeline", "rempts", inst)
    if fill or "svs" in inst:
        cnv["svs"] = app2db_fieldval("Timeline", "svs", inst)
    if fill or "preb" in inst:
        cnv["preb"] = app2db_fieldval("Timeline", "preb", inst)
    return cnv


# Convert the given Timeline inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_Timeline(inst):
    cnv = {}
    cnv["dsType"] = "Timeline"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["editors"] = db2app_fieldval("Timeline", "editors", inst)
    cnv["lmuid"] = db2app_fieldval("Timeline", "lmuid", inst)
    cnv["importid"] = db2app_fieldval("Timeline", "importid", inst)
    cnv["name"] = db2app_fieldval("Timeline", "name", inst)
    cnv["cname"] = db2app_fieldval("Timeline", "cname", inst)
    cnv["slug"] = db2app_fieldval("Timeline", "slug", inst)
    cnv["title"] = db2app_fieldval("Timeline", "title", inst)
    cnv["subtitle"] = db2app_fieldval("Timeline", "subtitle", inst)
    cnv["featured"] = db2app_fieldval("Timeline", "featured", inst)
    cnv["lang"] = db2app_fieldval("Timeline", "lang", inst)
    cnv["comment"] = db2app_fieldval("Timeline", "comment", inst)
    cnv["about"] = db2app_fieldval("Timeline", "about", inst)
    cnv["kwds"] = db2app_fieldval("Timeline", "kwds", inst)
    cnv["ctype"] = db2app_fieldval("Timeline", "ctype", inst)
    cnv["cids"] = db2app_fieldval("Timeline", "cids", inst)
    cnv["rempts"] = db2app_fieldval("Timeline", "rempts", inst)
    cnv["svs"] = db2app_fieldval("Timeline", "svs", inst)
    cnv["preb"] = db2app_fieldval("Timeline", "preb", inst)
    return cnv


# Convert the given TLComp inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_TLComp(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("TLComp", "importid", inst)
    if fill or "userid" in inst:
        cnv["userid"] = app2db_fieldval("TLComp", "userid", inst)
    if fill or "tlid" in inst:
        cnv["tlid"] = app2db_fieldval("TLComp", "tlid", inst)
    if fill or "username" in inst:
        cnv["username"] = app2db_fieldval("TLComp", "username", inst)
    if fill or "tlname" in inst:
        cnv["tlname"] = app2db_fieldval("TLComp", "tlname", inst)
    if fill or "data" in inst:
        cnv["data"] = app2db_fieldval("TLComp", "data", inst)
    return cnv


# Convert the given TLComp inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_TLComp(inst):
    cnv = {}
    cnv["dsType"] = "TLComp"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("TLComp", "importid", inst)
    cnv["userid"] = db2app_fieldval("TLComp", "userid", inst)
    cnv["tlid"] = db2app_fieldval("TLComp", "tlid", inst)
    cnv["username"] = db2app_fieldval("TLComp", "username", inst)
    cnv["tlname"] = db2app_fieldval("TLComp", "tlname", inst)
    cnv["data"] = db2app_fieldval("TLComp", "data", inst)
    return cnv


# Convert the given DayCount inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_DayCount(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("DayCount", "importid", inst)
    if fill or "tstamp" in inst:
        cnv["tstamp"] = app2db_fieldval("DayCount", "tstamp", inst)
    if fill or "rtype" in inst:
        cnv["rtype"] = app2db_fieldval("DayCount", "rtype", inst)
    if fill or "detail" in inst:
        cnv["detail"] = app2db_fieldval("DayCount", "detail", inst)
    return cnv


# Convert the given DayCount inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_DayCount(inst):
    cnv = {}
    cnv["dsType"] = "DayCount"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("DayCount", "importid", inst)
    cnv["tstamp"] = db2app_fieldval("DayCount", "tstamp", inst)
    cnv["rtype"] = db2app_fieldval("DayCount", "rtype", inst)
    cnv["detail"] = db2app_fieldval("DayCount", "detail", inst)
    return cnv


# Convert the given AppService inst dict from app values to db values.  Removes
# the dsType field to avoid trying to write it to the db.
def app2db_AppService(inst, fill=True):
    cnv = {}
    cnv["dsId"] = None
    if "dsId" in inst:
        cnv["dsId"] = app2db_fieldval(None, "dsId", inst)
    if fill or "created" in inst:
        cnv["created"] = app2db_fieldval(None, "created", inst)
    if fill or "modified" in inst:
        cnv["modified"] = app2db_fieldval(None, "modified", inst)
    if fill or "batchconv" in inst:
        cnv["batchconv"] = app2db_fieldval(None, "batchconv", inst)
    if fill or "importid" in inst:
        cnv["importid"] = app2db_fieldval("AppService", "importid", inst)
    if fill or "name" in inst:
        cnv["name"] = app2db_fieldval("AppService", "name", inst)
    if fill or "ckey" in inst:
        cnv["ckey"] = app2db_fieldval("AppService", "ckey", inst)
    if fill or "csec" in inst:
        cnv["csec"] = app2db_fieldval("AppService", "csec", inst)
    if fill or "data" in inst:
        cnv["data"] = app2db_fieldval("AppService", "data", inst)
    return cnv


# Convert the given AppService inst dict from db values to app values.  Adds the
# dsType field for general app processing.
def db2app_AppService(inst):
    cnv = {}
    cnv["dsType"] = "AppService"
    cnv["dsId"] = db2app_fieldval(None, "dsId", inst)
    cnv["created"] = db2app_fieldval(None, "created", inst)
    cnv["modified"] = db2app_fieldval(None, "modified", inst)
    cnv["batchconv"] = db2app_fieldval(None, "batchconv", inst)
    cnv["importid"] = db2app_fieldval("AppService", "importid", inst)
    cnv["name"] = db2app_fieldval("AppService", "name", inst)
    cnv["ckey"] = db2app_fieldval("AppService", "ckey", inst)
    cnv["csec"] = db2app_fieldval("AppService", "csec", inst)
    cnv["data"] = db2app_fieldval("AppService", "data", inst)
    return cnv


def dblogmsg(op, entity, res):
    log_summary_flds = {
        "AppUser": ["email", "name"],
        "Point": ["editors", "date", "text"],
        "Timeline": ["name"],
        "TLComp": ["userid", "tlid", "username", "tlname"],
        "DayCount": ["tstamp", "rtype"],
        "AppService": ["name"]}
    if res:
        if op != "QRY":  # query is already a list, listify anything else
            res = [res]
        for obj in res:
            msg = "db" + op + " " + entity + " " + obj["dsId"]
            if entity in log_summary_flds:
                for field in log_summary_flds[entity]:
                    msg += " " + str(obj[field])[0:60]
            logging.info(msg)
    else:  # no res, probably a delete
        logging.info("db" + op + " " + entity + " -no obj details-")


# Write a new AppUser row, using the given field values or defaults.
def insert_new_AppUser(cnx, cursor, fields):
    fields = app2db_AppUser(fields)
    stmt = (
        "INSERT INTO AppUser (created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(email)s, %(phash)s, %(status)s, %(actsends)s, %(actcode)s, %(accessed)s, %(name)s, %(title)s, %(web)s, %(lang)s, %(settings)s, %(started)s, %(completed)s, %(remtls)s, %(built)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["AppUser"]["importid"]["dv"]),
        'email': fields.get("email", entdefs["AppUser"]["email"]["dv"]),
        'phash': fields.get("phash", entdefs["AppUser"]["phash"]["dv"]),
        'status': fields.get("status", entdefs["AppUser"]["status"]["dv"]),
        'actsends': fields.get("actsends", entdefs["AppUser"]["actsends"]["dv"]),
        'actcode': fields.get("actcode", entdefs["AppUser"]["actcode"]["dv"]),
        'accessed': fields.get("accessed", entdefs["AppUser"]["accessed"]["dv"]),
        'name': fields.get("name", entdefs["AppUser"]["name"]["dv"]),
        'title': fields.get("title", entdefs["AppUser"]["title"]["dv"]),
        'web': fields.get("web", entdefs["AppUser"]["web"]["dv"]),
        'lang': fields.get("lang", entdefs["AppUser"]["lang"]["dv"]),
        'settings': fields.get("settings", entdefs["AppUser"]["settings"]["dv"]),
        'started': fields.get("started", entdefs["AppUser"]["started"]["dv"]),
        'completed': fields.get("completed", entdefs["AppUser"]["completed"]["dv"]),
        'remtls': fields.get("remtls", entdefs["AppUser"]["remtls"]["dv"]),
        'built': fields.get("built", entdefs["AppUser"]["built"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppUser(fields)
    dblogmsg("ADD", "AppUser", fields)
    return fields


# Update the specified AppUser row with the given field values.
def update_existing_AppUser(context, fields):
    fields = app2db_AppUser(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE AppUser SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("AppUser" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppUser(result)
    dblogmsg("UPD", "AppUser", result)
    entcache.cache_remove(result)
    return result


# Write a new Point row, using the given field values or defaults.
def insert_new_Point(cnx, cursor, fields):
    fields = app2db_Point(fields)
    stmt = (
        "INSERT INTO Point (created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats) "
        "VALUES (%(created)s, %(modified)s, %(editors)s, %(srctl)s, %(lmuid)s, %(importid)s, %(source)s, %(date)s, %(text)s, %(refs)s, %(qtype)s, %(communities)s, %(regions)s, %(categories)s, %(tags)s, %(codes)s, %(srclang)s, %(translations)s, %(pic)s, %(stats)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'editors': fields.get("editors", entdefs["Point"]["editors"]["dv"]),
        'srctl': fields.get("srctl", entdefs["Point"]["srctl"]["dv"]),
        'lmuid': fields.get("lmuid", entdefs["Point"]["lmuid"]["dv"]),
        'importid': fields.get("importid", entdefs["Point"]["importid"]["dv"]),
        'source': fields.get("source", entdefs["Point"]["source"]["dv"]),
        'date': fields.get("date", entdefs["Point"]["date"]["dv"]),
        'text': fields.get("text", entdefs["Point"]["text"]["dv"]),
        'refs': fields.get("refs", entdefs["Point"]["refs"]["dv"]),
        'qtype': fields.get("qtype", entdefs["Point"]["qtype"]["dv"]),
        'communities': fields.get("communities", entdefs["Point"]["communities"]["dv"]),
        'regions': fields.get("regions", entdefs["Point"]["regions"]["dv"]),
        'categories': fields.get("categories", entdefs["Point"]["categories"]["dv"]),
        'tags': fields.get("tags", entdefs["Point"]["tags"]["dv"]),
        'codes': fields.get("codes", entdefs["Point"]["codes"]["dv"]),
        'srclang': fields.get("srclang", entdefs["Point"]["srclang"]["dv"]),
        'translations': fields.get("translations", entdefs["Point"]["translations"]["dv"]),
        'pic': fields.get("pic", entdefs["Point"]["pic"]["dv"]),
        'stats': fields.get("stats", entdefs["Point"]["stats"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Point(fields)
    dblogmsg("ADD", "Point", fields)
    return fields


# Update the specified Point row with the given field values.
def update_existing_Point(context, fields):
    fields = app2db_Point(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE Point SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("Point" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_Point(result)
    dblogmsg("UPD", "Point", result)
    entcache.cache_remove(result)
    return result


# Write a new Timeline row, using the given field values or defaults.
def insert_new_Timeline(cnx, cursor, fields):
    fields = app2db_Timeline(fields)
    stmt = (
        "INSERT INTO Timeline (created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) "
        "VALUES (%(created)s, %(modified)s, %(editors)s, %(lmuid)s, %(importid)s, %(name)s, %(cname)s, %(slug)s, %(title)s, %(subtitle)s, %(featured)s, %(lang)s, %(comment)s, %(about)s, %(kwds)s, %(ctype)s, %(cids)s, %(rempts)s, %(svs)s, %(preb)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'editors': fields.get("editors", entdefs["Timeline"]["editors"]["dv"]),
        'lmuid': fields.get("lmuid", entdefs["Timeline"]["lmuid"]["dv"]),
        'importid': fields.get("importid", entdefs["Timeline"]["importid"]["dv"]),
        'name': fields.get("name", entdefs["Timeline"]["name"]["dv"]),
        'cname': fields.get("cname", entdefs["Timeline"]["cname"]["dv"]),
        'slug': fields.get("slug", entdefs["Timeline"]["slug"]["dv"]),
        'title': fields.get("title", entdefs["Timeline"]["title"]["dv"]),
        'subtitle': fields.get("subtitle", entdefs["Timeline"]["subtitle"]["dv"]),
        'featured': fields.get("featured", entdefs["Timeline"]["featured"]["dv"]),
        'lang': fields.get("lang", entdefs["Timeline"]["lang"]["dv"]),
        'comment': fields.get("comment", entdefs["Timeline"]["comment"]["dv"]),
        'about': fields.get("about", entdefs["Timeline"]["about"]["dv"]),
        'kwds': fields.get("kwds", entdefs["Timeline"]["kwds"]["dv"]),
        'ctype': fields.get("ctype", entdefs["Timeline"]["ctype"]["dv"]),
        'cids': fields.get("cids", entdefs["Timeline"]["cids"]["dv"]),
        'rempts': fields.get("rempts", entdefs["Timeline"]["rempts"]["dv"]),
        'svs': fields.get("svs", entdefs["Timeline"]["svs"]["dv"]),
        'preb': fields.get("preb", entdefs["Timeline"]["preb"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_Timeline(fields)
    dblogmsg("ADD", "Timeline", fields)
    return fields


# Update the specified Timeline row with the given field values.
def update_existing_Timeline(context, fields):
    fields = app2db_Timeline(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE Timeline SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("Timeline" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_Timeline(result)
    dblogmsg("UPD", "Timeline", result)
    entcache.cache_remove(result)
    return result


# Write a new TLComp row, using the given field values or defaults.
def insert_new_TLComp(cnx, cursor, fields):
    fields = app2db_TLComp(fields)
    stmt = (
        "INSERT INTO TLComp (created, modified, importid, userid, tlid, username, tlname, data) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(userid)s, %(tlid)s, %(username)s, %(tlname)s, %(data)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["TLComp"]["importid"]["dv"]),
        'userid': fields.get("userid", entdefs["TLComp"]["userid"]["dv"]),
        'tlid': fields.get("tlid", entdefs["TLComp"]["tlid"]["dv"]),
        'username': fields.get("username", entdefs["TLComp"]["username"]["dv"]),
        'tlname': fields.get("tlname", entdefs["TLComp"]["tlname"]["dv"]),
        'data': fields.get("data", entdefs["TLComp"]["data"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_TLComp(fields)
    dblogmsg("ADD", "TLComp", fields)
    return fields


# Update the specified TLComp row with the given field values.
def update_existing_TLComp(context, fields):
    fields = app2db_TLComp(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE TLComp SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("TLComp" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_TLComp(result)
    dblogmsg("UPD", "TLComp", result)
    entcache.cache_remove(result)
    return result


# Write a new DayCount row, using the given field values or defaults.
def insert_new_DayCount(cnx, cursor, fields):
    fields = app2db_DayCount(fields)
    stmt = (
        "INSERT INTO DayCount (created, modified, importid, tstamp, rtype, detail) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(tstamp)s, %(rtype)s, %(detail)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["DayCount"]["importid"]["dv"]),
        'tstamp': fields.get("tstamp", entdefs["DayCount"]["tstamp"]["dv"]),
        'rtype': fields.get("rtype", entdefs["DayCount"]["rtype"]["dv"]),
        'detail': fields.get("detail", entdefs["DayCount"]["detail"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_DayCount(fields)
    dblogmsg("ADD", "DayCount", fields)
    return fields


# Update the specified DayCount row with the given field values.
def update_existing_DayCount(context, fields):
    fields = app2db_DayCount(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE DayCount SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("DayCount" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_DayCount(result)
    dblogmsg("UPD", "DayCount", result)
    entcache.cache_remove(result)
    return result


# Write a new AppService row, using the given field values or defaults.
def insert_new_AppService(cnx, cursor, fields):
    fields = app2db_AppService(fields)
    stmt = (
        "INSERT INTO AppService (created, modified, importid, name, ckey, csec, data) "
        "VALUES (%(created)s, %(modified)s, %(importid)s, %(name)s, %(ckey)s, %(csec)s, %(data)s)")
    data = {
        'created': fields.get("created"),
        'modified': fields.get("modified"),
        'importid': fields.get("importid", entdefs["AppService"]["importid"]["dv"]),
        'name': fields.get("name", entdefs["AppService"]["name"]["dv"]),
        'ckey': fields.get("ckey", entdefs["AppService"]["ckey"]["dv"]),
        'csec': fields.get("csec", entdefs["AppService"]["csec"]["dv"]),
        'data': fields.get("data", entdefs["AppService"]["data"]["dv"])}
    cursor.execute(stmt, data)
    fields["dsId"] = cursor.lastrowid
    cnx.commit()
    fields = db2app_AppService(fields)
    dblogmsg("ADD", "AppService", fields)
    return fields


# Update the specified AppService row with the given field values.
def update_existing_AppService(context, fields):
    fields = app2db_AppService(fields, fill=False)
    dsId = int(fields["dsId"])  # Verify int value
    stmt = ""
    for field in fields:  # only updating the fields passed in
        if stmt:
            stmt += ", "
        stmt += field + "=(%(" + field + ")s)"
    stmt = "UPDATE AppService SET " + stmt + " WHERE dsId=" + str(dsId)
    if context["vck"] != "override":
        stmt += " AND modified=\"" + context["vck"] + "\""
    data = {}
    for field in fields:
        data[field] = fields[field]
    context["cursor"].execute(stmt, data)
    if context["cursor"].rowcount < 1 and context["vck"] != "override":
        raise ValueError("AppService" + str(dsId) + " update received outdated version check value " + context["vck"] + ".")
    context["cnx"].commit()
    result = context["existing"]
    for field in fields:
        result[field] = fields[field]
    result = db2app_AppService(result)
    dblogmsg("UPD", "AppService", result)
    entcache.cache_put(result)
    return result


# Write the given dict/object based on the dsType.  Binary field values must
# be base64.b64encode.  Unspecified fields are set to default values for a
# new instance, and left alone on update.  For update, the verification
# check value must match the modified value of the existing instance.
def write_entity(inst, vck="1234-12-12T00:00:00Z"):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError("Database connection failed.")
    try:
        cursor = cnx.cursor()
        try:
            entity = inst.get("dsType", None)
            dsId = inst.get("dsId", 0)
            if dsId:
                existing = verify_timestamp_fields(entity, dsId, inst, vck)
                context = {"cnx":cnx, "cursor":cursor, "vck":vck,
                           "existing":existing}
                if entity == "AppUser":
                    return update_existing_AppUser(context, inst)
                if entity == "Point":
                    return update_existing_Point(context, inst)
                if entity == "Timeline":
                    return update_existing_Timeline(context, inst)
                if entity == "TLComp":
                    return update_existing_TLComp(context, inst)
                if entity == "DayCount":
                    return update_existing_DayCount(context, inst)
                if entity == "AppService":
                    return update_existing_AppService(context, inst)
                raise ValueError("Cannot modify unknown entity dsType " +
                                 str(entity))
            # No existing instance to update.  Insert new.
            initialize_timestamp_fields(inst, vck)
            if entity == "AppUser":
                return insert_new_AppUser(cnx, cursor, inst)
            if entity == "Point":
                return insert_new_Point(cnx, cursor, inst)
            if entity == "Timeline":
                return insert_new_Timeline(cnx, cursor, inst)
            if entity == "TLComp":
                return insert_new_TLComp(cnx, cursor, inst)
            if entity == "DayCount":
                return insert_new_DayCount(cnx, cursor, inst)
            if entity == "AppService":
                return insert_new_AppService(cnx, cursor, inst)
            raise ValueError("Cannot create unknown entity dsType " +
                             str(entity))
        except mysql.connector.Error as e:
            raise ValueError(str(e) or "No mysql error text")  # see note 1
        finally:
            cursor.close()
    finally:
        cnx.close()


def delete_entity(entity, dsId):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError("Database connection failed.")
    try:
        cursor = cnx.cursor()
        try:
            stmt = "DELETE FROM " + entity + " WHERE dsId=" + str(dsId)
            cursor.execute(stmt)
            cnx.commit()
            dblogmsg("DEL", entity + " " + str(dsId), None)
            # if cache cleanup is needed that is up to caller
        except mysql.connector.Error as e:
            raise ValueError(str(e) or "No mysql error text")  # see note 1
        finally:
            cursor.close()
    finally:
        cnx.close()


def query_AppUser(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built"
    query += " FROM AppUser " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, email, phash, status, actsends, actcode, accessed, name, title, web, lang, settings, started, completed, remtls, built) in cursor:
        inst = {"dsType": "AppUser", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "email": email, "phash": phash, "status": status, "actsends": actsends, "actcode": actcode, "accessed": accessed, "name": name, "title": title, "web": web, "lang": lang, "settings": settings, "started": started, "completed": completed, "remtls": remtls, "built": built}
        inst = db2app_AppUser(inst)
        res.append(inst)
    dblogmsg("QRY", "AppUser", res)
    return res


def query_Point(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats"
    query += " FROM Point " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, editors, srctl, lmuid, importid, source, date, text, refs, qtype, communities, regions, categories, tags, codes, srclang, translations, pic, stats) in cursor:
        inst = {"dsType": "Point", "dsId": dsId, "created": created, "modified": modified, "editors": editors, "srctl": srctl, "lmuid": lmuid, "importid": importid, "source": source, "date": date, "text": text, "refs": refs, "qtype": qtype, "communities": communities, "regions": regions, "categories": categories, "tags": tags, "codes": codes, "srclang": srclang, "translations": translations, "pic": pic, "stats": stats}
        inst = db2app_Point(inst)
        res.append(inst)
    dblogmsg("QRY", "Point", res)
    return res


def query_Timeline(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb"
    query += " FROM Timeline " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, editors, lmuid, importid, name, cname, slug, title, subtitle, featured, lang, comment, about, kwds, ctype, cids, rempts, svs, preb) in cursor:
        inst = {"dsType": "Timeline", "dsId": dsId, "created": created, "modified": modified, "editors": editors, "lmuid": lmuid, "importid": importid, "name": name, "cname": cname, "slug": slug, "title": title, "subtitle": subtitle, "featured": featured, "lang": lang, "comment": comment, "about": about, "kwds": kwds, "ctype": ctype, "cids": cids, "rempts": rempts, "svs": svs, "preb": preb}
        inst = db2app_Timeline(inst)
        res.append(inst)
    dblogmsg("QRY", "Timeline", res)
    return res


def query_TLComp(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, userid, tlid, username, tlname, data"
    query += " FROM TLComp " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, userid, tlid, username, tlname, data) in cursor:
        inst = {"dsType": "TLComp", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "userid": userid, "tlid": tlid, "username": username, "tlname": tlname, "data": data}
        inst = db2app_TLComp(inst)
        res.append(inst)
    dblogmsg("QRY", "TLComp", res)
    return res


def query_DayCount(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, tstamp, rtype, detail"
    query += " FROM DayCount " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, tstamp, rtype, detail) in cursor:
        inst = {"dsType": "DayCount", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "tstamp": tstamp, "rtype": rtype, "detail": detail}
        inst = db2app_DayCount(inst)
        res.append(inst)
    dblogmsg("QRY", "DayCount", res)
    return res


def query_AppService(cnx, cursor, where):
    query = "SELECT dsId, created, modified, "
    query += "importid, name, ckey, csec, data"
    query += " FROM AppService " + where
    cursor.execute(query)
    res = []
    for (dsId, created, modified, importid, name, ckey, csec, data) in cursor:
        inst = {"dsType": "AppService", "dsId": dsId, "created": created, "modified": modified, "importid": importid, "name": name, "ckey": ckey, "csec": csec, "data": data}
        inst = db2app_AppService(inst)
        res.append(inst)
    dblogmsg("QRY", "AppService", res)
    return res


# Fetch all instances of the specified entity kind for the given WHERE
# clause.  The WHERE clause should include a LIMIT, and should only match on
# indexed fields and/or declared query indexes.  For speed and general
# compatibility, only one inequality operator should be used in the match.
def query_entity(entity, where):
    cnx = get_mysql_connector()
    if not cnx:
        raise ValueError("Database connection failed.")
    try:
        cursor = cnx.cursor()
        try:
            if entity == "AppUser":
                return query_AppUser(cnx, cursor, where)
            if entity == "Point":
                return query_Point(cnx, cursor, where)
            if entity == "Timeline":
                return query_Timeline(cnx, cursor, where)
            if entity == "TLComp":
                return query_TLComp(cnx, cursor, where)
            if entity == "DayCount":
                return query_DayCount(cnx, cursor, where)
            if entity == "AppService":
                return query_AppService(cnx, cursor, where)
        except mysql.connector.Error as e:
            raise ValueError(str(e) or "No mysql error text")  # see note 1
        finally:
            cursor.close()
    finally:
        cnx.close()


def visible_AppUser_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        if fld == "email" and audience != "private":
            continue
        if fld == "phash":
            continue
        if fld == "status" and audience != "private":
            continue
        if fld == "actsends":
            continue
        if fld == "actcode":
            continue
        filtobj[fld] = val
    return filtobj


def visible_Point_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        if fld == "pic":
            if obj["pic"]:
                val = obj["dsId"]
            else:
                val = ""
        filtobj[fld] = val
    return filtobj


def visible_Timeline_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


def visible_TLComp_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


def visible_DayCount_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


def visible_AppService_fields(obj, audience):
    filtobj = {}
    for fld, val in obj.items():
        if fld == "importid":
            continue
        filtobj[fld] = val
    return filtobj


# Return a copied object with only the fields appropriate to the audience.
# Specifying audience="private" includes peronal info.  The given obj is
# assumed to already have been through db2app conversion.  Image fields are
# converted to dsId values for separate download.
def visible_fields(obj, audience="public"):
    if obj["dsType"] == "AppUser":
        return visible_AppUser_fields(obj, audience)
    if obj["dsType"] == "Point":
        return visible_Point_fields(obj, audience)
    if obj["dsType"] == "Timeline":
        return visible_Timeline_fields(obj, audience)
    if obj["dsType"] == "TLComp":
        return visible_TLComp_fields(obj, audience)
    if obj["dsType"] == "DayCount":
        return visible_DayCount_fields(obj, audience)
    if obj["dsType"] == "AppService":
        return visible_AppService_fields(obj, audience)
    raise ValueError("Unknown object dsType: " + obj["dsType"])


