move sql to new model

src/main/kotlin/com/restapi/AppAccessManager.kt

@@ -46,7 +46,15 @@ class AppAccessManager : AccessManager {
             }
 
             val isAllowed = currentRoles().count { allowedRoles.contains(it) } > 0
-            logger.warn("entity - $entity, action $action, user roles = ${currentRoles()}, allowed = $allowedRoles, isAllowed? $isAllowed, enforce? ${appConfig.enforceRoleRestriction()}")
+            logger.debug(
+                "entity - {}, action {}, user roles = {}, allowed = {}, isAllowed? {}, enforce? {}",
+                entity,
+                action,
+                currentRoles(),
+                allowedRoles,
+                isAllowed,
+                appConfig.enforceRoleRestriction()
+            )
             if (isAllowed || !appConfig.enforceRoleRestriction() || allowedRoles.isEmpty()) {
                 //if role is allowed, or enforcement is turned off or no roles are explicitly allowed
                 handler.handle(ctx)

src/main/kotlin/com/restapi/Main.kt

@@ -75,7 +75,6 @@ fun main(args: Array<String>) {
             cfg.accessManager(AppAccessManager())
         }
         .routes {
-
             path("/auth") {
                 //for testing, development only
                 get("/endpoint") {
@@ -149,6 +148,7 @@ fun main(args: Array<String>) {
 
                     val key = "$AUTH_TOKEN${authUser.userName}"
                     val found = redis.llen(key)
+                    logger.warn("for user ${authUser.userName}, found from redis, $key => $found entries")
                     val foundOldAt = (0..found)
                         .mapNotNull { redis.lindex(key, it) }
                         .map { objectMapper.readValue<AuthTokenResponse>(it) }
@@ -208,7 +208,6 @@ fun main(args: Array<String>) {
                     }
                 }
             }
-
             before("/api/*") { ctx ->
 
                 NaiveRateLimit.requestPerTimeUnit(
@@ -243,8 +242,6 @@ fun main(args: Array<String>) {
                 }
 
             }
-
-
             path("/api") {
                 post("/audit/{action}") {
                     logger.warn("User ${currentUser()} of tenant ${currentTenant()} has performed ${it.pathParam("action")} @ ${LocalDateTime.now()}")

src/main/kotlin/com/restapi/controllers/Entities.kt

@@ -3,7 +3,7 @@ package com.restapi.controllers
 import com.restapi.domain.*
 import com.restapi.domain.Session.currentUser
 import com.restapi.domain.Session.database
-import com.restapi.domain.Session.findByEntityAndId
+import com.restapi.domain.Session.findDataModelByEntityAndUniqId
 import com.restapi.integ.Scripting
 import io.ebean.CallableSql
 import io.ebean.RawSqlBuilder
@@ -18,9 +18,27 @@ import java.time.LocalDateTime
 import java.time.LocalTime
 import java.time.format.DateTimeFormatter
 
-data class Query(
+enum class QueryParamType {
+    STRING, NUMBER, DATETIME, DATE
+}
+
+data class QueryParam(val type: QueryParamType, val value: String) {
+    fun getValue(): Any {
+        return when (type) {
+            QueryParamType.STRING -> value
+            QueryParamType.NUMBER -> if (value.matches(Regex("\\d+"))) value.toLong() else value.toDouble()
+            QueryParamType.DATETIME -> LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))
+            QueryParamType.DATE -> LocalDate.parse(value, DateTimeFormatter.ofPattern("yyyy-MM-dd"))
+        }
+    }
+}
+
+data class RawQuery(
     val sql: String,
-    val params: Map<String, Any>
+    val params: Map<String, QueryParam>
+)
+data class QueryById(
+    val params: List<QueryParam>
 )
 
 enum class ResultType {
@@ -33,7 +51,7 @@ data class StoredProcedure(val input: Map<String, Any>, val output: Map<String,
 object Entities {
     private val logger = LoggerFactory.getLogger("Entities")
     fun delete(ctx: Context) {
-        val e = database.findByEntityAndId(ctx.pathParam("entity"), ctx.pathParam("id"))
+        val e = database.findDataModelByEntityAndUniqId(ctx.pathParam("entity"), ctx.pathParam("id"))
         e.deletedBy = Session.currentUser()
         e.deletedOn = LocalDateTime.now()
         e.update()
@@ -41,7 +59,7 @@ object Entities {
     }
 
     fun patch(ctx: Context) {
-        val e = database.findByEntityAndId(ctx.pathParam("entity"), ctx.pathParam("id"))
+        val e = database.findDataModelByEntityAndUniqId(ctx.pathParam("entity"), ctx.pathParam("id"))
         val pv = ctx.bodyAsClass<Map<String, Any>>()
         pv.forEach { (key, value) ->
             e.data[key] = value;
@@ -52,7 +70,7 @@ object Entities {
 
     fun update(ctx: Context) {
         val purgeExisting = ctx.queryParam("purge")?.toBooleanStrictOrNull() == true
-        val e = database.findByEntityAndId(ctx.pathParam("entity"), ctx.pathParam("id"))
+        val e = database.findDataModelByEntityAndUniqId(ctx.pathParam("entity"), ctx.pathParam("id"))
 
         val newData = ctx.bodyAsClass<Map<String, Any>>()
         if (purgeExisting) {
@@ -73,7 +91,7 @@ object Entities {
     }
 
     private fun approveOrReject(ctx: Context, rejected: ApprovalStatus) {
-        val e = database.findByEntityAndId(ctx.pathParam("entity"), ctx.pathParam("id"))
+        val e = database.findDataModelByEntityAndUniqId(ctx.pathParam("entity"), ctx.pathParam("id"))
         val reject = ctx.bodyAsClass<RejectAction>()
         e.approvalStatus = rejected
         e.comments.add(Comments(text = reject.reason, by = currentUser()))
@@ -131,14 +149,15 @@ object Entities {
     }
 
     fun sqlQueryRaw(ctx: Context) {
-        val sql = ctx.bodyAsClass<Query>()
+        val sql = ctx.bodyAsClass<RawQuery>()
+        logger.warn("running sql ${sql.sql}, with params ${sql.params}")
         ctx.json(
             database.find(DataModel::class.java)
                 .setRawSql(
                     RawSqlBuilder.parse(sql.sql).create()
                 ).apply {
                     sql.params.forEach { (t, u) ->
-                        setParameter(t, u)
+                        setParameter(t, u.getValue())
                     }
                 }
                 .findList()
@@ -147,18 +166,23 @@ object Entities {
     }
 
     fun sqlQueryById(ctx: Context) {
-        val sql = ctx.bodyAsClass<Query>()
+        val sql = ctx.bodyAsClass<QueryById>()
-        val query = database.findByEntityAndId(ctx.pathParam("entity"), ctx.pathParam("id"))
+        val sqlId = ctx.pathParam("id")
+        logger.warn("running sqlId $sqlId, with params ${sql.params}")
 
-        val querySql = query.data["sql"] as String? ?: throw NotFoundResponse()
+        val entity = ctx.pathParam("entity")
+        val query = database.find(SqlModel::class.java)
+            .where()
+            .eq("entityName", entity)
+            .eq("sqlId", sqlId)
+            .findOne() ?: throw NotFoundResponse("sql not found for $entity, $sqlId")
 
         ctx.json(
             database.find(DataModel::class.java)
-                .setRawSql(
+                .setRawSql(RawSqlBuilder.parse(query.sql).create())
-                    RawSqlBuilder.parse(querySql).create()
+                .apply {
-                ).apply {
+                    sql.params.forEachIndexed { index, entry ->
-                    sql.params.forEach { (t, u) ->
+                        setParameter(index+1, entry.getValue())
-                        setParameter(t, u)
                     }
                 }
                 .findList()
@@ -168,7 +192,7 @@ object Entities {
 
     fun view(it: Context) {
         it.json(
-            database.findByEntityAndId(it.pathParam("entity"), it.pathParam("id"))
+            database.findDataModelByEntityAndUniqId(it.pathParam("entity"), it.pathParam("id"))
         )
     }
 

src/main/kotlin/com/restapi/domain/db.kt

@@ -163,10 +163,10 @@ object Session {
     fun currentToken() = currentUser.get().token
     fun jwk() = keypair.toParams(JsonWebKey.OutputControlLevel.PUBLIC_ONLY)
 
-    fun Database.findByEntityAndId(entity: String, id: String): DataModel {
+    fun Database.findDataModelByEntityAndUniqId(entity: String, uniqId: String): DataModel {
         return find(DataModel::class.java)
             .where()
-            .eq("uniqueIdentifier", id)
+            .eq("uniqueIdentifier", uniqId)
             .eq("entityName", entity)
             .findOne() ?: throw DataNotFoundException
     }

src/main/kotlin/com/restapi/domain/models.kt

@@ -169,6 +169,18 @@ enum class JobType {
     SCRIPT, DB
 }
 
+@Entity
+@Index(unique = true, name = "sql_unique_id", columnNames = ["entity_name", "sql_id", "tenant_id"])
+open class SqlModel : BaseTenantModel(){
+
+    @JsonDeserialize(using = SafeStringDeserializer::class)
+    var sqlId: String = ""
+
+    var entityName: String = ""
+
+    @Column(columnDefinition = "text")
+    var sql: String = ""
+}
 @Entity
 open class JobModel : BaseTenantModel() {
     @Index(unique = true)

src/main/resources/dbmigration/1.0.2.1__vehicle_queries.sql

@@ -0,0 +1,4 @@
+insert into sql_model(tenant_id, entity_name, sql_id, sql, created_by, modified_by)
+values ('compegence', 'vehicle', 'SQL0001',
+        'select sys_pk,deleted_on,current_approval_level,required_approval_levels,deleted,version,created_at,modified_at,deleted_by,approval_status,tags,comments,tenant_id,unique_identifier,entity_name,data,created_by,modified_by from entity_model where created_at >= ?',
+        'system', 'system')

src/main/resources/dbmigration/1.0.2.sql

@@ -0,0 +1,27 @@
+-- apply changes
+create table sql_model (
+  sys_pk                        bigint generated by default as identity not null,
+  deleted_on                    timestamp,
+  current_approval_level        integer default 0 not null,
+  required_approval_levels      integer default 0 not null,
+  sql                           text not null,
+  deleted                       boolean default false not null,
+  version                       integer default 1 not null,
+  created_at                    timestamp default 'now()' not null,
+  modified_at                   timestamp default 'now()' not null,
+  deleted_by                    varchar(255),
+  approval_status               varchar(8) default 'APPROVED' not null,
+  tags                          varchar[] default '{}' not null,
+  comments                      jsonb default '[]' not null,
+  tenant_id                     varchar(255) not null,
+  sql_id                        varchar(255) not null,
+  entity_name                   varchar(255) not null,
+  created_by                    varchar(255) not null,
+  modified_by                   varchar(255) not null,
+  constraint ck_sql_model_approval_status check ( approval_status in ('PENDING','APPROVED','REJECTED')),
+  constraint sql_unique_id unique (entity_name,sql_id,tenant_id),
+  constraint pk_sql_model primary key (sys_pk)
+);
+
+ALTER TABLE sql_model ADD FOREIGN KEY(tenant_id) REFERENCES tenant_model(name);
+ALTER TABLE sql_model ADD FOREIGN KEY(entity_name) REFERENCES entity_model(name);

src/main/resources/dbmigration/model/1.0.2.model.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<migration xmlns="http://ebean-orm.github.io/xml/ns/dbmigration">
+    <changeSet type="apply">
+        <createTable name="sql_model" pkName="pk_sql_model">
+            <column name="sys_pk" type="bigint" primaryKey="true"/>
+            <column name="deleted_on" type="localdatetime"/>
+            <column name="deleted_by" type="varchar"/>
+            <column name="current_approval_level" type="integer" defaultValue="0" notnull="true"/>
+            <column name="required_approval_levels" type="integer" defaultValue="0" notnull="true"/>
+            <column name="approval_status" type="varchar(8)" defaultValue="'APPROVED'" notnull="true" checkConstraint="check ( approval_status in ('PENDING','APPROVED','REJECTED'))" checkConstraintName="ck_sql_model_approval_status"/>
+            <column name="tags" type="varchar[]" defaultValue="'{}'" notnull="true"/>
+            <column name="comments" type="jsonb" defaultValue="'[]'" notnull="true"/>
+            <column name="tenant_id" type="varchar" notnull="true"/>
+            <column name="sql_id" type="varchar" notnull="true"/>
+            <column name="entity_name" type="varchar" notnull="true"/>
+            <column name="sql" type="text" notnull="true"/>
+            <column name="deleted" type="boolean" defaultValue="false" notnull="true"/>
+            <column name="version" type="integer" defaultValue="1" notnull="true"/>
+            <column name="created_at" type="localdatetime" defaultValue="'now()'" notnull="true"/>
+            <column name="modified_at" type="localdatetime" defaultValue="'now()'" notnull="true"/>
+            <column name="created_by" type="varchar" notnull="true"/>
+            <column name="modified_by" type="varchar" notnull="true"/>
+            <uniqueConstraint name="sql_unique_id" columnNames="entity_name,sql_id,tenant_id" oneToOne="false" nullableColumns=""/>
+        </createTable>
+    </changeSet>
+</migration>

src/main/resources/logback.xml

@@ -6,10 +6,10 @@
     </appender>
 
     <!-- SQL and bind values -->
-    <logger name="io.ebean.SQL" level="INFO"/>
+    <logger name="io.ebean.SQL" level="TRACE"/>
 
     <!-- Transaction Commit and Rollback events -->
-    <logger name="io.ebean.TXN" level="WARN"/>
+    <logger name="io.ebean.TXN" level="DEBUG"/>
     <logger name="io.ebean.SUM" level="WARN"/>
 
     <root level="info">