From 97898da28a3f6aa05f3560104549a1e1fa698d2e Mon Sep 17 00:00:00 2001 From: JosephTLockwood Date: Wed, 19 Jul 2023 16:56:31 -0400 Subject: [PATCH 1/2] Attempt at auto inserting dataset in sql database --- .../extensions/common/DatasetExtensions.kt | 123 ++++++++++++++++++ .../common/DatasetExtensions.properties | 9 ++ 2 files changed, 132 insertions(+) diff --git a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt index 5b31d60..50bc199 100644 --- a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt +++ b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt @@ -3,8 +3,11 @@ package org.imdc.extensions.common import com.inductiveautomation.ignition.common.Dataset import com.inductiveautomation.ignition.common.PyUtilities import com.inductiveautomation.ignition.common.TypeUtilities +import com.inductiveautomation.ignition.common.datasource.DatasourceMeta import com.inductiveautomation.ignition.common.script.PyArgParser +import com.inductiveautomation.ignition.common.script.builtin.AbstractDBUtilities import com.inductiveautomation.ignition.common.script.builtin.KeywordArgs +import com.inductiveautomation.ignition.common.script.builtin.SProcCall import com.inductiveautomation.ignition.common.script.hints.ScriptArg import com.inductiveautomation.ignition.common.script.hints.ScriptFunction import com.inductiveautomation.ignition.common.util.DatasetBuilder @@ -28,6 +31,7 @@ import org.python.core.PyType import org.python.core.PyUnicode import java.io.File import java.math.BigDecimal +import java.sql.SQLException import java.util.Date import kotlin.jvm.optionals.getOrElse import kotlin.math.max @@ -252,6 +256,45 @@ object DatasetExtensions { ) } + @Suppress("unused") + @ScriptFunction(docBundlePrefix = "DatasetExtensions") + @KeywordArgs( + names = ["dataset", "table", "database", "tx", "getKey", "skipAudit"], + types = [Dataset::class, String::class, String::class, String::class, Boolean::class, Boolean::class], + ) + fun toSQL(args: Array, keywords: Array): Int { + val parserArgs = PyArgParser.parseArgs( + args, + keywords, + arrayOf("dataset", "table", "database", "tx", "getKey", "skipAudit"), + Array(6) { Any::class.java }, + "toSQL", + ) + val dataset = parserArgs.requirePyObject("dataset").toJava() + val table = parserArgs.requireString("table") + val database = parserArgs.requireString("database") + val tx = parserArgs.requireString("tx") + val getKey = parserArgs.getBoolean("getKey").orElse(false) + val skipAudit = parserArgs.getBoolean("skipAudit").orElse(false) + val columnHeaders = dataset.columnIndices.joinToString(",") { dataset.getColumnName(it) } + var rowsAffected = -1 + for (row in dataset.rowIndices) { + val rowValues = dataset.columnIndices.map { col -> + dataset[row, col] + } + val placeholders = List(rowValues.size) { "?" }.joinToString(",") + rowsAffected = CustomDBUtilities().runPrepUpdate( + "INSERT INTO $table ($columnHeaders) VALUES ($placeholders)", + rowValues, + database, + tx, + getKey, + skipAudit, + ) + } + return rowsAffected + } + @Suppress("unused") @ScriptFunction(docBundlePrefix = "DatasetExtensions") @KeywordArgs( @@ -503,4 +546,84 @@ object DatasetExtensions { PyFloat.TYPE -> Double::class.java else -> toJava>() } + + class CustomDBUtilities : AbstractDBUtilities() { + fun runPrepUpdate( + p0: String?, + p1: List, + p2: String?, + p3: String?, + p4: Boolean, + p5: Boolean, + ): Int { + try { + runPrepUpdate( + arrayOf(Py.java2py(p0), Py.java2py(p1), Py.java2py(p2), Py.java2py(p3), Py.java2py(p4), Py.java2py(p5)), + arrayOf(), + ) + } catch (e: SQLException) { + // Handle the exception + return -1 + } + return 0 + } + + override fun _runUpdateQuery( + p0: String?, + p1: String?, + p2: String?, + p3: Boolean, + p4: Boolean, + ): Int { + TODO("Not yet implemented") + } + + override fun _runPrepStmt( + p0: String?, + p1: String?, + p2: String?, + p3: Boolean, + p4: Boolean, + p5: Array?, + ): Int { + TODO("Not yet implemented") + } + + override fun _runPrepQuery( + p0: String?, + p1: String?, + p2: String?, + p3: Array?, + ): Dataset { + TODO("Not yet implemented") + } + + override fun _runQuery(p0: String?, p1: String?, p2: String?): Dataset { + TODO("Not yet implemented") + } + + override fun _findDatasources(): MutableList { + TODO("Not yet implemented") + } + + override fun _beginTransaction(p0: String?, p1: Int, p2: Long): String { + TODO("Not yet implemented") + } + + override fun _commitTransaction(p0: String?) { + TODO("Not yet implemented") + } + + override fun _rollbackTransaction(p0: String?) { + TODO("Not yet implemented") + } + + override fun _closeTransaction(p0: String?) { + TODO("Not yet implemented") + } + + override fun _call(p0: SProcCall?) { + TODO("Not yet implemented") + } + } } diff --git a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties index b7a5530..cbaaed3 100644 --- a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties +++ b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties @@ -48,6 +48,15 @@ columnsEqual.param.ignoreCase=Pass True if the column names should be compared c columnsEqual.param.includeTypes=Pass True if the column types must match as well. Defaults to True. columnsEqual.returns=True if the two datasets have the same columns, per additional parameters. +toSql.desc=Insert a dataset into a sql database table +toSql.param.dataset=The dataset to insert. Must not be null. +toSql.param.table=The name of the table to insert into. +toSql.param.database=The name of the database connection to execute against. +toSql.param.tx=A transaction identifier. If omitted, the update will be executed in its own transaction +toSql.param.getKey=A flag indicating whether or not the result should be the number of rows affected(getKey=0) or the newly generated key value that was created as a result of the update (getKey=1). Not all databases support automatic retrieval of generated keys. [optional] +toSql.param.skipAudit=A flag which, if set to true, will cause the prep update to skip the audit system. Useful for some queries that have fields which won't fit into the audit log. [optional] + + builder.desc=Creates a new dataset using supplied column names and types. builder.param.**columns=Optional. Keyword arguments can be supplied to predefine column names and types. The value of the argument should be string "typecode" (see system.dataset.fromCSV) or a Java or Python class instance. builder.returns=A DatasetBuilder object. Use addRow(value, ...) to add new values, and build() to construct the final dataset. \ From 973ee2d134ba6b6407af24fce0cdd8facbf684ab Mon Sep 17 00:00:00 2001 From: JosephTLockwood Date: Thu, 20 Jul 2023 13:13:08 -0400 Subject: [PATCH 2/2] Update to List return of String Query and List of Args --- .../extensions/common/DatasetExtensions.kt | 142 ++++-------------- .../common/DatasetExtensions.properties | 12 +- .../common/DatasetExtensionsTests.kt | 10 ++ 3 files changed, 41 insertions(+), 123 deletions(-) diff --git a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt index 50bc199..29c5fe1 100644 --- a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt +++ b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt @@ -3,11 +3,8 @@ package org.imdc.extensions.common import com.inductiveautomation.ignition.common.Dataset import com.inductiveautomation.ignition.common.PyUtilities import com.inductiveautomation.ignition.common.TypeUtilities -import com.inductiveautomation.ignition.common.datasource.DatasourceMeta import com.inductiveautomation.ignition.common.script.PyArgParser -import com.inductiveautomation.ignition.common.script.builtin.AbstractDBUtilities import com.inductiveautomation.ignition.common.script.builtin.KeywordArgs -import com.inductiveautomation.ignition.common.script.builtin.SProcCall import com.inductiveautomation.ignition.common.script.hints.ScriptArg import com.inductiveautomation.ignition.common.script.hints.ScriptFunction import com.inductiveautomation.ignition.common.util.DatasetBuilder @@ -31,7 +28,6 @@ import org.python.core.PyType import org.python.core.PyUnicode import java.io.File import java.math.BigDecimal -import java.sql.SQLException import java.util.Date import kotlin.jvm.optionals.getOrElse import kotlin.math.max @@ -54,7 +50,8 @@ object DatasetExtensions { ) val dataset = parsedArgs.requirePyObject("dataset").toJava() val mapper = parsedArgs.requirePyObject("mapper") - val preserveColumnTypes = parsedArgs.getBoolean("preserveColumnTypes").filter { it }.isPresent + val preserveColumnTypes = + parsedArgs.getBoolean("preserveColumnTypes").filter { it }.isPresent val columnTypes = if (preserveColumnTypes) { dataset.columnTypes @@ -62,7 +59,8 @@ object DatasetExtensions { List(dataset.columnCount) { Any::class.java } } - val builder = DatasetBuilder.newBuilder().colNames(dataset.columnNames).colTypes(columnTypes) + val builder = + DatasetBuilder.newBuilder().colNames(dataset.columnNames).colTypes(columnTypes) for (row in dataset.rowIndices) { val columnValues = Array(dataset.columnCount) { col -> @@ -148,7 +146,11 @@ object DatasetExtensions { return printDataset(appendable, dataset, includeTypes) } - internal fun printDataset(appendable: Appendable, dataset: Dataset, includeTypes: Boolean = false) { + internal fun printDataset( + appendable: Appendable, + dataset: Dataset, + includeTypes: Boolean = false, + ) { val typeNames = List(dataset.columnCount) { column -> if (includeTypes) { dataset.getColumnType(column).simpleName @@ -259,40 +261,28 @@ object DatasetExtensions { @Suppress("unused") @ScriptFunction(docBundlePrefix = "DatasetExtensions") @KeywordArgs( - names = ["dataset", "table", "database", "tx", "getKey", "skipAudit"], - types = [Dataset::class, String::class, String::class, String::class, Boolean::class, Boolean::class], + names = ["dataset", "table"], + types = [Dataset::class, String::class], ) - fun toSQL(args: Array, keywords: Array): Int { + fun insertDataset(args: Array, keywords: Array): List { val parserArgs = PyArgParser.parseArgs( args, keywords, - arrayOf("dataset", "table", "database", "tx", "getKey", "skipAudit"), - Array(6) { Any::class.java }, - "toSQL", + arrayOf("dataset", "table"), + Array(2) { Any::class.java }, + "insertDataset", ) val dataset = parserArgs.requirePyObject("dataset").toJava() val table = parserArgs.requireString("table") - val database = parserArgs.requireString("database") - val tx = parserArgs.requireString("tx") - val getKey = parserArgs.getBoolean("getKey").orElse(false) - val skipAudit = parserArgs.getBoolean("skipAudit").orElse(false) - val columnHeaders = dataset.columnIndices.joinToString(",") { dataset.getColumnName(it) } - var rowsAffected = -1 - for (row in dataset.rowIndices) { - val rowValues = dataset.columnIndices.map { col -> - dataset[row, col] - } - val placeholders = List(rowValues.size) { "?" }.joinToString(",") - rowsAffected = CustomDBUtilities().runPrepUpdate( - "INSERT INTO $table ($columnHeaders) VALUES ($placeholders)", - rowValues, - database, - tx, - getKey, - skipAudit, - ) + + val columnNames = dataset.columnNames.joinToString(", ") + val valuePlaceholders = dataset.columnIndices.joinToString { "?" } + val insertString = "INSERT INTO $table ($columnNames) VALUES " + dataset.rowIndices.joinToString { "($valuePlaceholders)" } + val valueList = dataset.rowIndices.flatMap { row -> + dataset.columnIndices.map { col -> dataset[row, col] } } - return rowsAffected + + return listOf(insertString, valueList) } @Suppress("unused") @@ -336,7 +326,8 @@ object DatasetExtensions { val sheet = workbook.getSheetAt(sheetNumber) val headerRow = parsedArgs.getInteger("headerRow").orElse(-1) - val firstRow = parsedArgs.getInteger("firstRow").orElseGet { max(sheet.firstRowNum, headerRow + 1) } + val firstRow = parsedArgs.getInteger("firstRow") + .orElseGet { max(sheet.firstRowNum, headerRow + 1) } val lastRow = parsedArgs.getInteger("lastRow").orElseGet { sheet.lastRowNum } val dataRange = firstRow..lastRow @@ -352,7 +343,8 @@ object DatasetExtensions { val firstColumn = parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() } val lastColumn = - parsedArgs.getInteger("lastColumn").map { it + 1 }.orElseGet { columnRow.lastCellNum.toInt() } + parsedArgs.getInteger("lastColumn").map { it + 1 } + .orElseGet { columnRow.lastCellNum.toInt() } if (firstColumn >= lastColumn) { throw Py.ValueError("firstColumn ($firstColumn) must be less than lastColumn ($lastColumn)") } @@ -546,84 +538,4 @@ object DatasetExtensions { PyFloat.TYPE -> Double::class.java else -> toJava>() } - - class CustomDBUtilities : AbstractDBUtilities() { - fun runPrepUpdate( - p0: String?, - p1: List, - p2: String?, - p3: String?, - p4: Boolean, - p5: Boolean, - ): Int { - try { - runPrepUpdate( - arrayOf(Py.java2py(p0), Py.java2py(p1), Py.java2py(p2), Py.java2py(p3), Py.java2py(p4), Py.java2py(p5)), - arrayOf(), - ) - } catch (e: SQLException) { - // Handle the exception - return -1 - } - return 0 - } - - override fun _runUpdateQuery( - p0: String?, - p1: String?, - p2: String?, - p3: Boolean, - p4: Boolean, - ): Int { - TODO("Not yet implemented") - } - - override fun _runPrepStmt( - p0: String?, - p1: String?, - p2: String?, - p3: Boolean, - p4: Boolean, - p5: Array?, - ): Int { - TODO("Not yet implemented") - } - - override fun _runPrepQuery( - p0: String?, - p1: String?, - p2: String?, - p3: Array?, - ): Dataset { - TODO("Not yet implemented") - } - - override fun _runQuery(p0: String?, p1: String?, p2: String?): Dataset { - TODO("Not yet implemented") - } - - override fun _findDatasources(): MutableList { - TODO("Not yet implemented") - } - - override fun _beginTransaction(p0: String?, p1: Int, p2: Long): String { - TODO("Not yet implemented") - } - - override fun _commitTransaction(p0: String?) { - TODO("Not yet implemented") - } - - override fun _rollbackTransaction(p0: String?) { - TODO("Not yet implemented") - } - - override fun _closeTransaction(p0: String?) { - TODO("Not yet implemented") - } - - override fun _call(p0: SProcCall?) { - TODO("Not yet implemented") - } - } } diff --git a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties index cbaaed3..3d4f46b 100644 --- a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties +++ b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties @@ -48,14 +48,10 @@ columnsEqual.param.ignoreCase=Pass True if the column names should be compared c columnsEqual.param.includeTypes=Pass True if the column types must match as well. Defaults to True. columnsEqual.returns=True if the two datasets have the same columns, per additional parameters. -toSql.desc=Insert a dataset into a sql database table -toSql.param.dataset=The dataset to insert. Must not be null. -toSql.param.table=The name of the table to insert into. -toSql.param.database=The name of the database connection to execute against. -toSql.param.tx=A transaction identifier. If omitted, the update will be executed in its own transaction -toSql.param.getKey=A flag indicating whether or not the result should be the number of rows affected(getKey=0) or the newly generated key value that was created as a result of the update (getKey=1). Not all databases support automatic retrieval of generated keys. [optional] -toSql.param.skipAudit=A flag which, if set to true, will cause the prep update to skip the audit system. Useful for some queries that have fields which won't fit into the audit log. [optional] - +insertDataset.desc=Insert a dataset into a sql database table +insertDataset.param.dataset=The dataset to insert. Must not be null. +insertDataset.param.table=The name of the table to insert into. Must not be null. +insertDataset.returns=List where 0 is String Query and 1 is List of args builder.desc=Creates a new dataset using supplied column names and types. builder.param.**columns=Optional. Keyword arguments can be supplied to predefine column names and types. The value of the argument should be string "typecode" (see system.dataset.fromCSV) or a Java or Python class instance. diff --git a/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt b/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt index c3c8215..8f84440 100644 --- a/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt +++ b/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt @@ -3,6 +3,7 @@ package org.imdc.extensions.common import com.inductiveautomation.ignition.common.BasicDataset import com.inductiveautomation.ignition.common.Dataset import com.inductiveautomation.ignition.common.util.DatasetBuilder +import io.kotest.assertions.asClue import io.kotest.assertions.withClue import io.kotest.engine.spec.tempfile import io.kotest.matchers.shouldBe @@ -103,6 +104,15 @@ class DatasetExtensionsTests : JythonTest( } } + context("Insert Dataset") { + test("Dataset to String") { + eval>("utils.insertDataset(dataset, 'testing')").asClue { + it[0] shouldBe "INSERT INTO testing (a, b, c) VALUES (?, ?, ?), (?, ?, ?)" + it[1] shouldBe listOf(1, 3.14, "pi", 2, 6.28, "tau") + } + } + } + context("Filter tests") { test("Constant filter") { eval("utils.filter(dataset, lambda **kwargs: False)").asClue {