Skip to content

Attempt at auto inserting dataset in sql database #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JosephTLockwood wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Attempt at auto inserting dataset in sql database #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
97898da
Attempt at auto inserting dataset in sql database
JosephTLockwood Jul 19, 2023
973ee2d
Update to List return of String Query and List of Args
JosephTLockwood Jul 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 40 additions & 5 deletions common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,17 @@ object DatasetExtensions {
)
val dataset = parsedArgs.requirePyObject("dataset").toJava<Dataset>()
val mapper = parsedArgs.requirePyObject("mapper")
val preserveColumnTypes = parsedArgs.getBoolean("preserveColumnTypes").filter { it }.isPresent
val preserveColumnTypes =
parsedArgs.getBoolean("preserveColumnTypes").filter { it }.isPresent

val columnTypes = if (preserveColumnTypes) {
dataset.columnTypes
} else {
List(dataset.columnCount) { Any::class.java }
}

val builder = DatasetBuilder.newBuilder().colNames(dataset.columnNames).colTypes(columnTypes)
val builder =
DatasetBuilder.newBuilder().colNames(dataset.columnNames).colTypes(columnTypes)

for (row in dataset.rowIndices) {
val columnValues = Array<PyObject>(dataset.columnCount) { col ->
Expand Down Expand Up @@ -144,7 +146,11 @@ object DatasetExtensions {
return printDataset(appendable, dataset, includeTypes)
}

internal fun printDataset(appendable: Appendable, dataset: Dataset, includeTypes: Boolean = false) {
internal fun printDataset(
appendable: Appendable,
dataset: Dataset,
includeTypes: Boolean = false,
) {
val typeNames = List<String>(dataset.columnCount) { column ->
if (includeTypes) {
dataset.getColumnType(column).simpleName
Expand Down Expand Up @@ -252,6 +258,33 @@ object DatasetExtensions {
)
}

@Suppress("unused")
@ScriptFunction(docBundlePrefix = "DatasetExtensions")
@KeywordArgs(
names = ["dataset", "table"],
types = [Dataset::class, String::class],
)
fun insertDataset(args: Array<PyObject>, keywords: Array<String>): List<Any> {
val parserArgs = PyArgParser.parseArgs(
args,
keywords,
arrayOf("dataset", "table"),
Array(2) { Any::class.java },
"insertDataset",
)
val dataset = parserArgs.requirePyObject("dataset").toJava<Dataset>()
val table = parserArgs.requireString("table")

val columnNames = dataset.columnNames.joinToString(", ")
val valuePlaceholders = dataset.columnIndices.joinToString { "?" }
val insertString = "INSERT INTO $table ($columnNames) VALUES " + dataset.rowIndices.joinToString { "($valuePlaceholders)" }
val valueList = dataset.rowIndices.flatMap { row ->
dataset.columnIndices.map { col -> dataset[row, col] }
}

return listOf(insertString, valueList)
}

@Suppress("unused")
@ScriptFunction(docBundlePrefix = "DatasetExtensions")
@KeywordArgs(
Expand Down Expand Up @@ -293,7 +326,8 @@ object DatasetExtensions {
val sheet = workbook.getSheetAt(sheetNumber)

val headerRow = parsedArgs.getInteger("headerRow").orElse(-1)
val firstRow = parsedArgs.getInteger("firstRow").orElseGet { max(sheet.firstRowNum, headerRow + 1) }
val firstRow = parsedArgs.getInteger("firstRow")
.orElseGet { max(sheet.firstRowNum, headerRow + 1) }
val lastRow = parsedArgs.getInteger("lastRow").orElseGet { sheet.lastRowNum }

val dataRange = firstRow..lastRow
Expand All @@ -309,7 +343,8 @@ object DatasetExtensions {
val firstColumn =
parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() }
val lastColumn =
parsedArgs.getInteger("lastColumn").map { it + 1 }.orElseGet { columnRow.lastCellNum.toInt() }
parsedArgs.getInteger("lastColumn").map { it + 1 }
.orElseGet { columnRow.lastCellNum.toInt() }
if (firstColumn >= lastColumn) {
throw Py.ValueError("firstColumn ($firstColumn) must be less than lastColumn ($lastColumn)")
}
Expand Down
5 changes: 5 additions & 0 deletions common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ columnsEqual.param.ignoreCase=Pass True if the column names should be compared c
columnsEqual.param.includeTypes=Pass True if the column types must match as well. Defaults to True.
columnsEqual.returns=True if the two datasets have the same columns, per additional parameters.

insertDataset.desc=Insert a dataset into a sql database table
insertDataset.param.dataset=The dataset to insert. Must not be null.
insertDataset.param.table=The name of the table to insert into. Must not be null.
insertDataset.returns=List where 0 is String Query and 1 is List of args

builder.desc=Creates a new dataset using supplied column names and types.
builder.param.**columns=Optional. Keyword arguments can be supplied to predefine column names and types. The value of the argument should be string "typecode" (see system.dataset.fromCSV) or a Java or Python class instance.
builder.returns=A DatasetBuilder object. Use <code>addRow(value, ...)</code> to add new values, and <code>build()</code> to construct the final dataset. \
Expand Down
10 changes: 10 additions & 0 deletions common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package org.imdc.extensions.common
import com.inductiveautomation.ignition.common.BasicDataset
import com.inductiveautomation.ignition.common.Dataset
import com.inductiveautomation.ignition.common.util.DatasetBuilder
import io.kotest.assertions.asClue
import io.kotest.assertions.withClue
import io.kotest.engine.spec.tempfile
import io.kotest.matchers.shouldBe
Expand Down Expand Up @@ -103,6 +104,15 @@ class DatasetExtensionsTests : JythonTest(
}
}

context("Insert Dataset") {
test("Dataset to String") {
eval<List<Any>>("utils.insertDataset(dataset, 'testing')").asClue {
it[0] shouldBe "INSERT INTO testing (a, b, c) VALUES (?, ?, ?), (?, ?, ?)"
it[1] shouldBe listOf(1, 3.14, "pi", 2, 6.28, "tau")
}
}
}

context("Filter tests") {
test("Constant filter") {
eval<Dataset>("utils.filter(dataset, lambda **kwargs: False)").asClue {
Expand Down