From cce57cce0aa1df69e1da9d21a9b504e55af56a3f Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Fri, 24 Jul 2026 16:58:57 +0400 Subject: [PATCH 01/10] Add SQL type mapping documentation and SQLite metadata probe for testing --- .../kotlinx/dataframe/io/db/_sqliteProbe.kt | 129 ++++ .../kotlinx/dataframe/io/db/jdbcTypesTest.kt | 630 +++++++++++++++--- docs/StardustDocs/d.tree | 9 + .../StardustDocs/topics/readSqlTypeMapping.md | 54 ++ .../topics/readSqlTypeMapping_DuckDB.md | 123 ++++ .../topics/readSqlTypeMapping_H2.md | 113 ++++ .../topics/readSqlTypeMapping_MariaDB.md | 119 ++++ .../topics/readSqlTypeMapping_MsSql.md | 113 ++++ .../topics/readSqlTypeMapping_MySQL.md | 114 ++++ .../topics/readSqlTypeMapping_PostgreSQL.md | 171 +++++ .../topics/readSqlTypeMapping_SQLite.md | 176 +++++ 11 files changed, 1673 insertions(+), 78 deletions(-) create mode 100644 dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_DuckDB.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_H2.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_MariaDB.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_MsSql.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_MySQL.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_PostgreSQL.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_SQLite.md diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt new file mode 100644 index 0000000000..e84e1b3ebe --- /dev/null +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt @@ -0,0 +1,129 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +import org.junit.Ignore +import org.junit.Test +import java.io.File +import java.nio.file.Files +import java.sql.DriverManager +import java.sql.Types + +/** + * Probe used to figure out what the Xerial SQLite JDBC driver reports in the metadata + * for common declared column types (SQLite type affinity is applied at storage level, but + * the driver preserves the declared type name in `sqlTypeName`). + * + * Not part of the regular suite; enable ad-hoc if you need to refresh the mapping. + */ +@Ignore("Investigation probe — not a real test") +class SqliteMetadataProbe { + + @Test + fun `dump getObject class vs metadata jdbcType for problematic types`() { + val file = Files.createTempFile("dataframe_sqlite_probe2_", ".db").toFile() + file.deleteOnExit() + val url = "jdbc:sqlite:${file.absolutePath}" + + // (declaredType, sql-literal, description) + val cases = listOf( + Triple("BOOLEAN", "1", "boolean-as-int"), + Triple("BOOLEAN", "0", "boolean-as-int-false"), + Triple("DATE", "'2020-01-15'", "date-as-iso-text"), + Triple("DATE", "1579046400", "date-as-unix-int"), + Triple("DATETIME", "'2020-01-15 10:30:00'", "datetime-as-iso-text"), + Triple("TIMESTAMP", "'2020-01-15T10:30:00Z'", "timestamp-as-iso-text"), + Triple("TIMESTAMP", "1579083000", "timestamp-as-unix-int"), + Triple("NUMERIC", "1.5", "numeric-as-real"), + Triple("NUMERIC", "42", "numeric-as-int"), + Triple("DECIMAL(10,5)", "1.5", "decimal-as-real"), + Triple("DATE", "2460146.5", "date-as-julian-real"), + Triple("TIMESTAMP", "2460146.5", "timestamp-as-julian-real"), + ) + + DriverManager.getConnection(url).use { conn -> + cases.forEachIndexed { i, (decl, literal, _) -> + conn.createStatement().execute("CREATE TABLE t$i (col $decl)") + conn.createStatement().execute("INSERT INTO t$i (col) VALUES ($literal)") + } + for ((i, case) in cases.withIndex()) { + val (decl, literal, desc) = case + conn.prepareStatement("SELECT col FROM t$i").executeQuery().use { rs -> + val md = rs.metaData + val jdbc = md.getColumnType(1) + val jdbcName = jdbcName(jdbc) + val cls = md.getColumnClassName(1) + rs.next() + val got = rs.getObject(1) + val gotClass = got?.javaClass?.name + println( + "%-25s declared=%-14s literal=%-24s meta.jdbc=%-9s meta.class=%-22s getObject.class=%s value=%s".format( + desc, decl, literal, jdbcName, cls, gotClass, got, + ), + ) + } + } + } + file.delete() + } + + @Test + fun `dump metadata for common declared types`() { + val file = Files.createTempFile("dataframe_sqlite_probe_", ".db").toFile() + file.deleteOnExit() + val url = "jdbc:sqlite:${file.absolutePath}" + + val declaredTypes = listOf( + "INTEGER", "INT", "TINYINT", "SMALLINT", "MEDIUMINT", "BIGINT", "UNSIGNED BIG INT", + "INT2", "INT8", + "REAL", "DOUBLE", "DOUBLE PRECISION", "FLOAT", + "NUMERIC", "DECIMAL(10,5)", + "BOOLEAN", + "DATE", "DATETIME", "TIMESTAMP", + "TEXT", "VARCHAR(255)", "CHAR(10)", "CLOB", "NVARCHAR(10)", "NCHAR(10)", + "BLOB", + "", // no declared type + "CUSTOM_TYPE", // unknown → NUMERIC affinity + ) + + DriverManager.getConnection(url).use { conn -> + declaredTypes.forEachIndexed { i, decl -> + val col = "col$i ${if (decl.isEmpty()) "" else decl}".trim() + conn.createStatement().execute("CREATE TABLE t$i ($col)") + // Insert one representative value so the driver can infer the actual storage class. + conn.createStatement().execute("INSERT INTO t$i (col$i) VALUES (${sampleValueFor(decl)})") + } + for ((i, decl) in declaredTypes.withIndex()) { + conn.prepareStatement("SELECT col$i FROM t$i").executeQuery().use { rs -> + val md = rs.metaData + val jdbc = md.getColumnType(1) + val jdbcName = jdbcName(jdbc) + val typeName = md.getColumnTypeName(1) + val cls = md.getColumnClassName(1) + println( + "declared=%-24s -> sqlTypeName=%-16s jdbcType=%-8s (%s) classNm=%s".format( + "\"$decl\"", typeName, jdbc.toString(), jdbcName, cls, + ), + ) + } + } + } + file.delete() + } + + private fun jdbcName(t: Int): String = + Types::class.java.fields.firstOrNull { it.getInt(null) == t }?.name ?: "?" + + private fun sampleValueFor(decl: String): String { + val u = decl.uppercase() + return when { + u.contains("BLOB") -> "x'00'" + u.contains("BOOLEAN") -> "1" + u.contains("DATE") || u.contains("TIMESTAMP") || u.contains("TIME") -> "'2020-01-01'" + u.contains("CHAR") || u.contains("TEXT") || u.contains("CLOB") -> "'x'" + u.contains("REAL") || u.contains("DOUB") || u.contains("FLOA") || + u.contains("NUMERIC") || u.contains("DECIMAL") -> "1.5" + u.contains("INT") -> "1" + u.isEmpty() -> "1" + else -> "'x'" + } + } +} diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt index 56758ec27b..2b86bd102c 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt @@ -3,113 +3,587 @@ package org.jetbrains.kotlinx.dataframe.io.db import io.kotest.matchers.shouldBe -import org.jetbrains.kotlinx.dataframe.io.db.JdbcTypesTest.MySqlDBTypes.BIGINT_UNSIGNED import org.junit.Test import org.junit.experimental.runners.Enclosed import org.junit.runner.RunWith +import org.postgresql.geometric.PGbox +import org.postgresql.geometric.PGcircle +import org.postgresql.geometric.PGline +import org.postgresql.geometric.PGlseg +import org.postgresql.geometric.PGpath +import org.postgresql.geometric.PGpoint +import org.postgresql.geometric.PGpolygon +import org.postgresql.util.PGInterval +import org.postgresql.util.PGmoney +import java.math.BigDecimal import java.math.BigInteger +import java.sql.Blob +import java.sql.Clob +import java.sql.NClob +import java.sql.Ref +import java.sql.RowId +import java.sql.SQLXML +import java.sql.Time +import java.sql.Types +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.util.Date import kotlin.reflect.KType +import kotlin.reflect.full.withNullability import kotlin.reflect.typeOf +import kotlin.time.Instant +import kotlin.uuid.Uuid +import kotlinx.datetime.LocalDate as KotlinLocalDate +import kotlinx.datetime.LocalDateTime as KotlinLocalDateTime +import kotlinx.datetime.LocalTime as KotlinLocalTime -// TODO: complete and enhance (#1736) +/** + * Non-integration tests for [DbType.getExpectedJdbcType] and related type-mapping logic. + * + * Each DB owns a [TypeMapping] list that acts as the source of truth for its SQL → Kotlin type + * mapping. The list is exercised for both nullable and non-nullable columns. + * + * See https://github.com/Kotlin/dataframe/issues/1736. + */ @RunWith(Enclosed::class) class JdbcTypesTest { - abstract class ColumnType( - val sqlTypeName: String, - val jdbcType: Int, - val javaClassName: String, - val isNullable: Boolean, - val expectedKotlinType: KType, - ) { - fun mockkColMetaData() = - TableColumnMetadata( - "name", - sqlTypeName, - jdbcType, - 10, - javaClassName, - isNullable, - ) + class DefaultDbTypeTypes { + + // A concrete DbType whose behavior is exactly the default one from the base class. + private object DefaultDbType : DbType("default") { + override val driverClassName: String get() = "does.not.matter" + + override fun isSystemTable(tableMetadata: TableMetadata): Boolean = false + + override fun buildTableMetadata(tables: java.sql.ResultSet): TableMetadata = + TableMetadata("t", null, null) + } + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(DefaultDbType, commonJdbcTypeMappings) + } + + @Test + fun `TIMESTAMP with LocalDateTime driver class maps to java_time_LocalDateTime`() { + assertMappings(DefaultDbType, listOf(timestampAsLocalDateTime)) + } + + @Test + fun `BINARY with UUID driver class maps to UUID`() { + assertMappings(DefaultDbType, listOf(binaryAsUuid)) + } + + @Test + fun `Types_OTHER with byte array javaClassName maps to ByteArray`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "[B", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `Types_OTHER with generic Object maps to Any`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + } } - class MariaDBTypes { + class MariaDbTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MariaDb, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MariaDB-specific overrides`() { + assertMappings(MariaDb, mariaDbSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MariaDb.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MariaDb) } } - class MySqlDBTypes { + class MySqlTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MySql, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MySQL-specific overrides`() { + assertMappings(MySql, mySqlSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MySql.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MySql) } } + /** + * SQLite is dynamically typed: it has only 5 storage classes (NULL, INTEGER, REAL, TEXT, BLOB) + * and picks one per row based on the value, guided by "type affinity" derived from the + * declared column type. The Xerial JDBC driver reports metadata based on the actual stored + * value, so `getExpectedJdbcType` sees driver-specific `jdbcType` and `javaClassName` combos + * that differ from other databases. The tests below reflect that. + */ class SqliteTypes { - // Taken from #964 - - object LONGVARCHAR_1 : ColumnType( - "LONGVARCHAR", - -2, - "java.lang.Object", - false, - typeOf(), - ) - - object LONGVARCHAR_2 : ColumnType( - "LONGVARCHAR", - 12, - "java.lang.String", - true, - typeOf(), - ) - - val customTypes: List = listOf( - LONGVARCHAR_1, - LONGVARCHAR_2, - ) - - @Test - fun `SQLite custom types`() { - val sqliteCustom = Sqlite( - mapOf("LONGVARCHAR" to typeOf()), + @Test + fun `INTEGER affinity — declared int-like types map to Int or Long`() { + assertMappings(Sqlite.default, sqliteIntegerAffinityMappings) + } + + @Test + fun `REAL affinity — declared real-like types map to Double`() { + assertMappings(Sqlite.default, sqliteRealAffinityMappings) + } + + @Test + fun `TEXT affinity — declared text-like types map to String`() { + assertMappings(Sqlite.default, sqliteTextAffinityMappings) + } + + @Test + fun `BLOB affinity — declared BLOB maps to ByteArray`() { + assertMappings(Sqlite.default, sqliteBlobAffinityMappings) + } + + @Test + fun `NUMERIC affinity — declared numeric-like types map by declared type`() { + assertMappings(Sqlite.default, sqliteNumericAffinityMappings) + } + + @Test + fun `BOOLEAN declared type resolves to Boolean`() { + // Xerial reports Types.BOOLEAN metadata even though values are stored as INTEGER; + // the schema type is Boolean, and `getValueFromResultSet` converts each row via + // `rs.getBoolean` (see Sqlite.getValueFromResultSet). + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "BOOLEAN", + jdbcType = Types.BOOLEAN, + javaClassName = "java.lang.Integer", + isNullable = false, + ), + ) shouldBe typeOf() + + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "BOOLEAN", + jdbcType = Types.BOOLEAN, + javaClassName = "java.lang.Integer", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unrecognised declared type is treated by NUMERIC affinity`() { + // For an unknown declared type Xerial applies NUMERIC affinity and reports the + // jdbcType of the actual stored value; for a text sample that is VARCHAR. + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "CUSTOM_TYPE", + jdbcType = Types.VARCHAR, + javaClassName = "java.lang.String", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `customTypesMap overrides the default mapping by SQL type name`() { + val custom = Sqlite( + customTypesMap = mapOf( + "INTEGER" to typeOf(), + "MY_TYPE" to typeOf(), + ), + ) + // INTEGER is normally Int, but is overridden to Long + custom.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "INTEGER", + jdbcType = Types.INTEGER, + javaClassName = "java.lang.Integer", + isNullable = true, + ), + ) shouldBe typeOf() + // Custom type name is respected as-is + custom.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MY_TYPE", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(Sqlite.default) + } + } + + class PostgreSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(PostgreSql, commonJdbcTypeMappings) + } + + @Test + fun `PGobject types map to their PGobject Kotlin types`() { + assertMappings(PostgreSql, postgreSqlSpecificMappings) + } + + @Test + fun `PGobject lookup is case-insensitive`() { + PostgreSql.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "POINT", + jdbcType = Types.OTHER, + javaClassName = "org.postgresql.geometric.PGpoint", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(PostgreSql) + } + } + + class MsSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MsSql, commonJdbcTypeMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MsSql) + } + } + + class H2Types { + + @Test + fun `Regular mode uses default type mappings`() { + assertMappings(H2(H2.Mode.Regular), commonJdbcTypeMappings) + } + + @Test + fun `MySql mode delegates to MySQL-specific overrides`() { + assertMappings(H2(H2.Mode.MySql), mySqlSpecificMappings) + } + + @Test + fun `MariaDb mode delegates to MariaDB-specific overrides`() { + assertMappings(H2(H2.Mode.MariaDb), mariaDbSpecificMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(H2(H2.Mode.Regular)) + } + } +} + +// -------------------- Type mapping model & helpers -------------------- + +/** + * A single row in the JDBC → Kotlin type mapping table. + * + * @property sqlTypeName the human-readable SQL type name (e.g. "BIGINT") + * @property jdbcType a constant from [java.sql.Types] + * @property javaClassName the JDBC-reported class name for this column (as returned by + * [java.sql.ResultSetMetaData.getColumnClassName]) + * @property expectedType the expected non-nullable Kotlin type + */ +internal data class TypeMapping( + val sqlTypeName: String, + val jdbcType: Int, + val javaClassName: String, + val expectedType: KType, +) + +internal const val UNKNOWN_JDBC_TYPE: Int = -9999 + +/** + * Test helper that constructs a [TableColumnMetadata] with sensible defaults. + * Not a mock — a lightweight factory to keep test call sites readable. + */ +internal fun createColumnMetadata( + name: String = "col", + sqlTypeName: String, + jdbcType: Int, + size: Int = 10, + javaClassName: String, + isNullable: Boolean, +): TableColumnMetadata = + TableColumnMetadata( + name = name, + sqlTypeName = sqlTypeName, + jdbcType = jdbcType, + size = size, + javaClassName = javaClassName, + isNullable = isNullable, + ) + +/** + * Verifies each mapping resolves correctly for both nullable and non-nullable columns. + * Runs the full type-resolution pipeline (`getExpectedJdbcType` → `getPreprocessedValueType`) + * and compares against the **final DataFrame column type**, which is what the reference + * documentation describes. + */ +internal fun assertMappings(dbType: DbType, mappings: List) { + mappings.forEach { m -> + listOf(false, true).forEach { isNullable -> + val meta = createColumnMetadata( + sqlTypeName = m.sqlTypeName, + jdbcType = m.jdbcType, + javaClassName = m.javaClassName, + isNullable = isNullable, ) - customTypes.forEach { type -> - sqliteCustom.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + val jdbcType = dbType.getExpectedJdbcType(meta) + val finalType = dbType.getPreprocessedValueType(meta, jdbcType) + finalType shouldBe m.expectedType.withNullability(isNullable) } } } + +internal fun assertUnknownMapsToString(dbType: DbType) { + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() +} + +// -------------------- Type mapping tables -------------------- + +/** + * The default SQL → Kotlin type mapping applied by [DbType]. + * Every DB that does not override the given entry falls through to this table. + */ +internal val commonJdbcTypeMappings: List = listOf( + TypeMapping("BIT", Types.BIT, "java.lang.Boolean", typeOf()), + TypeMapping("TINYINT", Types.TINYINT, "java.lang.Integer", typeOf()), + TypeMapping("SMALLINT", Types.SMALLINT, "java.lang.Integer", typeOf()), + TypeMapping("INTEGER", Types.INTEGER, "java.lang.Integer", typeOf()), + TypeMapping("BIGINT", Types.BIGINT, "java.lang.Long", typeOf()), + TypeMapping("FLOAT", Types.FLOAT, "java.lang.Float", typeOf()), + TypeMapping("REAL", Types.REAL, "java.lang.Float", typeOf()), + TypeMapping("DOUBLE", Types.DOUBLE, "java.lang.Double", typeOf()), + TypeMapping("NUMERIC", Types.NUMERIC, "java.math.BigDecimal", typeOf()), + TypeMapping("DECIMAL", Types.DECIMAL, "java.math.BigDecimal", typeOf()), + TypeMapping("CHAR", Types.CHAR, "java.lang.String", typeOf()), + TypeMapping("VARCHAR", Types.VARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGVARCHAR", Types.LONGVARCHAR, "java.lang.String", typeOf()), + TypeMapping("NCHAR", Types.NCHAR, "java.lang.String", typeOf()), + TypeMapping("NVARCHAR", Types.NVARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGNVARCHAR", Types.LONGNVARCHAR, "java.lang.String", typeOf()), + TypeMapping("DATE", Types.DATE, "java.sql.Date", typeOf()), + TypeMapping("TIME", Types.TIME, "java.sql.Time", typeOf