diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt index 778bd02539..7e96c5fb1d 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt @@ -357,6 +357,11 @@ public class Sqlite( Types.DECIMAL, Types.NUMERIC -> return jdbcToDfConverterFor(expectedKType) } + // 5) CLOB — stored as String. + if ("CLOB" in declaredUpper) { + return jdbcToDfConverterFor(expectedKType) + } + // 5) Fallback — delegate to the base [DbType] end-to-end pipeline. return fallbackConverter(tableColumnMetadata) } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt index e77ee3fff9..fa5a82ef2a 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt @@ -6,113 +6,598 @@ import io.kotest.matchers.shouldBe import org.junit.Test import org.junit.experimental.runners.Enclosed import org.junit.runner.RunWith +import org.postgresql.geometric.PGbox +import org.postgresql.geometric.PGcircle +import org.postgresql.geometric.PGline +import org.postgresql.geometric.PGlseg +import org.postgresql.geometric.PGpath +import org.postgresql.geometric.PGpoint +import org.postgresql.geometric.PGpolygon +import org.postgresql.util.PGInterval +import org.postgresql.util.PGmoney +import java.math.BigDecimal import java.math.BigInteger +import java.sql.Blob +import java.sql.Clob +import java.sql.NClob +import java.sql.Ref +import java.sql.RowId +import java.sql.SQLXML +import java.sql.Time +import java.sql.Types +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.util.Date import kotlin.reflect.KType +import kotlin.reflect.full.withNullability import kotlin.reflect.typeOf +import kotlin.time.Instant +import kotlin.uuid.Uuid +import kotlinx.datetime.LocalDate as KotlinLocalDate +import kotlinx.datetime.LocalDateTime as KotlinLocalDateTime +import kotlinx.datetime.LocalTime as KotlinLocalTime -// TODO: complete and enhance (#1736) +/** + * Non-integraion tests for [DbType.getExpectedJdbcType] and related type-mapping logic. + * + * For each DB type implemented in Kotlin Dataframe, checks mapping between the all of SQL types and their + * expected types for a [column][org.jetbrains.kotlinx.dataframe.DataColumn]. + * + * Each DB owns a [TypeMapping] list that acts as the source of truth for its SQL → Kotlin type + * mapping. The list is exercised for both nullable and non-nullable columns. + * + * Note that for all DBs (except SQLite), drivers report canonical name, erasing type aliases. + */ @RunWith(Enclosed::class) class JdbcTypesTest { - abstract class ColumnType( - val sqlTypeName: String, - val jdbcType: Int, - val javaClassName: String, - val isNullable: Boolean, - val expectedKotlinType: KType, - ) { - fun mockkColMetaData() = - TableColumnMetadata( - "name", - sqlTypeName, - jdbcType, - 10, - javaClassName, - isNullable, - ) + class DefaultDbTypeTypes { + + // A concrete DbType whose behavior is exactly the default one from the base class. + private object DefaultDbType : DbType("default") { + override val driverClassName: String get() = "does.not.matter" + + override fun isSystemTable(tableMetadata: TableMetadata): Boolean = false + + override fun buildTableMetadata(tables: java.sql.ResultSet): TableMetadata = TableMetadata("t", null, null) + } + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(DefaultDbType, commonJdbcTypeMappings) + } + + @Test + fun `TIMESTAMP with LocalDateTime driver class maps to java_time_LocalDateTime`() { + assertMappings(DefaultDbType, listOf(timestampAsLocalDateTime)) + } + + @Test + fun `BINARY with UUID driver class maps to UUID`() { + assertMappings(DefaultDbType, listOf(binaryAsUuid)) + } + + @Test + fun `Types_OTHER with byte array javaClassName maps to ByteArray`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "[B", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `Types_OTHER with generic Object maps to Any`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + } } - class MariaDBTypes { + class MariaDbTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MariaDb, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MariaDB-specific overrides`() { + assertMappings(MariaDb, mariaDbSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MariaDb.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MariaDb) } } - class MySqlDBTypes { + class MySqlTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MySql, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MySQL-specific overrides`() { + assertMappings(MySql, mySqlSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MySql.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MySql) } } + /** + * SQLite is dynamically typed: it has only 5 storage classes (NULL, INTEGER, REAL, TEXT, BLOB) + * and picks one per row based on the value, guided by "type affinity" derived from the + * declared column type. The Xerial JDBC driver reports metadata based on the actual stored + * value, so `getExpectedJdbcType` sees driver-specific `jdbcType` and `javaClassName` combos + * that differ from other databases. The tests below reflect that. + */ class SqliteTypes { - // Taken from #964 - - object LONGVARCHAR_NON_NULL : ColumnType( - "LONGVARCHAR", - -2, - "java.lang.Object", - false, - typeOf(), - ) - - object LONGVARCHAR_NULLABLE : ColumnType( - "LONGVARCHAR", - 12, - "java.lang.String", - true, - typeOf(), - ) - - @Test - fun `identity forType with non-null T resolves to non-null String`() { - // User's KType is authoritative; column nullability is not applied on top of it. - val sqliteCustom = Sqlite.withCustomConverters { - forType("LONGVARCHAR") + @Test + fun `INTEGER affinity — declared int-like types map to Int or Long`() { + assertMappings(Sqlite.default, sqliteIntegerAffinityMappings) + } + + @Test + fun `REAL affinity — declared real-like types map to Double`() { + assertMappings(Sqlite.default, sqliteRealAffinityMappings) + } + + @Test + fun `TEXT affinity — declared text-like types map to String`() { + assertMappings(Sqlite.default, sqliteTextAffinityMappings) + } + + @Test + fun `BLOB affinity — declared BLOB maps to ByteArray`() { + assertMappings(Sqlite.default, sqliteBlobAffinityMappings) + } + + @Test + fun `NUMERIC affinity — declared numeric-like types map by declared type`() { + assertMappings(Sqlite.default, sqliteNumericAffinityMappings) + } + + @Test + fun `BOOLEAN declared type resolves to Boolean`() { + // SQLite has no native boolean storage — values are kept as INTEGER (0/1). The Xerial + // driver still reports Types.BOOLEAN in metadata, but `rs.getObject` returns Integer. + // The final DataFrame column type is `Boolean`, produced by a preprocessor that runs + // `convertToBoolean` on each raw value (see `generateConverter`). + assertMappings( + Sqlite.default, + listOf( + TypeMapping("BOOLEAN", Types.BOOLEAN, "java.lang.Integer", typeOf()), + ), + ) + } + + @Test + fun `unrecognised declared type is treated by NUMERIC affinity`() { + // For an unknown declared type Xerial applies NUMERIC affinity and reports the + // jdbcType of the actual stored value; for a text sample that is VARCHAR. + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "CUSTOM_TYPE", + jdbcType = Types.VARCHAR, + javaClassName = "java.lang.String", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `custom type mapping overrides the default mapping by SQL type name`() { + // The identity `forType(name)` overload pins the resulting DataFrame column type + // to `T`. Nullability is part of `T` (there is no implicit widening from the column's + // isNullable metadata) — declare `T` as nullable explicitly if you want nullable. + val custom = Sqlite.withCustomConverters { + forType("INTEGER") + forType("MY_TYPE") } - sqliteCustom.getExpectedJdbcType(LONGVARCHAR_NON_NULL.mockkColMetaData()) shouldBe - LONGVARCHAR_NON_NULL.expectedKotlinType + // INTEGER is normally Int, but is overridden to Long? — regardless of column nullability. + listOf(false, true).forEach { isNullable -> + val meta = createColumnMetadata( + sqlTypeName = "INTEGER", + jdbcType = Types.INTEGER, + javaClassName = "java.lang.Integer", + isNullable = isNullable, + ) + custom.getPreprocessedValueType(meta, custom.getExpectedJdbcType(meta)) shouldBe typeOf() + } + // Custom type name is respected as-is: `MY_TYPE` -> String + val meta = createColumnMetadata( + sqlTypeName = "MY_TYPE", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = false, + ) + custom.getPreprocessedValueType(meta, custom.getExpectedJdbcType(meta)) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(Sqlite.default) + } + } + + class PostgreSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(PostgreSql, commonJdbcTypeMappings) + } + + @Test + fun `PGobject types map to their PGobject Kotlin types`() { + assertMappings(PostgreSql, postgreSqlSpecificMappings) } @Test - fun `identity forType with nullable T resolves to nullable String`() { - val sqliteCustom = Sqlite.withCustomConverters { - forType("LONGVARCHAR") + fun `PGobject lookup is case-insensitive`() { + PostgreSql.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "POINT", + jdbcType = Types.OTHER, + javaClassName = "org.postgresql.geometric.PGpoint", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unrecognised PGobject types fall back to Any`() { + // Composite / hstore / PostGIS types come through the driver as generic PGobject + // with Types.OTHER. Because they are not in PostgreSql's pgObjectTypes lookup + // table, the mapping falls through to the default DbType handler, which returns + // Any for Types.OTHER + non-`[B` javaClassName. + listOf( + // Anonymous composite / user-defined ROW. + "record", + // hstore extension. + "hstore", + // PostGIS types. + "geometry", + "geography", + ).forEach { typeName -> + PostgreSql.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = typeName, + jdbcType = Types.OTHER, + javaClassName = "org.postgresql.util.PGobject", + isNullable = false, + ), + ) shouldBe typeOf() } - sqliteCustom.getExpectedJdbcType(LONGVARCHAR_NULLABLE.mockkColMetaData()) shouldBe - LONGVARCHAR_NULLABLE.expectedKotlinType + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(PostgreSql) } } + + class MsSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MsSql, commonJdbcTypeMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MsSql) + } + } + + class H2Types { + + @Test + fun `Regular mode uses default type mappings`() { + assertMappings(H2(H2.Mode.Regular), commonJdbcTypeMappings) + } + + @Test + fun `MySql mode delegates to MySQL-specific overrides`() { + assertMappings(H2(H2.Mode.MySql), mySqlSpecificMappings) + } + + @Test + fun `MariaDb mode delegates to MariaDB-specific overrides`() { + assertMappings(H2(H2.Mode.MariaDb), mariaDbSpecificMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(H2(H2.Mode.Regular)) + } + } +} + +// -------------------- Type mapping model & helpers -------------------- + +/** + * JDBC column metadata → Kotlin type mapping. + * Nullability is omitted ([expectedType] should be non-nullable). + * + * @property sqlTypeName the jdbc reported SQL type name (e.g. "BIGINT") + * @property jdbcType the jdbc reported SQL type constant from [java.sql.Types] + * @property javaClassName the JDBC-reported class name for this column (as returned by + * [java.sql.ResultSetMetaData.getColumnClassName]) + * @property expectedType the expected non-nullable Kotlin type + */ +internal data class TypeMapping( + val sqlTypeName: String, + val jdbcType: Int, + val javaClassName: String, + val expectedType: KType, +) + +internal const val UNKNOWN_JDBC_TYPE: Int = -9999 + +/** + * Test helper that constructs a [TableColumnMetadata] with sensible defaults. + * Not a mock — a lightweight factory to keep test call sites readable. + */ +internal fun createColumnMetadata( + name: String = "col", + sqlTypeName: String, + jdbcType: Int, + size: Int = 10, + javaClassName: String, + isNullable: Boolean, +): TableColumnMetadata = + TableColumnMetadata( + name = name, + sqlTypeName = sqlTypeName, + jdbcType = jdbcType, + size = size, + javaClassName = javaClassName, + isNullable = isNullable, + ) + +/** + * Verifies each mapping resolves correctly for both nullable and non-nullable columns. + * Runs the full type-resolution pipeline (`getExpectedJdbcType` → `getPreprocessedValueType`) + * and compares against the **final DataFrame column type**, which is what the reference + * documentation describes. + */ +internal fun assertMappings(dbType: DbType, mappings: List) { + mappings.forEach { m -> + listOf(false, true).forEach { isNullable -> + val meta = createColumnMetadata( + sqlTypeName = m.sqlTypeName, + jdbcType = m.jdbcType, + javaClassName = m.javaClassName, + isNullable = isNullable, + ) + val jdbcType = dbType.getExpectedJdbcType(meta) + val finalType = dbType.getPreprocessedValueType(meta, jdbcType) + finalType shouldBe m.expectedType.withNullability(isNullable) + } + } +} + +internal fun assertUnknownMapsToString(dbType: DbType) { + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() } + +// -------------------- Type mapping tables -------------------- + +/** + * The default SQL → Kotlin type mapping applied by [DbType]. + * Every DB that does not override the given entry falls through to this table. + */ +internal val commonJdbcTypeMappings: List = listOf( + TypeMapping("BIT", Types.BIT, "java.lang.Boolean", typeOf()), + TypeMapping("TINYINT", Types.TINYINT, "java.lang.Integer", typeOf()), + TypeMapping("SMALLINT", Types.SMALLINT, "java.lang.Integer", typeOf()), + TypeMapping("INTEGER", Types.INTEGER, "java.lang.Integer", typeOf()), + TypeMapping("BIGINT", Types.BIGINT, "java.lang.Long", typeOf()), + TypeMapping("FLOAT", Types.FLOAT, "java.lang.Float", typeOf()), + TypeMapping("REAL", Types.REAL, "java.lang.Float", typeOf()), + TypeMapping("DOUBLE", Types.DOUBLE, "java.lang.Double", typeOf()), + TypeMapping("NUMERIC", Types.NUMERIC, "java.math.BigDecimal", typeOf()), + TypeMapping("DECIMAL", Types.DECIMAL, "java.math.BigDecimal", typeOf()), + TypeMapping("CHAR", Types.CHAR, "java.lang.String", typeOf()), + TypeMapping("VARCHAR", Types.VARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGVARCHAR", Types.LONGVARCHAR, "java.lang.String", typeOf()), + TypeMapping("NCHAR", Types.NCHAR, "java.lang.String", typeOf()), + TypeMapping("NVARCHAR", Types.NVARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGNVARCHAR", Types.LONGNVARCHAR, "java.lang.String", typeOf()), + TypeMapping("DATE", Types.DATE, "java.sql.Date", typeOf()), + TypeMapping("TIME", Types.TIME, "java.sql.Time", typeOf