diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml index 60d5d7b364..f0bb6a13b7 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml @@ -408,6 +408,17 @@ ${assertj.version} test + + + + org.junit.platform + junit-platform-launcher + test + @@ -416,12 +427,6 @@ org.apache.maven.plugins maven-surefire-plugin - - - listener - org.apache.amoro.listener.AmoroRunListener - - -verbose:class diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java index dbf8238e77..79d03b98d2 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java @@ -23,12 +23,19 @@ import static org.apache.flink.table.api.config.TableConfigOptions.TABLE_DYNAMIC_TABLE_OPTIONS_ENABLED; import org.apache.amoro.BasicTableTestHelper; +import org.apache.amoro.MockAmoroManagementServer; +import org.apache.amoro.TableFormat; import org.apache.amoro.TableTestHelper; +import org.apache.amoro.TestAms; +import org.apache.amoro.UnifiedCatalog; +import org.apache.amoro.api.CatalogMeta; import org.apache.amoro.catalog.CatalogTestHelper; -import org.apache.amoro.catalog.TableTestBase; import org.apache.amoro.flink.catalog.factories.CatalogFactoryOptions; import org.apache.amoro.flink.write.MixedFormatRowDataTaskWriterFactory; import org.apache.amoro.io.reader.GenericKeyedDataReader; +import org.apache.amoro.mixed.CatalogLoader; +import org.apache.amoro.mixed.MixedFormatCatalog; +import org.apache.amoro.properties.CatalogMetaProperties; import org.apache.amoro.scan.CombinedScanTask; import org.apache.amoro.scan.KeyedTableScanTask; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; @@ -37,6 +44,14 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.table.KeyedTable; +import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableBuilder; +import org.apache.amoro.table.TableMetaStore; +import org.apache.amoro.table.UnkeyedTable; +import org.apache.amoro.utils.CatalogUtil; +import org.apache.amoro.utils.MixedTableUtil; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.state.StateBackend; @@ -67,13 +82,15 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.io.WriteResult; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.nio.file.Files; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.Arrays; @@ -84,14 +101,24 @@ import java.util.Set; import java.util.concurrent.ExecutionException; -public class FlinkTestBase extends TableTestBase { +/** + * JUnit 5 base class for Flink integration tests in this module. + * + *

This class intentionally no longer extends {@code TableTestBase}/{@code CatalogTestBase} + * (which remain on JUnit 4 until the closing PR of the umbrella migration). The catalog/table + * lifecycle is re-implemented here against the same {@link CatalogTestHelper}/{@link + * TableTestHelper} contracts so that children can stay clean Jupiter classes; for parameterized + * children that cannot pass helpers through a constructor, call {@link + * #initFlinkTestBase(CatalogTestHelper, TableTestHelper)} from the {@code @ParameterizedTest} + * method body. + */ +public class FlinkTestBase { private static final Logger LOG = LoggerFactory.getLogger(FlinkTestBase.class); - @ClassRule - public static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = - MiniClusterResource.createWithClassloaderCheckDisabled(); + protected static final TestAms TEST_AMS = new TestAms(); - @Rule public TestName name = new TestName(); + protected static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = + MiniClusterResource.createWithClassloaderCheckDisabled(); public static String metastoreUri; @@ -113,14 +140,228 @@ public class FlinkTestBase extends TableTestBase { public static InternalCatalogBuilder catalogBuilder; + private CatalogTestHelper catalogTestHelper; + private TableTestHelper tableTestHelper; + private CatalogMeta catalogMeta; + private MixedFormatCatalog mixedFormatCatalog; + private UnifiedCatalog unifiedCatalog; + private org.apache.iceberg.catalog.Catalog icebergCatalog; + private MixedTable mixedTable; + private TableMetaStore tableMetaStore; + private File tempRoot; + + /** + * No-arg constructor for parameterized children that pass helpers via {@link #initFlinkTestBase}. + */ + public FlinkTestBase() {} + public FlinkTestBase(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { - super(catalogTestHelper, tableTestHelper); + this.catalogTestHelper = catalogTestHelper; + this.tableTestHelper = tableTestHelper; + } + + @BeforeAll + public static void startFlinkBaseClassResources() throws Exception { + TEST_AMS.before(); + MINI_CLUSTER_RESOURCE.before(); } - @Before - public void before() throws Exception { + @AfterAll + public static void stopFlinkBaseClassResources() { + try { + MINI_CLUSTER_RESOURCE.after(); + } finally { + TEST_AMS.after(); + } + } + + @BeforeEach + public void setUpFlinkTestBaseLifecycle() throws Exception { + if (catalogTestHelper == null) { + // Parameterized child: the @ParameterizedTest body must call initFlinkTestBase(...). + return; + } + initLifecycle(); + } + + @AfterEach + public void tearDownFlinkTestBaseLifecycle() { + teardownLifecycle(); + } + + /** + * Initializer used by {@code @ParameterizedTest} children. Sets the helpers (since the parameters + * are received by the test method, not the constructor) and runs the catalog/table setup. + */ + protected void initFlinkTestBase( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + this.catalogTestHelper = catalogTestHelper; + this.tableTestHelper = tableTestHelper; + initLifecycle(); + } + + private void initLifecycle() throws Exception { + tempRoot = Files.createTempDirectory("flink-test-base").toFile(); + String baseDir = tempRoot.getPath(); + if (!SystemUtils.IS_OS_UNIX) { + baseDir = "file:/" + baseDir.replace("\\", "/"); + } + catalogMeta = catalogTestHelper.buildCatalogMeta(baseDir); + catalogMeta.putToCatalogProperties(CatalogMetaProperties.AMS_URI, TEST_AMS.getServerUrl()); + getAmsHandler().createCatalog(catalogMeta); metastoreUri = getCatalogUri(); catalogBuilder = InternalCatalogBuilder.builder().amsUri(metastoreUri); + if (tableTestHelper != null) { + createTestTable(); + } + } + + private void teardownLifecycle() { + if (tableTestHelper != null && unifiedCatalog != null) { + try { + unifiedCatalog.dropTable( + tableTestHelper.id().getDatabase(), tableTestHelper.id().getTableName(), true); + } catch (Exception e) { + LOG.warn("dropTable failed", e); + } + try { + unifiedCatalog.dropDatabase(TableTestHelper.TEST_DB_NAME); + } catch (Exception e) { + // ignore + } + } + if (catalogMeta != null) { + try { + getAmsHandler().dropCatalog(catalogMeta.getCatalogName()); + } catch (Exception e) { + LOG.warn("dropCatalog failed", e); + } + } + if (tempRoot != null) { + try { + FileUtils.deleteDirectory(tempRoot); + } catch (Exception e) { + LOG.warn("Failed to clean temp directory {}", tempRoot, e); + } + } + catalogMeta = null; + mixedFormatCatalog = null; + unifiedCatalog = null; + icebergCatalog = null; + mixedTable = null; + tableMetaStore = null; + tempRoot = null; + } + + private void createTestTable() { + this.tableMetaStore = CatalogUtil.buildMetaStore(getCatalogMeta()); + getUnifiedCatalog().createDatabase(TableTestHelper.TEST_DB_NAME); + TableFormat format = getTestFormat(); + if (format.in(TableFormat.MIXED_HIVE, TableFormat.MIXED_ICEBERG)) { + createMixedFormatTable(); + } else if (TableFormat.ICEBERG.equals(format)) { + createIcebergFormatTable(); + } + } + + private void createMixedFormatTable() { + TableBuilder tableBuilder = + getMixedFormatCatalog() + .newTableBuilder(TableTestHelper.TEST_TABLE_ID, tableTestHelper.tableSchema()); + tableBuilder.withProperties(tableTestHelper.tableProperties()); + if (isKeyedTable()) { + tableBuilder.withPrimaryKeySpec(tableTestHelper.primaryKeySpec()); + } + if (isPartitionedTable()) { + tableBuilder.withPartitionSpec(tableTestHelper.partitionSpec()); + } + mixedTable = tableBuilder.create(); + } + + private void createIcebergFormatTable() { + getIcebergCatalog() + .createTable( + org.apache.iceberg.catalog.TableIdentifier.of( + TableTestHelper.TEST_DB_NAME, TableTestHelper.TEST_TABLE_NAME), + tableTestHelper.tableSchema(), + tableTestHelper.partitionSpec(), + tableTestHelper.tableProperties()); + mixedTable = + (MixedTable) + getUnifiedCatalog() + .loadTable(TableTestHelper.TEST_DB_NAME, TableTestHelper.TEST_TABLE_NAME) + .originalTable(); + } + + public static MockAmoroManagementServer.AmsHandler getAmsHandler() { + return TEST_AMS.getAmsHandler(); + } + + protected MixedFormatCatalog getMixedFormatCatalog() { + if (mixedFormatCatalog == null) { + mixedFormatCatalog = CatalogLoader.load(getCatalogUri()); + } + return mixedFormatCatalog; + } + + protected void refreshMixedFormatCatalog() { + this.mixedFormatCatalog = CatalogLoader.load(getCatalogUri()); + } + + protected String getCatalogUri() { + return TEST_AMS.getServerUrl() + "/" + catalogMeta.getCatalogName(); + } + + protected CatalogMeta getCatalogMeta() { + return catalogMeta; + } + + protected TableFormat getTestFormat() { + return catalogTestHelper.tableFormat(); + } + + protected org.apache.iceberg.catalog.Catalog getIcebergCatalog() { + if (icebergCatalog == null) { + icebergCatalog = catalogTestHelper.buildIcebergCatalog(catalogMeta); + } + return icebergCatalog; + } + + protected UnifiedCatalog getUnifiedCatalog() { + if (unifiedCatalog == null) { + unifiedCatalog = catalogTestHelper.buildUnifiedCatalog(catalogMeta); + } + return unifiedCatalog; + } + + protected MixedTable getMixedTable() { + return mixedTable; + } + + protected UnkeyedTable getBaseStore() { + return MixedTableUtil.baseStore(mixedTable); + } + + protected TableMetaStore getTableMetaStore() { + return this.tableMetaStore; + } + + protected boolean isKeyedTable() { + return tableTestHelper.primaryKeySpec() != null + && tableTestHelper.primaryKeySpec().primaryKeyExisted(); + } + + protected boolean isPartitionedTable() { + return tableTestHelper.partitionSpec() != null + && tableTestHelper.partitionSpec().isPartitioned(); + } + + protected TableTestHelper tableTestHelper() { + return tableTestHelper; + } + + protected CatalogTestHelper catalogTestHelper() { + return catalogTestHelper; } public void config() { @@ -129,6 +370,10 @@ public void config() { props.put(CatalogFactoryOptions.AMS_URI.key(), metastoreUri); } + protected int defaultParallelism() { + return 1; + } + public static void prepare() throws Exception { KAFKA_CONTAINER.start(); } @@ -163,7 +408,7 @@ protected StreamExecutionEnvironment getEnv() { env = StreamExecutionEnvironment.getExecutionEnvironment( MiniClusterResource.DISABLE_CLASSLOADER_CHECK_CONFIG); - env.setParallelism(1); + env.setParallelism(defaultParallelism()); env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE); env.getCheckpointConfig().setCheckpointInterval(300); env.getCheckpointConfig() @@ -307,12 +552,18 @@ protected static void commit(KeyedTable keyedTable, WriteResult result, boolean } } + /** + * 3-arg variant of the keyed task writer factory with a default mask of 3. The 4-arg variant + * lives under a different name ({@link #createKeyedTaskWriterWithMask}) to avoid the inherited + * static signature clashing with the {@code FlinkTaskWriterBaseTest} interface's default method + * in classes that {@code implements} that interface. + */ protected static TaskWriter createKeyedTaskWriter( KeyedTable keyedTable, RowType rowType, boolean base) { - return createKeyedTaskWriter(keyedTable, rowType, base, 3); + return createKeyedTaskWriterWithMask(keyedTable, rowType, base, 3); } - protected static TaskWriter createKeyedTaskWriter( + protected static TaskWriter createKeyedTaskWriterWithMask( KeyedTable keyedTable, RowType rowType, boolean base, long mask) { MixedFormatRowDataTaskWriterFactory taskWriterFactory = new MixedFormatRowDataTaskWriterFactory(keyedTable, rowType, base); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/TestFlinkSchemaUtil.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/TestFlinkSchemaUtil.java index 0eeee1bc9d..62c438285e 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/TestFlinkSchemaUtil.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/TestFlinkSchemaUtil.java @@ -21,8 +21,8 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableSchema; import org.apache.iceberg.Schema; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Map; @@ -55,6 +55,6 @@ public void testFlinkSchemaToIcebergSchema() { TableSchema fromIcebergSchema = FlinkSchemaUtil.toSchema(icebergSchema, new ArrayList<>(), extraOptions); - Assert.assertEquals(flinkSchema, fromIcebergSchema); + Assertions.assertEquals(flinkSchema, fromIcebergSchema); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkAmoroCatalogITCase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkAmoroCatalogITCase.java index 92b2a286a1..69cdb99014 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkAmoroCatalogITCase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkAmoroCatalogITCase.java @@ -27,7 +27,6 @@ import org.apache.amoro.formats.paimon.PaimonHadoopCatalogTestHelper; import org.apache.amoro.formats.paimon.PaimonHiveCatalogTestHelper; import org.apache.amoro.formats.paimon.PaimonTable; -import org.apache.amoro.hive.TestHMS; import org.apache.flink.table.api.TableResult; import org.apache.flink.table.catalog.AbstractCatalog; import org.apache.flink.table.catalog.Catalog; @@ -35,42 +34,48 @@ import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.types.Row; import org.apache.paimon.table.FileStoreTable; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; -/** ITCase for Flink UnifiedCatalog based on AmoroCatalogTestBase */ -@RunWith(value = Parameterized.class) +/** ITCase for Flink UnifiedCatalog based on AmoroCatalogTestBase. */ public class FlinkAmoroCatalogITCase extends AmoroCatalogITCaseBase { - static final TestHMS TEST_HMS = new TestHMS(); AbstractCatalog flinkCatalog; - public FlinkAmoroCatalogITCase(AmoroCatalogTestHelper catalogTestHelper) { - super(catalogTestHelper); + static Stream parameters() { + return Stream.of( + Arguments.of(PaimonHiveCatalogTestHelper.defaultHelper()), + Arguments.of(PaimonHadoopCatalogTestHelper.defaultHelper())); } - @Parameterized.Parameters(name = "{0}") - public static Object[] parameters() { - return new Object[] { - PaimonHiveCatalogTestHelper.defaultHelper(), PaimonHadoopCatalogTestHelper.defaultHelper() - }; + @BeforeAll + public static void startTestHms() throws Exception { + TEST_HMS.before(); } - @BeforeClass - public static void beforeAll() throws Exception { - TEST_HMS.before(); + @AfterAll + public static void stopTestHms() { + TEST_HMS.after(); } - @Before - public void setup() throws Exception { - createDatabase(); - createTable(); + @AfterEach + public void teardown() { + if (flinkCatalog != null) { + flinkCatalog.close(); + } + } + + private void setUpForParam(AmoroCatalogTestHelper catalogTestHelper) throws Exception { + initAmoroCatalog(catalogTestHelper); + catalogTestHelper.createDatabase(TEST_DB_NAME); + catalogTestHelper.createTable(TEST_DB_NAME, TEST_TABLE_NAME); String catalog = "unified_catalog"; exec( "CREATE CATALOG %s WITH ('type'='unified', 'metastore.url'='%s')", @@ -83,32 +88,10 @@ public void setup() throws Exception { assertEquals(catalog, flinkCatalog.getName()); } - @After - public void teardown() { - TEST_HMS.after(); - if (flinkCatalog != null) { - flinkCatalog.close(); - } - } - - public void createDatabase() { - try { - catalogTestHelper.createDatabase(TEST_DB_NAME); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public void createTable() { - try { - catalogTestHelper.createTable(TEST_DB_NAME, TEST_TABLE_NAME); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Test - public void testTableExists() throws Exception { + @ParameterizedTest(name = "{0}") + @MethodSource("parameters") + public void testTableExists(AmoroCatalogTestHelper catalogTestHelper) throws Exception { + setUpForParam(catalogTestHelper); CatalogBaseTable catalogBaseTable = flinkCatalog.getTable(new ObjectPath(TEST_DB_NAME, TEST_TABLE_NAME)); assertNotNull(catalogBaseTable); @@ -120,8 +103,10 @@ public void testTableExists() throws Exception { catalogBaseTable.getUnresolvedSchema().getColumns().size()); } - @Test - public void testInsertAndQuery() throws Exception { + @ParameterizedTest(name = "{0}") + @MethodSource("parameters") + public void testInsertAndQuery(AmoroCatalogTestHelper catalogTestHelper) throws Exception { + setUpForParam(catalogTestHelper); exec("INSERT INTO %s SELECT 1, 'Lily', 1234567890", TEST_TABLE_NAME); TableResult tableResult = exec("select * from %s /*+OPTIONS('monitor-interval'='1s')*/ ", TEST_TABLE_NAME); @@ -132,8 +117,11 @@ public void testInsertAndQuery() throws Exception { assertEquals(Row.of(1, "Lily", 1234567890).toString(), actualRow.toString()); } - @Test - public void testSwitchCurrentCatalog() { + @ParameterizedTest(name = "{0}") + @MethodSource("parameters") + public void testSwitchCurrentCatalog(AmoroCatalogTestHelper catalogTestHelper) + throws Exception { + setUpForParam(catalogTestHelper); String memCatalog = "mem_catalog"; exec("create catalog %s with('type'='generic_in_memory')", memCatalog); exec( diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkUnifiedCatalogITCase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkUnifiedCatalogITCase.java index 6e9a654bf5..756e3776d1 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkUnifiedCatalogITCase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/FlinkUnifiedCatalogITCase.java @@ -34,44 +34,49 @@ import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.ObjectPath; -import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.flink.types.Row; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; -@RunWith(value = Parameterized.class) public class FlinkUnifiedCatalogITCase extends CatalogITCaseBase { static final TestHMS TEST_HMS = new TestHMS(); AbstractCatalog flinkCatalog; TableIdentifier identifier; - public FlinkUnifiedCatalogITCase(CatalogTestHelper catalogTestHelper) { - super(catalogTestHelper, new BasicTableTestHelper(true, false)); + static Stream parameters() { + return Stream.of( + Arguments.of(new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf())), + Arguments.of(new HiveCatalogTestHelper(TableFormat.MIXED_ICEBERG, TEST_HMS.getHiveConf())), + Arguments.of(new HiveCatalogTestHelper(TableFormat.ICEBERG, TEST_HMS.getHiveConf()))); } - @Parameterized.Parameters(name = "catalogTestHelper = {0}") - public static Object[][] parameters() { - return new Object[][] { - {new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf())}, - {new HiveCatalogTestHelper(TableFormat.MIXED_ICEBERG, TEST_HMS.getHiveConf())}, - {new HiveCatalogTestHelper(TableFormat.ICEBERG, TEST_HMS.getHiveConf())} - }; + @BeforeAll + public static void startTestHms() throws Exception { + TEST_HMS.before(); } - @BeforeClass - public static void beforeAll() throws Exception { - TEST_HMS.before(); + @AfterAll + public static void stopTestHms() { + TEST_HMS.after(); + } + + @AfterEach + public void teardown() { + if (flinkCatalog != null) { + flinkCatalog.close(); + } } - @Before - public void setup() throws Exception { + private void setUpForParam(CatalogTestHelper catalogTestHelper) throws Exception { + initCatalogITCase(catalogTestHelper, new BasicTableTestHelper(true, false)); String catalog = "unified_catalog"; exec("CREATE CATALOG %s WITH ('type'='unified', 'ams.uri'='%s')", catalog, getCatalogUri()); exec("USE CATALOG %s", catalog); @@ -83,16 +88,10 @@ public void setup() throws Exception { identifier = tableTestHelper().id(); } - @After - public void teardown() { - TEST_HMS.after(); - if (flinkCatalog != null) { - flinkCatalog.close(); - } - } - - @Test - public void testTableExists() throws TableNotExistException { + @ParameterizedTest(name = "catalogTestHelper = {0}") + @MethodSource("parameters") + public void testTableExists(CatalogTestHelper catalogTestHelper) throws Exception { + setUpForParam(catalogTestHelper); CatalogBaseTable catalogBaseTable = flinkCatalog.getTable(new ObjectPath(identifier.getDatabase(), identifier.getTableName())); assertNotNull(catalogBaseTable); @@ -101,8 +100,10 @@ public void testTableExists() throws TableNotExistException { catalogBaseTable.getUnresolvedSchema().getColumns().size()); } - @Test - public void testInsertAndQuery() throws Exception { + @ParameterizedTest(name = "catalogTestHelper = {0}") + @MethodSource("parameters") + public void testInsertAndQuery(CatalogTestHelper catalogTestHelper) throws Exception { + setUpForParam(catalogTestHelper); exec( "INSERT INTO %s SELECT 1, 'Lily', 1234567890, TO_TIMESTAMP('2020-01-01 01:02:03')", identifier.getTableName()); @@ -116,8 +117,10 @@ public void testInsertAndQuery() throws Exception { Row.of(1, "Lily", 1234567890L, "2020-01-01T01:02:03").toString(), actualRow.toString()); } - @Test - public void testSwitchCurrentCatalog() { + @ParameterizedTest(name = "catalogTestHelper = {0}") + @MethodSource("parameters") + public void testSwitchCurrentCatalog(CatalogTestHelper catalogTestHelper) throws Exception { + setUpForParam(catalogTestHelper); String memCatalog = "mem_catalog"; exec("create catalog %s with('type'='generic_in_memory')", memCatalog); exec( diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalog.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalog.java index 3536511d8f..b19fcb4c9c 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalog.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalog.java @@ -29,16 +29,22 @@ import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_DATA_TYPE; import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_EXPR; +import org.apache.amoro.MockAmoroManagementServer; import org.apache.amoro.TableFormat; import org.apache.amoro.TableTestHelper; +import org.apache.amoro.TestAms; +import org.apache.amoro.api.CatalogMeta; import org.apache.amoro.catalog.BasicCatalogTestHelper; -import org.apache.amoro.catalog.CatalogTestBase; import org.apache.amoro.flink.MiniClusterResource; import org.apache.amoro.flink.catalog.factories.CatalogFactoryOptions; +import org.apache.amoro.mixed.CatalogLoader; +import org.apache.amoro.mixed.MixedFormatCatalog; +import org.apache.amoro.properties.CatalogMetaProperties; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableIdentifier; +import org.apache.commons.io.FileUtils; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.state.StateBackend; @@ -52,18 +58,17 @@ import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.CollectionUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; +import java.io.File; +import java.nio.file.Files; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -74,28 +79,26 @@ /** * Test cases for mixed catalog factories, including: * CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER, - * CatalogFactoryOptions.LEGACY_MIXED_IDENTIFIER + * CatalogFactoryOptions.LEGACY_MIXED_IDENTIFIER. + * + *

This test no longer extends the still-JUnit-4 {@code CatalogTestBase}; the bits of catalog + * lifecycle that were pulled in from there are inlined so the class can be a clean Jupiter test. */ -@RunWith(value = Parameterized.class) -public class TestMixedCatalog extends CatalogTestBase { - private String catalogName; - private String catalogFactoryType; +public class TestMixedCatalog { private static final Logger LOG = LoggerFactory.getLogger(TestMixedCatalog.class); - public TestMixedCatalog(String catalogFactoryType) { - super(new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG)); - this.catalogFactoryType = catalogFactoryType; - this.catalogName = catalogFactoryType + "_catalog"; - } + protected static final TestAms TEST_AMS = new TestAms(); - @Parameterized.Parameters(name = "catalogFactoryType = {0}") - public static Object[] parameters() { - return new Object[] { - CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER - }; - } + private final BasicCatalogTestHelper testHelper = + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG); + + private CatalogMeta catalogMeta; + private MixedFormatCatalog mixedFormatCatalog; + private File tempRoot; + + private String catalogName; + private String catalogFactoryType; - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); protected Map props; private static final String DB = TableTestHelper.TEST_DB_NAME; @@ -103,8 +106,56 @@ public static Object[] parameters() { private volatile StreamExecutionEnvironment env = null; private volatile StreamTableEnvironment tEnv = null; - @Before - public void before() throws Exception { + @BeforeAll + public static void startTestAms() throws Exception { + TEST_AMS.before(); + } + + @AfterAll + public static void stopTestAms() { + TEST_AMS.after(); + } + + @AfterEach + public void afterEach() { + try { + if (catalogName != null) { + sql("DROP TABLE IF EXISTS " + catalogName + "." + DB + "." + TABLE); + sql("USE CATALOG default_catalog"); + sql("DROP DATABASE IF EXISTS " + catalogName + "." + DB); + Assertions.assertTrue( + CollectionUtil.isNullOrEmpty(getMixedFormatCatalog().listDatabases())); + sql("DROP CATALOG " + catalogName); + } + } catch (Throwable t) { + LOG.warn("Test teardown failed", t); + } + if (catalogMeta != null) { + try { + getAmsHandler().dropCatalog(catalogMeta.getCatalogName()); + } catch (Exception e) { + LOG.warn("dropCatalog failed", e); + } + catalogMeta = null; + } + if (tempRoot != null) { + try { + FileUtils.deleteDirectory(tempRoot); + } catch (Exception e) { + // ignore + } + tempRoot = null; + } + mixedFormatCatalog = null; + } + + private void setUpForParam(String catalogFactoryType) throws Exception { + this.catalogFactoryType = catalogFactoryType; + this.catalogName = catalogFactoryType + "_catalog"; + tempRoot = Files.createTempDirectory("test-mixed-catalog").toFile(); + catalogMeta = testHelper.buildCatalogMeta(tempRoot.getPath()); + catalogMeta.putToCatalogProperties(CatalogMetaProperties.AMS_URI, TEST_AMS.getServerUrl()); + getAmsHandler().createCatalog(catalogMeta); props = Maps.newHashMap(); props.put("type", catalogFactoryType); props.put(CatalogFactoryOptions.AMS_URI.key(), getCatalogUri()); @@ -113,27 +164,49 @@ public void before() throws Exception { sql("CREATE DATABASE " + catalogName + "." + DB); } - @After - public void after() { - sql("DROP TABLE IF EXISTS " + catalogName + "." + DB + "." + TABLE); - // Switch away from the catalog before dropping the database. - // Flink 1.19+ rejects dropping the currently active database. - sql("USE CATALOG default_catalog"); - sql("DROP DATABASE IF EXISTS " + catalogName + "." + DB); - Assert.assertTrue(CollectionUtil.isNullOrEmpty(getMixedFormatCatalog().listDatabases())); - sql("DROP CATALOG " + catalogName); + static MockAmoroManagementServer.AmsHandler getAmsHandler() { + return TEST_AMS.getAmsHandler(); + } + + protected MixedFormatCatalog getMixedFormatCatalog() { + if (mixedFormatCatalog == null) { + mixedFormatCatalog = CatalogLoader.load(getCatalogUri()); + } + return mixedFormatCatalog; + } + + protected String getCatalogUri() { + return TEST_AMS.getServerUrl() + "/" + catalogMeta.getCatalogName(); + } + + static Stream parameters() { + return Stream.of( + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER); } - @Test - public void testMixedCatalog() { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testMixedCatalog(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); String[] catalogs = getTableEnv().listCatalogs(); - Assert.assertArrayEquals( + Assertions.assertArrayEquals( Arrays.stream(catalogs).sorted().toArray(), Stream.of("default_catalog", catalogName).sorted().toArray()); } - @Test - public void testDDL() { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testDDL(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); sql( "CREATE TABLE " + catalogName @@ -150,16 +223,22 @@ public void testDDL() { sql("USE " + catalogName + "." + DB); sql("SHOW tables"); - Assert.assertTrue( + Assertions.assertTrue( getMixedFormatCatalog() .loadTable(TableIdentifier.of(catalogName, DB, TABLE)) .isKeyedTable()); } - @Test - public void testComputeIndex() { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testComputeIndex(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); // if compute column before any physical column, will throw exception. - Assert.assertThrows( + Assertions.assertThrows( org.apache.flink.table.api.TableException.class, () -> sql( @@ -190,8 +269,14 @@ public void testComputeIndex() { + ") "); } - @Test - public void testDDLWithVirtualColumn() throws IOException { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testDDLWithVirtualColumn(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); // create mixed-format table with compute columns and watermark under mixed-format catalog // org.apache.iceberg.flink.TypeToFlinkType will convert Timestamp to Timestamp(6), so we cast // datatype manually @@ -221,28 +306,34 @@ public void testDDLWithVirtualColumn() throws IOException { Arrays.stream(computedIndex) .forEach( x -> { - Assert.assertTrue( + Assertions.assertTrue( properties.containsKey(compoundKey(FLINK_PREFIX, COMPUTED_COLUMNS, x, NAME))); - Assert.assertTrue( + Assertions.assertTrue( properties.containsKey(compoundKey(FLINK_PREFIX, COMPUTED_COLUMNS, x, EXPR))); - Assert.assertTrue( + Assertions.assertTrue( properties.containsKey( compoundKey(FLINK_PREFIX, COMPUTED_COLUMNS, x, DATA_TYPE))); }); - Assert.assertTrue( + Assertions.assertTrue( properties.containsKey(compoundKey(FLINK_PREFIX, WATERMARK, WATERMARK_ROWTIME))); - Assert.assertTrue( + Assertions.assertTrue( properties.containsKey(compoundKey(FLINK_PREFIX, WATERMARK, WATERMARK_STRATEGY_EXPR))); - Assert.assertTrue( + Assertions.assertTrue( properties.containsKey(compoundKey(FLINK_PREFIX, WATERMARK, WATERMARK_STRATEGY_DATA_TYPE))); List result = sql("DESC " + catalogName + "." + DB + "." + TABLE + ""); - Assert.assertEquals(6, result.size()); + Assertions.assertEquals(6, result.size()); } - @Test - public void testDMLWithVirtualColumn() throws IOException { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testDMLWithVirtualColumn(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); // create mixed-format table with compute columns under mixed-format catalog sql( "CREATE TABLE " @@ -277,8 +368,14 @@ public void testDMLWithVirtualColumn() throws IOException { checkRows(rows); } - @Test - public void testReadNotMatchColumn() throws IOException { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testReadNotMatchColumn(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); // create mixed-format table with compute columns under mixed-format catalog sql( "CREATE TABLE " @@ -306,12 +403,12 @@ public void testReadNotMatchColumn() throws IOException { .set(compoundKey(FLINK_PREFIX, COMPUTED_COLUMNS, 2, EXPR), afterExpr) .commit(); - Assert.assertNotEquals( + Assertions.assertNotEquals( beforeExpr, amoroTable.properties().get(compoundKey(FLINK_PREFIX, COMPUTED_COLUMNS, 2, EXPR))); // property for expr do not match any columns in amoro, will throw exception. - Assert.assertThrows( + Assertions.assertThrows( IllegalStateException.class, () -> sql("DESC " + catalogName + "." + DB + "." + TABLE + "")); amoroTable @@ -323,8 +420,14 @@ public void testReadNotMatchColumn() throws IOException { sql("DESC " + catalogName + "." + DB + "." + TABLE + ""); } - @Test - public void testDML() throws IOException { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testDML(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); sql( "CREATE TABLE default_catalog.default_database." + TABLE @@ -374,19 +477,19 @@ public void testDML() throws IOException { + " /*+ OPTIONS(" + "'streaming'='false'" + ") */"); - Assert.assertEquals(1, rows.size()); + Assertions.assertEquals(1, rows.size()); sql("DROP TABLE default_catalog.default_database." + TABLE); } private void checkRows(List rows) { - Assert.assertEquals(1, rows.size()); + Assertions.assertEquals(1, rows.size()); int id = (int) rows.get(0).getField("id"); int computeId = (int) rows.get(0).getField("compute_id"); - Assert.assertEquals(1, id); + Assertions.assertEquals(1, id); // computeId should be id+5 - Assert.assertEquals(id + 5, computeId); - Assert.assertEquals(4, rows.get(0).getFieldNames(true).size()); + Assertions.assertEquals(id + 5, computeId); + Assertions.assertEquals(4, rows.get(0).getFieldNames(true).size()); } protected List sql(String query, Object... args) { @@ -505,8 +608,14 @@ private void insertValue() { sql("DROP TABLE default_catalog.default_database." + TABLE); } - @Test - public void testAlterUnKeyTable() throws Exception { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testAlterUnKeyTable(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); sql( "CREATE TABLE " + catalogName @@ -534,12 +643,18 @@ public void testAlterUnKeyTable() throws Exception { + "SET ( 'write.metadata.delete-after-commit.enabled' = 'false')"); Map unKeyTableProperties = getMixedFormatCatalog().loadTable(TableIdentifier.of(catalogName, DB, TABLE)).properties(); - Assert.assertEquals( + Assertions.assertEquals( unKeyTableProperties.get("write.metadata.delete-after-commit.enabled"), "false"); } - @Test - public void testAlterKeyTable() throws Exception { + @ParameterizedTest(name = "catalogFactoryType = {0}") + @ValueSource( + strings = { + CatalogFactoryOptions.MIXED_ICEBERG_IDENTIFIER, + CatalogFactoryOptions.MIXED_HIVE_IDENTIFIER + }) + public void testAlterKeyTable(String catalogFactoryType) throws Exception { + setUpForParam(catalogFactoryType); sql( "CREATE TABLE " + catalogName @@ -584,8 +699,8 @@ public void testAlterKeyTable() throws Exception { Map keyTableProperties = getMixedFormatCatalog().loadTable(TableIdentifier.of(catalogName, DB, TABLE)).properties(); - Assert.assertEquals(keyTableProperties.get("self-optimizing.enabled"), "true"); - Assert.assertEquals(keyTableProperties.get("self-optimizing.group"), "flink"); - Assert.assertEquals(keyTableProperties.get("write.upsert.enabled"), "true"); + Assertions.assertEquals(keyTableProperties.get("self-optimizing.enabled"), "true"); + Assertions.assertEquals(keyTableProperties.get("self-optimizing.group"), "flink"); + Assertions.assertEquals(keyTableProperties.get("write.upsert.enabled"), "true"); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalogTablePartitions.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalogTablePartitions.java index e951acce03..754bdfcc38 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalogTablePartitions.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalogTablePartitions.java @@ -44,8 +44,9 @@ import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; import org.apache.flink.types.RowKind; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.LinkedList; @@ -61,8 +62,8 @@ public TestMixedCatalogTablePartitions() { new BasicTableTestHelper(true, true)); } - public void before() throws Exception { - super.before(); + @BeforeEach + public void configProps() { super.config(); } @@ -105,7 +106,7 @@ public void testListPartitionsUnKeyedTable() throws TableNotPartitionedException new CatalogPartitionSpec(ImmutableMap.of("dt", "2023-10-02")); expected.add(partitionSpec1); expected.add(partitionSpec2); - Assert.assertEquals("Should produce the expected catalog partition specs.", list, expected); + Assertions.assertEquals(list, expected, "Should produce the expected catalog partition specs."); } @Test @@ -149,8 +150,8 @@ public void testListPartitionsKeyedTable() throws TableNotPartitionedException { new CatalogPartitionSpec(ImmutableMap.of("dt", "2023-10-02")); expected.add(partitionSpec1); expected.add(partitionSpec2); - Assert.assertEquals( - "Should produce the expected catalog partition specs.", partitionList, expected); + Assertions.assertEquals( + partitionList, expected, "Should produce the expected catalog partition specs."); } @Test @@ -203,21 +204,21 @@ public void testListPartitionsByFilter() new CatalogPartitionSpec(ImmutableMap.of("dt", "2023-10-01", "name", "mark")); expected.add(partitionSpec1); expected.add(partitionSpec2); - Assert.assertEquals("Should produce the expected catalog partition specs.", list, expected); + Assertions.assertEquals(list, expected, "Should produce the expected catalog partition specs."); List listCatalogPartitionSpec = mixedCatalog.listPartitions( objectPath, new CatalogPartitionSpec(ImmutableMap.of("dt", "2023-10-01", "name", "Gerry"))); - Assert.assertEquals( - "Should produce the expected catalog partition specs.", listCatalogPartitionSpec.size(), 1); + Assertions.assertEquals( + listCatalogPartitionSpec.size(), 1, "Should produce the expected catalog partition specs."); try { mixedCatalog.listPartitions( objectPath, new CatalogPartitionSpec(ImmutableMap.of("dt", "2023-10-01", "name1", "Gerry"))); } catch (Exception e) { - Assert.assertTrue(e instanceof PartitionSpecInvalidException); + Assertions.assertTrue(e instanceof PartitionSpecInvalidException); } } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/ByteArraySetSerializerTest.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/ByteArraySetSerializerTest.java index 8f7aae0606..e2a8151b68 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/ByteArraySetSerializerTest.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/ByteArraySetSerializerTest.java @@ -18,8 +18,8 @@ package org.apache.amoro.flink.lookup; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,8 +38,8 @@ public void testByteArraySetSerializer() { byteSet.add(new ByteArrayWrapper(data, data.length)); byte[] serialized = ByteArraySetSerializer.serialize(byteSet); Set actualSet = ByteArraySetSerializer.deserialize(serialized); - Assert.assertEquals(byteSet.size(), actualSet.size()); - Assert.assertEquals(byteSet, actualSet); + Assertions.assertEquals(byteSet.size(), actualSet.size()); + Assertions.assertEquals(byteSet, actualSet); } @Test @@ -56,13 +56,13 @@ public void testPerformance() { totalSize += 4 + tmp.length; } LOG.info("added {} items process time: {}", num, System.currentTimeMillis() - start); - Assert.assertEquals(num, byteArraySet.size()); + Assertions.assertEquals(num, byteArraySet.size()); start = System.currentTimeMillis(); byte[] serialized = ByteArraySetSerializer.serialize(byteArraySet); long cost = System.currentTimeMillis() - start; assert serialized != null; - Assert.assertEquals(totalSize, serialized.length); + Assertions.assertEquals(totalSize, serialized.length); LOG.info( "serialized cost: {}, num= {}, result byte array size={}.", cost, num, serialized.length); @@ -70,14 +70,14 @@ public void testPerformance() { Set actualSet = ByteArraySetSerializer.deserialize(serialized); cost = System.currentTimeMillis() - start; LOG.info("deserialized cost: {}, num= {}, set size={}.", cost, num, actualSet.size()); - Assert.assertEquals(byteArraySet, actualSet); + Assertions.assertEquals(byteArraySet, actualSet); // exists sb = new StringBuilder(); start = System.currentTimeMillis(); for (int i = 0; i < num; i++) { sb.append(i); - Assert.assertTrue( + Assertions.assertTrue( actualSet.contains( new ByteArrayWrapper(sb.toString().getBytes(), sb.toString().getBytes().length))); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/TestKVTable.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/TestKVTable.java index 3dc0551cc6..d0d6f9d78c 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/TestKVTable.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/TestKVTable.java @@ -20,7 +20,7 @@ import static org.apache.amoro.flink.table.descriptors.MixedFormatValidator.LOOKUP_CACHE_TTL_AFTER_WRITE; import static org.apache.amoro.flink.table.descriptors.MixedFormatValidator.ROCKSDB_WRITING_THREADS; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.amoro.flink.lookup.filter.RowDataPredicate; import org.apache.amoro.flink.lookup.filter.RowDataPredicateExpressionVisitor; @@ -50,14 +50,10 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.flink.FlinkSchemaUtil; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.TestName; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,16 +71,15 @@ import java.util.stream.Collectors; @SuppressWarnings("OptionalUsedAsFieldOrParameterType") -@RunWith(value = Parameterized.class) public class TestKVTable extends TestRowDataPredicateBase { private static final Logger LOG = LoggerFactory.getLogger(TestKVTable.class); - @Rule public TemporaryFolder temp = new TemporaryFolder(); - @Rule public TestName name = new TestName(); + @TempDir public java.nio.file.Path temp; + private final Configuration config = new Configuration(); private final List primaryKeys = Lists.newArrayList("id", "grade"); private final List primaryKeysDisorder = Lists.newArrayList("grade", "num", "id"); - private final boolean guavaCacheEnabled; + private boolean guavaCacheEnabled; private final Schema mixedTableSchema = new Schema( @@ -94,25 +89,19 @@ public class TestKVTable extends TestRowDataPredicateBase { private String dbPath; - @Parameterized.Parameters(name = "guavaCacheEnabled = {0}") - public static Object[][] parameters() { - return new Object[][] {{true}, {false}}; - } - - public TestKVTable(boolean guavaCacheEnabled) { + private void setUpForParam(boolean guavaCacheEnabled) throws IOException { this.guavaCacheEnabled = guavaCacheEnabled; - } - - @Before - public void before() throws IOException { - dbPath = temp.newFolder().getPath(); + dbPath = java.nio.file.Files.createTempDirectory(temp, "kv-table").toFile().getPath(); if (!guavaCacheEnabled) { config.set(MixedFormatValidator.LOOKUP_CACHE_MAX_ROWS, 0L); } } - @Test - public void testRowDataSerializer() throws IOException { + @ParameterizedTest(name = "guavaCacheEnabled = {0}") + @ValueSource(booleans = {true, false}) + public void testRowDataSerializer(boolean guavaCacheEnabled) throws IOException { + setUpForParam(guavaCacheEnabled); + BinaryRowDataSerializer binaryRowDataSerializer = new BinaryRowDataSerializer(3); GenericRowData genericRowData = (GenericRowData) row(1, "2", 3); @@ -126,10 +115,10 @@ public void testRowDataSerializer() throws IOException { BinaryRowData desRowData = binaryRowDataSerializer.deserialize(new DataInputDeserializer(view.getCopyOfBuffer())); - Assert.assertNotNull(desRowData); - Assert.assertEquals(record.getInt(0), desRowData.getInt(0)); - Assert.assertEquals(record.getInt(1), desRowData.getInt(1)); - Assert.assertEquals(record.getInt(2), desRowData.getInt(2)); + Assertions.assertNotNull(desRowData); + Assertions.assertEquals(record.getInt(0), desRowData.getInt(0)); + Assertions.assertEquals(record.getInt(1), desRowData.getInt(1)); + Assertions.assertEquals(record.getInt(2), desRowData.getInt(2)); // test join key rowData binaryRowDataSerializer = new BinaryRowDataSerializer(2); @@ -149,11 +138,14 @@ public void testRowDataSerializer() throws IOException { view.clear(); binaryRowDataSerializer.serialize(binaryRowData1, view); byte[] rowBytes1 = view.getCopyOfBuffer(); - Assert.assertArrayEquals(rowBytes1, rowBytes); + Assertions.assertArrayEquals(rowBytes1, rowBytes); } - @Test - public void testInitialUniqueKeyTable() throws IOException { + @ParameterizedTest(name = "guavaCacheEnabled = {0}") + @ValueSource(booleans = {true, false}) + public void testInitialUniqueKeyTable(boolean guavaCacheEnabled) throws IOException { + setUpForParam(guavaCacheEnabled); + config.setInteger(ROCKSDB_WRITING_THREADS, 5); List joinKeys = Lists.newArrayList("id", "grade"); try (UniqueIndexTable uniqueIndexTable = (UniqueIndexTable) createTable(joinKeys)) { @@ -212,8 +204,11 @@ public void testInitialUniqueKeyTable() throws IOException { } } - @Test - public void testSecondaryKeysMapping() throws IOException { + @ParameterizedTest(name = "guavaCacheEnabled = {0}") + @ValueSource(booleans = {true, false}) + public void testSecondaryKeysMapping(boolean guavaCacheEnabled) throws IOException { + setUpForParam(guavaCacheEnabled); + // primary keys are id and grade. List joinKeys = Lists.newArrayList("grade", "id"); try (SecondaryIndexTable secondaryIndexTable = @@ -252,8 +247,11 @@ public void testSecondaryKeysMapping() throws IOException { } } - @Test - public void testInitialSecondaryKeyTable() throws IOException { + @ParameterizedTest(name = "guavaCacheEnabled = {0}") + @ValueSource(booleans = {true, false}) + public void testInitialSecondaryKeyTable(boolean guavaCacheEnabled) throws IOException { + setUpForParam(guavaCacheEnabled); + config.setInteger(ROCKSDB_WRITING_THREADS, 10); config.set(LOOKUP_CACHE_TTL_AFTER_WRITE, Duration.ofMinutes(1000)); // primary keys are id and grade. @@ -303,8 +301,11 @@ private void writeAndAssert(SecondaryIndexTable secondaryIndexTable) throws IOEx assertTableSet(secondaryIndexTable, row(3), row(3, "3", 5), row(3, "4", 4)); } - @Test - public void testCacheExpired() throws InterruptedException { + @ParameterizedTest(name = "guavaCacheEnabled = {0}") + @ValueSource(booleans = {true, false}) + public void testCacheExpired(boolean guavaCacheEnabled) throws Exception { + setUpForParam(guavaCacheEnabled); + Cache cache = CacheBuilder.newBuilder().expireAfterWrite(Duration.ofSeconds(1)).build(); cache.put(1, 1); @@ -318,21 +319,24 @@ public void testCacheExpired() throws InterruptedException { } return v; }); - Assert.assertEquals(Integer.valueOf(1), cache.getIfPresent(1)); - Assert.assertEquals(Integer.valueOf(2), cache.getIfPresent(2)); + Assertions.assertEquals(Integer.valueOf(1), cache.getIfPresent(1)); + Assertions.assertEquals(Integer.valueOf(2), cache.getIfPresent(2)); Thread.sleep(1001); - Assert.assertEquals(2, cache.size()); - Assert.assertNull(cache.getIfPresent(1)); - Assert.assertNull(cache.getIfPresent(2)); + Assertions.assertEquals(2, cache.size()); + Assertions.assertNull(cache.getIfPresent(1)); + Assertions.assertNull(cache.getIfPresent(2)); cache.cleanUp(); cache.put(3, 3); - Assert.assertEquals(1, cache.size()); - Assert.assertNull(cache.getIfPresent(1)); - Assert.assertEquals(Integer.valueOf(3), cache.getIfPresent(3)); + Assertions.assertEquals(1, cache.size()); + Assertions.assertNull(cache.getIfPresent(1)); + Assertions.assertEquals(Integer.valueOf(3), cache.getIfPresent(3)); } - @Test - public void testPredicate() throws IOException { + @ParameterizedTest(name = "guavaCacheEnabled = {0}") + @ValueSource(booleans = {true, false}) + public void testPredicate(boolean guavaCacheEnabled) throws IOException { + setUpForParam(guavaCacheEnabled); + String filter = "id >= 2 and num < 5 and num > 2"; Optional rowDataPredicate = generatePredicate(filter); @@ -388,8 +392,11 @@ public void testPredicate() throws IOException { row(4, "4", 4)); } - @Test - public void testSecondaryIndexPredicate() throws IOException { + @ParameterizedTest(name = "guavaCacheEnabled = {0}") + @ValueSource(booleans = {true, false}) + public void testSecondaryIndexPredicate(boolean guavaCacheEnabled) throws IOException { + setUpForParam(guavaCacheEnabled); + String filter = "id >= 2 and num < 5 and num > 2"; Optional rowDataPredicate = generatePredicate(filter); @@ -500,12 +507,12 @@ private void assertTable(KVTable table, RowData... rows) throws IOExcep RowData key = rows[i], expected = rows[i + 1]; List values = table.get(key); - Assert.assertNotNull(values); + Assertions.assertNotNull(values); if (expected == null) { - Assert.assertEquals(0, values.size()); + Assertions.assertEquals(0, values.size()); continue; } - Assert.assertEquals(expected.toString(), 1, values.size()); + Assertions.assertEquals(1, values.size(), expected.toString()); RowData actual = values.get(0); assertRecord(expected, actual); } @@ -515,10 +522,10 @@ private void assertTableSet(KVTable table, RowData key, RowData... expe throws IOException { List values = table.get(key); if (expects == null) { - Assert.assertEquals(0, values.size()); + Assertions.assertEquals(0, values.size()); return; } - Assert.assertEquals(expects.length, values.size()); + Assertions.assertEquals(expects.length, values.size()); values = values.stream().sorted(compare()).collect(Collectors.toList()); List expectsAfterSort = Arrays.stream(expects).sorted(compare()).collect(Collectors.toList()); @@ -546,16 +553,16 @@ private void assertRecord(RowData expected, RowData actual) { switch (j) { case 0: case 2: - Assert.assertEquals( - String.format("expected:%s, actual:%s.", expected.toString(), actual), + Assertions.assertEquals( expected.getInt(j), - binaryRowData.getInt(j)); + binaryRowData.getInt(j), + String.format("expected:%s, actual:%s.", expected.toString(), actual)); break; case 1: - Assert.assertEquals( - String.format("expected:%s, actual:%s.", expected, actual), + Assertions.assertEquals( expected.getString(j), - binaryRowData.getString(j)); + binaryRowData.getString(j), + String.format("expected:%s, actual:%s.", expected, actual)); break; } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateAllFieldTypes.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateAllFieldTypes.java index b48aa9bfeb..c97ed3470c 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateAllFieldTypes.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateAllFieldTypes.java @@ -18,9 +18,9 @@ package org.apache.amoro.flink.lookup.filter; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.amoro.flink.util.DateTimeUtils; import org.apache.flink.table.api.DataTypes; @@ -33,8 +33,8 @@ import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.types.DataType; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.math.BigDecimal; import java.sql.Timestamp; @@ -53,7 +53,7 @@ public class TestRowDataPredicateAllFieldTypes extends TestRowDataPredicateBase List columns = new ArrayList<>(); protected ResolvedSchema schema; - @Before + @BeforeEach public void setUp() { columns.add(0, Column.physical("f0", DataTypes.INT())); columns.add(1, Column.physical("f1", DataTypes.STRING())); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateBase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateBase.java index c50d2d38a1..d4e18a714c 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateBase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateBase.java @@ -36,7 +36,7 @@ import org.apache.flink.table.planner.expressions.RexNodeExpression; import org.apache.flink.table.planner.plan.utils.RexNodeToExpressionConverter; import org.apache.flink.table.types.logical.RowType; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; import java.util.Collections; import java.util.List; @@ -47,7 +47,7 @@ public abstract class TestRowDataPredicateBase { public static StreamExecutionEnvironment env; public static TableEnvironment tEnv; - @Before + @BeforeEach public void init() { env = StreamExecutionEnvironment.getExecutionEnvironment(); tEnv = StreamTableEnvironment.create(env); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateExpressionVisitor.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateExpressionVisitor.java index e69689dbbd..66d9716bc6 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateExpressionVisitor.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/lookup/filter/TestRowDataPredicateExpressionVisitor.java @@ -18,9 +18,9 @@ package org.apache.amoro.flink.lookup.filter; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.catalog.Column; @@ -29,8 +29,8 @@ import org.apache.flink.table.data.StringData; import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.types.DataType; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; @@ -47,7 +47,7 @@ public class TestRowDataPredicateExpressionVisitor extends TestRowDataPredicateB List columns = new ArrayList<>(); ResolvedSchema schema; - @Before + @BeforeEach public void setUp() { columns.add(0, Column.physical("id", DataTypes.INT())); columns.add(1, Column.physical("name", DataTypes.STRING())); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java index ef5d08a68b..2f6334ec2d 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java @@ -47,8 +47,8 @@ import org.apache.iceberg.Table; import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.io.WriteResult; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.time.LocalDateTime; @@ -145,7 +145,7 @@ public void testUnkeyedTableDataStream() throws Exception { GenericRowData.of( o.getInt(0), o.getString(1), o.getLong(2), o.getTimestamp(3, 6)))); - Assert.assertEquals(new HashSet<>(expectedRecords), rowData); + Assertions.assertEquals(new HashSet<>(expectedRecords), rowData); } @Test @@ -198,7 +198,7 @@ public void testUnkeyedStreamingRead() throws Exception { } jobClient.cancel(); - Assert.assertEquals(new HashSet<>(expectedRecords), rowData); + Assertions.assertEquals(new HashSet<>(expectedRecords), rowData); } @Test @@ -272,7 +272,7 @@ public void testUnkeyedSnapshotRead() throws Exception { } jobClient.cancel(); - Assert.assertEquals(new HashSet<>(expectedRecords), rowData); + Assertions.assertEquals(new HashSet<>(expectedRecords), rowData); CloseableIterator resultIterator = FlinkSource.forRowData() @@ -299,6 +299,6 @@ public void testUnkeyedSnapshotRead() throws Exception { o.getInt(0), o.getString(1), o.getLong(2), o.getTimestamp(3, 6)))); expectedRecords = DataUtil.toRowData(s1); - Assert.assertEquals(new HashSet<>(expectedRecords), rowData); + Assertions.assertEquals(new HashSet<>(expectedRecords), rowData); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java index 859776b94c..3d881000e3 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java @@ -22,8 +22,8 @@ import org.apache.amoro.flink.read.hybrid.split.MixedFormatSplit; import org.apache.amoro.scan.ChangeTableIncrementalScan; import org.apache.iceberg.Snapshot; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; @@ -37,7 +37,7 @@ public void testPlanSplitFromKeyedTable() { testKeyedTable.changeTable().refresh(); List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); - Assert.assertEquals(7, splitList.size()); + Assertions.assertEquals(7, splitList.size()); } @Test @@ -47,7 +47,7 @@ public void testIncrementalChangelog() throws IOException { List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); - Assert.assertEquals(7, splitList.size()); + Assertions.assertEquals(7, splitList.size()); long startSnapshotId = testKeyedTable.changeTable().currentSnapshot().snapshotId(); writeUpdate(); @@ -67,6 +67,6 @@ public void testIncrementalChangelog() throws IOException { List changeSplits = FlinkSplitPlanner.planChangeTable(changeTableScan, new AtomicInteger()); - Assert.assertEquals(1, changeSplits.size()); + Assertions.assertEquals(1, changeSplits.size()); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java index ecdf4c47e9..b21b0d0d19 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java @@ -81,11 +81,10 @@ import org.apache.iceberg.io.WriteResult; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,7 +108,6 @@ public class TestMixedFormatSource extends TestRowDataReaderFunction implements private static final long serialVersionUID = 7418812854449034756L; private static final int PARALLELISM = 1; - @Rule public final MiniClusterWithClientResource miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() @@ -119,13 +117,23 @@ public class TestMixedFormatSource extends TestRowDataReaderFunction implements .withHaLeadershipControl() .build()); + @BeforeEach + public void startMiniClusterResource() throws Exception { + miniClusterResource.before(); + } + + @AfterEach + public void stopMiniClusterResource() { + miniClusterResource.after(); + } + protected KeyedTable testFailoverTable; protected static final String SINK_TABLE_NAME = "test_sink_exactly_once"; protected static final TableIdentifier FAIL_TABLE_ID = TableIdentifier.of( TableTestHelper.TEST_CATALOG_NAME, TableTestHelper.TEST_DB_NAME, SINK_TABLE_NAME); - @Before + @BeforeEach public void testSetup() throws IOException { MixedFormatCatalog testCatalog = getMixedFormatCatalog(); @@ -145,7 +153,7 @@ public void testSetup() throws IOException { } } - @After + @AfterEach public void dropTable() { miniClusterResource.cancelAllJobs(); getMixedFormatCatalog().dropTable(FAIL_TABLE_ID, true); @@ -266,7 +274,7 @@ public void testDimTaskManagerFailover() throws Exception { Thread.sleep(1000); LOG.info("wait for watermark after failover"); } - Assert.assertEquals(Long.MAX_VALUE, WatermarkAwareFailWrapper.getWatermarkAfterFailover()); + Assertions.assertEquals(Long.MAX_VALUE, WatermarkAwareFailWrapper.getWatermarkAfterFailover()); } @Test @@ -359,7 +367,7 @@ public void testMixedFormatContinuousSourceWithEmptyChangeInInit() throws Except List actualResult = collectRecordsFromUnboundedStream(clientAndIterator, baseData.size()); - Assert.assertEquals(new HashSet<>(baseData), new HashSet<>(actualResult)); + Assertions.assertEquals(new HashSet<>(baseData), new HashSet<>(actualResult)); LOG.info( "begin write update_before update_after data and commit new snapshot to change table."); @@ -369,7 +377,7 @@ public void testMixedFormatContinuousSourceWithEmptyChangeInInit() throws Except actualResult = collectRecordsFromUnboundedStream(clientAndIterator, excepts2().length * 2); jobClient.cancel(); - Assert.assertEquals(new HashSet<>(updateRecords()), new HashSet<>(actualResult)); + Assertions.assertEquals(new HashSet<>(updateRecords()), new HashSet<>(actualResult)); getMixedFormatCatalog().dropTable(tableId, true); } @@ -429,7 +437,7 @@ public void testMixedFormatSourceEnumeratorWithChangeExpired() throws Exception commit(table, result, false); for (DataFile dataFile : changeDataFiles) { - Assert.assertTrue(table.io().exists(dataFile.path().toString())); + Assertions.assertTrue(table.io().exists(dataFile.path().toString())); } final Duration monitorInterval = Duration.ofSeconds(1); @@ -446,12 +454,12 @@ public void testMixedFormatSourceEnumeratorWithChangeExpired() throws Exception List actualResult = collectRecordsFromUnboundedStream(clientAndIterator, changeData.size()); - Assert.assertEquals(new HashSet<>(changeData), new HashSet<>(actualResult)); + Assertions.assertEquals(new HashSet<>(changeData), new HashSet<>(actualResult)); // expire changeTable snapshots DeleteFiles deleteFiles = table.changeTable().newDelete(); for (DataFile dataFile : changeDataFiles) { - Assert.assertTrue(table.io().exists(dataFile.path().toString())); + Assertions.assertTrue(table.io().exists(dataFile.path().toString())); deleteFiles.deleteFile(dataFile); } deleteFiles.commit(); @@ -472,7 +480,7 @@ public void testMixedFormatSourceEnumeratorWithChangeExpired() throws Exception actualResult = collectRecordsFromUnboundedStream(clientAndIterator, excepts2().length * 2); jobClient.cancel(); - Assert.assertEquals(new HashSet<>(updateRecords()), new HashSet<>(actualResult)); + Assertions.assertEquals(new HashSet<>(updateRecords()), new HashSet<>(actualResult)); getMixedFormatCatalog().dropTable(tableId, true); } @@ -532,7 +540,7 @@ public void testMixedFormatSourceEnumeratorWithBaseExpired() throws Exception { commit(table, result, true); for (DataFile dataFile : baseDataFiles) { - Assert.assertTrue(table.io().exists(dataFile.path().toString())); + Assertions.assertTrue(table.io().exists(dataFile.path().toString())); } final Duration monitorInterval = Duration.ofSeconds(1); @@ -549,12 +557,12 @@ public void testMixedFormatSourceEnumeratorWithBaseExpired() throws Exception { List actualResult = collectRecordsFromUnboundedStream(clientAndIterator, baseData.size()); - Assert.assertEquals(new HashSet<>(baseData), new HashSet<>(actualResult)); + Assertions.assertEquals(new HashSet<>(baseData), new HashSet<>(actualResult)); // expire baseTable snapshots DeleteFiles deleteFiles = table.baseTable().newDelete(); for (DataFile dataFile : baseDataFiles) { - Assert.assertTrue(table.io().exists(dataFile.path().toString())); + Assertions.assertTrue(table.io().exists(dataFile.path().toString())); deleteFiles.deleteFile(dataFile); } deleteFiles.commit(); @@ -575,7 +583,7 @@ public void testMixedFormatSourceEnumeratorWithBaseExpired() throws Exception { actualResult = collectRecordsFromUnboundedStream(clientAndIterator, excepts2().length * 2); jobClient.cancel(); - Assert.assertEquals(new HashSet<>(updateRecords()), new HashSet<>(actualResult)); + Assertions.assertEquals(new HashSet<>(updateRecords()), new HashSet<>(actualResult)); getMixedFormatCatalog().dropTable(tableId, true); } @@ -682,7 +690,7 @@ private boolean equalsRecords(List expected, List tableRecords try { RowData[] expectedArray = sortRowDataCollection(expected); RowData[] actualArray = sortRowDataCollection(tableRecords); - Assert.assertArrayEquals(expectedArray, actualArray); + Assertions.assertArrayEquals(expectedArray, actualArray); return true; } catch (Throwable e) { return false; @@ -813,12 +821,12 @@ private List collectRecordsFromUnboundedStream( } } - Assert.assertEquals( + Assertions.assertEquals( + numElements, + result.size(), String.format( "The stream ended before reaching the requested %d records. Only %d records were received, received list:%s.", - numElements, result.size(), Arrays.toString(result.toArray())), - numElements, - result.size()); + numElements, result.size(), Arrays.toString(result.toArray()))); return result; } @@ -855,7 +863,8 @@ private ClientAndIterator executeAndCollectWithClient( env.fromSource( mixedFormatSource, WatermarkStrategy.noWatermarks(), "MixedFormatParallelSource") .setParallelism(PARALLELISM); - return DataStreamUtils.collectWithClient(source, "job_" + name.getMethodName()); + return DataStreamUtils.collectWithClient( + source, "job_" + getClass().getSimpleName() + "_" + System.nanoTime()); } private static GenericRowData convert(RowData row) { @@ -1113,7 +1122,7 @@ public void processWatermark(Watermark mark) throws Exception { } if (failoverHappened) { LOG.info("failover happened, watermark: {}", mark); - Assert.assertEquals(Long.MAX_VALUE, mark.getTimestamp()); + Assertions.assertEquals(Long.MAX_VALUE, mark.getTimestamp()); if (watermarkAfterFailover == -1) { watermarkAfterFailover = mark.getTimestamp(); } else { diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaConsumer.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaConsumer.java index 029ebf3e75..4a7b83ddfa 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaConsumer.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaConsumer.java @@ -35,10 +35,10 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,12 +54,12 @@ public class TestKafkaConsumer extends TestBaseLog { private static final Logger LOG = LoggerFactory.getLogger(TestKafkaConsumer.class); - @BeforeClass + @BeforeAll public static void prepare() throws Exception { KAFKA_CONTAINER.start(); } - @AfterClass + @AfterAll public static void shutdown() throws Exception { KAFKA_CONTAINER.close(); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaSourceReader.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaSourceReader.java index 2747d9ec59..2dfc42b391 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaSourceReader.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestKafkaSourceReader.java @@ -23,7 +23,7 @@ import static org.apache.amoro.flink.kafka.testutils.KafkaContainerTest.readRecordsBytes; import static org.apache.amoro.flink.shuffle.RowKindUtil.transformFromFlinkRowKind; import static org.apache.amoro.flink.table.descriptors.MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.amoro.flink.kafka.testutils.KafkaContainerTest; import org.apache.amoro.flink.read.source.log.kafka.LogKafkaPartitionSplit; @@ -51,12 +51,11 @@ import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,23 +76,21 @@ public class TestKafkaSourceReader { private static final int NUM_RECORDS_PER_SPLIT = 10; private static final int TOTAL_NUM_RECORDS = NUM_RECORDS_PER_SPLIT * NUM_SPLITS; - @Rule public TestName testName = new TestName(); - private static final byte[] JOB_ID = IdGenerator.generateUpstreamId(); - @BeforeClass + @BeforeAll public static void prepare() throws Exception { KAFKA_CONTAINER.start(); } - @AfterClass + @AfterAll public static void shutdown() throws Exception { KAFKA_CONTAINER.close(); } - @Before - public void initData() throws Exception { - topic = TestUtil.getUtMethodName(testName); + @BeforeEach + public void initData(TestInfo testInfo) throws Exception { + topic = TestUtil.getUtMethodName(testInfo); KafkaContainerTest.createTopics(KAFKA_PARTITION_NUMS, 1, topic); write(topic, TOTAL_NUM_RECORDS); } @@ -233,16 +230,16 @@ public void emitWatermark(Watermark watermark) {} public void validate() { assertEquals( - String.format("Should be %d distinct elements in total", TOTAL_NUM_RECORDS), TOTAL_NUM_RECORDS, - consumedValues.size()); + consumedValues.size(), + String.format("Should be %d distinct elements in total", TOTAL_NUM_RECORDS)); assertEquals( - String.format("Should be %d elements in total", TOTAL_NUM_RECORDS), TOTAL_NUM_RECORDS, - count); - assertEquals("The min value should be 0", 0, min); + count, + String.format("Should be %d elements in total", TOTAL_NUM_RECORDS)); + assertEquals(0, min, "The min value should be 0"); assertEquals( - "The max value should be " + (TOTAL_NUM_RECORDS - 1), TOTAL_NUM_RECORDS - 1, max); + TOTAL_NUM_RECORDS - 1, max, "The max value should be " + (TOTAL_NUM_RECORDS - 1)); } public int count() { diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestLogKafkaPartitionSplitReader.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestLogKafkaPartitionSplitReader.java index 1c45d4abb9..53efe9f865 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestLogKafkaPartitionSplitReader.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hidden/kafka/TestLogKafkaPartitionSplitReader.java @@ -21,7 +21,7 @@ import static org.apache.amoro.flink.kafka.testutils.KafkaContainerTest.KAFKA_CONTAINER; import static org.apache.amoro.flink.kafka.testutils.KafkaContainerTest.readRecordsBytes; import static org.apache.amoro.flink.shuffle.RowKindUtil.transformFromFlinkRowKind; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.amoro.flink.kafka.testutils.KafkaConfigGenerate; import org.apache.amoro.flink.kafka.testutils.KafkaContainerTest; @@ -51,10 +51,10 @@ import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,7 +79,7 @@ public class TestLogKafkaPartitionSplitReader { private static Map> splitsByOwners; private static final byte[] JOB_ID = IdGenerator.generateUpstreamId(); - @BeforeClass + @BeforeAll public static void prepare() throws Exception { KAFKA_CONTAINER.start(); @@ -89,12 +89,12 @@ public static void prepare() throws Exception { splitsByOwners = getSplitsByOwners(earliestOffsets); } - @AfterClass + @AfterAll public static void shutdown() throws Exception { KAFKA_CONTAINER.close(); } - @Before + @BeforeEach public void initData() throws Exception { // |0 1 2 3 4 5 6 7 8 9 Flip 10 11 12 13 14| 15 16 17 18 19 write(TOPIC1, 0); @@ -222,9 +222,10 @@ private void assignSplitsAndFetchUntilFinish( numConsumedRecords.forEach( (splitId, recordCount) -> { assertEquals( - String.format("%s should have %d records.", splits.get(splitId), expectedRecordCount), expectedRecordCount, - (long) recordCount); + (long) recordCount, + String.format( + "%s should have %d records.", splits.get(splitId), expectedRecordCount)); }); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java index eef62b8dbe..092c00b79e 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java @@ -31,8 +31,8 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.groups.SplitEnumeratorMetricGroup; import org.apache.flink.table.data.RowData; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +67,7 @@ public void testSingleParallelism() { } } - Assert.assertEquals(splitList.size(), actual.size()); + Assertions.assertEquals(splitList.size(), actual.size()); } @Test @@ -90,7 +90,7 @@ public void testMultiParallelism() { } } - Assert.assertEquals(splitList.size(), actual.size()); + Assertions.assertEquals(splitList.size(), actual.size()); } @Test @@ -156,7 +156,7 @@ public String toString() { .map(treeNode -> new long[] {treeNode.mask(), treeNode.index()}) .toArray(value -> new long[actualNodes.size()][]); - Assert.assertArrayEquals(expectNodes, result); + Assertions.assertArrayEquals(expectNodes, result); } @Test diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java index 71c85609ae..2f552c52dd 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java @@ -21,8 +21,8 @@ import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.split.MixedFormatSplit; import org.apache.amoro.flink.read.hybrid.split.MixedFormatSplitState; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Collection; import java.util.List; @@ -36,8 +36,8 @@ public class TestSplitAssignerAwaiting extends TestShuffleSplitAssigner { public void testEmpty() { ShuffleSplitAssigner splitAssigner = instanceSplitAssigner(1); Split split = splitAssigner.getNext(0); - Assert.assertNotNull(split); - Assert.assertEquals(Split.Status.UNAVAILABLE, split.status()); + Assertions.assertNotNull(split); + Assertions.assertEquals(Split.Status.UNAVAILABLE, split.status()); } @Test @@ -91,11 +91,11 @@ private void assertAvailableFuture(ShuffleSplitAssigner assigner, Runnable addSp // calling isAvailable again should return the same object reference // note that thenAccept will return a new future. // we want to assert the same instance on the assigner returned future - Assert.assertSame(future, assigner.isAvailable()); + Assertions.assertSame(future, assigner.isAvailable()); // now add some splits addSplitsRunnable.run(); - Assert.assertTrue(futureCompleted.get()); + Assertions.assertTrue(futureCompleted.get()); for (int i = 0; i < 1; ++i) { assertGetNext(assigner, Split.Status.AVAILABLE); @@ -106,21 +106,21 @@ private void assertAvailableFuture(ShuffleSplitAssigner assigner, Runnable addSp private void assertGetNext(ShuffleSplitAssigner assigner, Split.Status expectedStatus) { Split result = assigner.getNext(0); - Assert.assertEquals(expectedStatus, result.status()); + Assertions.assertEquals(expectedStatus, result.status()); switch (expectedStatus) { case AVAILABLE: - Assert.assertNotNull(result.split()); + Assertions.assertNotNull(result.split()); break; case UNAVAILABLE: - Assert.assertNull(result.split()); + Assertions.assertNull(result.split()); break; default: - Assert.fail("Unknown status: " + expectedStatus); + Assertions.fail("Unknown status: " + expectedStatus); } } private void assertSnapshot(ShuffleSplitAssigner assigner, int splitCount) { Collection stateBeforeGet = assigner.state(); - Assert.assertEquals(splitCount, stateBeforeGet.size()); + Assertions.assertEquals(splitCount, stateBeforeGet.size()); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java index 3c614e921d..638abdfa90 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java @@ -21,8 +21,8 @@ import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.reader.TestRowDataReaderFunction; import org.apache.amoro.flink.read.hybrid.split.MixedFormatSplit; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +53,7 @@ public void testSingleParallelism() throws IOException { } } - Assert.assertEquals(splitList.size(), actual.size()); + Assertions.assertEquals(splitList.size(), actual.size()); } } @@ -77,7 +77,7 @@ public void testMultiParallelism() throws IOException { } } - Assert.assertEquals(splitList.size(), actual.size()); + Assertions.assertEquals(splitList.size(), actual.size()); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestContinuousSplitPlannerImpl.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestContinuousSplitPlannerImpl.java index ff41fb5319..e6e6b27d64 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestContinuousSplitPlannerImpl.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestContinuousSplitPlannerImpl.java @@ -33,7 +33,7 @@ import org.apache.flink.types.RowKind; import org.apache.iceberg.flink.FlinkSchemaUtil; import org.apache.iceberg.io.TaskWriter; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +60,7 @@ public TestContinuousSplitPlannerImpl( new BasicTableTestHelper(true, true)); } - @Before + @BeforeEach public void init() throws IOException { testKeyedTable = getMixedTable().asKeyedTable(); // write base diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumStateSerializer.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumStateSerializer.java index 1c1f52616f..04d51245a6 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumStateSerializer.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumStateSerializer.java @@ -25,8 +25,8 @@ import org.apache.amoro.flink.read.hybrid.split.MixedFormatSplit; import org.apache.amoro.flink.read.hybrid.split.TemporalJoinSplits; import org.apache.flink.api.connector.source.SplitEnumeratorContext; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,12 +60,12 @@ public void testMixedFormatEnumState() throws IOException { new MixedFormatSourceEnumStateSerializer(); byte[] ser = mixedFormatSourceEnumStateSerializer.serialize(expect); - Assert.assertNotNull(ser); + Assertions.assertNotNull(ser); MixedFormatSourceEnumState actual = mixedFormatSourceEnumStateSerializer.deserialize(1, ser); - Assert.assertEquals(expect.pendingSplits().size(), actual.pendingSplits().size()); - Assert.assertEquals( + Assertions.assertEquals(expect.pendingSplits().size(), actual.pendingSplits().size()); + Assertions.assertEquals( Objects.requireNonNull(expect.shuffleSplitRelation()).length, Objects.requireNonNull(actual.shuffleSplitRelation()).length); @@ -86,10 +86,10 @@ public void testMixedFormatEnumState() throws IOException { } } - Assert.assertEquals(splitList.size(), actualSplits.size()); + Assertions.assertEquals(splitList.size(), actualSplits.size()); TemporalJoinSplits temporalJoinSplits = actual.temporalJoinSplits(); - Assert.assertEquals(expect.temporalJoinSplits(), temporalJoinSplits); + Assertions.assertEquals(expect.temporalJoinSplits(), temporalJoinSplits); } } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumerator.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumerator.java index 2849bf3ca3..396300cc9e 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumerator.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestMixedFormatSourceEnumerator.java @@ -44,9 +44,9 @@ import org.apache.flink.table.data.TimestampData; import org.apache.flink.types.RowKind; import org.apache.iceberg.io.TaskWriter; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.time.LocalDate; @@ -79,7 +79,7 @@ public TestMixedFormatSourceEnumerator() { protected static final LocalDateTime LDT = LocalDateTime.of(LocalDate.of(2022, 1, 1), LocalTime.of(0, 0, 0, 0)); - @Before + @BeforeEach public void init() throws IOException { testKeyedTable = getMixedTable().asKeyedTable(); // write change insert @@ -152,7 +152,7 @@ public void testReadersNumGreaterThanSplits() throws Exception { Collection pendingSplitsEmpty = enumerator.snapshotState(1).pendingSplits(); - Assert.assertEquals(splitCount, pendingSplitsEmpty.size()); + Assertions.assertEquals(splitCount, pendingSplitsEmpty.size()); // register readers, and let them request a split // 4 split, 5 subtask, one or more subtask will fetch empty split @@ -177,13 +177,13 @@ public void testReadersNumGreaterThanSplits() throws Exception { enumerator.addReader(4); enumerator.handleSourceEvent(4, new SplitRequestEvent()); - Assert.assertEquals(parallelism - splitCount, enumerator.getReadersAwaitingSplit().size()); - Assert.assertTrue(enumerator.snapshotState(2).pendingSplits().isEmpty()); + Assertions.assertEquals(parallelism - splitCount, enumerator.getReadersAwaitingSplit().size()); + Assertions.assertTrue(enumerator.snapshotState(2).pendingSplits().isEmpty()); } private void assertSnapshot(ShuffleSplitAssigner assigner, int splitCount) { Collection stateBeforeGet = assigner.state(); - Assert.assertEquals(splitCount, stateBeforeGet.size()); + Assertions.assertEquals(splitCount, stateBeforeGet.size()); } private ShuffleSplitAssigner instanceSplitAssigner( diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java index ba378491d1..c65aa1aae4 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java @@ -20,8 +20,8 @@ import org.apache.amoro.flink.read.hybrid.split.MixedFormatSplit; import org.apache.amoro.flink.read.hybrid.split.TemporalJoinSplits; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collection; @@ -72,7 +72,7 @@ public void round(List allSplit, Collection mixedForma CompletableFuture f4 = CompletableFuture.runAsync(() -> temporalJoinSplits.addSplitsBack(as2)); CompletableFuture.allOf(f1, f2, f3, f4).join(); - Assert.assertTrue(temporalJoinSplits.removeAndReturnIfAllFinished(allSplit)); + Assertions.assertTrue(temporalJoinSplits.removeAndReturnIfAllFinished(allSplit)); } static class TestMixedFormatSplit extends MixedFormatSplit { diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java index 61da6d3e71..77b371f8fd 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java @@ -21,7 +21,7 @@ import org.apache.amoro.BasicTableTestHelper; import org.apache.amoro.TableFormat; import org.apache.amoro.catalog.BasicCatalogTestHelper; -import org.apache.amoro.catalog.TableTestBase; +import org.apache.amoro.flink.FlinkTestBase; import org.apache.amoro.flink.read.MixedIncrementalLoader; import org.apache.amoro.flink.read.hybrid.enumerator.ContinuousSplitPlanner; import org.apache.amoro.flink.read.hybrid.enumerator.MergeOnReadIncrementalPlanner; @@ -42,34 +42,25 @@ import org.apache.iceberg.flink.data.RowDataUtil; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.TaskWriter; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; -@RunWith(value = Parameterized.class) -public class MixedIncrementalLoaderTest extends TableTestBase implements FlinkTaskWriterBaseTest { +public class MixedIncrementalLoaderTest extends FlinkTestBase implements FlinkTaskWriterBaseTest { - public MixedIncrementalLoaderTest(boolean partitionedTable) { - super( + private void setUpForParam(boolean partitionedTable) throws Exception { + initFlinkTestBase( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, partitionedTable)); + populateTable(); } - @Parameterized.Parameters(name = "partitionedTable = {0}") - public static Object[][] parameters() { - // todo mix hive test - return new Object[][] {{true}, {false}}; - } - - @Before - public void before() throws IOException { + private void populateTable() throws IOException { MixedTable mixedTable = getMixedTable(); TableSchema flinkPartialSchema = TableSchema.builder() @@ -109,8 +100,10 @@ public void before() throws IOException { } } - @Test - public void testMOR() { + @ParameterizedTest(name = "partitionedTable = {0}") + @ValueSource(booleans = {true, false}) + public void testMOR(boolean partitionedTable) throws Exception { + setUpForParam(partitionedTable); KeyedTable keyedTable = getMixedTable().asKeyedTable(); List expressions = Lists.newArrayList(Expressions.greaterThan("op_time", "2022-06-20T10:10:11.0")); @@ -154,9 +147,9 @@ public void testMOR() { } } if (isPartitionedTable()) { - Assert.assertEquals(6, actuals.size()); + Assertions.assertEquals(6, actuals.size()); } else { - Assert.assertEquals(9, actuals.size()); + Assertions.assertEquals(9, actuals.size()); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java index a86e2afda8..13169f245e 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java @@ -41,8 +41,8 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.TaskWriter; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -179,7 +179,7 @@ public void testReadNodesUpMoved() throws IOException { } protected void assertArrayEquals(RowData[] excepts, List actual) { - Assert.assertArrayEquals(excepts, sortRowDataCollection(actual)); + Assertions.assertArrayEquals(excepts, sortRowDataCollection(actual)); } protected RowData[] sortRowDataCollection(Collection records) { @@ -207,7 +207,7 @@ protected void writeUpdateWithSpecifiedMaskOne() throws IOException { protected void writeUpdateWithSpecifiedMask(List input, KeyedTable table, long mask) throws IOException { // write change update - TaskWriter taskWriter = createKeyedTaskWriter(table, ROW_TYPE, false, mask); + TaskWriter taskWriter = createKeyedTaskWriterWithMask(table, ROW_TYPE, false, mask); for (RowData record : input) { taskWriter.write(record); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestMixedFormatSplitSerializer.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestMixedFormatSplitSerializer.java index f1a6d115c3..d372753566 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestMixedFormatSplitSerializer.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestMixedFormatSplitSerializer.java @@ -21,8 +21,8 @@ import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.reader.TestRowDataReaderFunction; import org.apache.flink.util.FlinkRuntimeException; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; @@ -60,7 +60,7 @@ private void assertSerializedSplitEquals(List expected) { }) .collect(Collectors.toList()); - Assert.assertArrayEquals( + Assertions.assertArrayEquals( expected.toArray(new MixedFormatSplit[0]), contents.stream() .map( @@ -84,6 +84,6 @@ public void testNullableSplit() throws IOException { MixedFormatSplit actual = serializer.deserialize(1, ser); - Assert.assertNull(actual); + Assertions.assertNull(actual); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestLogRecordV1.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestLogRecordV1.java index bafb9c28cc..9e8bac740a 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestLogRecordV1.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestLogRecordV1.java @@ -18,8 +18,8 @@ package org.apache.amoro.flink.shuffle; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.amoro.data.ChangeAction; import org.apache.amoro.log.Bytes; @@ -35,8 +35,8 @@ import org.apache.flink.table.data.StringData; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; @@ -83,7 +83,7 @@ public void testLogDataSerialize() throws IOException { byte[] bytes = logDataJsonSerialization.serialize(logData); - Assert.assertNotNull(bytes); + Assertions.assertNotNull(bytes); String actualJson = new String(Bytes.subByte(bytes, 18, bytes.length - 18)); String expected = "{\"f_boolean\":true,\"f_int\":1,\"f_long\":123456789,\"f_list_string\":[null,\"b\",null,\"c\",null]}"; @@ -93,7 +93,7 @@ public void testLogDataSerialize() throws IOException { new LogDataJsonDeserialization<>( userSchema, LogRecordV1.factory, LogRecordV1.arrayFactory, LogRecordV1.mapFactory); LogData result = logDataJsonDeserialization.deserialize(bytes); - Assert.assertNotNull(result); + Assertions.assertNotNull(result); check(logData, result); } @@ -118,7 +118,7 @@ public void testLogDataSerializeNullList() throws IOException { byte[] bytes = logDataJsonSerialization.serialize(logData); - Assert.assertNotNull(bytes); + Assertions.assertNotNull(bytes); String actualJson = new String(Bytes.subByte(bytes, 18, bytes.length - 18)); String expected = "{\"f_boolean\":true,\"f_int\":1,\"f_long\":123456789,\"f_list_string\":[null,null,null]}"; @@ -128,7 +128,7 @@ public void testLogDataSerializeNullList() throws IOException { new LogDataJsonDeserialization<>( userSchema, LogRecordV1.factory, LogRecordV1.arrayFactory, LogRecordV1.mapFactory); LogData result = logDataJsonDeserialization.deserialize(bytes); - Assert.assertNotNull(result); + Assertions.assertNotNull(result); check(logData, result); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestRoundRobinShuffleRulePolicy.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestRoundRobinShuffleRulePolicy.java index c870eb2287..125deb1e5e 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestRoundRobinShuffleRulePolicy.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/shuffle/TestRoundRobinShuffleRulePolicy.java @@ -27,144 +27,154 @@ import org.apache.flink.api.common.functions.Partitioner; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.table.data.RowData; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestRoundRobinShuffleRulePolicy extends FlinkTestBase { - public TestRoundRobinShuffleRulePolicy(boolean keyedTable, boolean partitionedTable) { - super( + static Stream parameters() { + return Stream.of( + Arguments.of(true, true), + Arguments.of(true, false), + Arguments.of(false, true), + Arguments.of(false, false)); + } + + private void setUpForParam(boolean keyedTable, boolean partitionedTable) throws Exception { + initFlinkTestBase( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(keyedTable, partitionedTable)); } - @Parameterized.Parameters(name = "keyedTable = {0}, partitionedTable = {1}") - public static Object[][] parameters() { - return new Object[][] { - {true, true}, - {true, false}, - {false, true}, - {false, false} - }; - } + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testPrimaryKeyPartitionedTable(boolean keyedTable, boolean partitionedTable) + throws Exception { + setUpForParam(keyedTable, partitionedTable); - @Test - public void testPrimaryKeyPartitionedTable() throws Exception { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); ShuffleHelper helper = ShuffleHelper.build(getMixedTable(), getMixedTable().schema(), FLINK_ROW_TYPE); RoundRobinShuffleRulePolicy policy = new RoundRobinShuffleRulePolicy(helper, 5, 2); Map> subTaskTreeNodes = policy.getSubtaskTreeNodes(); - Assert.assertEquals(subTaskTreeNodes.size(), 5); + Assertions.assertEquals(subTaskTreeNodes.size(), 5); subTaskTreeNodes .values() .forEach( nodes -> { - Assert.assertEquals(nodes.size(), 2); - Assert.assertTrue(nodes.contains(DataTreeNode.of(1, 0))); - Assert.assertTrue(nodes.contains(DataTreeNode.of(1, 1))); + Assertions.assertEquals(nodes.size(), 2); + Assertions.assertTrue(nodes.contains(DataTreeNode.of(1, 0))); + Assertions.assertTrue(nodes.contains(DataTreeNode.of(1, 1))); }); KeySelector keySelector = policy.generateKeySelector(); Partitioner partitioner = policy.generatePartitioner(); - Assert.assertEquals( + Assertions.assertEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(1, "hello2", "2022-10-11T10:10:11.0")), 5)); - Assert.assertNotEquals( + Assertions.assertNotEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(1, "hello2", "2022-10-12T10:10:11.0")), 5)); - Assert.assertNotEquals( + Assertions.assertNotEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(2, "hello2", "2022-10-11T10:10:11.0")), 5)); } - @Test - public void testPrimaryKeyTableWithoutPartition() throws Exception { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeFalse(isPartitionedTable()); + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testPrimaryKeyTableWithoutPartition(boolean keyedTable, boolean partitionedTable) + throws Exception { + setUpForParam(keyedTable, partitionedTable); + + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeFalse(isPartitionedTable()); ShuffleHelper helper = ShuffleHelper.build(getMixedTable(), getMixedTable().schema(), FLINK_ROW_TYPE); RoundRobinShuffleRulePolicy policy = new RoundRobinShuffleRulePolicy(helper, 5, 2); Map> subTaskTreeNodes = policy.getSubtaskTreeNodes(); - Assert.assertEquals(subTaskTreeNodes.size(), 5); - Assert.assertEquals( + Assertions.assertEquals(subTaskTreeNodes.size(), 5); + Assertions.assertEquals( subTaskTreeNodes.get(0), Sets.newHashSet(DataTreeNode.of(7, 0), DataTreeNode.of(7, 5))); - Assert.assertEquals( + Assertions.assertEquals( subTaskTreeNodes.get(1), Sets.newHashSet(DataTreeNode.of(7, 1), DataTreeNode.of(7, 6))); - Assert.assertEquals( + Assertions.assertEquals( subTaskTreeNodes.get(2), Sets.newHashSet(DataTreeNode.of(7, 2), DataTreeNode.of(7, 7))); - Assert.assertEquals(subTaskTreeNodes.get(3), Sets.newHashSet(DataTreeNode.of(7, 3))); - Assert.assertEquals(subTaskTreeNodes.get(4), Sets.newHashSet(DataTreeNode.of(7, 4))); + Assertions.assertEquals(subTaskTreeNodes.get(3), Sets.newHashSet(DataTreeNode.of(7, 3))); + Assertions.assertEquals(subTaskTreeNodes.get(4), Sets.newHashSet(DataTreeNode.of(7, 4))); KeySelector keySelector = policy.generateKeySelector(); Partitioner partitioner = policy.generatePartitioner(); - Assert.assertEquals( + Assertions.assertEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(1, "hello2", "2022-10-11T10:10:11.0")), 5)); - Assert.assertEquals( + Assertions.assertEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(1, "hello2", "2022-10-12T10:10:11.0")), 5)); - Assert.assertNotEquals( + Assertions.assertNotEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(2, "hello2", "2022-10-11T10:10:11.0")), 5)); } - @Test - public void testPartitionedTableWithoutPrimaryKey() throws Exception { - Assume.assumeFalse(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testPartitionedTableWithoutPrimaryKey(boolean keyedTable, boolean partitionedTable) + throws Exception { + setUpForParam(keyedTable, partitionedTable); + + Assumptions.assumeFalse(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); ShuffleHelper helper = ShuffleHelper.build(getMixedTable(), getMixedTable().schema(), FLINK_ROW_TYPE); RoundRobinShuffleRulePolicy policy = new RoundRobinShuffleRulePolicy(helper, 5, 2); Map> subTaskTreeNodes = policy.getSubtaskTreeNodes(); - Assert.assertEquals(subTaskTreeNodes.size(), 5); + Assertions.assertEquals(subTaskTreeNodes.size(), 5); subTaskTreeNodes .values() .forEach( nodes -> { - Assert.assertEquals(nodes.size(), 1); - Assert.assertTrue(nodes.contains(DataTreeNode.of(0, 0))); + Assertions.assertEquals(nodes.size(), 1); + Assertions.assertTrue(nodes.contains(DataTreeNode.of(0, 0))); }); KeySelector keySelector = policy.generateKeySelector(); Partitioner partitioner = policy.generatePartitioner(); - Assert.assertEquals( + Assertions.assertEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(1, "hello2", "2022-10-11T10:10:11.0")), 5)); - Assert.assertEquals( + Assertions.assertEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( keySelector.getKey(createRowData(2, "hello2", "2022-10-11T10:10:11.0")), 5)); - Assert.assertNotEquals( + Assertions.assertNotEquals( partitioner.partition( keySelector.getKey(createRowData(1, "hello", "2022-10-11T10:10:11.0")), 5), partitioner.partition( diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/AmoroCatalogITCaseBase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/AmoroCatalogITCaseBase.java index 6ab283e097..6617de5dbb 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/AmoroCatalogITCaseBase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/AmoroCatalogITCaseBase.java @@ -20,11 +20,12 @@ import static org.apache.flink.table.api.config.TableConfigOptions.TABLE_DYNAMIC_TABLE_OPTIONS_ENABLED; +import org.apache.amoro.AmoroCatalog; import org.apache.amoro.TestAms; import org.apache.amoro.flink.MiniClusterResource; -import org.apache.amoro.formats.AmoroCatalogTestBase; import org.apache.amoro.formats.AmoroCatalogTestHelper; import org.apache.amoro.hive.TestHMS; +import org.apache.commons.io.FileUtils; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.state.StateBackend; @@ -37,31 +38,105 @@ import org.apache.flink.table.api.TableResult; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.test.util.MiniClusterWithClientResource; -import org.junit.ClassRule; - -import java.io.IOException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Files; + +/** + * JUnit 5 base class for Flink unified-catalog integration tests. + * + *

This class no longer extends the still-JUnit-4 {@code AmoroCatalogTestBase}; the catalog + * lifecycle is re-implemented here against the {@link AmoroCatalogTestHelper} contract so children + * can be clean Jupiter classes. Parameterized children construct without arguments and call {@link + * #initAmoroCatalog(AmoroCatalogTestHelper)} from the {@code @ParameterizedTest} method body. + */ +public class AmoroCatalogITCaseBase { + private static final Logger LOG = LoggerFactory.getLogger(AmoroCatalogITCaseBase.class); -public class AmoroCatalogITCaseBase extends AmoroCatalogTestBase { - static final TestHMS TEST_HMS = new TestHMS(); + protected static final TestHMS TEST_HMS = new TestHMS(); public static final String TEST_DB_NAME = "test_db"; public static final String TEST_TABLE_NAME = "test_table"; - @ClassRule - public static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = + protected static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = MiniClusterResource.createWithClassloaderCheckDisabled(); - @ClassRule public static TestAms TEST_AMS = new TestAms(); + protected static final TestAms TEST_AMS = new TestAms(); + + protected AmoroCatalogTestHelper catalogTestHelper; + protected AmoroCatalog amoroCatalog; + protected Object originalCatalog; + + private File tempRoot; private volatile StreamTableEnvironment tEnv = null; private volatile StreamExecutionEnvironment env = null; + /** No-arg constructor for parameterized children. */ + public AmoroCatalogITCaseBase() {} + public AmoroCatalogITCaseBase(AmoroCatalogTestHelper catalogTestHelper) { - super(catalogTestHelper); + this.catalogTestHelper = catalogTestHelper; + } + + @BeforeAll + public static void startBaseClassResources() throws Exception { + TEST_AMS.before(); + MINI_CLUSTER_RESOURCE.before(); + } + + @AfterAll + public static void stopBaseClassResources() { + try { + MINI_CLUSTER_RESOURCE.after(); + } finally { + TEST_AMS.after(); + } + } + + @BeforeEach + public void setUpAmoroCatalog() throws Exception { + if (catalogTestHelper == null) { + // Parameterized child: must call initAmoroCatalog(...) inside the test body. + return; + } + initLifecycle(); + } + + @AfterEach + public void tearDownAmoroCatalog() { + if (catalogTestHelper != null) { + try { + catalogTestHelper.clean(); + } catch (Throwable t) { + LOG.warn("Failed to clean catalog helper", t); + } + } + if (tempRoot != null) { + try { + FileUtils.deleteDirectory(tempRoot); + } catch (Throwable t) { + LOG.warn("Failed to delete temp dir {}", tempRoot, t); + } + tempRoot = null; + } + } + + protected void initAmoroCatalog(AmoroCatalogTestHelper catalogTestHelper) throws Exception { + this.catalogTestHelper = catalogTestHelper; + initLifecycle(); } - @Override - public void setupCatalog() throws IOException { - super.setupCatalog(); + private void initLifecycle() throws Exception { + tempRoot = Files.createTempDirectory("amoro-catalog-it").toFile(); + catalogTestHelper.initWarehouse(tempRoot.getPath()); + this.amoroCatalog = catalogTestHelper.amoroCatalog(); + this.originalCatalog = catalogTestHelper.originalCatalog(); catalogTestHelper.initHiveConf(TEST_HMS.getHiveConf()); TEST_AMS.getAmsHandler().createCatalog(catalogTestHelper.getCatalogMeta()); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/CatalogITCaseBase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/CatalogITCaseBase.java index 84500df822..8d1e67a2aa 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/CatalogITCaseBase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/CatalogITCaseBase.java @@ -20,10 +20,25 @@ import static org.apache.flink.table.api.config.TableConfigOptions.TABLE_DYNAMIC_TABLE_OPTIONS_ENABLED; +import org.apache.amoro.MockAmoroManagementServer; +import org.apache.amoro.TableFormat; import org.apache.amoro.TableTestHelper; +import org.apache.amoro.TestAms; +import org.apache.amoro.UnifiedCatalog; +import org.apache.amoro.api.CatalogMeta; import org.apache.amoro.catalog.CatalogTestHelper; -import org.apache.amoro.catalog.TableTestBase; import org.apache.amoro.flink.MiniClusterResource; +import org.apache.amoro.mixed.CatalogLoader; +import org.apache.amoro.mixed.MixedFormatCatalog; +import org.apache.amoro.properties.CatalogMetaProperties; +import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableBuilder; +import org.apache.amoro.table.TableMetaStore; +import org.apache.amoro.table.UnkeyedTable; +import org.apache.amoro.utils.CatalogUtil; +import org.apache.amoro.utils.MixedTableUtil; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.state.StateBackend; @@ -36,19 +51,246 @@ import org.apache.flink.table.api.TableResult; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.test.util.MiniClusterWithClientResource; -import org.junit.ClassRule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class CatalogITCaseBase extends TableTestBase { +import java.io.File; +import java.nio.file.Files; - @ClassRule - public static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = +/** + * JUnit 5 base class for Flink catalog integration tests that exercise the mixed-format catalog + * lifecycle but do not need the streaming-write helpers from FlinkTestBase. Implementations of + * FlinkTaskWriterBaseTest extend this directly so that no static {@code createKeyedTaskWriter} + * inherited from FlinkTestBase clashes with the interface's default methods. + */ +public abstract class CatalogITCaseBase { + private static final Logger LOG = LoggerFactory.getLogger(CatalogITCaseBase.class); + + protected static final TestAms TEST_AMS = new TestAms(); + + protected static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = MiniClusterResource.createWithClassloaderCheckDisabled(); + private CatalogTestHelper catalogTestHelper; + private TableTestHelper tableTestHelper; + private CatalogMeta catalogMeta; + private MixedFormatCatalog mixedFormatCatalog; + private UnifiedCatalog unifiedCatalog; + private org.apache.iceberg.catalog.Catalog icebergCatalog; + private MixedTable mixedTable; + private TableMetaStore tableMetaStore; + private File tempRoot; + private volatile StreamTableEnvironment tEnv = null; private volatile StreamExecutionEnvironment env = null; + /** No-arg constructor for parameterized children that supply helpers in the test method body. */ + public CatalogITCaseBase() {} + public CatalogITCaseBase(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { - super(catalogTestHelper, tableTestHelper); + this.catalogTestHelper = catalogTestHelper; + this.tableTestHelper = tableTestHelper; + } + + @BeforeAll + public static void startBaseClassResources() throws Exception { + TEST_AMS.before(); + MINI_CLUSTER_RESOURCE.before(); + } + + @AfterAll + public static void stopBaseClassResources() { + try { + MINI_CLUSTER_RESOURCE.after(); + } finally { + TEST_AMS.after(); + } + } + + @BeforeEach + public void setUpCatalogITCase() throws Exception { + if (catalogTestHelper == null) { + return; + } + initLifecycle(); + } + + @AfterEach + public void tearDownCatalogITCase() { + teardownLifecycle(); + } + + protected void initCatalogITCase( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + this.catalogTestHelper = catalogTestHelper; + this.tableTestHelper = tableTestHelper; + initLifecycle(); + } + + private void initLifecycle() throws Exception { + tempRoot = Files.createTempDirectory("catalog-it").toFile(); + String baseDir = tempRoot.getPath(); + if (!SystemUtils.IS_OS_UNIX) { + baseDir = "file:/" + baseDir.replace("\\", "/"); + } + catalogMeta = catalogTestHelper.buildCatalogMeta(baseDir); + catalogMeta.putToCatalogProperties(CatalogMetaProperties.AMS_URI, TEST_AMS.getServerUrl()); + getAmsHandler().createCatalog(catalogMeta); + if (tableTestHelper != null) { + createTestTable(); + } + } + + private void teardownLifecycle() { + if (tableTestHelper != null && unifiedCatalog != null) { + try { + unifiedCatalog.dropTable( + tableTestHelper.id().getDatabase(), tableTestHelper.id().getTableName(), true); + } catch (Exception e) { + LOG.warn("dropTable failed", e); + } + try { + unifiedCatalog.dropDatabase(TableTestHelper.TEST_DB_NAME); + } catch (Exception e) { + // ignore + } + } + if (catalogMeta != null) { + try { + getAmsHandler().dropCatalog(catalogMeta.getCatalogName()); + } catch (Exception e) { + LOG.warn("dropCatalog failed", e); + } + } + if (tempRoot != null) { + try { + FileUtils.deleteDirectory(tempRoot); + } catch (Exception e) { + LOG.warn("Failed to clean temp directory {}", tempRoot, e); + } + } + catalogMeta = null; + mixedFormatCatalog = null; + unifiedCatalog = null; + icebergCatalog = null; + mixedTable = null; + tableMetaStore = null; + tempRoot = null; + } + + private void createTestTable() { + this.tableMetaStore = CatalogUtil.buildMetaStore(getCatalogMeta()); + getUnifiedCatalog().createDatabase(TableTestHelper.TEST_DB_NAME); + TableFormat format = getTestFormat(); + if (format.in(TableFormat.MIXED_HIVE, TableFormat.MIXED_ICEBERG)) { + createMixedFormatTable(); + } else if (TableFormat.ICEBERG.equals(format)) { + createIcebergFormatTable(); + } + } + + private void createMixedFormatTable() { + TableBuilder tableBuilder = + getMixedFormatCatalog() + .newTableBuilder(TableTestHelper.TEST_TABLE_ID, tableTestHelper.tableSchema()); + tableBuilder.withProperties(tableTestHelper.tableProperties()); + if (isKeyedTable()) { + tableBuilder.withPrimaryKeySpec(tableTestHelper.primaryKeySpec()); + } + if (isPartitionedTable()) { + tableBuilder.withPartitionSpec(tableTestHelper.partitionSpec()); + } + mixedTable = tableBuilder.create(); + } + + private void createIcebergFormatTable() { + getIcebergCatalog() + .createTable( + org.apache.iceberg.catalog.TableIdentifier.of( + TableTestHelper.TEST_DB_NAME, TableTestHelper.TEST_TABLE_NAME), + tableTestHelper.tableSchema(), + tableTestHelper.partitionSpec(), + tableTestHelper.tableProperties()); + mixedTable = + (MixedTable) + getUnifiedCatalog() + .loadTable(TableTestHelper.TEST_DB_NAME, TableTestHelper.TEST_TABLE_NAME) + .originalTable(); + } + + public static MockAmoroManagementServer.AmsHandler getAmsHandler() { + return TEST_AMS.getAmsHandler(); + } + + protected MixedFormatCatalog getMixedFormatCatalog() { + if (mixedFormatCatalog == null) { + mixedFormatCatalog = CatalogLoader.load(getCatalogUri()); + } + return mixedFormatCatalog; + } + + protected void refreshMixedFormatCatalog() { + this.mixedFormatCatalog = CatalogLoader.load(getCatalogUri()); + } + + protected String getCatalogUri() { + return TEST_AMS.getServerUrl() + "/" + catalogMeta.getCatalogName(); + } + + protected CatalogMeta getCatalogMeta() { + return catalogMeta; + } + + protected TableFormat getTestFormat() { + return catalogTestHelper.tableFormat(); + } + + protected org.apache.iceberg.catalog.Catalog getIcebergCatalog() { + if (icebergCatalog == null) { + icebergCatalog = catalogTestHelper.buildIcebergCatalog(catalogMeta); + } + return icebergCatalog; + } + + protected UnifiedCatalog getUnifiedCatalog() { + if (unifiedCatalog == null) { + unifiedCatalog = catalogTestHelper.buildUnifiedCatalog(catalogMeta); + } + return unifiedCatalog; + } + + protected MixedTable getMixedTable() { + return mixedTable; + } + + protected UnkeyedTable getBaseStore() { + return MixedTableUtil.baseStore(mixedTable); + } + + protected TableMetaStore getTableMetaStore() { + return this.tableMetaStore; + } + + protected boolean isKeyedTable() { + return tableTestHelper.primaryKeySpec() != null + && tableTestHelper.primaryKeySpec().primaryKeyExisted(); + } + + protected boolean isPartitionedTable() { + return tableTestHelper.partitionSpec() != null + && tableTestHelper.partitionSpec().isPartitioned(); + } + + protected TableTestHelper tableTestHelper() { + return tableTestHelper; + } + + protected CatalogTestHelper catalogTestHelper() { + return catalogTestHelper; } protected TableResult exec(String query, Object... args) { @@ -67,7 +309,6 @@ protected StreamTableEnvironment getTableEnv() { StreamTableEnvironment.create( getEnv(), EnvironmentSettings.newInstance().inStreamingMode().build()); Configuration configuration = tEnv.getConfig().getConfiguration(); - // set low-level key-value options configuration.setString(TABLE_DYNAMIC_TABLE_OPTIONS_ENABLED.key(), "true"); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/LookupITCase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/LookupITCase.java index 9a1a21e09d..9b096eca81 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/LookupITCase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/LookupITCase.java @@ -33,10 +33,10 @@ import org.apache.flink.util.CloseableIterator; import org.apache.iceberg.flink.FlinkSchemaUtil; import org.apache.iceberg.io.TaskWriter; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.HashSet; @@ -54,7 +54,7 @@ public LookupITCase() { new BasicTableTestHelper(true, false)); } - @Before + @BeforeEach public void setup() throws IOException { List dbs = getMixedFormatCatalog().listDatabases(); if (dbs.isEmpty()) { @@ -83,7 +83,7 @@ public void setup() throws IOException { TableIdentifier.of(getCatalogName(), db, "L"), Lists.newArrayList(DataUtil.toRowData(1))); } - @After + @AfterEach public void drop() { exec("drop table mixed_catalog.%s.L", db); exec("drop table mixed_catalog.%s.DIM", db); @@ -129,7 +129,7 @@ public void testLookup() throws Exception { } } - Assert.assertEquals(expected, actual.size()); + Assertions.assertEquals(expected, actual.size()); List expects = new LinkedList<>(); expects.add(new Object[] {1, "a"}); expects.add(new Object[] {2, "c"}); @@ -137,7 +137,7 @@ public void testLookup() throws Exception { expects.add(new Object[] {4, "e"}); expects.add(new Object[] {5, "f"}); expects.add(new Object[] {6, null}); - Assert.assertEquals(DataUtil.toRowSet(expects), actual); + Assertions.assertEquals(DataUtil.toRowSet(expects), actual); } @Override @@ -171,7 +171,7 @@ private void writeAndCommit( boolean upsertEnabled) throws IOException { MixedTable mixedTable = getMixedFormatCatalog().loadTable(table); - Assert.assertNotNull(mixedTable); + Assertions.assertNotNull(mixedTable); RowType rowType = FlinkSchemaUtil.convert(mixedTable.schema()); for (RowData rowData : expected) { try (TaskWriter taskWriter = diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java index 8d2d4bed08..9878a68144 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java @@ -45,12 +45,10 @@ import org.apache.flink.types.RowKind; import org.apache.flink.util.CloseableIterator; import org.apache.iceberg.io.TaskWriter; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,8 +66,6 @@ public class TestJoin extends FlinkTestBase { public static final Logger LOG = LoggerFactory.getLogger(TestJoin.class); - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - private static final String DB = TableTestHelper.TEST_DB_NAME; private static final String TABLE = "test_keyed"; private static final TableIdentifier TABLE_ID = @@ -81,14 +77,13 @@ public TestJoin() { new BasicTableTestHelper(false, false)); } - @Before - public void before() throws Exception { - super.before(); + @BeforeEach + public void configProps() { super.config(); } - @After - public void after() { + @AfterEach + public void dropTestTable() { getMixedFormatCatalog().dropTable(TABLE_ID, true); } @@ -151,7 +146,7 @@ public void testRightEmptyLookupJoin() throws Exception { expected.add(new Object[] {"d", 1000022L, null, null}); expected.add(new Object[] {"e", 1000021L, null, null}); expected.add(new Object[] {"e", 1000016L, null, null}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); } @Test @@ -246,7 +241,7 @@ public void testLookupJoin() throws Exception { expected.add(new Object[] {"f", 6L, 324, "lily"}); expected.add(new Object[] {"g", 8L, null, null}); expected.add(new Object[] {"h", 9L, null, null}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); } @Test @@ -342,7 +337,7 @@ public void testLookupJoinWithPartialFields() throws Exception { expected.add(new Object[] {"f", 6L, "lily"}); expected.add(new Object[] {"g", 8L, null}); expected.add(new Object[] {"h", 9L, null}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); } private void writeChange(KeyedTable keyedTable, RowType rowType) { diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java index 05fc24eb23..7582f2edad 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java @@ -49,26 +49,19 @@ import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; import org.apache.flink.util.CloseableIterator; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.TestName; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -76,14 +69,11 @@ import java.util.Map; import java.util.Set; -@RunWith(Parameterized.class) public class TestKeyed extends FlinkTestBase { public static final Logger LOG = LoggerFactory.getLogger(TestKeyed.class); - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - @Rule public TestName testName = new TestName(); - @ClassRule public static TestHMS TEST_HMS = new TestHMS(); + static final TestHMS TEST_HMS = new TestHMS(); private static final String DB = TableTestHelper.TEST_TABLE_ID.getDatabase(); private static final String TABLE = "test_keyed"; @@ -94,68 +84,67 @@ public class TestKeyed extends FlinkTestBase { private final Map tableProperties = new HashMap<>(); public boolean isHive; - public TestKeyed( - CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) { - super(catalogTestHelper, tableTestHelper); - this.isHive = isHive; - } - - @Parameterized.Parameters(name = "{0}, {1}, {2}") - public static Collection parameters() { - return Arrays.asList( - new Object[][] { - { + static java.util.stream.Stream parameters() { + return java.util.stream.Stream.of( + Arguments.of( new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), new HiveTableTestHelper(true, true), - true - }, - { + true), + Arguments.of( new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), new HiveTableTestHelper(true, true), - true - }, - { + true), + Arguments.of( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true), - false - }, - { + false), + Arguments.of( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true), - false - } - }); + false)); } - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { + TEST_HMS.before(); FlinkTestBase.prepare(); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { - FlinkTestBase.shutdown(); + try { + FlinkTestBase.shutdown(); + } finally { + TEST_HMS.after(); + } } - @Before - public void before() throws Exception { + @AfterEach + public void dropTestTable() { + if (db != null) { + sql("DROP TABLE IF EXISTS mixed_catalog." + db + "." + TABLE); + } + } + + private void setUpForParam( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + this.isHive = isHive; if (isHive) { db = HiveTableTestHelper.TEST_DB_NAME; } else { db = DB; } - super.before(); - prepareLog(); - super.config(); + initFlinkTestBase(catalogTestHelper, tableTestHelper); + prepareLog(testInfo); + config(); } - @After - public void after() { - sql("DROP TABLE IF EXISTS mixed_catalog." + db + "." + TABLE); - } - - private void prepareLog() { - topic = TestUtil.getUtMethodName(testName) + isHive; + private void prepareLog(TestInfo testInfo) { + topic = TestUtil.getUtMethodName(testInfo) + isHive; tableProperties.clear(); tableProperties.put(ENABLE_LOG_STORE, "true"); tableProperties.put(LOG_STORE_MESSAGE_TOPIC, topic); @@ -165,8 +154,15 @@ private void prepareLog() { LOG_STORE_ADDRESS, KafkaContainerTest.KAFKA_CONTAINER.getBootstrapServers()); } - @Test - public void testSinkSourceFile() throws IOException { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testSinkSourceFile( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); List data = new LinkedList<>(); data.add( @@ -293,11 +289,19 @@ public void testSinkSourceFile() throws IOException { LocalDateTime.parse("2022-06-17T10:10:11.0").atZone(ZoneId.systemDefault()).toInstant() }); - Assert.assertTrue(CollectionUtils.isEqualCollection(DataUtil.toRowList(expected), actual)); + Assertions.assertTrue(CollectionUtils.isEqualCollection(DataUtil.toRowList(expected), actual)); } - @Test - public void testUnpartitionLogSinkSource() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testUnpartitionLogSinkSource( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a"}); data.add(new Object[] {1000015, "b"}); @@ -358,12 +362,20 @@ public void testUnpartitionLogSinkSource() throws Exception { actual.add(row); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } - @Test - public void testUnpartitionLogSinkSourceWithSelectedFields() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testUnpartitionLogSinkSourceWithSelectedFields( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); data.add(new Object[] {1000015, "b", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -435,12 +447,20 @@ public void testUnpartitionLogSinkSourceWithSelectedFields() throws Exception { expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } - @Test - public void testUnPartitionDoubleSink() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testUnPartitionDoubleSink( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a"}); data.add(new Object[] {1000015, "b"}); @@ -480,7 +500,7 @@ public void testUnPartitionDoubleSink() throws Exception { + ") */" + "select id, name from input"); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(data), new HashSet<>( sql( @@ -503,12 +523,19 @@ public void testUnPartitionDoubleSink() throws Exception { actual.add(iterator.next()); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } - @Test - public void testPartitionSinkFile() throws IOException { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionSinkFile( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -551,7 +578,7 @@ public void testPartitionSinkFile() throws IOException { + ")*/" + " select * from input"); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(data), new HashSet<>( sql( @@ -564,8 +591,15 @@ public void testPartitionSinkFile() throws IOException { + ") */"))); } - @Test - public void testSinkSourceFileWithoutSelectPK() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testSinkSourceFileWithoutSelectPK( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -633,11 +667,18 @@ public void testSinkSourceFileWithoutSelectPK() throws Exception { expected.add(new Object[] {"d", LocalDateTime.parse("2022-06-18T10:10:11.0")}); expected.add(new Object[] {"e", LocalDateTime.parse("2022-06-18T10:10:11.0")}); - Assert.assertEquals(DataUtil.toRowSet(expected), new HashSet<>(actual)); + Assertions.assertEquals(DataUtil.toRowSet(expected), new HashSet<>(actual)); } - @Test - public void testFileUpsert() { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testFileUpsert( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); List data = new LinkedList<>(); data.add( @@ -722,7 +763,7 @@ public void testFileUpsert() { new Object[] {RowKind.INSERT, 1000021, "e", LocalDateTime.parse("2022-06-17T10:11:11.0")}); expected.add( new Object[] {RowKind.INSERT, 1000021, "f", LocalDateTime.parse("2022-06-17T10:10:11.0")}); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(expected), new HashSet<>( sql( @@ -735,8 +776,15 @@ public void testFileUpsert() { + ") */"))); } - @Test - public void testFileCDC() { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testFileCDC( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); List data = new LinkedList<>(); data.add( @@ -846,7 +894,7 @@ public void testFileCDC() { new Object[] {RowKind.INSERT, 1000031, "f", LocalDateTime.parse("2022-06-17T10:10:11.0")}); expected.add( new Object[] {RowKind.INSERT, 1000032, "e", LocalDateTime.parse("2022-06-17T10:10:11.0")}); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(expected), new HashSet<>( sql( @@ -859,8 +907,15 @@ public void testFileCDC() { + ") */"))); } - @Test - public void testFileUpsertWithSamePrimaryKey() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testFileUpsertWithSamePrimaryKey( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); List data = new LinkedList<>(); data.add( @@ -937,13 +992,21 @@ public void testFileUpsertWithSamePrimaryKey() throws Exception { DataUtil.groupByPrimaryKey(DataUtil.toRowList(expected), 0); for (Object key : actualMap.keySet()) { - Assert.assertTrue( + Assertions.assertTrue( CollectionUtils.isEqualCollection(actualMap.get(key), expectedMap.get(key))); } } - @Test - public void testPartitionLogSinkSource() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionLogSinkSource( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); data.add(new Object[] {1000015, "b", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -1005,12 +1068,20 @@ public void testPartitionLogSinkSource() throws Exception { actual.add(row); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } - @Test - public void testPartitionLogSinkSourceWithSelectedFields() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionLogSinkSourceWithSelectedFields( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); data.add(new Object[] {1000015, "b", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -1082,12 +1153,20 @@ public void testPartitionLogSinkSourceWithSelectedFields() throws Exception { expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } - @Test - public void testPartitionDoubleSink() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionDoubleSink( + CatalogTestHelper catalogTestHelper, + TableTestHelper tableTestHelper, + boolean isHive, + TestInfo testInfo) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive, testInfo); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); data.add(new Object[] {1000015, "b", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -1131,7 +1210,7 @@ public void testPartitionDoubleSink() throws Exception { + ") */" + "select * from input"); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(data), new HashSet<>( sql( @@ -1157,7 +1236,7 @@ public void testPartitionDoubleSink() throws Exception { actual.add(row); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java index 659f4e955f..893a24e99d 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java @@ -35,10 +35,10 @@ import org.apache.flink.util.CloseableIterator; import org.apache.iceberg.flink.FlinkSchemaUtil; import org.apache.iceberg.io.TaskWriter; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; @@ -57,7 +57,7 @@ public TestLookupSecondary() { new BasicTableTestHelper(true, false)); } - @Before + @BeforeEach public void setup() throws IOException { List dbs = getMixedFormatCatalog().listDatabases(); if (dbs.isEmpty()) { @@ -96,7 +96,7 @@ public void setup() throws IOException { true); } - @After + @AfterEach public void drop() { exec("drop table mixed_catalog.%s.L", db); exec("drop table mixed_catalog.%s.DIM_2", db); @@ -128,7 +128,7 @@ public void testLookup() throws Exception { } } - Assert.assertEquals(expected, actual.size()); + Assertions.assertEquals(expected, actual.size()); List rows = expects.stream() .map( @@ -137,7 +137,7 @@ public void testLookup() throws Exception { ? Row.ofKind((RowKind) r[0], ArrayUtils.subarray(r, 1, r.length)) : Row.of(r)) .collect(Collectors.toList()); - Assert.assertEquals( + Assertions.assertEquals( rows.stream().sorted(Comparator.comparing(Row::toString)).collect(Collectors.toList()), actual.stream().sorted(Comparator.comparing(Row::toString)).collect(Collectors.toList())); } @@ -173,7 +173,7 @@ private void writeAndCommit( boolean upsertEnabled) throws IOException { MixedTable mixedTable = getMixedFormatCatalog().loadTable(table); - Assert.assertNotNull(mixedTable); + Assertions.assertNotNull(mixedTable); RowType rowType = FlinkSchemaUtil.convert(mixedTable.schema()); for (RowData rowData : expected) { try (TaskWriter taskWriter = diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java index 5bbc2412e2..735a8e9299 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java @@ -32,41 +32,44 @@ import org.apache.amoro.hive.catalog.HiveTableTestHelper; import org.apache.amoro.table.MixedTable; import org.apache.iceberg.UpdateProperties; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; -import java.util.Arrays; -import java.util.Collection; import java.util.Map; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestTableRefresh extends FlinkTestBase { - @ClassRule public static TestHMS TEST_HMS = new TestHMS(); + static final TestHMS TEST_HMS = new TestHMS(); - public TestTableRefresh(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { - super(catalogTestHelper, tableTestHelper); - } - - @Parameterized.Parameters(name = "{0}, {1}") - public static Collection parameters() { - return Arrays.asList( - new Object[][] { - { + static Stream parameters() { + return Stream.of( + Arguments.of( new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(true, true) - }, - { + new HiveTableTestHelper(true, true)), + Arguments.of( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), - new BasicTableTestHelper(true, true) - } - }); + new BasicTableTestHelper(true, true))); + } + + @BeforeAll + public static void startTestHms() throws Exception { + TEST_HMS.before(); + } + + @AfterAll + public static void stopTestHms() { + TEST_HMS.after(); } - @Test - public void testRefresh() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testRefresh(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) + throws Exception { + initFlinkTestBase(catalogTestHelper, tableTestHelper); MixedFormatTableLoader tableLoader = MixedFormatTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); @@ -82,7 +85,7 @@ public void testRefresh() { mixedTable.refresh(); Map properties = mixedTable.properties(); - Assert.assertEquals(String.valueOf(catchUp), properties.get(LOG_STORE_CATCH_UP.key())); - Assert.assertEquals(catchUpTs, properties.get(LOG_STORE_CATCH_UP_TIMESTAMP.key())); + Assertions.assertEquals(String.valueOf(catchUp), properties.get(LOG_STORE_CATCH_UP.key())); + Assertions.assertEquals(catchUpTs, properties.get(LOG_STORE_CATCH_UP_TIMESTAMP.key())); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java index ccab05a761..fd8d4e76e8 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java @@ -49,23 +49,16 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.types.Types; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.IOException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -73,11 +66,8 @@ import java.util.Map; import java.util.Set; -@RunWith(Parameterized.class) public class TestUnkeyed extends FlinkTestBase { - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - private static final String TABLE = "test_unkeyed"; private static final String DB = TableTestHelper.TEST_TABLE_ID.getDatabase(); @@ -86,49 +76,51 @@ public class TestUnkeyed extends FlinkTestBase { private String db; private String topic; - @ClassRule public static TestHMS TEST_HMS = new TestHMS(); + static final TestHMS TEST_HMS = new TestHMS(); public boolean isHive; - public TestUnkeyed( - CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) { - super(catalogTestHelper, tableTestHelper); - this.isHive = isHive; - } - - @Parameterized.Parameters(name = "{0}, {1}, {2}") - public static Collection parameters() { - return Arrays.asList( - new Object[][] { - { + static java.util.stream.Stream parameters() { + return java.util.stream.Stream.of( + Arguments.of( new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), new HiveTableTestHelper(true, true), - true - }, - { + true), + Arguments.of( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true), - false - }, - { + false), + Arguments.of( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true), - false - } - }); + false)); } - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { + TEST_HMS.before(); KAFKA_CONTAINER.start(); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { - KAFKA_CONTAINER.close(); + try { + KAFKA_CONTAINER.close(); + } finally { + TEST_HMS.after(); + } } - @Before - public void before() throws Exception { + @AfterEach + public void dropTestTable() { + if (db != null) { + sql("DROP TABLE IF EXISTS mixed_catalog." + db + "." + TABLE); + } + } + + private void setUpForParam( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + this.isHive = isHive; if (isHive) { catalog = HiveTableTestHelper.TEST_CATALOG_NAME; db = HiveTableTestHelper.TEST_DB_NAME; @@ -136,19 +128,19 @@ public void before() throws Exception { catalog = TEST_CATALOG_NAME; db = DB; } - super.before(); + initFlinkTestBase(catalogTestHelper, tableTestHelper); mixedFormatCatalog = getMixedFormatCatalog(); topic = String.join(".", catalog, db, TABLE); - super.config(); + config(); } - @After - public void after() { - sql("DROP TABLE IF EXISTS mixed_catalog." + db + "." + TABLE); - } + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testUnPartitionDDL( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); - @Test - public void testUnPartitionDDL() throws IOException { sql("CREATE CATALOG mixed_catalog WITH %s", toWithClause(props)); sql( @@ -172,11 +164,16 @@ public void testUnPartitionDDL() throws IOException { Types.NestedField.optional(6, "height", Types.FloatType.get()), Types.NestedField.optional(7, "speed", Types.DoubleType.get()), Types.NestedField.optional(8, "ts", Types.TimestampType.withoutZone())); - Assert.assertEquals(required.asStruct(), table.schema().asStruct()); + Assertions.assertEquals(required.asStruct(), table.schema().asStruct()); } - @Test - public void testPartitionDDL() throws IOException { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionDDL( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + sql("CREATE CATALOG mixed_catalog WITH %s", toWithClause(props)); sql( @@ -199,14 +196,19 @@ public void testPartitionDDL() throws IOException { Types.NestedField.optional(7, "speed", Types.DoubleType.get()), Types.NestedField.optional(8, "ts", Types.TimestampType.withoutZone())); MixedTable table = mixedFormatCatalog.loadTable(TableIdentifier.of(catalog, db, TABLE)); - Assert.assertEquals(required.asStruct(), table.schema().asStruct()); + Assertions.assertEquals(required.asStruct(), table.schema().asStruct()); PartitionSpec requiredSpec = PartitionSpec.builderFor(required).identity("ts").build(); - Assert.assertEquals(requiredSpec, table.spec()); + Assertions.assertEquals(requiredSpec, table.spec()); } - @Test - public void testUnkeyedWatermarkSet() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testUnkeyedWatermarkSet( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -263,13 +265,18 @@ public void testUnkeyedWatermarkSet() throws Exception { actual.add(iterator.next()); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } - @Test - public void testSinkBatchRead() throws IOException { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testSinkBatchRead( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); data.add(new Object[] {1000015, "b", LocalDateTime.parse("2022-06-17T10:08:11.0")}); @@ -310,7 +317,7 @@ public void testSinkBatchRead() throws IOException { Iterable snapshots = table.asUnkeyedTable().snapshots(); Snapshot s = snapshots.iterator().next(); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(data), new HashSet<>( sql( @@ -327,8 +334,13 @@ public void testSinkBatchRead() throws IOException { + ")*/"))); } - @Test - public void testSinkStreamRead() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testSinkStreamRead( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a"}); data.add(new Object[] {1000015, "b"}); @@ -374,7 +386,7 @@ public void testSinkStreamRead() throws Exception { } } resultWithEarliestPosition.getJobClient().ifPresent(TestUtil::cancelJob); - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); // verify in latest scan-startup-mode file read TableResult resultWithLatestPosition = @@ -413,16 +425,21 @@ public void testSinkStreamRead() throws Exception { try (CloseableIterator iterator = resultWithLatestPosition.collect()) { sql("insert into mixed_catalog." + db + "." + TABLE + " select * from appendInput"); for (int i = 0; i < appendData.size(); i++) { - Assert.assertTrue("Should have more records", iterator.hasNext()); + Assertions.assertTrue(iterator.hasNext(), "Should have more records"); actual.add(iterator.next()); } } resultWithLatestPosition.getJobClient().ifPresent(TestUtil::cancelJob); - Assert.assertEquals(DataUtil.toRowSet(appendData), actual); + Assertions.assertEquals(DataUtil.toRowSet(appendData), actual); } - @Test - public void testLogSinkSource() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testLogSinkSource( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + String topic = this.topic + "testLogSinkSource"; KafkaContainerTest.createTopics(KAFKA_PARTITION_NUMS, 1, topic); @@ -488,14 +505,19 @@ public void testLogSinkSource() throws Exception { actual.add(iterator.next()); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); KafkaContainerTest.deleteTopics(topic); } - @Test - public void testUnpartitionLogSinkSourceWithSelectedFields() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testUnpartitionLogSinkSourceWithSelectedFields( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", LocalDateTime.parse("2022-06-17T10:10:11.0")}); data.add(new Object[] {1000015, "b", LocalDateTime.parse("2022-06-17T10:10:11.0")}); @@ -570,13 +592,18 @@ public void testUnpartitionLogSinkSourceWithSelectedFields() throws Exception { expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); } - @Test - public void testUnPartitionDoubleSink() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testUnPartitionDoubleSink( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + String topic = this.topic + "testUnPartitionDoubleSink"; KafkaContainerTest.createTopics(KAFKA_PARTITION_NUMS, 1, topic); @@ -624,7 +651,7 @@ public void testUnPartitionDoubleSink() throws Exception { + ") */" + "select id, name from input"); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(data), sqlSet( "select * from mixed_catalog." @@ -646,13 +673,18 @@ public void testUnPartitionDoubleSink() throws Exception { actual.add(iterator.next()); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); KafkaContainerTest.deleteTopics(topic); } - @Test - public void testPartitionSinkBatchRead() throws IOException { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionSinkBatchRead( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", "2022-05-17"}); data.add(new Object[] {1000015, "b", "2022-05-17"}); @@ -702,7 +734,7 @@ public void testPartitionSinkBatchRead() throws IOException { Iterable snapshots = table.asUnkeyedTable().snapshots(); Snapshot s = snapshots.iterator().next(); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(expected), sqlSet( "select * from mixed_catalog." @@ -716,7 +748,7 @@ public void testPartitionSinkBatchRead() throws IOException { + "'" + ", 'streaming'='false'" + ")*/")); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(expected), sqlSet( "select * from mixed_catalog." @@ -732,8 +764,13 @@ public void testPartitionSinkBatchRead() throws IOException { + ")*/")); } - @Test - public void testPartitionSinkStreamRead() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionSinkStreamRead( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", "2022-05-17"}); data.add(new Object[] {1000015, "b", "2022-05-17"}); @@ -814,11 +851,16 @@ public void testPartitionSinkStreamRead() throws Exception { } } result.getJobClient().ifPresent(TestUtil::cancelJob); - Assert.assertEquals(new HashSet<>(expected), actual); + Assertions.assertEquals(new HashSet<>(expected), actual); } - @Test - public void testPartitionLogSinkSource() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionLogSinkSource( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + String topic = this.topic + "testUnKeyedPartitionLogSinkSource"; KafkaContainerTest.createTopics(KAFKA_PARTITION_NUMS, 1, topic); @@ -885,14 +927,19 @@ public void testPartitionLogSinkSource() throws Exception { actual.add(iterator.next()); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); KafkaContainerTest.deleteTopics(topic); } - @Test - public void testPartitionLogSinkSourceWithSelectedFields() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionLogSinkSourceWithSelectedFields( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + String topic = this.topic + "testPartitionLogSinkSourceWithSelectedFields"; KafkaContainerTest.createTopics(KAFKA_PARTITION_NUMS, 1, topic); @@ -969,14 +1016,19 @@ public void testPartitionLogSinkSourceWithSelectedFields() throws Exception { expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); expected.add(new Object[] {1000007, LocalDateTime.parse("2022-06-18T10:10:11.0")}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); KafkaContainerTest.deleteTopics(topic); } - @Test - public void testPartitionDoubleSink() throws Exception { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionDoubleSink( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + String topic = this.topic + "testUnkeyedPartitionDoubleSink"; KafkaContainerTest.createTopics(KAFKA_PARTITION_NUMS, 1, topic); @@ -1024,7 +1076,7 @@ public void testPartitionDoubleSink() throws Exception { + ") */" + "select * from input"); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(data), sqlSet( "select * from mixed_catalog." @@ -1045,7 +1097,7 @@ public void testPartitionDoubleSink() throws Exception { actual.add(iterator.next()); } } - Assert.assertEquals(DataUtil.toRowSet(data), actual); + Assertions.assertEquals(DataUtil.toRowSet(data), actual); result.getJobClient().ifPresent(TestUtil::cancelJob); KafkaContainerTest.deleteTopics(topic); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java index 026b20e65e..b51b18f790 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java @@ -24,7 +24,6 @@ import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.catalog.CatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.MiniClusterResource; import org.apache.amoro.flink.util.DataUtil; import org.apache.amoro.hive.TestHMS; import org.apache.amoro.hive.catalog.HiveCatalogTestHelper; @@ -32,79 +31,79 @@ import org.apache.flink.table.api.ApiExpression; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Table; -import org.apache.flink.test.util.MiniClusterWithClientResource; -import org.junit.After; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.LinkedList; import java.util.List; -@RunWith(Parameterized.class) public class TestUnkeyedOverwrite extends FlinkTestBase { private static final Logger LOGGER = LoggerFactory.getLogger(TestUnkeyedOverwrite.class); - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - - @ClassRule - public static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = - MiniClusterResource.createWithClassloaderCheckDisabled(); - private static final String TABLE = "test_unkeyed"; private static final String DB = TableTestHelper.TEST_TABLE_ID.getDatabase(); private String db; public boolean isHive; - @ClassRule public static TestHMS TEST_HMS = new TestHMS(); + static final TestHMS TEST_HMS = new TestHMS(); + + static java.util.stream.Stream parameters() { + return java.util.stream.Stream.of( + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(true, true), + true), + Arguments.of( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(true, true), + false)); + } - public TestUnkeyedOverwrite( - CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) { - super(catalogTestHelper, tableTestHelper); - this.isHive = isHive; + @BeforeAll + public static void startTestHms() throws Exception { + TEST_HMS.before(); + } + + @AfterAll + public static void stopTestHms() { + TEST_HMS.after(); } - @Parameterized.Parameters(name = "{0}, {1}, {2}") - public static Object[] parameters() { - return new Object[][] { - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(true, true), - true - }, - { - new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), - new BasicTableTestHelper(true, true), - false - } - }; + @AfterEach + public void dropTestTable() { + if (db != null) { + sql("DROP TABLE IF EXISTS mixed_catalog." + db + "." + TABLE); + } } - public void before() throws Exception { + private void setUpForParam( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + this.isHive = isHive; if (isHive) { db = HiveTableTestHelper.TEST_DB_NAME; } else { db = DB; } - super.before(); - super.config(); + initFlinkTestBase(catalogTestHelper, tableTestHelper); + config(); } - @After - public void after() { - sql("DROP TABLE IF EXISTS mixed_catalog." + db + "." + TABLE); - } + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testInsertOverwrite( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); - @Test - public void testInsertOverwrite() throws IOException { List data = new LinkedList<>(); data.add(new Object[] {1000004, "a"}); data.add(new Object[] {1000015, "b"}); @@ -135,7 +134,7 @@ public void testInsertOverwrite() throws IOException { sql("insert overwrite mixed_catalog." + db + "." + TABLE + " select * from input"); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(data), sqlSet( "select * from mixed_catalog." @@ -147,8 +146,13 @@ public void testInsertOverwrite() throws IOException { + ") */")); } - @Test - public void testPartitionInsertOverwrite() throws IOException { + @ParameterizedTest(name = "{0}, {1}, {2}") + @MethodSource("parameters") + public void testPartitionInsertOverwrite( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper, boolean isHive) + throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper, isHive); + List data = new LinkedList<>(); data.add(new Object[] {1000004, "a", "2022-05-17"}); data.add(new Object[] {1000015, "b", "2022-05-17"}); @@ -193,7 +197,7 @@ public void testPartitionInsertOverwrite() throws IOException { + TABLE + " PARTITION (dt='2022-05-18') select id, name from input where dt = '2022-05-19'"); - Assert.assertEquals( + Assertions.assertEquals( DataUtil.toRowSet(expected), sqlSet( "select id, name, '2022-05-19' from mixed_catalog." diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java index ae0dbe8c77..434339a2fb 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java @@ -51,12 +51,10 @@ import org.apache.flink.types.RowKind; import org.apache.flink.util.CloseableIterator; import org.apache.iceberg.io.TaskWriter; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,8 +72,6 @@ public class TestWatermark extends FlinkTestBase { public static final Logger LOG = LoggerFactory.getLogger(TestWatermark.class); - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - private static final String DB = TableTestHelper.TEST_TABLE_ID.getDatabase(); private static final String TABLE = "test_keyed"; @@ -85,14 +81,13 @@ public TestWatermark() { new BasicTableTestHelper(true, true)); } - @Before - public void before() throws Exception { - super.before(); + @BeforeEach + public void configProps() { super.config(); } - @After - public void after() { + @AfterEach + public void dropTestTable() { sql("DROP TABLE IF EXISTS mixed_catalog." + DB + "." + TABLE); } @@ -157,7 +152,7 @@ public void testWatermark() throws Exception { op.waitWatermark(); - Assert.assertTrue(op.watermark > Long.MIN_VALUE); + Assertions.assertTrue(op.watermark > Long.MIN_VALUE); } @Test @@ -224,7 +219,7 @@ public void testSelectWatermarkField() throws Exception { List expected = new LinkedList<>(); expected.add(new Object[] {true, LocalDateTime.parse("2022-06-17T10:08:11")}); - Assert.assertEquals(DataUtil.toRowSet(expected), actual); + Assertions.assertEquals(DataUtil.toRowSet(expected), actual); } public static class WatermarkTestOperator extends AbstractStreamOperator diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/DataUtil.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/DataUtil.java index f71476ad31..db569cdf37 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/DataUtil.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/DataUtil.java @@ -33,7 +33,7 @@ import org.apache.iceberg.data.IcebergGenerics; import org.apache.iceberg.data.Record; import org.apache.iceberg.io.CloseableIterable; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import java.time.Instant; import java.time.LocalDateTime; @@ -80,16 +80,16 @@ public static List toRowList(Collection data) { } public static void assertEqual(Collection expected, Collection actual) { - Assert.assertEquals( + Assertions.assertEquals( CollectionUtil.isNullOrEmpty(expected), CollectionUtil.isNullOrEmpty(actual)); if (expected == null) { return; } - Assert.assertEquals(expected.size(), actual.size()); + Assertions.assertEquals(expected.size(), actual.size()); for (Iterator i1 = expected.iterator(), i2 = actual.iterator(); i1.hasNext(); ) { Object[] actualRow = i2.next(); System.out.println(ArrayUtils.toString(actualRow)); - Assert.assertArrayEquals(i1.next(), actualRow); + Assertions.assertArrayEquals(i1.next(), actualRow); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestCompatibleFlinkPropertyUtil.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestCompatibleFlinkPropertyUtil.java index dfe12e2f82..cbebfffe05 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestCompatibleFlinkPropertyUtil.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestCompatibleFlinkPropertyUtil.java @@ -20,26 +20,26 @@ import org.apache.amoro.flink.table.descriptors.MixedFormatValidator; import org.apache.flink.configuration.Configuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class TestCompatibleFlinkPropertyUtil { @Test public void testGetNewProperty() { Configuration config = new Configuration(); - Assert.assertEquals( + Assertions.assertEquals( MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE.defaultValue(), CompatibleFlinkPropertyUtil.propertyAsBoolean( config, MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE)); config.setBoolean(MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE, true); - Assert.assertTrue( + Assertions.assertTrue( CompatibleFlinkPropertyUtil.propertyAsBoolean( config, MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE)); config.setBoolean( MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE_LEGACY, false); - Assert.assertTrue( + Assertions.assertTrue( CompatibleFlinkPropertyUtil.propertyAsBoolean( config, MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE)); } @@ -49,7 +49,7 @@ public void testGetLegacyProperty() { Configuration config = new Configuration(); config.setBoolean( MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE_LEGACY, true); - Assert.assertTrue( + Assertions.assertTrue( CompatibleFlinkPropertyUtil.propertyAsBoolean( config, MixedFormatValidator.MIXED_FORMAT_LOG_CONSISTENCY_GUARANTEE_ENABLE)); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestUtil.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestUtil.java index 6888add512..bdda4f638b 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestUtil.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestUtil.java @@ -20,7 +20,7 @@ import org.apache.flink.api.common.JobStatus; import org.apache.flink.core.execution.JobClient; -import org.junit.rules.TestName; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,13 +28,14 @@ public class TestUtil { public static final Logger LOG = LoggerFactory.getLogger(TestUtil.class); - /** get ut method name without parameters. */ - public static String getUtMethodName(TestName testName) { - int i = testName.getMethodName().indexOf("["); + /** Returns the test method name without any parameter index suffix. */ + public static String getUtMethodName(TestInfo testInfo) { + String name = testInfo.getTestMethod().map(java.lang.reflect.Method::getName).orElse("unknown"); + int i = name.indexOf("["); if (i == -1) { - return testName.getMethodName(); + return name; } - return testName.getMethodName().substring(0, i); + return name.substring(0, i); } public static void cancelJob(JobClient jobClient) { diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java index 610a7854b0..f9e144c19f 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java @@ -41,7 +41,7 @@ import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.io.WriteResult; import org.apache.iceberg.types.TypeUtil; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +60,7 @@ default void testWriteAndReadMixedFormatTable( RowType rowType = (RowType) flinkTableSchema.toRowDataType().getLogicalType(); try (TaskWriter taskWriter = createTaskWriter(mixedTable, rowType)) { - Assert.assertNotNull(taskWriter); + Assertions.assertNotNull(taskWriter); writeAndCommit(expected, taskWriter, mixedTable); @@ -96,8 +96,8 @@ default void assertRecords( selectedSchema, flinkTableSchema); } - Assert.assertEquals(1, records.size()); - Assert.assertEquals(expected, records.get(0)); + Assertions.assertEquals(1, records.size()); + Assertions.assertEquals(expected, records.get(0)); } /** For asserting unkeyed table records. */ @@ -121,7 +121,7 @@ default void writeAndCommit( WriteResult writerResult = taskWriter.complete(); boolean writeToBase = mixedTable.isUnkeyedTable(); commit(mixedTable, writerResult, writeToBase); - Assert.assertEquals(upsertEnabled ? 2 : 1, writerResult.dataFiles().length); + Assertions.assertEquals(upsertEnabled ? 2 : 1, writerResult.dataFiles().length); } default boolean upsertEnabled() { diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/MixedFormatFileWriterITCase.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/MixedFormatFileWriterITCase.java index 133d9acbb6..f3f9e11eb5 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/MixedFormatFileWriterITCase.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/MixedFormatFileWriterITCase.java @@ -56,9 +56,9 @@ import org.apache.flink.table.data.TimestampData; import org.apache.iceberg.DataFile; import org.apache.iceberg.Snapshot; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,7 +90,7 @@ public MixedFormatFileWriterITCase() { new BasicTableTestHelper(true, true)); } - @Before + @BeforeEach public void setup() { this.latchId = UUID.randomUUID().toString(); // We wait for two successful checkpoints in sources before shutting down. This ensures that @@ -293,7 +293,7 @@ public static void checkResult(KeyedTable keyedTable, int exceptedSize) { long maxTxIdInSnapshot = -1; for (DataFile addedFile : snapshot.addedDataFiles(keyedTable.io())) { String path = addedFile.path().toString(); - Assert.assertFalse(paths.contains(path)); + Assertions.assertFalse(paths.contains(path)); paths.add(path); LOG.info("add file: {}", addedFile.path()); @@ -301,11 +301,11 @@ public static void checkResult(KeyedTable keyedTable, int exceptedSize) { minTxIdInSnapshot = Math.min(minTxIdInSnapshot, txId); maxTxIdInSnapshot = Math.max(maxTxIdInSnapshot, txId); } - Assert.assertTrue(maxTxId <= minTxIdInSnapshot); + Assertions.assertTrue(maxTxId <= minTxIdInSnapshot); maxTxId = maxTxIdInSnapshot; } - Assert.assertEquals(exceptedSize, TestMixedFormatSource.tableRecords(keyedTable).size()); + Assertions.assertEquals(exceptedSize, TestMixedFormatSource.tableRecords(keyedTable).size()); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java index 057765df1c..3e0648f30f 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java @@ -23,7 +23,7 @@ import org.apache.amoro.TableFormat; import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.CatalogTestHelper; -import org.apache.amoro.catalog.TableTestBase; +import org.apache.amoro.flink.FlinkTestBase; import org.apache.amoro.flink.read.AdaptHiveFlinkParquetReaders; import org.apache.amoro.hive.TestHMS; import org.apache.amoro.hive.catalog.HiveCatalogTestHelper; @@ -50,12 +50,13 @@ import org.apache.iceberg.io.WriteResult; import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.AdaptHiveParquet; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.math.BigDecimal; @@ -67,189 +68,250 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; -@RunWith(Parameterized.class) -public class TestAdaptHiveWriter extends TableTestBase { +public class TestAdaptHiveWriter extends FlinkTestBase { - @ClassRule public static TestHMS TEST_HMS = new TestHMS(); + static final TestHMS TEST_HMS = new TestHMS(); - public TestAdaptHiveWriter(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { - super(catalogTestHelper, tableTestHelper); + @BeforeAll + public static void startTestHms() throws Exception { + TEST_HMS.before(); } - @Parameterized.Parameters(name = "{0}, {1}") - public static Object[] parameters() { - return new Object[][] { - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(true, true) - }, - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(true, false) - }, - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(false, true) - }, - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(false, false) - }, - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(true, true, FILE_FORMAT_ORC) - }, - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(true, false, FILE_FORMAT_ORC) - }, - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(false, true, FILE_FORMAT_ORC) - }, - { - new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), - new HiveTableTestHelper(false, false, FILE_FORMAT_ORC) - } - }; + @AfterAll + public static void stopTestHms() { + TEST_HMS.after(); } - @Test - public void testKeyedTableWriteTypeFromOperateKind() { - Assume.assumeTrue(isKeyedTable()); + static Stream parameters() { + return Stream.of( + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(true, true)), + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(true, false)), + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(false, true)), + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(false, false)), + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(true, true, FILE_FORMAT_ORC)), + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(true, false, FILE_FORMAT_ORC)), + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(false, true, FILE_FORMAT_ORC)), + Arguments.of( + new HiveCatalogTestHelper(TableFormat.MIXED_HIVE, TEST_HMS.getHiveConf()), + new HiveTableTestHelper(false, false, FILE_FORMAT_ORC))); + } + + private void setUpForParam(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) + throws Exception { + initFlinkTestBase(catalogTestHelper, tableTestHelper); + } + + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testKeyedTableWriteTypeFromOperateKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeTrue(isKeyedTable()); MixedTable testKeyedHiveTable = getMixedTable(); FlinkTaskWriterBuilder builder = FlinkTaskWriterBuilder.buildFor(testKeyedHiveTable) .withFlinkSchema(FlinkSchemaUtil.convert(testKeyedHiveTable.schema())); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(ChangeLocationKind.INSTANT) instanceof FlinkChangeTaskWriter); - Assert.assertTrue(builder.buildWriter(BaseLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); - Assert.assertTrue(builder.buildWriter(HiveLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); + Assertions.assertTrue( + builder.buildWriter(BaseLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); + Assertions.assertTrue( + builder.buildWriter(HiveLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.APPEND) instanceof FlinkChangeTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.OVERWRITE) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.MINOR_OPTIMIZE) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.MAJOR_OPTIMIZE) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.FULL_OPTIMIZE) instanceof FlinkBaseTaskWriter); } - @Test - public void testUnKeyedTableWriteTypeFromOperateKind() { - Assume.assumeFalse(isKeyedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnKeyedTableWriteTypeFromOperateKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeFalse(isKeyedTable()); MixedTable testHiveTable = getMixedTable(); FlinkTaskWriterBuilder builder = FlinkTaskWriterBuilder.buildFor(testHiveTable) .withFlinkSchema(FlinkSchemaUtil.convert(testHiveTable.schema())); - Assert.assertTrue(builder.buildWriter(BaseLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); - Assert.assertTrue(builder.buildWriter(HiveLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); + Assertions.assertTrue( + builder.buildWriter(BaseLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); + Assertions.assertTrue( + builder.buildWriter(HiveLocationKind.INSTANT) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.APPEND) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.OVERWRITE) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.MAJOR_OPTIMIZE) instanceof FlinkBaseTaskWriter); - Assert.assertTrue( + Assertions.assertTrue( builder.buildWriter(WriteOperationKind.FULL_OPTIMIZE) instanceof FlinkBaseTaskWriter); } - @Test - public void testKeyedTableChangeWriteByLocationKind() throws IOException { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testKeyedTableChangeWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); testWrite(getMixedTable(), ChangeLocationKind.INSTANT, geneRowData(), "change"); } - @Test - public void testKeyedTableBaseWriteByLocationKind() throws IOException { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testKeyedTableBaseWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); testWrite(getMixedTable(), BaseLocationKind.INSTANT, geneRowData(), "base"); } - @Test - public void testKeyedTableHiveWriteByLocationKind() throws IOException { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testKeyedTableHiveWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); testWrite(getMixedTable(), HiveLocationKind.INSTANT, geneRowData(), "hive"); } - @Test - public void testUnPartitionKeyedTableChangeWriteByLocationKind() throws IOException { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeFalse(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnPartitionKeyedTableChangeWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeFalse(isPartitionedTable()); testWrite(getMixedTable(), ChangeLocationKind.INSTANT, geneRowData(), "change"); } - @Test - public void testUnPartitionKeyedTableBaseWriteByLocationKind() throws IOException { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeFalse(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnPartitionKeyedTableBaseWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeFalse(isPartitionedTable()); testWrite(getMixedTable(), BaseLocationKind.INSTANT, geneRowData(), "base"); } - @Test - public void testUnPartitionKeyedTableHiveWriteByLocationKind() throws IOException { - Assume.assumeTrue(isKeyedTable()); - Assume.assumeFalse(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnPartitionKeyedTableHiveWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeTrue(isKeyedTable()); + Assumptions.assumeFalse(isPartitionedTable()); testWrite(getMixedTable(), HiveLocationKind.INSTANT, geneRowData(), "hive"); } - @Test - public void testUnKeyedTableChangeWriteByLocationKind() throws IOException { - Assume.assumeFalse(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnKeyedTableChangeWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeFalse(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); try { testWrite(getMixedTable(), ChangeLocationKind.INSTANT, geneRowData(), "change"); } catch (Exception e) { - Assert.assertTrue(e instanceof IllegalArgumentException); + Assertions.assertTrue(e instanceof IllegalArgumentException); } } - @Test - public void testUnKeyedTableBaseWriteByLocationKind() throws IOException { - Assume.assumeFalse(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnKeyedTableBaseWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeFalse(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); testWrite(getMixedTable(), BaseLocationKind.INSTANT, geneRowData(), "base"); } - @Test - public void testUnKeyedTableHiveWriteByLocationKind() throws IOException { - Assume.assumeFalse(isKeyedTable()); - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnKeyedTableHiveWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeFalse(isKeyedTable()); + Assumptions.assumeTrue(isPartitionedTable()); testWrite(getMixedTable(), HiveLocationKind.INSTANT, geneRowData(), "hive"); } - @Test - public void testUnPartitionUnKeyedTableChangeWriteByLocationKind() throws IOException { - Assume.assumeFalse(isKeyedTable()); - Assume.assumeFalse(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnPartitionUnKeyedTableChangeWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeFalse(isKeyedTable()); + Assumptions.assumeFalse(isPartitionedTable()); try { testWrite(getMixedTable(), ChangeLocationKind.INSTANT, geneRowData(), "change"); } catch (Exception e) { - Assert.assertTrue(e instanceof IllegalArgumentException); + Assertions.assertTrue(e instanceof IllegalArgumentException); } } - @Test - public void testUnPartitionUnKeyedTableBaseWriteByLocationKind() throws IOException { - Assume.assumeFalse(isKeyedTable()); - Assume.assumeFalse(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnPartitionUnKeyedTableBaseWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeFalse(isKeyedTable()); + Assumptions.assumeFalse(isPartitionedTable()); testWrite(getMixedTable(), BaseLocationKind.INSTANT, geneRowData(), "base"); } - @Test - public void testUnPartitionUnKeyedTableHiveWriteByLocationKind() throws IOException { - Assume.assumeFalse(isKeyedTable()); - Assume.assumeFalse(isPartitionedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnPartitionUnKeyedTableHiveWriteByLocationKind( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws Exception { + setUpForParam(catalogTestHelper, tableTestHelper); + + Assumptions.assumeFalse(isKeyedTable()); + Assumptions.assumeFalse(isPartitionedTable()); testWrite(getMixedTable(), HiveLocationKind.INSTANT, geneRowData(), "hive"); } @@ -266,7 +328,7 @@ public void testWrite( } WriteResult complete = changeWrite.complete(); Arrays.stream(complete.dataFiles()) - .forEach(s -> Assert.assertTrue(s.path().toString().contains(pathFeature))); + .forEach(s -> Assertions.assertTrue(s.path().toString().contains(pathFeature))); CloseableIterable concat = CloseableIterable.concat( Arrays.stream(complete.dataFiles()) @@ -285,7 +347,7 @@ public void testWrite( .collect(Collectors.toList())); Set result = new HashSet<>(); Iterators.addAll(result, concat.iterator()); - Assert.assertEquals(result, records.stream().collect(Collectors.toSet())); + Assertions.assertEquals(result, records.stream().collect(Collectors.toSet())); } private CloseableIterable readParquet(Schema schema, String path) { diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java index fbdf6a4b7e..a08bbb8825 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java @@ -30,8 +30,8 @@ import org.apache.amoro.table.MixedTable; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.iceberg.UpdateProperties; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.time.Duration; @@ -56,15 +56,17 @@ public void testTableProperties() { new AutomaticDoubleWriteStatus(tableLoader, Duration.ofSeconds(10)); status.open(); - Assert.assertFalse(status.isDoubleWrite()); + Assertions.assertFalse(status.isDoubleWrite()); status.processWatermark(new Watermark(System.currentTimeMillis() - 11 * 1000)); - Assert.assertFalse(status.isDoubleWrite()); - Assert.assertFalse(Boolean.parseBoolean(mixedTable.properties().get(LOG_STORE_CATCH_UP.key()))); + Assertions.assertFalse(status.isDoubleWrite()); + Assertions.assertFalse( + Boolean.parseBoolean(mixedTable.properties().get(LOG_STORE_CATCH_UP.key()))); status.processWatermark(new Watermark(System.currentTimeMillis() - 9 * 1000)); - Assert.assertTrue(status.isDoubleWrite()); - Assert.assertTrue(status.isDoubleWrite()); + Assertions.assertTrue(status.isDoubleWrite()); + Assertions.assertTrue(status.isDoubleWrite()); mixedTable.refresh(); - Assert.assertTrue(Boolean.parseBoolean(mixedTable.properties().get(LOG_STORE_CATCH_UP.key()))); + Assertions.assertTrue( + Boolean.parseBoolean(mixedTable.properties().get(LOG_STORE_CATCH_UP.key()))); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java index b71cf8496f..1aee767bf7 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java @@ -64,14 +64,12 @@ import org.apache.iceberg.types.TypeUtil; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.producer.ProducerConfig; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,53 +88,53 @@ import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestAutomaticLogWriter extends FlinkTestBase { private static final Logger LOG = LoggerFactory.getLogger(TestAutomaticLogWriter.class); public MixedFormatTableLoader tableLoader; public static final TestGlobalAggregateManager GLOBAL_AGGREGATE_MANGER = new TestGlobalAggregateManager(); - private final boolean isGapNone; - private final boolean logstoreEnabled; + private boolean isGapNone; + private boolean logstoreEnabled; - public TestAutomaticLogWriter(boolean isGapNone, boolean logstoreEnabled) { + public TestAutomaticLogWriter() { super( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); - this.isGapNone = isGapNone; - this.logstoreEnabled = logstoreEnabled; } - @Parameterized.Parameters(name = "isGapNone={0}, logstoreEnabled={1}") - public static Object[][] parameters() { - return new Object[][] { - {true, true}, - {false, false}, - {false, true}, - {true, false} - }; + static Stream parameters() { + return Stream.of( + Arguments.of(true, true), + Arguments.of(false, false), + Arguments.of(false, true), + Arguments.of(true, false)); } - @BeforeClass + @BeforeAll public static void prepare() throws Exception { KAFKA_CONTAINER.start(); } - @AfterClass + @AfterAll public static void shutdown() throws Exception { KAFKA_CONTAINER.close(); } - @Before - public void init() { + private void setUpForParam(boolean isGapNone, boolean logstoreEnabled) { + this.isGapNone = isGapNone; + this.logstoreEnabled = logstoreEnabled; tableLoader = MixedFormatTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); tableLoader.open(); } - @Test - public void testHasCaughtUp() throws Exception { + @ParameterizedTest(name = "isGapNone={0}, logstoreEnabled={1}") + @MethodSource("parameters") + public void testHasCaughtUp(boolean isGapNone, boolean logstoreEnabled) throws Exception { + setUpForParam(isGapNone, logstoreEnabled); + String topic = Thread.currentThread().getStackTrace()[1].getMethodName() + isGapNone + logstoreEnabled; @@ -225,7 +223,7 @@ public void testHasCaughtUp() throws Exception { List actual = MixedDataTestHelpers.readKeyedTable(testKeyedTable, null); Set expected = toRecords(DataUtil.toRowSet(expects)); - Assert.assertEquals(expected, new HashSet<>(actual)); + Assertions.assertEquals(expected, new HashSet<>(actual)); if (logstoreEnabled) { checkLogstoreDataAccuracy(topic, expects); } else { @@ -233,8 +231,11 @@ public void testHasCaughtUp() throws Exception { } } - @Test - public void testHasNotCaughtUp() throws Exception { + @ParameterizedTest(name = "isGapNone={0}, logstoreEnabled={1}") + @MethodSource("parameters") + public void testHasNotCaughtUp(boolean isGapNone, boolean logstoreEnabled) throws Exception { + setUpForParam(isGapNone, logstoreEnabled); + String topic = Thread.currentThread().getStackTrace()[1].getMethodName() + isGapNone + logstoreEnabled; byte[] jobId = IdGenerator.generateUpstreamId(); @@ -258,7 +259,7 @@ public void testHasNotCaughtUp() throws Exception { List expects = new LinkedList<>(); List results; testKeyedTable.refresh(); - Assert.assertFalse( + Assertions.assertFalse( Boolean.parseBoolean( testKeyedTable.properties().getOrDefault(LOG_STORE_CATCH_UP.key(), "false"))); try (TestOneInputStreamOperatorIntern harness = @@ -313,13 +314,13 @@ public void testHasNotCaughtUp() throws Exception { } // check expects accuracy. - Assert.assertEquals(3, results.size()); - results.forEach(result -> Assert.assertEquals(1, result.writeResult().dataFiles().length)); + Assertions.assertEquals(3, results.size()); + results.forEach(result -> Assertions.assertEquals(1, result.writeResult().dataFiles().length)); List expected = isGapNone ? expects : expects.subList(2, expects.size()); checkLogstoreDataAccuracy(topic, expected); testKeyedTable.refresh(); if (!isGapNone) { - Assert.assertTrue( + Assertions.assertTrue( Boolean.parseBoolean(testKeyedTable.properties().get(LOG_STORE_CATCH_UP.key()))); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java index 226d721155..65237270fb 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java @@ -34,39 +34,33 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.data.RowData; import org.apache.iceberg.data.Record; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.Set; -@RunWith(Parameterized.class) public class TestFlinkSink extends FlinkTestBase { - public TestFlinkSink(boolean isKeyed) { - super( + private void setUpForParam(boolean isKeyed) throws Exception { + initFlinkTestBase( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(isKeyed, false)); } - @Parameterized.Parameters(name = "{0}") - public static Collection parameters() { - return Arrays.asList(new Object[][] {{true}, {false}}); - } + @ParameterizedTest(name = "isKeyed = {0}") + @ValueSource(booleans = {true, false}) + public void testKeyedSink(boolean isKeyed) throws Exception { + setUpForParam(isKeyed); - @Test - public void testKeyedSink() throws Exception { - Assume.assumeTrue(isKeyedTable()); + Assumptions.assumeTrue(isKeyedTable()); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); KeyedTable testKeyedTable = getMixedTable().asKeyedTable(); @@ -135,12 +129,15 @@ public void testKeyedSink() throws Exception { List actual = MixedDataTestHelpers.readKeyedTable(testKeyedTable, null); Set expected = toRecords(DataUtil.toRowSet(data)); - Assert.assertEquals(expected, new HashSet<>(actual)); + Assertions.assertEquals(expected, new HashSet<>(actual)); } - @Test - public void testUnkeyedSink() throws Exception { - Assume.assumeFalse(isKeyedTable()); + @ParameterizedTest(name = "isKeyed = {0}") + @ValueSource(booleans = {true, false}) + public void testUnkeyedSink(boolean isKeyed) throws Exception { + setUpForParam(isKeyed); + + Assumptions.assumeFalse(isKeyedTable()); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); UnkeyedTable testTable = getMixedTable().asUnkeyedTable(); @@ -178,12 +175,15 @@ public void testUnkeyedSink() throws Exception { Set actual = DataUtil.read(testTable); Set expected = toRecords(DataUtil.toRowSet(data)); - Assert.assertEquals(expected, actual); + Assertions.assertEquals(expected, actual); } - @Test - public void testUnkeyedOverwrite() throws Exception { - Assume.assumeFalse(isKeyedTable()); + @ParameterizedTest(name = "isKeyed = {0}") + @ValueSource(booleans = {true, false}) + public void testUnkeyedOverwrite(boolean isKeyed) throws Exception { + setUpForParam(isKeyed); + + Assumptions.assumeFalse(isKeyedTable()); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); UnkeyedTable testTable = getMixedTable().asUnkeyedTable(); @@ -241,6 +241,6 @@ public void testUnkeyedOverwrite() throws Exception { Set actual = DataUtil.read(testTable); Set expected = toRecords(DataUtil.toRowSet(data)); - Assert.assertEquals(expected, actual); + Assertions.assertEquals(expected, actual); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileCommitter.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileCommitter.java index bc0bfe8236..3be8d28eb9 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileCommitter.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileCommitter.java @@ -38,8 +38,8 @@ import org.apache.iceberg.TableScan; import org.apache.iceberg.flink.sink.FlinkWriteResult; import org.apache.iceberg.io.CloseableIterable; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.List; @@ -84,8 +84,8 @@ public void checkChangeFiles(int fileCnt, int recordCnt, KeyedTable table) { actualFileCnt++; actualRecordCnt += fileScanTask.file().recordCount(); } - Assert.assertEquals(fileCnt, actualFileCnt); - Assert.assertEquals(recordCnt, actualRecordCnt); + Assertions.assertEquals(fileCnt, actualFileCnt); + Assertions.assertEquals(recordCnt, actualRecordCnt); } @Test @@ -124,8 +124,8 @@ private List prepareChangeFiles() throws Exception { testHarness.processElement(createRowData(3, "hello", "2020-10-13T10:10:11.0"), 1); testHarness.prepareSnapshotPreBarrier(checkpointId); - Assert.assertEquals(1, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertEquals(1, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 3, testHarness.extractOutputValues().get(0).writeResult().dataFiles().length); checkpointId = checkpointId + 1; @@ -143,8 +143,8 @@ private List prepareChangeFiles() throws Exception { testHarness.prepareSnapshotPreBarrier(checkpointId); // testHarness.extractOutputValues() compute the sum - Assert.assertEquals(2, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertEquals(2, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 4, testHarness.extractOutputValues().get(1).writeResult().dataFiles().length); changeFiles = testHarness.extractOutputValues(); } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileWriter.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileWriter.java index 7cf7a99eb5..ce6204fb3d 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileWriter.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestMixedFormatFileWriter.java @@ -41,34 +41,32 @@ import org.apache.iceberg.flink.sink.TaskWriterFactory; import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.io.WriteResult; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.HashMap; import java.util.List; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestMixedFormatFileWriter extends FlinkTestBase { public static final long TARGET_FILE_SIZE = 128 * 1024 * 1024; public MixedFormatTableLoader tableLoader; - private final boolean submitEmptySnapshots; - - @Parameterized.Parameters(name = "{0}, {1}") - public static Object[][] parameters() { - return new Object[][] { - {true, false}, - {true, true}, - {false, false}, - {false, true} - }; + private boolean submitEmptySnapshots; + + static Stream parameters() { + return Stream.of( + Arguments.of(true, false), + Arguments.of(true, true), + Arguments.of(false, false), + Arguments.of(false, true)); } - public TestMixedFormatFileWriter(boolean isKeyed, boolean submitEmptySnapshots) { - super( + private void setUpForParam(boolean isKeyed, boolean submitEmptySnapshots) throws Exception { + initFlinkTestBase( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(isKeyed, true)); this.submitEmptySnapshots = submitEmptySnapshots; @@ -133,44 +131,51 @@ public static TaskWriter createUnkeyedTaskWriter( return taskWriterFactory.create(); } - @Test - public void testInsertWrite() throws Exception { - Assume.assumeTrue(isKeyedTable()); + @ParameterizedTest(name = "isKeyed = {0}, submitEmptySnapshots = {1}") + @MethodSource("parameters") + public void testInsertWrite(boolean isKeyed, boolean submitEmptySnapshots) throws Exception { + setUpForParam(isKeyed, submitEmptySnapshots); + + Assumptions.assumeTrue(isKeyedTable()); tableLoader = MixedFormatTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = createMixedFormatStreamWriter(tableLoader)) { MixedFormatFileWriter fileWriter = (MixedFormatFileWriter) testHarness.getOneInputOperator(); - Assert.assertNotNull(fileWriter.getWriter()); + Assertions.assertNotNull(fileWriter.getWriter()); // The first checkpoint testHarness.processElement(createRowData(1, "hello", "2020-10-11T10:10:11.0"), 1); testHarness.processElement(createRowData(2, "hello", "2020-10-12T10:10:11.0"), 1); testHarness.processElement(createRowData(3, "hello", "2020-10-13T10:10:11.0"), 1); testHarness.prepareSnapshotPreBarrier(checkpointId); - Assert.assertNull(fileWriter.getWriter()); - Assert.assertEquals(1, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertNull(fileWriter.getWriter()); + Assertions.assertEquals(1, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 3, testHarness.extractOutputValues().get(0).writeResult().dataFiles().length); checkpointId = checkpointId + 1; // The second checkpoint testHarness.processElement(createRowData(1, "hello", "2020-10-12T10:10:11.0"), 1); - Assert.assertNotNull(fileWriter.getWriter()); + Assertions.assertNotNull(fileWriter.getWriter()); testHarness.processElement(createRowData(2, "hello", "2020-10-12T10:10:11.0"), 1); testHarness.processElement(createRowData(3, "hello", "2020-10-12T10:10:11.0"), 1); testHarness.prepareSnapshotPreBarrier(checkpointId); // testHarness.extractOutputValues() calculates the cumulative value List completedFiles = testHarness.extractOutputValues(); - Assert.assertEquals(2, completedFiles.size()); - Assert.assertEquals(3, completedFiles.get(1).writeResult().dataFiles().length); + Assertions.assertEquals(2, completedFiles.size()); + Assertions.assertEquals(3, completedFiles.get(1).writeResult().dataFiles().length); } } - @Test - public void testSnapshotMultipleTimes() throws Exception { + @ParameterizedTest(name = "isKeyed = {0}, submitEmptySnapshots = {1}") + @MethodSource("parameters") + public void testSnapshotMultipleTimes(boolean isKeyed, boolean submitEmptySnapshots) + throws Exception { + setUpForParam(isKeyed, submitEmptySnapshots); + long checkpointId = 1; long timestamp = 1; @@ -190,8 +195,8 @@ public void testSnapshotMultipleTimes() throws Exception { .map(FlinkWriteResult::writeResult) .collect(java.util.stream.Collectors.toList())) .build(); - Assert.assertEquals(0, result.deleteFiles().length); - Assert.assertEquals(expectedDataFiles, result.dataFiles().length); + Assertions.assertEquals(0, result.deleteFiles().length); + Assertions.assertEquals(expectedDataFiles, result.dataFiles().length); // snapshot again immediately. for (int i = 0; i < 5; i++) { @@ -204,15 +209,19 @@ public void testSnapshotMultipleTimes() throws Exception { .map(FlinkWriteResult::writeResult) .collect(java.util.stream.Collectors.toList())) .build(); - Assert.assertEquals(0, result.deleteFiles().length); - Assert.assertEquals(expectedDataFiles, result.dataFiles().length); + Assertions.assertEquals(0, result.deleteFiles().length); + Assertions.assertEquals(expectedDataFiles, result.dataFiles().length); } } } - @Test - public void testInsertWriteWithoutPk() throws Exception { - Assume.assumeFalse(isKeyedTable()); + @ParameterizedTest(name = "isKeyed = {0}, submitEmptySnapshots = {1}") + @MethodSource("parameters") + public void testInsertWriteWithoutPk(boolean isKeyed, boolean submitEmptySnapshots) + throws Exception { + setUpForParam(isKeyed, submitEmptySnapshots); + + Assumptions.assumeFalse(isKeyedTable()); tableLoader = MixedFormatTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = @@ -223,8 +232,8 @@ public void testInsertWriteWithoutPk() throws Exception { testHarness.processElement(createRowData(3, "hello", "2020-10-13T10:10:11.0"), 1); testHarness.prepareSnapshotPreBarrier(checkpointId); - Assert.assertEquals(1, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertEquals(1, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 3, testHarness.extractOutputValues().get(0).writeResult().dataFiles().length); checkpointId = checkpointId + 1; @@ -237,14 +246,17 @@ public void testInsertWriteWithoutPk() throws Exception { testHarness.prepareSnapshotPreBarrier(checkpointId); // testHarness.extractOutputValues() calculates the cumulative value List completedFiles = testHarness.extractOutputValues(); - Assert.assertEquals(2, completedFiles.size()); - Assert.assertEquals(1, completedFiles.get(1).writeResult().dataFiles().length); + Assertions.assertEquals(2, completedFiles.size()); + Assertions.assertEquals(1, completedFiles.get(1).writeResult().dataFiles().length); } } - @Test - public void testDeleteWrite() throws Exception { - Assume.assumeTrue(isKeyedTable()); + @ParameterizedTest(name = "isKeyed = {0}, submitEmptySnapshots = {1}") + @MethodSource("parameters") + public void testDeleteWrite(boolean isKeyed, boolean submitEmptySnapshots) throws Exception { + setUpForParam(isKeyed, submitEmptySnapshots); + + Assumptions.assumeTrue(isKeyedTable()); tableLoader = MixedFormatTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = @@ -260,8 +272,8 @@ public void testDeleteWrite() throws Exception { createRowData(1, "hello", "2020-10-11T10:10:11.0", RowKind.DELETE), 1); testHarness.prepareSnapshotPreBarrier(checkpointId); - Assert.assertEquals(1, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertEquals(1, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 3, testHarness.extractOutputValues().get(0).writeResult().dataFiles().length); checkpointId = checkpointId + 1; @@ -276,15 +288,18 @@ public void testDeleteWrite() throws Exception { testHarness.prepareSnapshotPreBarrier(checkpointId); // testHarness.extractOutputValues() calculates the cumulative value - Assert.assertEquals(2, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertEquals(2, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 3, testHarness.extractOutputValues().get(1).writeResult().dataFiles().length); } } - @Test - public void testUpdateWrite() throws Exception { - Assume.assumeTrue(isKeyedTable()); + @ParameterizedTest(name = "isKeyed = {0}, submitEmptySnapshots = {1}") + @MethodSource("parameters") + public void testUpdateWrite(boolean isKeyed, boolean submitEmptySnapshots) throws Exception { + setUpForParam(isKeyed, submitEmptySnapshots); + + Assumptions.assumeTrue(isKeyedTable()); tableLoader = MixedFormatTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = @@ -300,8 +315,8 @@ public void testUpdateWrite() throws Exception { createRowData(1, "hello", "2020-10-13T10:10:11.0", RowKind.UPDATE_AFTER), 1); testHarness.prepareSnapshotPreBarrier(checkpointId); - Assert.assertEquals(1, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertEquals(1, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 3, testHarness.extractOutputValues().get(0).writeResult().dataFiles().length); checkpointId = checkpointId + 1; @@ -317,15 +332,18 @@ public void testUpdateWrite() throws Exception { testHarness.prepareSnapshotPreBarrier(checkpointId); // testHarness.extractOutputValues() calculates the cumulative value - Assert.assertEquals(2, testHarness.extractOutputValues().size()); - Assert.assertEquals( + Assertions.assertEquals(2, testHarness.extractOutputValues().size()); + Assertions.assertEquals( 3, testHarness.extractOutputValues().get(1).writeResult().dataFiles().length); } } - @Test - public void testEmitEmptyResults() throws Exception { - Assume.assumeTrue(isKeyedTable()); + @ParameterizedTest(name = "isKeyed = {0}, submitEmptySnapshots = {1}") + @MethodSource("parameters") + public void testEmitEmptyResults(boolean isKeyed, boolean submitEmptySnapshots) throws Exception { + setUpForParam(isKeyed, submitEmptySnapshots); + + Assumptions.assumeTrue(isKeyedTable()); tableLoader = MixedFormatTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; long excepted = submitEmptySnapshots ? 1 : 0; @@ -334,13 +352,13 @@ public void testEmitEmptyResults() throws Exception { // The first checkpoint testHarness.prepareSnapshotPreBarrier(checkpointId); - Assert.assertEquals(excepted, testHarness.extractOutputValues().size()); + Assertions.assertEquals(excepted, testHarness.extractOutputValues().size()); checkpointId = checkpointId + 1; // The second checkpoint testHarness.prepareSnapshotPreBarrier(checkpointId); - Assert.assertEquals(excepted, testHarness.extractOutputValues().size()); + Assertions.assertEquals(excepted, testHarness.extractOutputValues().size()); } } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenKafkaProducer.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenKafkaProducer.java index 845b79eb16..88c992a87d 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenKafkaProducer.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenKafkaProducer.java @@ -25,7 +25,7 @@ import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.amoro.data.ChangeAction; import org.apache.amoro.flink.kafka.testutils.KafkaConfigGenerate; @@ -44,10 +44,10 @@ import org.apache.flink.types.RowKind; import org.apache.flink.util.InstantiationUtil; import org.apache.kafka.clients.producer.ProducerRecord; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,12 +59,12 @@ public class TestHiddenKafkaProducer extends TestBaseLog { private static final Logger LOG = LoggerFactory.getLogger(TestHiddenKafkaProducer.class); - @BeforeClass + @BeforeAll public static void prepare() throws Exception { KAFKA_CONTAINER.start(); } - @AfterClass + @AfterAll public static void shutdown() throws Exception { KAFKA_CONTAINER.close(); } @@ -170,7 +170,7 @@ public void testLogDataNullValueSerialize() throws IOException { byte[] bytes = logDataJsonSerialization.serialize(logData); - Assert.assertNotNull(bytes); + Assertions.assertNotNull(bytes); String actualJson = new String(Bytes.subByte(bytes, 18, bytes.length - 18)); String expected = @@ -179,7 +179,7 @@ public void testLogDataNullValueSerialize() throws IOException { LogDataJsonDeserialization logDataDeserialization = createLogDataDeserialization(); LogData result = logDataDeserialization.deserialize(bytes); - Assert.assertNotNull(result); + Assertions.assertNotNull(result); } @Test @@ -190,6 +190,6 @@ public void testLogDataJsonSerializationClassSerialize() byte[] bytes = InstantiationUtil.serializeObject(actual); LogDataJsonSerialization result = InstantiationUtil.deserializeObject(bytes, actual.getClass().getClassLoader()); - Assert.assertNotNull(result); + Assertions.assertNotNull(result); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenLogOperators.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenLogOperators.java index 9a42fd37fb..75d64605f3 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenLogOperators.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/hidden/kafka/TestHiddenLogOperators.java @@ -53,10 +53,10 @@ import org.apache.flink.table.data.TimestampData; import org.apache.flink.util.CloseableIterator; import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,12 +79,12 @@ public class TestHiddenLogOperators { public static final TestGlobalAggregateManager GLOBAL_AGGREGATE_MANGER = new TestGlobalAggregateManager(); - @BeforeClass + @BeforeAll public static void prepare() throws Exception { KAFKA_CONTAINER.start(); } - @AfterClass + @AfterAll public static void shutdown() throws Exception { KAFKA_CONTAINER.close(); } diff --git a/amoro-common/src/test/java/org/apache/amoro/listener/AmoroRunListener.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/listener/AmoroRunListener.java similarity index 52% rename from amoro-common/src/test/java/org/apache/amoro/listener/AmoroRunListener.java rename to amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/listener/AmoroRunListener.java index 543b0dc4f3..7453a9f6b5 100644 --- a/amoro-common/src/test/java/org/apache/amoro/listener/AmoroRunListener.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/listener/AmoroRunListener.java @@ -18,74 +18,79 @@ package org.apache.amoro.listener; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.runner.Description; -import org.junit.runner.Result; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; +import java.util.Objects; import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentHashMap; -public class AmoroRunListener extends RunListener { +public class AmoroRunListener implements TestExecutionListener { private static final Logger LOG = LoggerFactory.getLogger(AmoroRunListener.class); private long startTime; - private long singleTestStartTime; + private final Map singleTestStartTimes = new ConcurrentHashMap<>(); private final PriorityQueue testCaseQueue = new PriorityQueue<>(); @Override - public void testRunStarted(Description description) { + public void testPlanExecutionStarted(TestPlan testPlan) { startTime = System.currentTimeMillis(); LOG.info( - "{} Tests started! Number of Test case: {}", - description == null ? "Unknown" : description.getClassName(), - description == null ? 0 : description.testCount()); + "Tests started! Number of containers/tests in plan: {}", + testPlan.countTestIdentifiers(id -> true)); } @Override - public void testRunFinished(Result result) { + public void testPlanExecutionFinished(TestPlan testPlan) { long endTime = System.currentTimeMillis(); - LOG.info("Tests finished! Number of test case: {}", result.getRunCount()); long elapsedSeconds = (endTime - startTime) / 1000; + LOG.info("Tests finished! Number of test cases recorded: {}", testCaseQueue.size()); LOG.info("Elapsed time of tests execution: {} seconds", elapsedSeconds); int printNum = Math.min(testCaseQueue.size(), 50); LOG.info("Print the top cost test case method name:"); for (int i = 0; i < printNum; i++) { TestCase testCase = testCaseQueue.poll(); - Assert.assertNotNull(testCase); + Objects.requireNonNull(testCase); LOG.info("NO-{}, cost: {}ms, methodName:{}", i + 1, testCase.cost, testCase.methodName); } } @Override - public void testStarted(Description description) { - singleTestStartTime = System.currentTimeMillis(); - LOG.info("{} test is starting...", description.getMethodName()); - } - - @Override - public void testFinished(Description description) { - long cost = System.currentTimeMillis() - singleTestStartTime; - testCaseQueue.add(TestCase.of(cost, description.getMethodName())); - LOG.info("{} test is finished, cost {}ms...\n", description.getMethodName(), cost); + public void executionStarted(TestIdentifier testIdentifier) { + if (testIdentifier.isTest()) { + singleTestStartTimes.put(testIdentifier.getUniqueId(), System.currentTimeMillis()); + LOG.info("{} test is starting...", testIdentifier.getDisplayName()); + } } @Override - public void testFailure(Failure failure) { - LOG.info("{} test FAILED!!!", failure.getDescription().getMethodName()); + public void executionFinished( + TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + if (testIdentifier.isTest()) { + Long start = singleTestStartTimes.remove(testIdentifier.getUniqueId()); + long cost = start == null ? 0 : System.currentTimeMillis() - start; + String methodName = testIdentifier.getDisplayName(); + testCaseQueue.add(TestCase.of(cost, methodName)); + if (testExecutionResult.getStatus() == TestExecutionResult.Status.FAILED) { + LOG.info("{} test FAILED!!!", methodName); + } + LOG.info("{} test is finished, cost {}ms...\n", methodName, cost); + } } @Override - public void testIgnored(Description description) throws Exception { - super.testIgnored(description); - Ignore ignore = description.getAnnotation(Ignore.class); - LOG.info( - "@Ignore test method '{}', ignored reason '{}'.", - description.getMethodName(), - ignore.value()); + public void executionSkipped(TestIdentifier testIdentifier, String reason) { + if (testIdentifier.isTest()) { + LOG.info( + "@Disabled test method '{}', ignored reason '{}'.", + testIdentifier.getDisplayName(), + reason); + } } private static class TestCase implements Comparable { @@ -103,7 +108,7 @@ public static TestCase of(long cost, String methodName) { @Override public int compareTo(AmoroRunListener.TestCase that) { - Assert.assertNotNull(that); + Objects.requireNonNull(that); return that.cost.compareTo(cost); } } diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 0000000000..f6a11dc26c --- /dev/null +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.amoro.listener.AmoroRunListener