From 74ed3bc4598e23ab19bb86b7f885cc0a413e2ba8 Mon Sep 17 00:00:00 2001 From: Fred Feng Date: Sat, 24 Oct 2026 14:40:07 +1100 Subject: [PATCH] feat: add Microsoft Access import and export --- README.md | 6 +- docs/blogger/sqljam-en.md | 3 +- docs/blogger/sqljam-medium.md | 3 +- docs/blogger/sqljam-zh.md | 3 +- docs/developer-guide.md | 6 +- docs/index.html | 4 +- pom.xml | 7 + .../sqljam/face/service/DatabaseSession.java | 6 +- .../github/sqljam/face/service/JdbcUrls.java | 4 + .../sqljam/face/view/ConnectionForm.java | 4 + .../github/sqljam/face/view/ExportDialog.java | 2 + .../sqljam/impexp/AbstractFileImporter.java | 3 + .../java/com/github/sqljam/impexp/DbType.java | 43 ++ .../sqljam/impexp/DefaultMetaDataVisitor.java | 18 +- .../com/github/sqljam/impexp/Dialect.java | 34 + .../com/github/sqljam/impexp/Exporter.java | 5 +- .../sqljam/impexp/ImportExportHandler.java | 27 +- .../github/sqljam/impexp/SqlScriptRunner.java | 23 +- .../sqljam/impexp/db/AccessDialect.java | 582 ++++++++++++++++++ .../impexp/db/AccessMetaDataOperations.java | 298 +++++++++ .../github/sqljam/jdbc/ManualTransaction.java | 28 + .../sqljam/access/AccessDialectTest.java | 254 ++++++++ .../github/sqljam/access/AccessImportIT.java | 32 + .../github/sqljam/access/AccessParquetIT.java | 32 + .../github/sqljam/access/AccessScriptIT.java | 32 + .../github/sqljam/access/AccessSessionIT.java | 32 + .../sqljam/face/service/JdbcUrlsTest.java | 6 + .../github/sqljam/impexp/DdlScripterTest.java | 1 + .../com/github/sqljam/impexp/DialectTest.java | 1 + .../db/AccessMetaDataOperationsTest.java | 159 +++++ .../github/sqljam/it/AbstractImportIT.java | 24 +- .../github/sqljam/it/AbstractSessionIT.java | 2 +- .../java/com/github/sqljam/it/ItDatabase.java | 24 + .../java/com/github/sqljam/it/ItVerifier.java | 26 +- .../sqljam/jdbc/ManualTransactionTest.java | 28 + src/test/resources/access/fixture.sql | 72 +++ src/test/resources/it-databases-ci.properties | 2 +- 37 files changed, 1802 insertions(+), 34 deletions(-) create mode 100644 src/main/java/com/github/sqljam/impexp/db/AccessDialect.java create mode 100644 src/main/java/com/github/sqljam/impexp/db/AccessMetaDataOperations.java create mode 100644 src/test/java/com/github/sqljam/access/AccessDialectTest.java create mode 100644 src/test/java/com/github/sqljam/access/AccessImportIT.java create mode 100644 src/test/java/com/github/sqljam/access/AccessParquetIT.java create mode 100644 src/test/java/com/github/sqljam/access/AccessScriptIT.java create mode 100644 src/test/java/com/github/sqljam/access/AccessSessionIT.java create mode 100644 src/test/java/com/github/sqljam/impexp/db/AccessMetaDataOperationsTest.java create mode 100644 src/test/resources/access/fixture.sql diff --git a/README.md b/README.md index c953d4b..4510ea9 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ **Move tables across databases. Schema, data and all.** -SqlJam is a desktop import/export tool for relational and OLAP databases: MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, DuckDB, ClickHouse and more. +SqlJam is a desktop import/export tool for relational and OLAP databases: MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, Microsoft Access, DuckDB, ClickHouse and more. It copies tables (with indexes, constraints, sequences and partitions) **directly into another database** or into the same schema as copies, or saves them as a **self-describing export package** of SQL, **Parquet** or **CSV** files that can be imported later. Imports into a database run in **one transaction** which an error rolls back as a whole, and rows are **inserted, updated or skipped by key**. ![Java](https://img.shields.io/badge/Java-17-ED8B00?logo=openjdk&logoColor=white) @@ -101,6 +101,7 @@ Only **Java 17+** is needed to run SqlJam. JavaFX and all JDBC drivers are insid | MariaDB | 11.8 | 10.3 | | H2 | 2.3 | 2.x | | SQLite | 3.46 | 3.x | +| Microsoft Access (`.accdb`, `.mdb`) | 2016 | 2000 | | DuckDB (OLAP) | 1.5 | 1.x | | ClickHouse (OLAP) | 26.9 | 24.x | @@ -309,6 +310,7 @@ try (ManualTransaction transaction = new ManualTransaction(connection, DbType.PO | MySQL / Oracle `VECTOR` | `text` | same | `nvarchar(max)` | same | | ClickHouse `Nullable(String)` / `Array(T)` | `text` | `CLOB` | `nvarchar(max)` | `longtext` | | DuckDB `HUGEINT` / `UNION` | `numeric(38, 0)` / `text` | `NUMBER(38,0)` / `CLOB` | `decimal(38,0)` / `nvarchar(max)` | `decimal(38,0)` / `longtext` | +| Access `CURRENCY` / `YESNO` / `OLE` | `numeric(19, 4)` / `boolean` / `bytea` | `NUMBER(19,4)` / `NUMBER(1)` / `BLOB` | `decimal(19,4)` / `bit` / `varbinary(max)` | `decimal(19,4)` / `tinyint(1)` / `longblob` | ### Best practices @@ -438,7 +440,7 @@ Environment: Apple M2 Max, 32 GB, JDK 17. MySQL/PostgreSQL local, Oracle/SQL Ser | Why does Db2 refuse a wide table? | Rows wider than a 4K page need a table space of larger pages. Create one once, e.g. `CREATE BUFFERPOOL BP32K PAGESIZE 32K` and `CREATE TABLESPACE TS32K PAGESIZE 32K BUFFERPOOL BP32K`, Db2 then places wide tables there by itself | | Do names have to match in case? | No. An existing target table and its columns are matched without case, e.g. `OrderId` of the source writes the column `orderid` of PostgreSQL or `ORDERID` of Oracle | | Which rows were skipped? | Each skipped row is in the log of the import with its key and its error. **Errors only** shows them, **Save Log** writes the whole log to a file | -| What does a roll back undo? | Everything on PostgreSQL, SQL Server, Db2, SQLite and DuckDB, tables included. MySQL, MariaDB, Oracle and H2 commit DDL at once, there the created tables stay empty and keys and identity values are set after the commit | +| What does a roll back undo? | Everything on PostgreSQL, SQL Server, Db2, SQLite and DuckDB, tables included. MySQL, MariaDB, Oracle, H2 and Access commit DDL at once, there the created tables stay empty and keys and identity values are set after the commit | | Can I export only the structure? | Yes. Rows are optional. Pick content **Structure** (`ExportMode.DDL`) for a package or a direct import. Tables, keys, indexes, comments, sequences and partitions are created without rows | | Are partitioned tables kept? | Yes. The same database type creates the partitioning again with its partitions, bounds and rows: range, list, hash, key and subpartitions of MySQL and MariaDB (system time too), range, list, hash, default and nested partitions of PostgreSQL, range, list, hash, interval, automatic list, composite and reference partitions of Oracle, partition functions of SQL Server, range partitions of Db2 and partition keys of ClickHouse. Other databases get the tables with all rows. In the tree a partitioned table shows its partitioning next to its name and PostgreSQL partitions are listed under their table | | Can I generate scripts for an older server? | Yes, choose the target version (e.g. Oracle `11.2`, SQL Server `2008`) and a compatible dialect is used | diff --git a/docs/blogger/sqljam-en.md b/docs/blogger/sqljam-en.md index 573aa7c..251490f 100644 --- a/docs/blogger/sqljam-en.md +++ b/docs/blogger/sqljam-en.md @@ -8,7 +8,7 @@ **SqlJam**: *Move tables across databases. Schema, data and all.* -A Java 17 + JavaFX desktop tool that copies tables between relational and OLAP databases (**MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, DuckDB, ClickHouse** and more) in any direction: +A Java 17 + JavaFX desktop tool that copies tables between relational and OLAP databases (**MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, Microsoft Access, DuckDB, ClickHouse** and more) in any direction: **import directly** into a target database or into the same schema as copies, or save a self-describing **export package** of SQL, **Parquet** or **CSV** files and import it later. Imports into a database run in **one transaction** which an error rolls back, and rows are **inserted, updated or skipped by key**. ![SqlJam main window](https://raw.githubusercontent.com/paganini2008/sqljam/master/docs/assets/main-dark.png) @@ -101,6 +101,7 @@ Only **Java 17+** is needed to run SqlJam. JavaFX and the JDBC drivers are insid | SQL Server | 2008 to 2022 | | MariaDB | 10.3 to 11.x | | H2 / SQLite | 2.x / 3.x | +| Microsoft Access (.accdb, .mdb) | 2000 to 2019 | | DuckDB (OLAP) | 1.x | | ClickHouse (OLAP) | 24.x to 26.x | diff --git a/docs/blogger/sqljam-medium.md b/docs/blogger/sqljam-medium.md index cea9f43..3ee5986 100644 --- a/docs/blogger/sqljam-medium.md +++ b/docs/blogger/sqljam-medium.md @@ -8,7 +8,7 @@ Tables, indexes, constraints, sequences, partitions and rows, in one go. ## Overview -**SqlJam** is a Java 17 + JavaFX desktop tool that copies tables between relational and OLAP databases (**MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, DuckDB, ClickHouse** and more) in any direction. Import directly into a target database or into the same schema as copies, or save a self-describing **export package** of SQL, **Parquet** or **CSV** files and import it later. Imports into a database run in **one transaction** which an error rolls back, and rows are **inserted, updated or skipped by key**. +**SqlJam** is a Java 17 + JavaFX desktop tool that copies tables between relational and OLAP databases (**MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, Microsoft Access, DuckDB, ClickHouse** and more) in any direction. Import directly into a target database or into the same schema as copies, or save a self-describing **export package** of SQL, **Parquet** or **CSV** files and import it later. Imports into a database run in **one transaction** which an error rolls back, and rows are **inserted, updated or skipped by key**. Source: https://github.com/paganini2008/sqljam @@ -97,6 +97,7 @@ Databases: - SQL Server 2008 to 2022 - MariaDB 10.3 to 11.x - H2 2.x, SQLite 3.x +- Microsoft Access 2000 to 2019 (.accdb and .mdb files) - DuckDB 1.x and ClickHouse 24.x to 26.x (OLAP) - All JDBC drivers are bundled diff --git a/docs/blogger/sqljam-zh.md b/docs/blogger/sqljam-zh.md index da02c55..0ed633c 100644 --- a/docs/blogger/sqljam-zh.md +++ b/docs/blogger/sqljam-zh.md @@ -8,7 +8,7 @@ **SqlJam**：*Move tables across databases. Schema, data and all.* -一个 Java 17 + JavaFX 的桌面导入导出工具，支持关系型与 OLAP 数据库（**MySQL、MariaDB、PostgreSQL、Oracle、SQL Server、Db2、H2、SQLite、DuckDB、ClickHouse** 等）任意互导： +一个 Java 17 + JavaFX 的桌面导入导出工具，支持关系型与 OLAP 数据库（**MySQL、MariaDB、PostgreSQL、Oracle、SQL Server、Db2、H2、SQLite、Microsoft Access、DuckDB、ClickHouse** 等）任意互导： 可以**直接导入目标库**，也可以在同一个 schema 里复制表，或者导出为带描述文件的 **SQL / Parquet / CSV 导出包**，之后再成对导入。导入目标库默认在**一个事务**里完成，出错整体回滚，数据可以**按主键更新、插入或跳过**。 ![SqlJam 主界面](https://raw.githubusercontent.com/paganini2008/sqljam/master/docs/assets/main-dark.png) @@ -101,6 +101,7 @@ bin/ | SQL Server | 2008 ~ 2022 | | MariaDB | 10.3 ~ 11.x | | H2 / SQLite | 2.x / 3.x | +| Microsoft Access（.accdb、.mdb） | 2000 ~ 2019 | | DuckDB（OLAP） | 1.x | | ClickHouse（OLAP） | 24.x ~ 26.x | diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 92aa5e8..99e8b7f 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -46,7 +46,7 @@ mvn -DskipTests install ``` -The JDBC drivers of MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, DuckDB and ClickHouse come with it. +The JDBC drivers of MySQL, MariaDB, PostgreSQL, Oracle, SQL Server, Db2, H2, SQLite, Microsoft Access (UCanAccess), DuckDB and ClickHouse come with it. --- @@ -331,7 +331,7 @@ The key is the primary key of the table, or the chosen columns: columns of a uni |---|---|---| | PostgreSQL, SQLite, DuckDB | `ON CONFLICT … DO UPDATE` | `ON CONFLICT … DO NOTHING` | | MySQL, MariaDB | `ON DUPLICATE KEY UPDATE` | `ON DUPLICATE KEY UPDATE` without changes | -| Oracle, SQL Server, Db2 | `MERGE … WHEN MATCHED … WHEN NOT MATCHED` | `MERGE … WHEN NOT MATCHED` | +| Oracle, SQL Server, Db2, Access | `MERGE … WHEN MATCHED … WHEN NOT MATCHED` | `MERGE … WHEN NOT MATCHED` | | H2 | `MERGE INTO … KEY` | `MERGE … WHEN NOT MATCHED` | Names of an existing target table are matched without case, `OrderId` of the source writes `orderid` of PostgreSQL or `ORDERID` of Oracle. @@ -426,6 +426,8 @@ copier.setExportListener(new ExportListener() { | Dialect: types, literals, DDL, paging, upserts | `impexp.db.XxxDialect extends Dialect` | | Metadata: catalogs, schemas, tables, columns, keys | `impexp.db.XxxMetaDataOperations extends MetaDataOperations` | | Older versions | `XxxNNDialect extends XxxDialect`, chosen by `Dialect.forVersion` | +| Versions other than the server version, e.g. the file format of Access | `Dialect.readDatabaseVersion` | +| Tables which are emptied before they are dropped, drops of missing tables | `Dialect.getEmptyTableStatement`, `Dialect.isMissingDropError` | | Registration | A constant of `DbType` with driver, URL template and category | | Fixture and tests | `src/test/resources//fixture.sql`, a value of `ItDatabase`, the cross-database matrix runs it | diff --git a/docs/index.html b/docs/index.html index b1716e7..1393ff3 100644 --- a/docs/index.html +++ b/docs/index.html @@ -4,7 +4,7 @@ SqlJam · Move tables across databases - + @@ -191,7 +191,7 @@

SqlJam

MySQLMariaDBPostgreSQLOracle SQL ServerDb2H2SQLite - DuckDBClickHouse + AccessDuckDBClickHouse
Get Started diff --git a/pom.xml b/pom.xml index e320645..7612e0f 100644 --- a/pom.xml +++ b/pom.xml @@ -27,6 +27,7 @@ 0.11.7 12.1.5.0 1.5.6.0 + 5.1.8 0.10.0 1.18.48 com.github.sqljam.face.Launcher @@ -168,6 +169,12 @@ duckdb_jdbc ${duckdb.version} + + + io.github.spannm + ucanaccess + ${ucanaccess.version} + com.clickhouse clickhouse-jdbc diff --git a/src/main/java/com/github/sqljam/face/service/DatabaseSession.java b/src/main/java/com/github/sqljam/face/service/DatabaseSession.java index a373550..fa2739c 100644 --- a/src/main/java/com/github/sqljam/face/service/DatabaseSession.java +++ b/src/main/java/com/github/sqljam/face/service/DatabaseSession.java @@ -341,8 +341,8 @@ public DataPage getData(String catalog, String schema, String table, TableQuery totalRows = count != null ? count : 0; } try (Connection connection = getConnection(catalog)) { - Dialect dialect = this.dialect.forVersion(connection.getMetaData().getDatabaseMajorVersion(), - connection.getMetaData().getDatabaseMinorVersion()); + int[] version = this.dialect.readDatabaseVersion(connection.getMetaData()); + Dialect dialect = this.dialect.forVersion(version[0], version[1]); List primaryKeys = metaDataOperations.getPrimaryKeyInfos(connection.getMetaData(), getMetaCatalog(catalog), schema, table).stream() .sorted(Comparator.comparingInt(info -> TableMetaData.getInt(info, "KEY_SEQ"))) @@ -370,7 +370,7 @@ public DataPage getData(String catalog, String schema, String table, TableQuery while (rs.next()) { List row = new ArrayList<>(indexes.size()); for (int i : indexes) { - row.add(formatValue(JdbcUtils.getColumnValue(rs, i, rsmd.getColumnType(i), + row.add(formatValue(dialect.getColumnValue(rs, i, rsmd.getColumnType(i), rsmd.getColumnTypeName(i)))); } rows.add(row); diff --git a/src/main/java/com/github/sqljam/face/service/JdbcUrls.java b/src/main/java/com/github/sqljam/face/service/JdbcUrls.java index 116020a..51889c7 100644 --- a/src/main/java/com/github/sqljam/face/service/JdbcUrls.java +++ b/src/main/java/com/github/sqljam/face/service/JdbcUrls.java @@ -99,6 +99,10 @@ public static ConnectionProfile parse(String url) { case DUCKDB: return fill(profile, null, null, StringUtils.substringBefore(text.substring("jdbc:duckdb:".length()), "?")); + case ACCESS: + // jdbc:ucanaccess://path;option=value + return fill(profile, null, null, StringUtils.substringBefore(StringUtils.removeStartIgnoreCase( + text.substring("jdbc:ucanaccess:".length()), "//"), ";")); default: matcher = SERVER.matcher(text); if (!matcher.matches()) { diff --git a/src/main/java/com/github/sqljam/face/view/ConnectionForm.java b/src/main/java/com/github/sqljam/face/view/ConnectionForm.java index 5499cad..ec5f68a 100644 --- a/src/main/java/com/github/sqljam/face/view/ConnectionForm.java +++ b/src/main/java/com/github/sqljam/face/view/ConnectionForm.java @@ -244,6 +244,10 @@ private void browseFile() { chooser.setInitialDirectory(current.getParentFile()); } } + if (dbTypeCombo.getValue() == DbType.ACCESS) { + chooser.getExtensionFilters().addAll(new FileChooser.ExtensionFilter(DbType.ACCESS.getDisplayName(), + "*.accdb", "*.mdb"), new FileChooser.ExtensionFilter("*", "*.*")); + } File file = chooser.showOpenDialog(getScene() != null ? getScene().getWindow() : null); if (file != null) { String path = file.getAbsolutePath(); diff --git a/src/main/java/com/github/sqljam/face/view/ExportDialog.java b/src/main/java/com/github/sqljam/face/view/ExportDialog.java index d3a0091..4ba1c31 100644 --- a/src/main/java/com/github/sqljam/face/view/ExportDialog.java +++ b/src/main/java/com/github/sqljam/face/view/ExportDialog.java @@ -101,6 +101,8 @@ public class ExportDialog extends Dialog { VERSION_PRESETS.put(DbType.DB2, List.of("", "11.1", "11.5", "12.1")); VERSION_PRESETS.put(DbType.H2, List.of("", "2.2")); VERSION_PRESETS.put(DbType.SQLITE, List.of("", "3")); + // Access 2007, 2010 and 2016 + VERSION_PRESETS.put(DbType.ACCESS, List.of("", "12", "14", "16")); } private final AppContext context; diff --git a/src/main/java/com/github/sqljam/impexp/AbstractFileImporter.java b/src/main/java/com/github/sqljam/impexp/AbstractFileImporter.java index d5e9474..ef80d86 100644 --- a/src/main/java/com/github/sqljam/impexp/AbstractFileImporter.java +++ b/src/main/java/com/github/sqljam/impexp/AbstractFileImporter.java @@ -223,6 +223,9 @@ protected void runScript(File file) throws IOException, SQLException { if (existingSkipped) { runner.setSkippedError(dbType.createDialect()::isAlreadyExistsError); } + if (dbType != null) { + runner.setMissingDropError(dbType.createDialect()::isMissingDropError); + } runner.setBatchSize(batchSize); long fileBytes = file.length(); runner.setExportListener(new ForwardingListener(exportListener) { diff --git a/src/main/java/com/github/sqljam/impexp/DbType.java b/src/main/java/com/github/sqljam/impexp/DbType.java index 7b5913e..58e4c25 100644 --- a/src/main/java/com/github/sqljam/impexp/DbType.java +++ b/src/main/java/com/github/sqljam/impexp/DbType.java @@ -15,11 +15,14 @@ */ package com.github.sqljam.impexp; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; +import com.github.sqljam.impexp.db.AccessDialect; +import com.github.sqljam.impexp.db.AccessMetaDataOperations; import com.github.sqljam.impexp.db.ClickHouseDialect; import com.github.sqljam.impexp.db.ClickHouseMetaDataOperations; import com.github.sqljam.impexp.db.Db2Dialect; @@ -229,6 +232,44 @@ public boolean isFileBased() { } }, + ACCESS("Access", "net.ucanaccess.jdbc.UcanaccessDriver", 0, false, false, false, AccessDialect::new, + AccessMetaDataOperations::new) { + /** + * Database file of Microsoft Access (.accdb or .mdb), a missing file is created: .accdb in the format of + * Access 2016 (64-bit integers), .mdb in the format of Access 2002-2003 + */ + @Override + public String getUrl(String hostname, int port, String catalogName) { + Validate.notBlank(catalogName, "Database file must be required while connecting to Access database."); + return normalizeUrl("jdbc:ucanaccess://" + catalogName); + } + + @Override + public String normalizeUrl(String url) { + if (StringUtils.isBlank(url) || StringUtils.containsIgnoreCase(url, "newdatabaseversion=")) { + return url; + } + String file = StringUtils.substringBefore(url, ";"); + return url + ";newdatabaseversion=" + (file.toLowerCase(Locale.ENGLISH).endsWith(".mdb") ? "V2003" + : "V2016"); + } + + @Override + public boolean isCatalogSupported() { + return false; + } + + @Override + public boolean isFileBased() { + return true; + } + + @Override + public String getValidationQuery() { + return "SELECT 1 FROM (VALUES (0))"; + } + }, + DUCKDB("DuckDB", "org.duckdb.DuckDBDriver", 0, false, true, true, DuckDBDialect::new, DuckDBMetaDataOperations::new) { @Override @@ -451,6 +492,8 @@ public static DbType forUrl(String jdbcUrl) { return H2; } else if (url.startsWith("jdbc:sqlite:")) { return SQLITE; + } else if (url.startsWith("jdbc:ucanaccess:")) { + return ACCESS; } else if (url.startsWith("jdbc:duckdb:")) { return DUCKDB; } else if (url.startsWith("jdbc:clickhouse:") || url.startsWith("jdbc:ch:")) { diff --git a/src/main/java/com/github/sqljam/impexp/DefaultMetaDataVisitor.java b/src/main/java/com/github/sqljam/impexp/DefaultMetaDataVisitor.java index f38a7aa..e1bf7c8 100644 --- a/src/main/java/com/github/sqljam/impexp/DefaultMetaDataVisitor.java +++ b/src/main/java/com/github/sqljam/impexp/DefaultMetaDataVisitor.java @@ -16,7 +16,9 @@ package com.github.sqljam.impexp; import java.sql.SQLException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.ArrayUtils; @@ -36,6 +38,10 @@ public class DefaultMetaDataVisitor implements MetaDataVisitor { private final Exporter.ExportConfiguration configuration; private final DdlScripter ddlScripter; + /** + * Tables of a schema whose rows are deleted before the drops, the drops follow these deletes + */ + private final Map emptiedTables = new HashMap<>(); public DefaultMetaDataVisitor(Exporter.ExportConfiguration configuration, DdlScripter ddlScripter) { this.configuration = configuration; @@ -104,10 +110,18 @@ public void visit(TableMetaData metaData) throws SQLException { String dropStatement = configuration.getDialect().getDropTableStatement(metaData.getCatalogName(), metaData.getSchemaName(), configuration.getDialect().getTargetTableName(metaData.getTableName())); + String emptyStatement = configuration.getDialect().getEmptyTableStatement(metaData.getCatalogName(), + metaData.getSchemaName(), configuration.getDialect().getTargetTableName(metaData.getTableName())); if (configuration.getDialect().isForeignKeyOrderRequired()) { - // Tables are visited with referenced tables first, they are dropped in the reverse order + // Tables are visited with referenced tables first, they are dropped in the reverse order. Rows of + // all tables are deleted before the first drop (Access). Schema schema = getDdlScripter().getSchema(metaData.getCatalogName(), metaData.getSchemaName()); - schema.getBeforeStatements().add(0, dropStatement); + int emptied = emptiedTables.getOrDefault(schema, 0); + if (emptyStatement != null) { + schema.getBeforeStatements().add(0, emptyStatement); + emptiedTables.put(schema, ++emptied); + } + schema.getBeforeStatements().add(emptied, dropStatement); } else { table.getBeforeStatements().add(0, dropStatement); } diff --git a/src/main/java/com/github/sqljam/impexp/Dialect.java b/src/main/java/com/github/sqljam/impexp/Dialect.java index d6b6972..2ea185b 100644 --- a/src/main/java/com/github/sqljam/impexp/Dialect.java +++ b/src/main/java/com/github/sqljam/impexp/Dialect.java @@ -17,6 +17,7 @@ import java.math.BigDecimal; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -344,6 +345,14 @@ public Dialect forVersion(int majorVersion, int minorVersion) { return dialect; } + /** + * Major and minor version of the connected database by its metadata, versions of other kinds (e.g. the file + * format of Access) are read by the dialect of the database + */ + public int[] readDatabaseVersion(DatabaseMetaData databaseMetaData) throws SQLException { + return new int[]{databaseMetaData.getDatabaseMajorVersion(), databaseMetaData.getDatabaseMinorVersion()}; + } + /** * Whether the database version is greater than or equal to the given version. Unknown version is treated as the * latest version. @@ -526,6 +535,14 @@ public String getDefaultSchemaName(String catalog) { return null; } + /** + * Statement deleting the rows of a table before the tables are dropped, null if a drop does it (default). + * Access fails to drop a table with rows whose related table was dropped before. + */ + public String getEmptyTableStatement(String catalog, String schema, String tableName) { + return null; + } + public String getDropTableStatement(String catalog, String schema, String tableName) { return String.format("DROP TABLE IF EXISTS %s CASCADE", getQualifiedTableName(catalog, schema, tableName)); } @@ -1209,6 +1226,15 @@ public String getUpsertStatement(String catalog, String schema, String tableName Arrays.stream(columnNames).map(name -> "?").toArray()); } + /** + * Parameter of a value of the target column in an upsert statement, e.g. a cast which types the parameter + * + * @param targetTypeName type of the target column of the database, null if it is not known + */ + public String getParameterMarker(String targetTypeName) { + return "?"; + } + /** * Whether upserts may update indexed columns, an update of an indexed column of DuckDB is a delete and an * insert which fails for keys and foreign keys @@ -1247,6 +1273,14 @@ public Set getIndexedColumns(Connection connection, String catalog, Stri return columns; } + /** + * Whether the error of a drop statement only means that the object does not exist, for databases whose drops + * are written without IF EXISTS (Access). The statement is skipped then. + */ + public boolean isMissingDropError(String sql, SQLException e) { + return false; + } + /** * Whether the error means that the created object exists already: a table, an index, a key or a constraint. * Imports which do not drop tables skip such statements of existing tables. diff --git a/src/main/java/com/github/sqljam/impexp/Exporter.java b/src/main/java/com/github/sqljam/impexp/Exporter.java index ad76e32..def1e56 100644 --- a/src/main/java/com/github/sqljam/impexp/Exporter.java +++ b/src/main/java/com/github/sqljam/impexp/Exporter.java @@ -345,8 +345,9 @@ private void prepare() { } private void configureSourceDialect(DatabaseMetaData databaseMetaData) throws SQLException { - int majorVersion = databaseMetaData.getDatabaseMajorVersion(); - int minorVersion = databaseMetaData.getDatabaseMinorVersion(); + int[] version = configuration.getSourceDialect().readDatabaseVersion(databaseMetaData); + int majorVersion = version[0]; + int minorVersion = version[1]; // A version specified explicitly is kept if (configuration.getSourceDialect().getDatabaseMajorVersion() < 0) { configuration.setSourceDialect(configuration.getSourceDialect().forVersion(majorVersion, minorVersion)); diff --git a/src/main/java/com/github/sqljam/impexp/ImportExportHandler.java b/src/main/java/com/github/sqljam/impexp/ImportExportHandler.java index 1f22d25..6253a29 100644 --- a/src/main/java/com/github/sqljam/impexp/ImportExportHandler.java +++ b/src/main/java/com/github/sqljam/impexp/ImportExportHandler.java @@ -153,8 +153,8 @@ public Dialect configureDialect(Dialect dialect) { DatabaseMetaData databaseMetaData = connection.getMetaData(); // A version specified explicitly is kept if (dialect.getDatabaseMajorVersion() < 0) { - dialect = dialect.forVersion(databaseMetaData.getDatabaseMajorVersion(), - databaseMetaData.getDatabaseMinorVersion()); + int[] version = dialect.readDatabaseVersion(databaseMetaData); + dialect = dialect.forVersion(version[0], version[1]); } this.dialect = dialect; exportListener.onMessage(String.format("Target database: %s %s", @@ -302,6 +302,13 @@ private void executeStatements(String catalogName, String schemaName, List tableMetaData.isInsertableColumn(md.getColumnName(), tableMetaData.getDialect())) .collect(LinkedHashMap::new, (m, e) -> m.put(e.getColumnName(), null), LinkedHashMap::putAll); + // Labels of a query may differ in case from its metadata, e.g. ID and id of UCanAccess + Map columnsInLowerCase = new HashMap<>(); + template.keySet().forEach(name -> columnsInLowerCase.putIfAbsent(name.toLowerCase(Locale.ENGLISH), name)); for (Map data : dataList) { Map row = new LinkedHashMap<>(template); for (Map.Entry entry : data.entrySet()) { - if (row.containsKey(entry.getKey())) { - ColumnMetaData columnMetaData = tableMetaData.findColumnMetaData(entry.getKey()).orElse(null); - row.put(entry.getKey(), ScriptExportHandler.getJdbcValue(dialect, columnMetaData, - entry.getValue())); + String column = row.containsKey(entry.getKey()) ? entry.getKey() + : columnsInLowerCase.get(entry.getKey().toLowerCase(Locale.ENGLISH)); + if (column != null) { + ColumnMetaData columnMetaData = tableMetaData.findColumnMetaData(column).orElse(null); + row.put(column, ScriptExportHandler.getJdbcValue(dialect, columnMetaData, entry.getValue())); } } rowList.add(row); @@ -462,8 +473,10 @@ public void exportData(String catalogName, String[] updates = configuration.getWriteMode() == WriteMode.SKIP_EXISTING ? new String[0] : Arrays.stream(columns).filter(name -> !fixed.contains(name.toLowerCase(Locale.ENGLISH))) .toArray(String[]::new); + Map columnTypes = targetTypes; insertSql = dialect.getUpsertStatement(catalogName, schemaName, targetTableName, columns, keys, - updates); + updates, Arrays.stream(columns).map(name -> dialect.getParameterMarker(columnTypes.get( + name.toLowerCase(Locale.ENGLISH)))).toArray()); } executeQuietly(connection, dialect.getStatementBeforeInsert(catalogName, schemaName, targetTableName, identityIncluded)); diff --git a/src/main/java/com/github/sqljam/impexp/SqlScriptRunner.java b/src/main/java/com/github/sqljam/impexp/SqlScriptRunner.java index 6f3bbb9..a5b3d73 100644 --- a/src/main/java/com/github/sqljam/impexp/SqlScriptRunner.java +++ b/src/main/java/com/github/sqljam/impexp/SqlScriptRunner.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.function.BiPredicate; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -82,6 +83,10 @@ public class SqlScriptRunner { * Errors of statements which create existing objects, they are skipped, e.g. tables which are not dropped */ private Predicate skippedError; + /** + * Errors of drop statements of missing objects, they are skipped (drops without IF EXISTS of Access) + */ + private BiPredicate missingDropError; private ExportListener exportListener = ExportListener.NONE; private Statement statement; /** @@ -105,6 +110,13 @@ public void setSkippedError(Predicate skippedError) { this.skippedError = skippedError; } + /** + * Drop statements failing with these errors are skipped, the dropped object does not exist + */ + public void setMissingDropError(BiPredicate missingDropError) { + this.missingDropError = missingDropError; + } + public void setStopOnError(boolean stopOnError) { this.stopOnError = stopOnError; } @@ -277,14 +289,21 @@ private void executeBatch(List batch) throws SQLException { * keeps the transaction usable after a failure (PostgreSQL aborts the whole transaction otherwise) */ private boolean execute(String sql) throws SQLException { - Savepoint savepoint = (stopOnError && skippedError == null) || !transactional ? null - : setSavepointQuietly(); + Savepoint savepoint = (stopOnError && skippedError == null && missingDropError == null) || !transactional + ? null : setSavepointQuietly(); try { statement().execute(sql); executedCount++; releaseSavepoint(savepoint); return true; } catch (SQLException e) { + if (missingDropError != null && missingDropError.test(sql, e)) { + rollbackQuietly(savepoint); + if (log.isInfoEnabled()) { + log.info("Does not exist, skipped: {}", StringUtils.abbreviate(sql, 200)); + } + return false; + } if (skippedError != null && skippedError.test(e)) { // The object exists, the transaction goes on from the savepoint rollbackQuietly(savepoint); diff --git a/src/main/java/com/github/sqljam/impexp/db/AccessDialect.java b/src/main/java/com/github/sqljam/impexp/db/AccessDialect.java new file mode 100644 index 0000000..d1d6407 --- /dev/null +++ b/src/main/java/com/github/sqljam/impexp/db/AccessDialect.java @@ -0,0 +1,582 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.impexp.db; + +import java.math.BigDecimal; +import java.sql.Blob; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Arrays; +import java.util.Locale; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import com.github.sqljam.impexp.DbType; +import com.github.sqljam.impexp.Dialect; +import com.github.sqljam.impexp.StringHelper; + +/** + * @Description: AccessDialect generates ddl/dml of Microsoft Access files (.accdb, .mdb) through UCanAccess. Access + * has no schemas, sequences, partitions or comments. Type names are the ones of Access (INTEGER is 16-bit, + * LONG 32-bit, BYTE unsigned). Text columns hold 255 characters (MEMO beyond), + * decimals 28 digits, date and time columns are DATETIME. 64-bit integers (BIGINT) are kept by files of + * Access 2016 and later, older files get DECIMAL(28,1): UCanAccess stores BIGINT of them as 32-bit and + * limits DECIMAL of scale 0 to 18 digits. Names are quoted by backticks, explicit autonumbers are + * inserted with DISABLE AUTOINCREMENT. + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +public class AccessDialect extends Dialect { + + /** + * Characters of TEXT, longer text is MEMO + */ + static final int MAX_TEXT_LENGTH = 255; + /** + * Bytes of BINARY, longer binaries are OLE objects + */ + static final int MAX_BINARY_LENGTH = 255; + static final int MAX_NUMERIC_PRECISION = 28; + /** + * Precision of DECIMAL of scale 0 which UCanAccess keeps, larger precisions are stored as 18 + */ + static final int MAX_INTEGRAL_DECIMAL_PRECISION = 18; + /** + * File format of Access 2016 (VERSION_16), the first one with 64-bit integers + */ + static final int BIGINT_VERSION = 16; + private static final Pattern FILE_VERSION = Pattern.compile("VERSION_(\\d+)"); + /** + * UCanAccess writes a warning for every row it passes while looking for an updated or deleted row, kept here + * because loggers of java.util.logging are weakly referenced + */ + private static final Logger CURSOR_LOGGER = Logger.getLogger("net.ucanaccess.commands.AbstractCursorCommand"); + + static { + CURSOR_LOGGER.setLevel(Level.SEVERE); + } + + public AccessDialect() { + super(); + registerColumnType(Types.BIT, "YESNO"); + registerColumnType(Types.BOOLEAN, "YESNO"); + // Type names of UCanAccess are the ones of Access: INTEGER is 16-bit, LONG 32-bit, BYTE (and TINYINT) is + // unsigned 0-255, REAL is DOUBLE + registerColumnType(Types.TINYINT, "SMALLINT"); + registerColumnType(Types.SMALLINT, "SMALLINT"); + registerColumnType(Types.INTEGER, "LONG"); + registerColumnType(Types.BIGINT, "BIGINT"); + registerColumnType(Types.REAL, "SINGLE"); + registerColumnType(Types.FLOAT, "DOUBLE"); + registerColumnType(Types.DOUBLE, "DOUBLE"); + registerColumnType(Types.NUMERIC, "DECIMAL($p,$s)"); + registerColumnType(Types.DECIMAL, "DECIMAL($p,$s)"); + + registerColumnType(Types.DATE, "DATETIME"); + registerColumnType(Types.TIME, "DATETIME"); + registerColumnType(Types.TIMESTAMP, "DATETIME"); + // Access has no time zones, values are migrated in the local time zone + registerColumnType(Types.TIME_WITH_TIMEZONE, "DATETIME"); + registerColumnType(Types.TIMESTAMP_WITH_TIMEZONE, "DATETIME"); + + registerColumnType(Types.BINARY, "OLE"); + registerColumnType(Types.BINARY, MAX_BINARY_LENGTH, "BINARY($l)"); + registerColumnType(Types.VARBINARY, "OLE"); + // BINARY of UCanAccess pads values to its length + registerColumnType(Types.VARBINARY, MAX_BINARY_LENGTH, "VARBINARY($l)"); + registerColumnType(Types.LONGVARBINARY, "OLE"); + registerColumnType(Types.BLOB, "OLE"); + + for (int dataType : new int[]{Types.CHAR, Types.NCHAR, Types.VARCHAR, Types.NVARCHAR}) { + registerColumnType(dataType, "MEMO"); + registerColumnType(dataType, MAX_TEXT_LENGTH, "TEXT($l)"); + } + registerColumnType(Types.LONGVARCHAR, "MEMO"); + registerColumnType(Types.LONGNVARCHAR, "MEMO"); + registerColumnType(Types.CLOB, "MEMO"); + registerColumnType(Types.NCLOB, "MEMO"); + registerColumnType(Types.SQLXML, "MEMO"); + registerColumnType(Types.OTHER, "MEMO"); + registerColumnType(Types.JAVA_OBJECT, "MEMO"); + + registerReservedWords("add", "all", "alter", "and", "any", "as", "asc", "autoincrement", "between", + "binary", "bit", "boolean", "by", "byte", "char", "character", "column", "constraint", "counter", + "create", "currency", "database", "date", "datetime", "decimal", "delete", "desc", "disallow", + "distinct", "distinctrow", "double", "drop", "exists", "float", "foreign", "from", "general", "group", + "guid", "having", "ignore", "imp", "in", "index", "inner", "insert", "integer", "into", "is", "join", + "key", "left", "level", "like", "logical", "long", "longbinary", "longtext", "memo", "money", "mod", + "name", "not", "null", "number", "numeric", "ole", "on", "option", "or", "order", "outer", "owneraccess", + "parameters", "password", "percent", "pivot", "primary", "procedure", "real", "references", "right", + "select", "set", "short", "single", "smallint", "some", "table", "text", "time", "timestamp", "top", + "transform", "union", "unique", "update", "user", "value", "values", "varbinary", "varchar", "where", + "with", "yesno"); + } + + @Override + public DbType getDbType() { + return DbType.ACCESS; + } + + /** + * Version of the Access file format, e.g. 14 of "V2010 [VERSION_14]" or 16 of Access 2016. The version of + * UCanAccess metadata is the one of its HSQLDB. + */ + @Override + public int[] readDatabaseVersion(DatabaseMetaData databaseMetaData) throws SQLException { + Matcher matcher = FILE_VERSION.matcher(StringUtils.defaultString(databaseMetaData + .getDatabaseProductVersion())); + return matcher.find() ? new int[]{Integer.parseInt(matcher.group(1)), 0} : new int[]{-1, -1}; + } + + @Override + public String getOpenQuote() { + return "`"; + } + + @Override + public String getCloseQuote() { + return "`"; + } + + @Override + public int getMaxIdentifierLength() { + return 64; + } + + @Override + public int getMaxNumericPrecision() { + return MAX_NUMERIC_PRECISION; + } + + /** + * Numbers without precision, e.g. numeric of PostgreSQL, the 28 digits of Access with 10 decimals + */ + @Override + public String getUnboundedNumericTypeName() { + return "DECIMAL(28,10)"; + } + + /** + * Dates are stored as days and fractions of a day, milliseconds are kept + */ + @Override + public int getMaxFractionalSecondsPrecision() { + return 3; + } + + /** + * Whether the file keeps 64-bit integers: Access 2016 and later. A script for an unknown file is written for + * the latest format. + */ + boolean isBigIntSupported() { + return isVersionAtLeast(BIGINT_VERSION, 0); + } + + @Override + protected String getSpecialTypeName(int dataType, String sourceTypeName, int columnSize, int columnScale) { + String baseTypeName = sourceTypeName.replaceAll("\\(.*\\)", "").replace(" unsigned", "").trim(); + switch (baseTypeName) { + case "uuid": + case "uniqueidentifier": + return "TEXT(38)"; + case "json": + case "jsonb": + case "xml": + case "xmltype": + case "text": + case "mediumtext": + case "longtext": + case "tinytext": + return "MEMO"; + case "inet": + case "inet4": + case "inet6": + case "cidr": + case "macaddr": + return "TEXT(50)"; + case "enum": + case "set": + return "TEXT(255)"; + default: + break; + } + if (baseTypeName.startsWith("_") || dataType == Types.ARRAY || dataType == Types.STRUCT) { + return "MEMO"; + } + if (dataType == Types.BIGINT && !isBigIntSupported()) { + return "DECIMAL(28,1)"; + } + if (dataType == Types.BIT && columnSize > 1) { + // Bit strings are integers, LONG is signed 32-bit + return columnSize < 32 ? "LONG" : isBigIntSupported() ? "BIGINT" : "DECIMAL(28,1)"; + } + if ((dataType == Types.NUMERIC || dataType == Types.DECIMAL) && columnSize > 0 + && columnSize <= MAX_NUMERIC_PRECISION) { + int scale = Math.max(columnScale, 0); + // UCanAccess stores DECIMAL of scale 0 with 18 digits, one decimal keeps the larger integers + if (scale == 0 && columnSize > MAX_INTEGRAL_DECIMAL_PRECISION) { + return String.format("DECIMAL(%d,1)", Math.min(columnSize + 1, MAX_NUMERIC_PRECISION)); + } + } + if ((dataType == Types.NUMERIC || dataType == Types.DECIMAL) && columnSize > MAX_NUMERIC_PRECISION) { + // Numbers wider than 28 digits are kept as text + return "TEXT(100)"; + } + return null; + } + + @Override + public String getCreateUserStatement(String username, String password) { + return null; + } + + @Override + public String getCreateDatabaseStatement(String catalog, String username) { + return null; + } + + @Override + public String getCreateSchemaStatement(String catalog, String schema, String username) { + return null; + } + + /** + * Access has no schemas, tables are named without qualifier + */ + @Override + public String getQualifiedTableName(String catalog, String schema, String tableName) { + return getIdentifier(tableName); + } + + @Override + public String getSourceTableName(String catalog, String schema, String tableName) { + return quoteIdentifier(tableName); + } + + /** + * Statements of UCanAccess without IF NOT EXISTS: with it the table is created in its HSQLDB mirror only + */ + @Override + public String getCreateTableStatement(String catalog, String schema, String tableName) { + return String.format("CREATE TABLE %s", getQualifiedTableName(catalog, schema, tableName)); + } + + @Override + public String getCreateIndexStatement(String catalog, String schema, String tableName, boolean partition, + String[] columnNames, String indexName, boolean unique, String indexType) { + return String.format("CREATE %sINDEX %s ON %s (%s)", unique ? "UNIQUE " : "", quoteIdentifier(indexName), + getQualifiedTableName(catalog, schema, tableName), getIdentifiers(columnNames)); + } + + /** + * UCanAccess drops a table of the Access file by DROP TABLE only, with IF EXISTS or CASCADE it drops the table + * of its HSQLDB mirror and keeps the one of the file. A missing table is skipped by its error. + */ + @Override + public String getDropTableStatement(String catalog, String schema, String tableName) { + return String.format("DROP TABLE %s", getQualifiedTableName(catalog, schema, tableName)); + } + + /** + * Rows are deleted before the tables are dropped: UCanAccess deletes the rows of a dropped table and fails on + * the relationship index of a related table dropped before (Reference to missing table) + */ + @Override + public String getEmptyTableStatement(String catalog, String schema, String tableName) { + return String.format("DELETE FROM %s", getQualifiedTableName(catalog, schema, tableName)); + } + + /** + * Related tables are created referenced tables first and dropped in the reverse order + */ + @Override + public boolean isForeignKeyOrderRequired() { + return true; + } + + /** + * The table of DROP TABLE (or of the DELETE before it) does not exist: "object not found" of HSQLDB (42501) + */ + @Override + public boolean isMissingDropError(String sql, SQLException e) { + String statement = StringUtils.trimToEmpty(sql); + if (!StringUtils.startsWithIgnoreCase(statement, "DROP TABLE") + && !StringUtils.startsWithIgnoreCase(statement, "DELETE FROM")) { + return false; + } + for (Throwable cause = e; cause != null; cause = cause.getCause()) { + String message = StringUtils.defaultString(cause.getMessage()); + if (message.contains("object not found") || message.contains("does not exist")) { + return true; + } + } + return false; + } + + @Override + public String getSequenceNameStatement(String catalog, String schema, String tableName, String columnName) { + return null; + } + + @Override + public String getDefaultSequenceName(String catalog, String schema, String tableName, String columnName) { + return null; + } + + @Override + public String getAlterSequenceStartValueStatement(String catalog, String schema, String tableName, + String sequenceName, long startValue) { + return null; + } + + /** + * UCanAccess looks for an updated row by passing the rows of the table, LOB values are written in INSERT + * statements + */ + @Override + public boolean isLobSeparationSupported() { + return false; + } + + @Override + public boolean isSequenceSupported() { + return false; + } + + /** + * An autonumber continues after the largest value when it is enabled again + */ + @Override + public String getResetIdentityStatement(String catalog, String schema, String tableName, String columnName, + long startValue) { + return null; + } + + @Override + public String getCreateCommentStatement(String catalog, String schema, String tableName, String columnName, + String comment) { + return null; + } + + @Override + public String getCreateTableCommentStatement(String catalog, String schema, String tableName, String comment) { + return null; + } + + /** + * A table of Access has one autonumber + */ + @Override + public boolean isSingleIncrementalColumn() { + return true; + } + + /** + * Autonumber (COUNTER) is a 32-bit integer (LONG), other identities keep their type and their values: a column + * related to an autonumber must be a LONG + */ + @Override + public String getIncrementalColumnStatement(String catalog, String schema, String tableName, String columnName, + int dataType, String typeName, int columnSize, int columnScale, + String defaultValue, boolean nullable) { + String columnType = getColumnTypeName(dataType, typeName, columnSize, columnScale); + StringBuilder columnDef = new StringBuilder(StringHelper.textLeft(getIdentifier(columnName), 30)); + columnDef.append(StringHelper.textLeft("LONG".equals(columnType) ? "COUNTER" : columnType, 30)); + columnDef.append(" NOT NULL"); + return columnDef.toString().trim(); + } + + /** + * Calculated fields are not created by UCanAccess, generated columns become plain columns + */ + @Override + public String getGeneratedColumnStatement(String catalog, String schema, String tableName, String columnName, + int dataType, String typeName, int columnSize, int columnScale, + String expression, boolean stored) { + return StringHelper.textLeft(getIdentifier(columnName), 30) + getColumnTypeName(dataType, typeName, + columnSize, columnScale); + } + + @Override + public String getCreatePartitionTableStatement(String catalog, String schema, String tableName, + String inheritedTableName) { + return null; + } + + /** + * Explicit autonumbers are inserted while the autonumber of the table is disabled + */ + @Override + public String[] getStatementBeforeInsert(String catalog, String schema, String tableName, + boolean identityIncluded) { + if (!identityIncluded) { + return null; + } + return new String[]{"DISABLE AUTOINCREMENT ON " + getQualifiedTableName(catalog, schema, tableName)}; + } + + @Override + public String[] getStatementAfterInsert(String catalog, String schema, String tableName, + boolean identityIncluded) { + if (!identityIncluded) { + return null; + } + return new String[]{"ENABLE AUTOINCREMENT ON " + getQualifiedTableName(catalog, schema, tableName)}; + } + + @Override + public String getUpsertStatement(String catalog, String schema, String tableName, String[] columnNames, + String[] keyColumnNames, String[] updateColumnNames, Object[] values) { + StringBuilder sql = new StringBuilder("MERGE INTO ").append(getQualifiedTableName(catalog, schema, tableName)) + .append(" tgt USING (VALUES (").append(StringUtils.join(values, ",")).append(")) src (") + .append(getIdentifiers(columnNames)).append(") ON ").append(Arrays.stream(keyColumnNames) + .map(name -> "tgt." + getIdentifier(name) + " = src." + getIdentifier(name)) + .collect(Collectors.joining(" AND "))); + if (updateColumnNames.length > 0) { + sql.append(" WHEN MATCHED THEN UPDATE SET ").append(Arrays.stream(updateColumnNames).map(name -> "tgt." + + getIdentifier(name) + " = src." + getIdentifier(name)).collect(Collectors.joining(", "))); + } + return sql.append(" WHEN NOT MATCHED THEN INSERT (").append(getIdentifiers(columnNames)).append(") VALUES (") + .append(Arrays.stream(columnNames).map(name -> "src." + getIdentifier(name)) + .collect(Collectors.joining(", "))).append(")").toString(); + } + + /** + * Types of the target columns type the parameters of upserts + */ + @Override + public boolean isTargetTypeConversionRequired() { + return true; + } + + /** + * Date times are bound as timestamps: a LocalDateTime in VALUES of MERGE fails in UCanAccess (cardinality + * violation) + */ + @Override + public Object getTargetJdbcValue(Object value, String targetTypeName) { + if (value instanceof LocalDateTime) { + return Timestamp.valueOf((LocalDateTime) value); + } else if (value instanceof LocalDate) { + return Timestamp.valueOf(((LocalDate) value).atStartOfDay()); + } else if (value instanceof LocalTime) { + return Timestamp.valueOf(LocalDateTime.of(LocalDate.of(1899, 12, 30), (LocalTime) value)); + } + return value; + } + + /** + * A parameter of VALUES has no type for UCanAccess, bytes are bound to binary columns and OLE objects (BLOB) by + * a cast + */ + @Override + public String getParameterMarker(String targetTypeName) { + String type = StringUtils.defaultString(targetTypeName).toUpperCase(Locale.ENGLISH); + if ("BLOB".equals(type)) { + return "CAST(? AS BLOB)"; + } + return "VARBINARY".equals(type) || "BINARY".equals(type) ? "CAST(? AS VARBINARY(" + MAX_BINARY_LENGTH + "))" + : "?"; + } + + @Override + public String getCurrentTimestampExpression() { + return "NOW()"; + } + + @Override + public String getCurrentDateExpression() { + return "DATE()"; + } + + @Override + public String getBooleanLiteral(boolean value) { + return value ? "TRUE" : "FALSE"; + } + + @Override + public String getTimestampLiteral(LocalDateTime value) { + return "'" + formatTimestamp(value) + "'"; + } + + @Override + public String getDateLiteral(LocalDate value) { + return "'" + value + "'"; + } + + /** + * A time of Access is a date time of 1899-12-30, the day zero of Access + */ + @Override + public String getTimeLiteral(LocalTime value) { + return "'1899-12-30 " + formatTime(value) + "'"; + } + + /** + * Times are bound as date times of the day zero of Access + */ + @Override + public Object getJdbcValue(Object value, int sourceDataType, String sourceTypeName, int columnSize) { + Object result = super.getJdbcValue(value, sourceDataType, sourceTypeName, columnSize); + if (result instanceof LocalTime) { + return LocalDateTime.of(LocalDate.of(1899, 12, 30), (LocalTime) result); + } + return result; + } + + /** + * OLE objects are read as blobs: bytes of UCanAccess are the key of the object in its mirror, not the content + */ + @Override + public Object getColumnValue(ResultSet rs, int columnIndex, int columnType, String columnTypeName) + throws SQLException { + if (columnType == Types.BLOB || columnType == Types.LONGVARBINARY) { + Blob blob = rs.getBlob(columnIndex); + return blob != null ? blob.getBytes(1, (int) blob.length()) : null; + } + Object value = super.getColumnValue(rs, columnIndex, columnType, columnTypeName); + if (value instanceof BigDecimal && columnType == Types.NUMERIC + && rs.getMetaData().getPrecision(columnIndex) > MAX_NUMERIC_PRECISION) { + // SINGLE of a file is NUMERIC(100,7) for UCanAccess, decimals of Access have 28 digits at most + return ((BigDecimal) value).doubleValue(); + } + return value; + } + + /** + * Floating point literals are plain numbers, UCanAccess takes the exponent of 1.0E100 as a name + */ + @Override + public String getStringValue(String catalog, String schema, String tableName, String columnName, + Object columnValue) { + if ((columnValue instanceof Double || columnValue instanceof Float) + && Double.isFinite(((Number) columnValue).doubleValue())) { + return new BigDecimal(columnValue.toString()).toPlainString(); + } + return super.getStringValue(catalog, schema, tableName, columnName, columnValue); + } +} diff --git a/src/main/java/com/github/sqljam/impexp/db/AccessMetaDataOperations.java b/src/main/java/com/github/sqljam/impexp/db/AccessMetaDataOperations.java new file mode 100644 index 0000000..44c777f --- /dev/null +++ b/src/main/java/com/github/sqljam/impexp/db/AccessMetaDataOperations.java @@ -0,0 +1,298 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.impexp.db; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import com.github.sqljam.impexp.MetaDataOperations; +import com.github.sqljam.utils.CaseInsensitiveMap; +import io.github.spannm.jackcess.Column; +import io.github.spannm.jackcess.DataType; +import io.github.spannm.jackcess.Database; +import io.github.spannm.jackcess.PropertyMap; +import io.github.spannm.jackcess.Table; +import lombok.extern.slf4j.Slf4j; +import net.ucanaccess.jdbc.UcanaccessConnection; + +/** + * @Description: AccessMetaDataOperations reads Microsoft Access files by UCanAccess, completed by the column + * definitions of the file (Jackcess): the types of Access (TEXT, MEMO, CURRENCY, SINGLE, BYTE, GUID) as + * standard types which every database reads alike (VARCHAR, CLOB, DECIMAL(19,4), REAL, SMALLINT), + * autonumbers and the descriptions of tables and columns. Access has no catalogs, schemas or sequences. + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +@Slf4j +public class AccessMetaDataOperations extends MetaDataOperations { + + /** + * An Access file has no catalogs, an empty catalog name is used as the only one + */ + @Override + public List> getCatalogInfos(DatabaseMetaData databaseMetaData) throws SQLException { + Map catalogInfo = new CaseInsensitiveMap<>(); + catalogInfo.put("TABLE_CAT", ""); + List> catalogInfos = new ArrayList<>(); + catalogInfos.add(catalogInfo); + return catalogInfos; + } + + @Override + public List> getSchemaInfos(DatabaseMetaData databaseMetaData, String catalogName) { + return Collections.emptyList(); + } + + /** + * Tables with the descriptions of Access as remarks + */ + @Override + public List> getTableInfos(DatabaseMetaData databaseMetaData, String catalogName, + String schemaName) throws SQLException { + List> tableInfos = super.getTableInfos(databaseMetaData, null, null); + Database database = getDatabase(databaseMetaData.getConnection()); + if (database != null) { + for (Map tableInfo : tableInfos) { + Table table = getTable(database, (String) tableInfo.get("TABLE_NAME")); + if (table != null && StringUtils.isBlank((String) tableInfo.get("REMARKS"))) { + tableInfo.put("REMARKS", getDescription(table::getProperties)); + } + } + } + return tableInfos; + } + + /** + * Columns of UCanAccess (its HSQLDB types) are completed by the columns of the Access file: type, size, + * autonumber and description + */ + @Override + public List> getColumnInfos(DatabaseMetaData databaseMetaData, String catalogName, + String schemaName, String tableName) throws SQLException { + List> columnInfos = super.getColumnInfos(databaseMetaData, null, null, tableName); + Database database = getDatabase(databaseMetaData.getConnection()); + Table table = database != null ? getTable(database, tableName) : null; + if (table == null) { + return columnInfos; + } + for (Map columnInfo : columnInfos) { + Column column = getColumn(table, (String) columnInfo.get("COLUMN_NAME")); + if (column != null) { + normalizeColumn(columnInfo, column); + } + } + return columnInfos; + } + + /** + * Type of an Access column as the jdbc type with a standard type name, size and scale. Names of Access (TEXT, + * LONG, CURRENCY) mean other types in other databases (text of MySQL, LONG of Oracle). + */ + static void normalizeColumn(Map columnInfo, Column column) { + boolean autoNumber = column.isAutoNumber(); + switch (column.getType()) { + case BOOLEAN: + setType(columnInfo, Types.BOOLEAN, "BOOLEAN", 1, 0); + // Yes/No fields of Access are never null + columnInfo.put("IS_NULLABLE", "NO"); + columnInfo.put("NULLABLE", DatabaseMetaData.columnNoNulls); + break; + case BYTE: + // Unsigned 0-255, a small integer of other databases + setType(columnInfo, Types.SMALLINT, "SMALLINT", 3, 0); + break; + case INT: + setType(columnInfo, Types.SMALLINT, "SMALLINT", 5, 0); + break; + case LONG: + setType(columnInfo, Types.INTEGER, "INTEGER", 10, 0); + break; + case BIG_INT: + setType(columnInfo, Types.BIGINT, "BIGINT", 19, 0); + break; + case MONEY: + setType(columnInfo, Types.DECIMAL, "DECIMAL", 19, 4); + break; + case FLOAT: + setType(columnInfo, Types.REAL, "REAL", 7, 0); + break; + case DOUBLE: + setType(columnInfo, Types.DOUBLE, "DOUBLE", 15, 0); + break; + case NUMERIC: + setType(columnInfo, Types.DECIMAL, "DECIMAL", column.getPrecision(), column.getScale()); + break; + case SHORT_DATE_TIME: + case EXT_DATE_TIME: + setType(columnInfo, Types.TIMESTAMP, "TIMESTAMP", 23, 3); + break; + case TEXT: + setType(columnInfo, Types.VARCHAR, "VARCHAR", column.getLengthInUnits(), 0); + break; + case MEMO: + setType(columnInfo, Types.CLOB, "CLOB", Integer.MAX_VALUE, 0); + break; + case BINARY: + setType(columnInfo, Types.VARBINARY, "VARBINARY", column.getLength(), 0); + break; + case OLE: + setType(columnInfo, Types.BLOB, "BLOB", Integer.MAX_VALUE, 0); + break; + case GUID: + // Replication ids are generated by Access, they are not identities of other databases + setType(columnInfo, Types.VARCHAR, "VARCHAR", 38, 0); + autoNumber = false; + break; + default: + break; + } + columnInfo.put("IS_AUTOINCREMENT", autoNumber ? "YES" : "NO"); + columnInfo.put("COLUMN_DEF", autoNumber ? null : normalizeDefault((String) columnInfo.get("COLUMN_DEF"), + column.getType() == DataType.TEXT || column.getType() == DataType.MEMO)); + String description = getDescription(column::getProperties); + if (StringUtils.isBlank((String) columnInfo.get("REMARKS")) && description != null) { + columnInfo.put("REMARKS", description); + } + } + + /** + * Default value of Access as SQL: an expression starts with = (=Now()), strings are quoted by double quotes + * ("n/a") or by nothing (n/a, written by UCanAccess), Date() is the current date + */ + static String normalizeDefault(String defaultValue, boolean text) { + String value = StringUtils.trimToNull(defaultValue); + if (value == null) { + return null; + } + if (value.startsWith("=")) { + value = value.substring(1).trim(); + } + if (value.length() >= 2 && value.startsWith("\"") && value.endsWith("\"")) { + return "'" + value.substring(1, value.length() - 1).replace("\"\"", "\"").replace("'", "''") + "'"; + } + if (value.length() >= 2 && value.startsWith("'") && value.endsWith("'")) { + return value; + } + if (value.matches("(?i)date\\s*\\(\\s*\\)")) { + return "CURRENT_DATE"; + } + if (text && !value.matches("(?i)[a-z_]+\\s*\\(.*\\)")) { + return "'" + value.replace("'", "''") + "'"; + } + return value; + } + + private static void setType(Map columnInfo, int dataType, String typeName, int size, int scale) { + columnInfo.put("DATA_TYPE", dataType); + columnInfo.put("TYPE_NAME", typeName); + columnInfo.put("COLUMN_SIZE", size); + columnInfo.put("DECIMAL_DIGITS", scale); + } + + /** + * Indexes of the relationships of Access (hidden indexes named .rB...) are not indexes of the table + */ + @Override + public List> getIndexInfos(DatabaseMetaData databaseMetaData, String catalogName, + String schemaName, String tableName) throws SQLException { + return super.getIndexInfos(databaseMetaData, null, null, tableName).stream() + .filter(info -> !StringUtils.startsWith((String) info.get("INDEX_NAME"), ".")) + .collect(Collectors.toList()); + } + + @Override + public List> getPrimaryKeyInfos(DatabaseMetaData databaseMetaData, String catalogName, + String schemaName, String tableName) throws SQLException { + return super.getPrimaryKeyInfos(databaseMetaData, null, null, tableName); + } + + @Override + public List> getImportedKeyInfos(DatabaseMetaData databaseMetaData, String catalogName, + String schemaName, String tableName) throws SQLException { + return super.getImportedKeyInfos(databaseMetaData, null, null, tableName); + } + + @Override + public List> getSequenceInfos(DatabaseMetaData databaseMetaData, String catalogName, + String schemaName) { + return Collections.emptyList(); + } + + /** + * The Access file of a connection of UCanAccess, also of a pooled connection; null if it is not available + */ + static Database getDatabase(Connection connection) { + try { + if (connection instanceof UcanaccessConnection) { + return ((UcanaccessConnection) connection).getDbIO(); + } + if (connection != null && connection.isWrapperFor(UcanaccessConnection.class)) { + return connection.unwrap(UcanaccessConnection.class).getDbIO(); + } + } catch (SQLException e) { + if (log.isDebugEnabled()) { + log.debug("The Access file of the connection is not available: {}", e.getMessage()); + } + } + return null; + } + + private static Table getTable(Database database, String tableName) { + try { + return tableName != null ? database.getTable(tableName) : null; + } catch (IOException | RuntimeException e) { + if (log.isDebugEnabled()) { + log.debug("Unable to read the Access table {}: {}", tableName, e.getMessage()); + } + return null; + } + } + + private static Column getColumn(Table table, String columnName) { + return table.getColumns().stream().filter(column -> column.getName().equalsIgnoreCase(columnName)) + .findFirst().orElse(null); + } + + /** + * Properties of a table or a column, read from the Access file + */ + interface PropertySource { + PropertyMap getProperties() throws IOException; + } + + /** + * Description of Access (the remark of a table or a column), null if it has none + */ + static String getDescription(PropertySource source) { + try { + PropertyMap properties = source.getProperties(); + Object description = properties != null ? properties.getValue(PropertyMap.DESCRIPTION_PROP) : null; + return description != null ? StringUtils.trimToNull(description.toString()) : null; + } catch (IOException | RuntimeException e) { + return null; + } + } +} diff --git a/src/main/java/com/github/sqljam/jdbc/ManualTransaction.java b/src/main/java/com/github/sqljam/jdbc/ManualTransaction.java index 54cfb53..d26e617 100644 --- a/src/main/java/com/github/sqljam/jdbc/ManualTransaction.java +++ b/src/main/java/com/github/sqljam/jdbc/ManualTransaction.java @@ -23,8 +23,11 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; @@ -55,11 +58,20 @@ public class ManualTransaction implements AutoCloseable { private static final Pattern PLSQL_DML = Pattern.compile("^(BEGIN|DECLARE)\\b.*\\b(INSERT|UPDATE|DELETE|MERGE)\\b.*", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + /** + * DELETE of all rows of a table, which is emptied before it is dropped (Access) + */ + private static final Pattern EMPTY_TABLE = Pattern.compile("^DELETE\\s+FROM\\s+(\\S+)\\s*$", + Pattern.CASE_INSENSITIVE); + private static final Pattern DROP_TABLE = Pattern.compile("^DROP\\s+TABLE\\s+(\\S+)\\s*$", + Pattern.CASE_INSENSITIVE); + private final Connection connection; private final boolean transactionalDdl; private final Connection proxy; private final List deferredStatements = Collections.synchronizedList(new ArrayList<>()); private volatile boolean rowsWritten; + private final Set emptiedTables = Collections.synchronizedSet(new HashSet<>()); private volatile boolean rolledBack; private volatile boolean finished; @@ -247,6 +259,10 @@ private Object invokeStatement(Statement statement, Method method, Object[] args case "executeLargeUpdate": case "addBatch": if (isDdl(sql)) { + if (!emptiedTables.isEmpty() && !emptiedTables.remove(getTableName(DROP_TABLE, sql))) { + // DDL commits the rows deleted from a table which is not dropped + rowsWritten = true; + } if (shouldDefer()) { deferredStatements.add(sql); if (log.isInfoEnabled()) { @@ -254,6 +270,9 @@ private Object invokeStatement(Statement statement, Method method, Object[] args } return defaultResult(method.getReturnType()); } + } else if (getTableName(EMPTY_TABLE, sql) != null) { + // The rows are discarded with the table if it is dropped next + emptiedTables.add(getTableName(EMPTY_TABLE, sql)); } else if (isDml(sql)) { rowsWritten = true; } @@ -269,6 +288,15 @@ private boolean shouldDefer() { return !transactionalDdl && rowsWritten; } + /** + * Table of a statement of the pattern, unquoted and in lower case, null if the statement does not match + */ + static String getTableName(Pattern pattern, String sql) { + Matcher matcher = pattern.matcher(sql.trim()); + return matcher.matches() ? matcher.group(1).replaceAll("[`\"\\[\\]]", "").toLowerCase(Locale.ENGLISH) + : null; + } + static boolean isDdl(String sql) { String text = sql.trim(); return DDL.matcher(text).matches() || PLSQL_DDL.matcher(text).matches(); diff --git a/src/test/java/com/github/sqljam/access/AccessDialectTest.java b/src/test/java/com/github/sqljam/access/AccessDialectTest.java new file mode 100644 index 0000000..d199211 --- /dev/null +++ b/src/test/java/com/github/sqljam/access/AccessDialectTest.java @@ -0,0 +1,254 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.access; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Proxy; +import java.math.BigDecimal; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Map; + +import javax.sql.rowset.serial.SerialBlob; + +import org.junit.jupiter.api.Test; +import com.github.sqljam.impexp.DbType; +import com.github.sqljam.impexp.Dialect; +import com.github.sqljam.impexp.db.AccessDialect; + +/** + * @Description: AccessDialectTest + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +class AccessDialectTest { + + @Test + void ddl() { + AccessDialect dialect = new AccessDialect(); + assertEquals(DbType.ACCESS, dialect.getDbType()); + assertNull(dialect.getCreateUserStatement("u", "p")); + assertNull(dialect.getCreateDatabaseStatement("db", "u")); + assertNull(dialect.getCreateSchemaStatement(null, "s", "u")); + assertEquals("CREATE TABLE t", dialect.getCreateTableStatement("c", "s", "t")); + assertEquals("t", dialect.getSourceTableName("c", "s", "t")); + assertEquals("`order`", dialect.getSourceTableName("c", "s", "order")); + assertEquals("DROP TABLE t", dialect.getDropTableStatement(null, null, "t")); + assertEquals("DELETE FROM t", dialect.getEmptyTableStatement(null, null, "t")); + assertEquals("CREATE UNIQUE INDEX ux ON t (a,b)", dialect.getCreateIndexStatement(null, null, "t", false, + new String[]{"a", "b"}, "ux", true, null).replace(", ", ",")); + assertEquals("CREATE INDEX ix ON t (a)", dialect.getCreateIndexStatement(null, null, "t", false, + new String[]{"a"}, "ix", false, null)); + assertTrue(dialect.isForeignKeyOrderRequired()); + assertFalse(dialect.isLobSeparationSupported()); + assertFalse(dialect.isSequenceSupported()); + assertTrue(dialect.isSingleIncrementalColumn()); + assertEquals("id COUNTER NOT NULL", dialect.getIncrementalColumnStatement(null, null, "t", "id", + Types.INTEGER, "INTEGER", 10, 0, null, false).replaceAll("\\s+", " ")); + // Columns related to a BIGINT identity are BIGINT + assertEquals("id BIGINT NOT NULL", dialect.getIncrementalColumnStatement(null, null, "t", "id", + Types.BIGINT, "BIGINT", 19, 0, null, false).replaceAll("\\s+", " ")); + assertEquals("c LONG", dialect.getGeneratedColumnStatement(null, null, "t", "c", Types.INTEGER, "INTEGER", + 10, 0, "a * 2", true).replaceAll("\\s+", " ")); + assertNull(dialect.getResetIdentityStatement(null, null, "t", "id", 10)); + assertNull(dialect.getCreateCommentStatement(null, null, "t", "c", "x")); + assertNull(dialect.getCreateTableCommentStatement(null, null, "t", "x")); + assertNull(dialect.getSequenceNameStatement(null, null, "t", "id")); + assertNull(dialect.getDefaultSequenceName(null, null, "t", "id")); + assertNull(dialect.getAlterSequenceStartValueStatement(null, null, "t", "s", 1)); + assertNull(dialect.getCreatePartitionTableStatement(null, null, "p", "t")); + assertNull(dialect.getStatementBeforeInsert(null, null, "t", false)); + assertNull(dialect.getStatementAfterInsert(null, null, "t", false)); + assertArrayEquals(new String[]{"DISABLE AUTOINCREMENT ON t"}, dialect.getStatementBeforeInsert(null, null, + "t", true)); + assertArrayEquals(new String[]{"ENABLE AUTOINCREMENT ON t"}, dialect.getStatementAfterInsert(null, null, + "t", true)); + assertEquals("`order`", dialect.getIdentifier("order")); + assertEquals(64, dialect.getMaxIdentifierLength()); + assertEquals(3, dialect.getMaxFractionalSecondsPrecision()); + } + + @Test + void types() { + AccessDialect dialect = new AccessDialect(); + dialect.setSourceDbType(DbType.POSTGRESQL); + assertEquals("YESNO", dialect.getColumnTypeName(Types.BOOLEAN, "bool", 1, 0)); + assertEquals("SMALLINT", dialect.getColumnTypeName(Types.SMALLINT, "int2", 5, 0)); + assertEquals("LONG", dialect.getColumnTypeName(Types.INTEGER, "int4", 10, 0)); + assertEquals("BIGINT", dialect.getColumnTypeName(Types.BIGINT, "int8", 19, 0)); + assertEquals("SINGLE", dialect.getColumnTypeName(Types.REAL, "float4", 8, 0)); + assertEquals("DOUBLE", dialect.getColumnTypeName(Types.DOUBLE, "float8", 17, 0)); + assertEquals("DECIMAL(10,2)", dialect.getColumnTypeName(Types.NUMERIC, "numeric", 10, 2)); + assertEquals("DECIMAL(18,0)", dialect.getColumnTypeName(Types.NUMERIC, "numeric", 18, 0)); + assertEquals("DECIMAL(21,1)", dialect.getColumnTypeName(Types.NUMERIC, "numeric", 20, 0)); + assertEquals("DECIMAL(28,1)", dialect.getColumnTypeName(Types.NUMERIC, "numeric", 28, 0)); + assertEquals("TEXT(100)", dialect.getColumnTypeName(Types.NUMERIC, "numeric", 40, 2)); + assertEquals("DECIMAL(28,10)", dialect.getColumnTypeName(Types.NUMERIC, "numeric", 0, 0)); + assertEquals("DATETIME", dialect.getColumnTypeName(Types.TIMESTAMP, "timestamp", 29, 6)); + assertEquals("DATETIME", dialect.getColumnTypeName(Types.DATE, "date", 13, 0)); + assertEquals("DATETIME", dialect.getColumnTypeName(Types.TIME, "time", 15, 6)); + assertEquals("DATETIME", dialect.getColumnTypeName(Types.TIMESTAMP_WITH_TIMEZONE, "timestamptz", 35, 6)); + assertEquals("BIGINT", dialect.getColumnTypeName(Types.BIT, "bit", 64, 0)); + assertEquals("TEXT(20)", dialect.getColumnTypeName(Types.VARCHAR, "varchar", 20, 0)); + assertEquals("MEMO", dialect.getColumnTypeName(Types.VARCHAR, "varchar", 300, 0)); + assertEquals("MEMO", dialect.getColumnTypeName(Types.VARCHAR, "text", Integer.MAX_VALUE, 0)); + assertEquals("VARBINARY(16)", dialect.getColumnTypeName(Types.VARBINARY, "varbinary", 16, 0)); + assertEquals("OLE", dialect.getColumnTypeName(Types.BINARY, "bytea", Integer.MAX_VALUE, 0)); + assertEquals("TEXT(38)", dialect.getColumnTypeName(Types.OTHER, "uuid", 0, 0)); + assertEquals("MEMO", dialect.getColumnTypeName(Types.OTHER, "jsonb", 0, 0)); + assertEquals("TEXT(50)", dialect.getColumnTypeName(Types.OTHER, "inet", 0, 0)); + assertEquals("MEMO", dialect.getColumnTypeName(Types.ARRAY, "_int4", 0, 0)); + assertEquals("LONG", dialect.getColumnTypeName(Types.BIT, "bit", 8, 0)); + + AccessDialect mysql = new AccessDialect(); + mysql.setSourceDbType(DbType.MYSQL); + assertEquals("TEXT(255)", mysql.getColumnTypeName(Types.CHAR, "enum", 5, 0)); + assertEquals("MEMO", mysql.getColumnTypeName(Types.LONGVARCHAR, "mediumtext", 16777215, 0)); + + // Files of Access 2010 keep 32-bit integers only + Dialect access2010 = dialect.forVersion(14, 0); + assertEquals("DECIMAL(28,1)", access2010.getColumnTypeName(Types.BIGINT, "int8", 19, 0)); + assertEquals("BIGINT", dialect.forVersion(16, 0).getColumnTypeName(Types.BIGINT, "int8", 19, 0)); + } + + @Test + void readsFileVersion() throws SQLException { + AccessDialect dialect = new AccessDialect(); + assertArrayEquals(new int[]{14, 0}, dialect.readDatabaseVersion(metaData("V2010 [VERSION_14]"))); + assertArrayEquals(new int[]{16, 0}, dialect.readDatabaseVersion(metaData("V2016 [VERSION_16]"))); + assertArrayEquals(new int[]{-1, -1}, dialect.readDatabaseVersion(metaData(null))); + } + + @Test + void missingDropErrors() { + AccessDialect dialect = new AccessDialect(); + SQLException missing = new SQLException("UCAExc:::5.1.8", new SQLException( + "user lacks privilege or object not found: T")); + assertTrue(dialect.isMissingDropError("DROP TABLE t", missing)); + assertTrue(dialect.isMissingDropError(" delete from t", new SQLException("Table t does not exist"))); + assertFalse(dialect.isMissingDropError("INSERT INTO t VALUES (1)", missing)); + assertFalse(dialect.isMissingDropError("DROP TABLE t", new SQLException("Reference to missing table"))); + assertFalse(dialect.isMissingDropError(null, missing)); + } + + @Test + void dml() { + AccessDialect dialect = new AccessDialect(); + assertEquals("MERGE INTO t tgt USING (VALUES (1,'a')) src (id,v) ON tgt.id = src.id" + + " WHEN MATCHED THEN UPDATE SET tgt.v = src.v" + + " WHEN NOT MATCHED THEN INSERT (id,v) VALUES (src.id, src.v)", + dialect.getUpsertStatement(null, null, "t", new String[]{"id", "v"}, new String[]{"id"}, + new String[]{"v"}, new Object[]{"1", "'a'"})); + assertEquals("MERGE INTO t tgt USING (VALUES (1)) src (id) ON tgt.id = src.id" + + " WHEN NOT MATCHED THEN INSERT (id) VALUES (src.id)", + dialect.getUpsertStatement(null, null, "t", new String[]{"id"}, new String[]{"id"}, + new String[0], new Object[]{"1"})); + assertEquals("NOW()", dialect.getCurrentTimestampExpression()); + assertEquals("DATE()", dialect.getCurrentDateExpression()); + assertEquals("TRUE", dialect.getBooleanLiteral(true)); + assertEquals("FALSE", dialect.getBooleanLiteral(false)); + assertEquals("'2024-02-29'", dialect.getDateLiteral(LocalDate.of(2024, 2, 29))); + assertTrue(dialect.getTimestampLiteral(LocalDateTime.of(2024, 2, 29, 23, 59, 59)).startsWith( + "'2024-02-29 23:59:59")); + assertTrue(dialect.getTimeLiteral(LocalTime.of(8, 30)).startsWith("'1899-12-30 08:30:00")); + } + + @Test + void literalsOfNumbers() { + AccessDialect dialect = new AccessDialect(); + assertEquals("-12345678901234.5", dialect.getStringValue(null, null, "t", "c", -1.23456789012345E13)); + assertEquals("1" + "0".repeat(100), dialect.getStringValue(null, null, "t", "c", 1.0E100)); + assertEquals("3.5", dialect.getStringValue(null, null, "t", "c", 3.5f)); + assertEquals("NULL", dialect.getStringValue(null, null, "t", "c", Double.NaN)); + assertEquals("1.50", dialect.getStringValue(null, null, "t", "c", new BigDecimal("1.50"))); + assertEquals("'a'", dialect.getStringValue(null, null, "t", "c", "a")); + } + + @Test + void jdbcValues() { + AccessDialect dialect = new AccessDialect(); + assertEquals(LocalDateTime.of(1899, 12, 30, 8, 30), dialect.getJdbcValue(LocalTime.of(8, 30), Types.TIME, + "time", 8)); + assertEquals(1, dialect.getJdbcValue(1, Types.INTEGER, "int4", 10)); + // Parameters of MERGE: date times as timestamps, bytes by a cast + assertTrue(dialect.isTargetTypeConversionRequired()); + assertEquals(Timestamp.valueOf("2024-02-29 23:59:59"), dialect.getTargetJdbcValue(LocalDateTime.of(2024, 2, + 29, 23, 59, 59), "TIMESTAMP")); + assertEquals(Timestamp.valueOf("2024-02-29 00:00:00"), dialect.getTargetJdbcValue(LocalDate.of(2024, 2, 29), + "TIMESTAMP")); + assertEquals(Timestamp.valueOf("1899-12-30 08:30:00"), dialect.getTargetJdbcValue(LocalTime.of(8, 30), + "TIMESTAMP")); + assertEquals("a", dialect.getTargetJdbcValue("a", "VARCHAR")); + assertEquals("CAST(? AS BLOB)", dialect.getParameterMarker("BLOB")); + assertEquals("CAST(? AS VARBINARY(255))", dialect.getParameterMarker("varbinary")); + assertEquals("?", dialect.getParameterMarker("VARCHAR")); + assertEquals("?", dialect.getParameterMarker(null)); + } + + @Test + void columnValues() throws SQLException { + AccessDialect dialect = new AccessDialect(); + byte[] bytes = {1, 2, 3}; + // Bytes of an OLE column are the key of UCanAccess, the blob is the content + assertArrayEquals(bytes, (byte[]) dialect.getColumnValue(resultSet(Map.of("getBlob", new SerialBlob(bytes), + "getBytes", new byte[]{9})), 1, Types.BLOB, "BLOB")); + assertNull(dialect.getColumnValue(resultSet(Map.of()), 1, Types.LONGVARBINARY, "BLOB")); + // SINGLE of a file: NUMERIC(100,7) + assertEquals(3.5, dialect.getColumnValue(resultSet(Map.of("getObject", new BigDecimal("3.5000000"), + "getPrecision", 100)), 1, Types.NUMERIC, "NUMERIC")); + assertEquals(new BigDecimal("3.5000000"), dialect.getColumnValue(resultSet(Map.of("getObject", + new BigDecimal("3.5000000"), "getPrecision", 10)), 1, Types.NUMERIC, "NUMERIC")); + assertEquals("x", dialect.getColumnValue(resultSet(Map.of("getObject", "x")), 1, Types.VARCHAR, "VARCHAR")); + } + + private static DatabaseMetaData metaData(String productVersion) { + return (DatabaseMetaData) Proxy.newProxyInstance(AccessDialectTest.class.getClassLoader(), + new Class[]{DatabaseMetaData.class}, (proxy, method, args) -> + "getDatabaseProductVersion".equals(method.getName()) ? productVersion : null); + } + + /** + * Result set returning the values of the methods by name, e.g. getBlob, and getPrecision of its metadata + */ + private static ResultSet resultSet(Map values) { + ResultSetMetaData metaData = (ResultSetMetaData) Proxy.newProxyInstance( + AccessDialectTest.class.getClassLoader(), new Class[]{ResultSetMetaData.class}, + (proxy, method, args) -> values.getOrDefault(method.getName(), 0)); + return (ResultSet) Proxy.newProxyInstance(AccessDialectTest.class.getClassLoader(), + new Class[]{ResultSet.class}, (proxy, method, args) -> { + if ("getMetaData".equals(method.getName())) { + return metaData; + } + if ("wasNull".equals(method.getName())) { + return false; + } + return values.get(method.getName()); + }); + } +} diff --git a/src/test/java/com/github/sqljam/access/AccessImportIT.java b/src/test/java/com/github/sqljam/access/AccessImportIT.java new file mode 100644 index 0000000..7ccefce --- /dev/null +++ b/src/test/java/com/github/sqljam/access/AccessImportIT.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.access; + +import com.github.sqljam.it.AbstractImportIT; +import com.github.sqljam.it.ItDatabase; + +/** + * @Description: AccessImportIT + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +class AccessImportIT extends AbstractImportIT { + + AccessImportIT() { + super(ItDatabase.ACCESS); + } +} diff --git a/src/test/java/com/github/sqljam/access/AccessParquetIT.java b/src/test/java/com/github/sqljam/access/AccessParquetIT.java new file mode 100644 index 0000000..512992f --- /dev/null +++ b/src/test/java/com/github/sqljam/access/AccessParquetIT.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.access; + +import com.github.sqljam.it.AbstractParquetIT; +import com.github.sqljam.it.ItDatabase; + +/** + * @Description: AccessParquetIT + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +class AccessParquetIT extends AbstractParquetIT { + + AccessParquetIT() { + super(ItDatabase.ACCESS); + } +} diff --git a/src/test/java/com/github/sqljam/access/AccessScriptIT.java b/src/test/java/com/github/sqljam/access/AccessScriptIT.java new file mode 100644 index 0000000..fe4adca --- /dev/null +++ b/src/test/java/com/github/sqljam/access/AccessScriptIT.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.access; + +import com.github.sqljam.it.AbstractScriptIT; +import com.github.sqljam.it.ItDatabase; + +/** + * @Description: AccessScriptIT + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +class AccessScriptIT extends AbstractScriptIT { + + AccessScriptIT() { + super(ItDatabase.ACCESS); + } +} diff --git a/src/test/java/com/github/sqljam/access/AccessSessionIT.java b/src/test/java/com/github/sqljam/access/AccessSessionIT.java new file mode 100644 index 0000000..0a5b5be --- /dev/null +++ b/src/test/java/com/github/sqljam/access/AccessSessionIT.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.access; + +import com.github.sqljam.it.AbstractSessionIT; +import com.github.sqljam.it.ItDatabase; + +/** + * @Description: AccessSessionIT + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +class AccessSessionIT extends AbstractSessionIT { + + AccessSessionIT() { + super(ItDatabase.ACCESS); + } +} diff --git a/src/test/java/com/github/sqljam/face/service/JdbcUrlsTest.java b/src/test/java/com/github/sqljam/face/service/JdbcUrlsTest.java index 41ac976..8972633 100644 --- a/src/test/java/com/github/sqljam/face/service/JdbcUrlsTest.java +++ b/src/test/java/com/github/sqljam/face/service/JdbcUrlsTest.java @@ -68,6 +68,8 @@ void readsFiles() { assertEquals("H2 h2 9093 ~/shop", parts("jdbc:h2:tcp://h2:9093/~/shop")); assertEquals("SQLITE null 0 /data/app.sqlite", parts("jdbc:sqlite:/data/app.sqlite?date_class=TEXT")); assertEquals("DUCKDB null 0 /data/lake.duckdb", parts("jdbc:duckdb:/data/lake.duckdb")); + assertEquals("ACCESS null 0 /data/shop.accdb", parts("jdbc:ucanaccess:///data/shop.accdb;memory=false")); + assertEquals("ACCESS null 0 C:/data/old.mdb", parts("jdbc:ucanaccess://C:/data/old.mdb")); assertNull(parts("jdbc:informix-sqli://host:9088/db")); assertNull(parts(null)); // Being typed @@ -84,6 +86,10 @@ void buildsUrlsOfFields() { assertEquals("jdbc:h2:file:/data/shop", JdbcUrls.build(DbType.H2, "", 0, "/data/shop")); assertEquals("jdbc:h2:tcp://h2:9092//data/shop", JdbcUrls.build(DbType.H2, "h2", 9092, "/data/shop")); assertEquals("", JdbcUrls.build(DbType.SQLITE, null, 0, "")); + assertEquals("jdbc:ucanaccess:///data/shop.accdb;newdatabaseversion=V2016", JdbcUrls.build(DbType.ACCESS, + null, 0, "/data/shop.accdb")); + assertEquals("jdbc:ucanaccess:///data/old.mdb;newdatabaseversion=V2003", JdbcUrls.build(DbType.ACCESS, + null, 0, "/data/old.mdb")); assertEquals("", JdbcUrls.build(null, "host", 1, "db")); // Read back assertEquals("ORACLE ora 1521 DEMO", parts(JdbcUrls.build(DbType.ORACLE, "ora", 1521, "DEMO"))); diff --git a/src/test/java/com/github/sqljam/impexp/DdlScripterTest.java b/src/test/java/com/github/sqljam/impexp/DdlScripterTest.java index db2bfdb..8f8c44a 100644 --- a/src/test/java/com/github/sqljam/impexp/DdlScripterTest.java +++ b/src/test/java/com/github/sqljam/impexp/DdlScripterTest.java @@ -123,6 +123,7 @@ void dbTypeLookup() { assertEquals(DbType.H2, DbType.forUrl("JDBC:H2:mem:test")); assertEquals(DbType.SQLITE, DbType.forUrl("jdbc:sqlite:test.db")); assertEquals(DbType.DB2, DbType.forUrl("jdbc:db2://localhost:50000/test")); + assertEquals(DbType.ACCESS, DbType.forUrl("jdbc:ucanaccess:///data/shop.accdb")); assertNull(DbType.forUrl("jdbc:informix-sqli://localhost:9088/test")); assertNull(DbType.forUrl(" ")); } diff --git a/src/test/java/com/github/sqljam/impexp/DialectTest.java b/src/test/java/com/github/sqljam/impexp/DialectTest.java index 6c5aee4..c8335b5 100644 --- a/src/test/java/com/github/sqljam/impexp/DialectTest.java +++ b/src/test/java/com/github/sqljam/impexp/DialectTest.java @@ -318,6 +318,7 @@ void versionDialects() { assertEquals("SQLServerDialect", DbType.SQLSERVER.createDialect(16, 0).getClass().getSimpleName()); assertEquals("H2Dialect", DbType.H2.createDialect(2, 3).getClass().getSimpleName()); assertEquals("SQLiteDialect", DbType.SQLITE.createDialect(3, 46).getClass().getSimpleName()); + assertEquals("AccessDialect", DbType.ACCESS.createDialect(16, 0).getClass().getSimpleName()); assertEquals(11, DbType.ORACLE.createDialect(11, 2).getDatabaseMajorVersion()); // Settings are copied to the dialect of the version diff --git a/src/test/java/com/github/sqljam/impexp/db/AccessMetaDataOperationsTest.java b/src/test/java/com/github/sqljam/impexp/db/AccessMetaDataOperationsTest.java new file mode 100644 index 0000000..498d1b3 --- /dev/null +++ b/src/test/java/com/github/sqljam/impexp/db/AccessMetaDataOperationsTest.java @@ -0,0 +1,159 @@ +/* + * Copyright 2023-2026 Fred Feng + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.sqljam.impexp.db; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Types; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import io.github.spannm.jackcess.Column; +import io.github.spannm.jackcess.DataType; +import io.github.spannm.jackcess.Database; +import io.github.spannm.jackcess.DatabaseBuilder; +import io.github.spannm.jackcess.PropertyMap; +import io.github.spannm.jackcess.Table; + +/** + * @Description: AccessMetaDataOperationsTest + * @Author: Fred Feng + * @Date: 24/10/2026 + * @Version 1.0.0 + */ +class AccessMetaDataOperationsTest { + + @TempDir + File dir; + + @Test + void defaults() { + assertNull(AccessMetaDataOperations.normalizeDefault(null, true)); + assertNull(AccessMetaDataOperations.normalizeDefault(" ", true)); + assertEquals("'n/a'", AccessMetaDataOperations.normalizeDefault("\"n/a\"", true)); + assertEquals("'say \"hi\" it''s'", AccessMetaDataOperations.normalizeDefault("\"say \"\"hi\"\" it's\"", + true)); + assertEquals("'x'", AccessMetaDataOperations.normalizeDefault("'x'", true)); + assertEquals("Now()", AccessMetaDataOperations.normalizeDefault("=Now()", false)); + assertEquals("CURRENT_DATE", AccessMetaDataOperations.normalizeDefault("=Date()", false)); + // Written by UCanAccess without quotes + assertEquals("'n/a'", AccessMetaDataOperations.normalizeDefault("n/a", true)); + assertEquals("'it''s'", AccessMetaDataOperations.normalizeDefault("it's", true)); + assertEquals("UCase('a')", AccessMetaDataOperations.normalizeDefault("UCase('a')", true)); + assertEquals("0", AccessMetaDataOperations.normalizeDefault("0", false)); + } + + @Test + void columnsOfAccessFile() throws IOException { + try (Database database = DatabaseBuilder.create(Database.FileFormat.V2016, new File(dir, "types.accdb"))) { + Table table = DatabaseBuilder.newTable("t") + .addColumn(DatabaseBuilder.newColumn("c_counter", DataType.LONG).withAutoNumber(true)) + .addColumn(DatabaseBuilder.newColumn("c_yesno", DataType.BOOLEAN)) + .addColumn(DatabaseBuilder.newColumn("c_byte", DataType.BYTE)) + .addColumn(DatabaseBuilder.newColumn("c_int", DataType.INT)) + .addColumn(DatabaseBuilder.newColumn("c_bigint", DataType.BIG_INT)) + .addColumn(DatabaseBuilder.newColumn("c_money", DataType.MONEY)) + .addColumn(DatabaseBuilder.newColumn("c_float", DataType.FLOAT)) + .addColumn(DatabaseBuilder.newColumn("c_double", DataType.DOUBLE)) + .addColumn(DatabaseBuilder.newColumn("c_numeric", DataType.NUMERIC).withPrecision(12) + .withScale(3)) + .addColumn(DatabaseBuilder.newColumn("c_date", DataType.SHORT_DATE_TIME)) + .addColumn(DatabaseBuilder.newColumn("c_text", DataType.TEXT).withLengthInUnits(40) + .withProperty(PropertyMap.DESCRIPTION_PROP, "Name of it")) + .addColumn(DatabaseBuilder.newColumn("c_memo", DataType.MEMO)) + .addColumn(DatabaseBuilder.newColumn("c_binary", DataType.BINARY).withLength(16)) + .addColumn(DatabaseBuilder.newColumn("c_ole", DataType.OLE)) + .addColumn(DatabaseBuilder.newColumn("c_guid", DataType.GUID).withAutoNumber(true)) + .putProperty(PropertyMap.DESCRIPTION_PROP, "Types") + .toTable(database); + Map types = new LinkedHashMap<>(); + for (Column column : table.getColumns()) { + Map info = new HashMap<>(); + info.put("COLUMN_DEF", "\"x\""); + AccessMetaDataOperations.normalizeColumn(info, column); + types.put(column.getName(), info.get("TYPE_NAME") + "(" + info.get("COLUMN_SIZE") + "," + + info.get("DECIMAL_DIGITS") + ") " + info.get("DATA_TYPE") + " " + info.get("IS_AUTOINCREMENT") + + (info.get("REMARKS") != null ? " " + info.get("REMARKS") : "")); + if ("c_yesno".equals(column.getName())) { + assertEquals("NO", info.get("IS_NULLABLE")); + } + if ("c_counter".equals(column.getName())) { + assertNull(info.get("COLUMN_DEF")); + } else { + assertEquals("'x'", info.get("COLUMN_DEF")); + } + } + assertEquals("INTEGER(10,0) " + Types.INTEGER + " YES", types.get("c_counter")); + assertEquals("BOOLEAN(1,0) " + Types.BOOLEAN + " NO", types.get("c_yesno")); + assertEquals("SMALLINT(3,0) " + Types.SMALLINT + " NO", types.get("c_byte")); + assertEquals("SMALLINT(5,0) " + Types.SMALLINT + " NO", types.get("c_int")); + assertEquals("BIGINT(19,0) " + Types.BIGINT + " NO", types.get("c_bigint")); + assertEquals("DECIMAL(19,4) " + Types.DECIMAL + " NO", types.get("c_money")); + assertEquals("REAL(7,0) " + Types.REAL + " NO", types.get("c_float")); + assertEquals("DOUBLE(15,0) " + Types.DOUBLE + " NO", types.get("c_double")); + assertEquals("DECIMAL(12,3) " + Types.DECIMAL + " NO", types.get("c_numeric")); + assertEquals("TIMESTAMP(23,3) " + Types.TIMESTAMP + " NO", types.get("c_date")); + assertEquals("VARCHAR(40,0) " + Types.VARCHAR + " NO Name of it", types.get("c_text")); + assertEquals("CLOB(" + Integer.MAX_VALUE + ",0) " + Types.CLOB + " NO", types.get("c_memo")); + assertEquals("VARBINARY(16,0) " + Types.VARBINARY + " NO", types.get("c_binary")); + assertEquals("BLOB(" + Integer.MAX_VALUE + ",0) " + Types.BLOB + " NO", types.get("c_ole")); + // Replication ids are not identities + assertEquals("VARCHAR(38,0) " + Types.VARCHAR + " NO", types.get("c_guid")); + assertEquals("Types", AccessMetaDataOperations.getDescription(table::getProperties)); + } + } + + @Test + void descriptionsAndConnections() throws SQLException { + assertNull(AccessMetaDataOperations.getDescription(() -> { + throw new IOException("broken"); + })); + assertNull(AccessMetaDataOperations.getDescription(() -> null)); + assertNull(AccessMetaDataOperations.getDatabase(null)); + assertNull(AccessMetaDataOperations.getDatabase(connection(false))); + assertNull(AccessMetaDataOperations.getDatabase(connection(true))); + AccessMetaDataOperations operations = new AccessMetaDataOperations(); + assertEquals(1, operations.getCatalogInfos(null).size()); + assertEquals("", operations.getCatalogInfos(null).get(0).get("TABLE_CAT")); + assertEquals(0, operations.getSchemaInfos(null, "").size()); + assertEquals(0, operations.getSequenceInfos(null, "", null).size()); + } + + /** + * Connection of another driver, or a pool which fails to unwrap + */ + private static Connection connection(boolean failing) { + return (Connection) Proxy.newProxyInstance(AccessMetaDataOperationsTest.class.getClassLoader(), + new Class[]{Connection.class}, (proxy, method, args) -> { + if ("isWrapperFor".equals(method.getName())) { + if (failing) { + throw new SQLException("closed"); + } + return false; + } + return null; + }); + } +} diff --git a/src/test/java/com/github/sqljam/it/AbstractImportIT.java b/src/test/java/com/github/sqljam/it/AbstractImportIT.java index 77542af..d879650 100644 --- a/src/test/java/com/github/sqljam/it/AbstractImportIT.java +++ b/src/test/java/com/github/sqljam/it/AbstractImportIT.java @@ -25,6 +25,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; @@ -149,6 +150,20 @@ private void dropCopies() throws Exception { dialect.setTargetSchemaName(source.getSourceSchema()); try (Connection connection = source.getSourceConnection(); Statement statement = connection.createStatement()) { + // Rows of related Access tables are deleted before the first drop, like the imports do + for (String table : FIXTURE_TABLES) { + String sql = dialect.getEmptyTableStatement(source.getSourceCatalog(), source.getSourceSchema(), + source.getPrefix() + table + COPY_SUFFIX); + if (sql != null) { + try { + statement.execute(sql); + } catch (SQLException e) { + if (!dialect.isMissingDropError(sql, e)) { + throw e; + } + } + } + } for (String table : FIXTURE_TABLES) { // Names of copies follow the case of the source tables, e.g. SJH_EMP_CP of H2 String copyName = source.getPrefix() + table + COPY_SUFFIX; @@ -162,7 +177,14 @@ private void dropCopies() throws Exception { + "WHEN OTHERS THEN NULL; END;", (source.getPrefix() + table + COPY_SUFFIX) .toUpperCase(Locale.ENGLISH)); } - statement.execute(sql); + try { + statement.execute(sql); + } catch (SQLException e) { + // Drops of Access have no IF EXISTS + if (!dialect.isMissingDropError(sql, e)) { + throw e; + } + } } } } diff --git a/src/test/java/com/github/sqljam/it/AbstractSessionIT.java b/src/test/java/com/github/sqljam/it/AbstractSessionIT.java index e3091ff..fec3934 100644 --- a/src/test/java/com/github/sqljam/it/AbstractSessionIT.java +++ b/src/test/java/com/github/sqljam/it/AbstractSessionIT.java @@ -99,7 +99,7 @@ void browse() throws Exception { assertTrue(columns.size() >= 11); assertTrue(columns.get(0).isPrimaryKey(), "id is primary key"); assertTrue(columns.stream().anyMatch(column -> "employee name".equalsIgnoreCase( - column.getRemarks())) || source == ItDatabase.SQLITE); + column.getRemarks())) || source == ItDatabase.SQLITE || source == ItDatabase.ACCESS); List indexes = session.getIndexes(catalog, schema, emp); List foreignKeys = session.getForeignKeys(catalog, schema, emp); if (source == ItDatabase.CLICKHOUSE) { diff --git a/src/test/java/com/github/sqljam/it/ItDatabase.java b/src/test/java/com/github/sqljam/it/ItDatabase.java index c8a5be8..b26b474 100644 --- a/src/test/java/com/github/sqljam/it/ItDatabase.java +++ b/src/test/java/com/github/sqljam/it/ItDatabase.java @@ -20,6 +20,7 @@ import java.io.InputStreamReader; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.sql.Connection; import java.sql.Date; import java.sql.DriverManager; @@ -80,6 +81,13 @@ public String getUrl(String catalog) { } }, + ACCESS(DbType.ACCESS, "sja_") { + @Override + public String getUrl(String catalog) { + return DbType.ACCESS.getUrl(null, 0, getAccessFile(catalog == null).getAbsolutePath()); + } + }, + DUCKDB(DbType.DUCKDB, "sjd_") { @Override public String getUrl(String catalog) { @@ -128,6 +136,10 @@ static String getRaw(ItDatabase db, String key) { } } + static File getAccessFile(boolean source) { + return new File(DATA_DIR, source ? "src.accdb" : "dst.accdb"); + } + static File getDuckDBFile(boolean source) { return new File(DATA_DIR, source ? "duck-src.duckdb" : "duck-dst.duckdb"); } @@ -266,6 +278,9 @@ private ConnectionProfile createProfile(String catalog, String fileKey) { profile.setDatabase(new File(DATA_DIR, fileKey == null ? "src.sqlite" : "dst.sqlite") .getAbsolutePath()); break; + case ACCESS: + profile.setDatabase(getAccessFile(fileKey == null).getAbsolutePath()); + break; default: profile.setHostname(Settings.get(this, "host")); profile.setPort(Integer.parseInt(Settings.get(this, "port"))); @@ -317,6 +332,15 @@ public synchronized boolean isAvailable() { System.err.println("Database is not in the environment " + Settings.ENVIRONMENT + ": " + this); return false; } + if (this == ACCESS) { + // The fixture of Access is created in a new file, UCanAccess has no DROP TABLE IF EXISTS of the + // file. It is deleted before any connection: UCanAccess keeps a closed file open for a while. + try { + Files.deleteIfExists(getAccessFile(true).toPath()); + } catch (java.io.IOException e) { + throw new IllegalStateException(e); + } + } // The first connection of an inactive database activates it, which takes some seconds (Db2) DriverManager.setLoginTimeout(20); Exception error = null; diff --git a/src/test/java/com/github/sqljam/it/ItVerifier.java b/src/test/java/com/github/sqljam/it/ItVerifier.java index 75e9e4b..3ea6252 100644 --- a/src/test/java/com/github/sqljam/it/ItVerifier.java +++ b/src/test/java/com/github/sqljam/it/ItVerifier.java @@ -47,7 +47,6 @@ import com.github.sqljam.impexp.DbType; import com.github.sqljam.impexp.Dialect; import com.github.sqljam.impexp.MetaDataOperations; -import com.github.sqljam.jdbc.JdbcUtils; /** * @Description: ItVerifier verifies tables imported from a fixture database @@ -85,7 +84,8 @@ public ItVerifier(ItDatabase source, ItDatabase target, Connection connection, S this.operations = target.getDbType().createMetaDataOperations(); this.dialect = target.getDbType().createDialect(); DbType dbType = target.getDbType(); - this.catalog = dbType == DbType.ORACLE || dbType == DbType.SQLITE || dbType == DbType.DB2 ? null + this.catalog = dbType == DbType.ORACLE || dbType == DbType.SQLITE || dbType == DbType.DB2 + || dbType == DbType.ACCESS ? null : connection.getCatalog(); this.schema = schema; for (Map info : operations.getTableInfos(connection.getMetaData(), catalog, schema)) { @@ -149,7 +149,7 @@ private Object queryEmp(Map columns, String column, long id) thr ps.setLong(1, id); try (ResultSet rs = ps.executeQuery()) { assertTrue(rs.next(), "Row not found: " + id); - return JdbcUtils.getColumnValue(rs, 1, rs.getMetaData().getColumnType(1), + return dialect.getColumnValue(rs, 1, rs.getMetaData().getColumnType(1), rs.getMetaData().getColumnTypeName(1)); } } @@ -259,7 +259,9 @@ private static Boolean asBoolean(Object value) { } public void verifyComments() throws SQLException { - if (source == ItDatabase.SQLITE || target == ItDatabase.SQLITE) { + // SQLite has no comments, Access descriptions are not written by UCanAccess + if (source == ItDatabase.SQLITE || target == ItDatabase.SQLITE || source == ItDatabase.ACCESS + || target == ItDatabase.ACCESS) { return; } Map nameColumn = operations.getColumnInfos(metaData(), catalog, schema, table("emp")).stream() @@ -274,6 +276,12 @@ public void verifyIdentity() throws SQLException { if (!isConstraintSupported()) { return; } + // Autonumbers of Access are 32-bit, a 64-bit identity is a BIGINT column which keeps its values + if (target == ItDatabase.ACCESS && operations.getColumnInfos(metaData(), catalog, schema, table("dept")) + .stream().filter(info -> "id".equalsIgnoreCase((String) info.get("COLUMN_NAME"))) + .noneMatch(info -> "YES".equals(info.get("IS_AUTOINCREMENT")))) { + return; + } Map columns = columns("dept"); String insert = String.format("INSERT INTO %s (%s) VALUES ('New Dept')", ref("dept"), column(columns, "name")); @@ -284,7 +292,12 @@ public void verifyIdentity() throws SQLException { column(columns, "name")); try (Statement statement = connection.createStatement(); ResultSet rs = statement.executeQuery(query)) { assertTrue(rs.next()); - assertEquals(4L, rs.getLong(1), "Identity value after import"); + if (target == ItDatabase.ACCESS) { + // An autonumber of Access never reuses the numbers of deleted rows, e.g. of an earlier verification + assertTrue(rs.getLong(1) >= 4L, "Identity value after import: " + rs.getLong(1)); + } else { + assertEquals(4L, rs.getLong(1), "Identity value after import"); + } } } @@ -329,6 +342,7 @@ public void verifyTypes() throws Exception { try (Connection sourceConnection = source.getSourceConnection()) { MetaDataOperations sourceOperations = source.getDbType().createMetaDataOperations(); String sourceCatalog = source.getDbType() == DbType.ORACLE || source.getDbType() == DbType.SQLITE + || source.getDbType() == DbType.ACCESS || source.getDbType() == DbType.DB2 ? null : sourceConnection.getCatalog(); String typesTable = sourceOperations.getTableInfos(sourceConnection.getMetaData(), sourceCatalog, source.getSourceSchema()).stream().map(info -> (String) info.get("TABLE_NAME")) @@ -390,7 +404,7 @@ private List> readRows(Connection connection, MetaDataOperat while (rs.next()) { Map row = new HashMap<>(); for (int i = 1; i <= rsmd.getColumnCount(); i++) { - row.put(rsmd.getColumnLabel(i).toLowerCase(Locale.ENGLISH), JdbcUtils.getColumnValue(rs, i, + row.put(rsmd.getColumnLabel(i).toLowerCase(Locale.ENGLISH), dialect.getColumnValue(rs, i, rsmd.getColumnType(i), rsmd.getColumnTypeName(i))); } rows.add(row); diff --git a/src/test/java/com/github/sqljam/jdbc/ManualTransactionTest.java b/src/test/java/com/github/sqljam/jdbc/ManualTransactionTest.java index 1abc17d..5e88c49 100644 --- a/src/test/java/com/github/sqljam/jdbc/ManualTransactionTest.java +++ b/src/test/java/com/github/sqljam/jdbc/ManualTransactionTest.java @@ -17,6 +17,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -29,6 +30,7 @@ import java.sql.Statement; import java.util.List; import java.util.UUID; +import java.util.regex.Pattern; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -126,6 +128,32 @@ private static void assertSame(Connection expected, Connection actual) { assertTrue(expected == actual, "Statements return the transaction connection"); } + @Test + void emptiedTablesWhichAreDroppedWriteNoRows() throws Exception { + String url = url(); + try (Connection setup = DriverManager.getConnection(url); Statement statement = setup.createStatement()) { + statement.execute("CREATE TABLE old_t (id INT PRIMARY KEY)"); + statement.execute("CREATE TABLE kept (id INT PRIMARY KEY)"); + } + try (ManualTransaction transaction = new ManualTransaction(DriverManager.getConnection(url), false)) { + try (Statement statement = transaction.getConnection().createStatement()) { + // Access empties the tables before it drops them + statement.executeUpdate("DELETE FROM old_t"); + statement.execute("DROP TABLE old_t"); + statement.execute("CREATE TABLE new_t (id INT PRIMARY KEY)"); + assertEquals(List.of(), transaction.getDeferredStatements()); + // Rows deleted from a table which is kept would be committed by DDL + statement.executeUpdate("DELETE FROM kept"); + statement.execute("CREATE INDEX idx_kept ON kept (id)"); + } + assertEquals(List.of("CREATE INDEX idx_kept ON kept (id)"), transaction.getDeferredStatements()); + transaction.rollback(); + } + assertEquals("old_t", ManualTransaction.getTableName(Pattern.compile("^DELETE\\s+FROM\\s+(\\S+)$"), + "DELETE FROM [OLD_T]")); + assertNull(ManualTransaction.getTableName(Pattern.compile("^DROP\\s+TABLE\\s+(\\S+)$"), "DROP VIEW v")); + } + @Test void rollbackOfImporterRollsBackEverything() throws Exception { String url = url(); diff --git a/src/test/resources/access/fixture.sql b/src/test/resources/access/fixture.sql new file mode 100644 index 0000000..e26354c --- /dev/null +++ b/src/test/resources/access/fixture.sql @@ -0,0 +1,72 @@ +-- Tables are created in a new file (ItDatabase deletes the files of an earlier run) + +CREATE TABLE sja_dept ( + id COUNTER NOT NULL PRIMARY KEY, + name TEXT(100) NOT NULL, + budget DECIMAL(12,2) DEFAULT 0, + created_at DATETIME DEFAULT NOW() +); +CREATE UNIQUE INDEX uk_sja_dept_name ON sja_dept (name); + +CREATE TABLE sja_emp ( + id COUNTER NOT NULL PRIMARY KEY, + dept_id LONG, + name TEXT(100) NOT NULL, + email TEXT(200) NOT NULL, + salary DECIMAL(10,2), + active YESNO, + hired DATETIME, + updated_at DATETIME, + photo OLE, + profile MEMO, + note TEXT(255) DEFAULT 'n/a' +); +CREATE UNIQUE INDEX uk_sja_emp_email ON sja_emp (email); +CREATE INDEX idx_sja_emp_dept_name ON sja_emp (dept_id, name); +ALTER TABLE sja_emp ADD CONSTRAINT fk_sja_emp_dept FOREIGN KEY (dept_id) REFERENCES sja_dept (id) ON DELETE CASCADE; + +CREATE TABLE sja_emp_tag ( + emp_id LONG NOT NULL, + tag_name TEXT(50) NOT NULL, + PRIMARY KEY (emp_id, tag_name) +); +ALTER TABLE sja_emp_tag ADD CONSTRAINT fk_sja_emp_tag_emp FOREIGN KEY (emp_id) REFERENCES sja_emp (id); + +CREATE TABLE sja_sales ( + id LONG NOT NULL, + sale_year SMALLINT NOT NULL, + amount DECIMAL(10,2), + PRIMARY KEY (id, sale_year) +); + +CREATE TABLE sja_types ( + id LONG NOT NULL PRIMARY KEY, + c_yesno YESNO, + c_byte BYTE, + c_smallint SMALLINT, + c_integer SMALLINT, + c_long LONG, + c_bigint BIGINT, + c_currency CURRENCY, + c_single SINGLE, + c_double DOUBLE, + c_decimal DECIMAL(18,4), + c_decimal28 DECIMAL(28,10), + c_text TEXT(255), + c_text10 TEXT(10), + c_memo MEMO, + c_datetime DATETIME, + c_date DATETIME, + c_binary VARBINARY(16), + c_ole OLE +); +INSERT INTO sja_types (id, c_yesno, c_byte, c_smallint, c_integer, c_long, c_bigint, c_currency, c_single, c_double, c_decimal, + c_decimal28, c_text, c_text10, c_memo, c_datetime, c_date, c_binary, c_ole) +VALUES (1, TRUE, 255, -32768, 32767, -2147483648, 9223372036854775807, 1234.5678, 3.5, 2.718281828459045, + 12345678901234.1234, 123456789012345678.1234567891, 'text ''quoted'' \ 中文', 'short', 'memo 中文', + '2024-02-29 23:59:59', '2024-02-29 00:00:00', X'DEADBEEF', X'CAFEBABE'); +INSERT INTO sja_types (id, c_yesno, c_byte, c_smallint, c_integer, c_long, c_bigint, c_currency, c_single, c_double, c_decimal, + c_decimal28, c_text, c_text10, c_memo, c_datetime, c_date, c_binary, c_ole) +VALUES (2, FALSE, 0, 32767, -32768, 2147483647, -9223372036854775808, -9999.9999, -3.5, -12345678901234.5, -0.0001, 0, '', ' ', '', + '1970-01-01 00:00:00', '9999-12-31 00:00:00', X'00', X''); +INSERT INTO sja_types (id) VALUES (3); diff --git a/src/test/resources/it-databases-ci.properties b/src/test/resources/it-databases-ci.properties index c81ccc6..f2e755c 100644 --- a/src/test/resources/it-databases-ci.properties +++ b/src/test/resources/it-databases-ci.properties @@ -1,5 +1,5 @@ # Environment of GitHub Actions: no database servers, only the file databases run (mvn verify -Dit.env=ci). -# H2, SQLite and DuckDB are files in target/it-data, every database server is skipped. +# H2, SQLite, DuckDB and Access are files in target/it-data, every database server is skipped. h2.username=sa h2.password=