[DEV] update migraation model and manage sqlite support update with milisecond

This commit is contained in:
Edouard DUPIN 2023-11-24 00:07:52 +01:00
parent 01de431f5a
commit ed3bfa0604
11 changed files with 160 additions and 121 deletions

View File

@ -23,7 +23,7 @@ import jakarta.persistence.GenerationType;
public class DataFactory {
static final Logger LOGGER = LoggerFactory.getLogger(DataFactory.class);
public static String convertTypeInSQL(final Class<?> type, final String fieldName) throws Exception {
if (!"sqlite".equals(ConfigBaseVariable.getDBType())) {
if (type == Long.class || type == long.class) {
@ -126,21 +126,21 @@ public class DataFactory {
}
throw new DataAccessException("Imcompatible type of element in object for: " + type.getCanonicalName());
}
public static void createTablesSpecificType(final String tableName, final Field elem, final StringBuilder mainTableBuilder, final List<String> preOtherTables, final List<String> postOtherTables,
final boolean createIfNotExist, final boolean createDrop, final int fieldId, final Class<?> classModel) throws Exception {
final String name = AnnotationTools.getFieldName(elem);
final Integer limitSize = AnnotationTools.getLimitSize(elem);
final boolean notNull = AnnotationTools.getNotNull(elem);
final boolean primaryKey = AnnotationTools.isPrimaryKey(elem);
final GenerationType strategy = AnnotationTools.getStrategy(elem);
final boolean createTime = elem.getDeclaredAnnotationsByType(CreationTimestamp.class).length != 0;
final boolean updateTime = elem.getDeclaredAnnotationsByType(UpdateTimestamp.class).length != 0;
final String comment = AnnotationTools.getComment(elem);
final String defaultValue = AnnotationTools.getDefault(elem);
if (fieldId == 0) {
mainTableBuilder.append("\n\t\t`");
} else {
@ -193,12 +193,13 @@ public class DataFactory {
triggerBuilder.append(tableName);
triggerBuilder.append(" SET ");
triggerBuilder.append(name);
triggerBuilder.append(" = datetime('now') WHERE id = NEW.id; \n");
//triggerBuilder.append(" = datetime('now') WHERE id = NEW.id; \n");
triggerBuilder.append(" = strftime('%Y-%m-%d %H:%M:%f', 'now') WHERE id = NEW.id; \n");
triggerBuilder.append("END;");
postOtherTables.add(triggerBuilder.toString());
}
mainTableBuilder.append(" ");
}
} else {
@ -233,11 +234,11 @@ public class DataFactory {
mainTableBuilder.append("DEFAULT ");
mainTableBuilder.append(defaultValue);
mainTableBuilder.append(" ");
}
if (primaryKey && "sqlite".equals(ConfigBaseVariable.getDBType())) {
mainTableBuilder.append("PRIMARY KEY ");
}
if (strategy == GenerationType.IDENTITY) {
if (!"sqlite".equals(ConfigBaseVariable.getDBType())) {
@ -248,14 +249,14 @@ public class DataFactory {
} else if (strategy != null) {
throw new DataAccessException("Can not generate a stategy different of IDENTITY");
}
if (comment != null && !"sqlite".equals(ConfigBaseVariable.getDBType())) {
mainTableBuilder.append("COMMENT '");
mainTableBuilder.append(comment.replace('\'', '\''));
mainTableBuilder.append("' ");
}
}
private static boolean isFieldFromSuperClass(final Class<?> model, final String filedName) {
final Class<?> superClass = model.getSuperclass();
if (superClass == null) {
@ -275,14 +276,14 @@ public class DataFactory {
}
return false;
}
public static List<String> createTable(final Class<?> clazz) throws Exception {
return createTable(clazz, null);
}
public static List<String> createTable(final Class<?> clazz, final QueryOptions options) throws Exception {
final String tableName = AnnotationTools.getTableName(clazz, options);
boolean createDrop = false;
if (options != null) {
final Object data = options.get(QueryOptions.CREATE_DROP_TABLE);
@ -292,7 +293,7 @@ public class DataFactory {
LOGGER.error("'{}' ==> has not a Boolean value: {}", QueryOptions.CREATE_DROP_TABLE, data);
}
}
final boolean createIfNotExist = clazz.getDeclaredAnnotationsByType(DataIfNotExists.class).length != 0;
final List<String> preActionList = new ArrayList<>();
final List<String> postActionList = new ArrayList<>();
@ -312,7 +313,7 @@ public class DataFactory {
int fieldId = 0;
LOGGER.debug("===> TABLE `{}`", tableName);
final List<String> primaryKeys = new ArrayList<>();
for (final Field elem : clazz.getFields()) {
// DEtect the primary key (support only one primary key right now...
if (AnnotationTools.isPrimaryKey(elem)) {
@ -393,5 +394,5 @@ public class DataFactory {
preActionList.addAll(postActionList);
return preActionList;
}
}

View File

@ -16,17 +16,17 @@ import org.slf4j.LoggerFactory;
public class MigrationEngine {
final static Logger LOGGER = LoggerFactory.getLogger(MigrationEngine.class);
// List of order migrations
private final List<MigrationInterface> datas;
// initialization of the migration if the DB is not present...
private MigrationInterface init;
/** Migration engine constructor (empty). */
public MigrationEngine() {
this(new ArrayList<>(), null);
}
/** Migration engine constructor (specific mode).
* @param datas All the migration ordered.
* @param init Initialization migration model. */
@ -34,19 +34,19 @@ public class MigrationEngine {
this.datas = datas;
this.init = init;
}
/** Add a Migration in the list
* @param migration Migration to add. */
public void add(final MigrationInterface migration) {
this.datas.add(migration);
}
/** Set first initialization class
* @param migration migration class for first init. */
public void setInit(final MigrationInterface migration) {
this.init = migration;
}
/** Get the current version/migration name
* @return Model represent the last migration. If null then no migration has been done. */
public Migration getCurrentVersion() {
@ -74,7 +74,7 @@ public class MigrationEngine {
}
return null;
}
/** Process the automatic migration of the system The function wait the Administrator intervention to correct the bug.
* @param config SQL connection for the migration.
* @throws InterruptedException user interrupt the migration */
@ -84,18 +84,20 @@ public class MigrationEngine {
} catch (final Exception ex) {
ex.printStackTrace();
while (true) {
LOGGER.error("Fail to create the local DB SQL model for migaration ==> wait administrator interventions");
LOGGER.error("ERROR: {}", ex.getMessage());
LOGGER.error("========================================================================");
LOGGER.error("== Fail to migrate ==> wait administrator interventions ==");
LOGGER.error("========================================================================");
Thread.sleep(60 * 60 * 1000);
}
}
}
/** Process the automatic migration of the system
* @param config SQL connection for the migration
* @throws IOException Error if access on the DB */
public void migrateErrorThrow(final DBConfig config) throws MigrationException {
LOGGER.info("Execute migration ... [BEGIN]");
// check the integrity of the migrations:
for (final MigrationInterface elem : this.datas) {
if (elem == null) {
@ -116,7 +118,7 @@ public class MigrationEngine {
}
}
}
// STEP 1: Check the DB exist:
LOGGER.info("Verify existance of '{}'", config.getDbName());
boolean exist = DataAccess.isDBExist(config.getDbName());
@ -228,7 +230,7 @@ public class MigrationEngine {
}
LOGGER.info("Execute migration ... [ END ]");
}
public void migrateSingle(final DBEntry entry, final MigrationInterface elem, final int id, final int count) throws MigrationException {
LOGGER.info("---------------------------------------------------------");
LOGGER.info("-- Migrate: [{}/{}] {} [BEGIN]", id, count, elem.getName());
@ -247,8 +249,16 @@ public class MigrationEngine {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (elem.applyMigration(entry, log, migrationResult)) {
boolean ret = true;
try {
ret = elem.applyMigration(entry, log, migrationResult);
} catch (final Exception e) {
log.append("\nFail in the migration apply ");
log.append(e.getLocalizedMessage());
e.printStackTrace();
throw new MigrationException("Migration fial: '" + migrationResult.name + "' defect @" + migrationResult.stepId + "/" + migrationResult.count);
}
if (ret) {
migrationResult.terminated = true;
try {
DataAccess.update(migrationResult, migrationResult.id, List.of("terminated"));
@ -267,7 +277,7 @@ public class MigrationEngine {
}
LOGGER.info("Migrate: [{}/{}] {} [ END ]", id, count, elem.getName());
}
public void revertTo(final DBEntry entry, final String migrationName) {
final Migration currentVersion = getCurrentVersion();
final List<MigrationInterface> toApply = new ArrayList<>();
@ -290,10 +300,10 @@ public class MigrationEngine {
revertSingle(entry, elem, id, count);
}
}
public void revertSingle(final DBEntry entry, final MigrationInterface elem, final int id, final int count) {
LOGGER.info("Revert migration: {} [BEGIN]", elem.getName());
LOGGER.info("Revert migration: {} [ END ]", elem.getName());
}
}

View File

@ -7,20 +7,20 @@ public interface MigrationInterface {
/** Get Name of the migration
* @return Migration name */
String getName();
/** Migrate the system to a new version.
* @param entry DB interface for the migration.
* @param log Stored data in the BDD for the migration progression.
* @param migration Migration post data on each step...
* @return true if migration is finished. */
boolean applyMigration(DBEntry entry, StringBuilder log, Migration model);
boolean applyMigration(DBEntry entry, StringBuilder log, Migration model) throws Exception;
/** Remove a migration the system to the previous version.
* @param entry DB interface for the migration.
* @param log Stored data in the BDD for the migration progression.
* @return true if migration is finished. */
boolean revertMigration(DBEntry entry, StringBuilder log);
boolean revertMigration(DBEntry entry, StringBuilder log) throws Exception;
/** Get the number of step in the migration process.
* @return count of SQL access. */
int getNumberOfStep();

View File

@ -13,11 +13,13 @@ import org.kar.archidata.util.ConfigBaseVariable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
record Action(String action, List<String> filterDB) {
record Action(
String action,
List<String> filterDB) {
public Action(final String action) {
this(action, List.of());
}
public Action(final String action, final String filterDB) {
this(action, List.of(filterDB));
}
@ -26,26 +28,35 @@ record Action(String action, List<String> filterDB) {
public class MigrationSqlStep implements MigrationInterface {
final static Logger LOGGER = LoggerFactory.getLogger(MigrationSqlStep.class);
private final List<Action> actions = new ArrayList<>();
@Override
public String getName() {
return getClass().getCanonicalName();
}
public void display() {
for (int iii = 0; iii < this.actions.size(); iii++) {
final Action action = this.actions.get(iii);
LOGGER.info(" >>>> SQL ACTION : {}/{} ==> filter='{}'\n{}", iii, this.actions.size(), action.filterDB(), action.action());
}
}
public void generateStep() throws Exception {
throw new Exception("Forward is not implemented");
}
public void generateRevertStep() throws Exception {
throw new Exception("Backward is not implemented");
}
@Override
public boolean applyMigration(final DBEntry entry, final StringBuilder log, final Migration model) {
public boolean applyMigration(final DBEntry entry, final StringBuilder log, final Migration model) throws Exception {
generateStep();
for (int iii = 0; iii < this.actions.size(); iii++) {
log.append("action [" + (iii + 1) + "/" + this.actions.size() + "]\n");
LOGGER.info(" >>>> SQL ACTION : {}/{}", iii + 1, this.actions.size());
final Action action = this.actions.get(iii);
LOGGER.info("SQL request: ```{}``` on '{}' current={}", action.action(), action.filterDB(), ConfigBaseVariable.getDBType());
log.append("SQL: " + action.action() + " on " + action.filterDB() + "\n");
boolean isValid = true;
@ -96,30 +107,31 @@ public class MigrationSqlStep implements MigrationInterface {
}
return true;
}
@Override
public boolean revertMigration(final DBEntry entry, final StringBuilder log) {
public boolean revertMigration(final DBEntry entry, final StringBuilder log) throws Exception {
generateRevertStep();
return false;
}
public void addAction(final String action) {
this.actions.add(new Action(action));
}
public void addAction(final String action, final String filterdBType) {
this.actions.add(new Action(action, filterdBType));
}
public void addClass(final Class<?> clazz) throws Exception {
final List<String> tmp = DataFactory.createTable(clazz);
for (final String elem : tmp) {
this.actions.add(new Action(elem));
}
}
@Override
public int getNumberOfStep() {
return this.actions.size();
}
}

View File

@ -33,30 +33,30 @@ public class TestSimpleTable {
private static final String DATA_INJECTED_2 = "dsqfsdfqsdfsqdf";
private static Long idOfTheObject = null;
private static Timestamp startAction = null;
@BeforeAll
public static void configureWebServer() throws Exception {
ConfigBaseVariable.dbType = "sqlite";
ConfigBaseVariable.dbHost = "memory";
// for test we need to connect all time the DB
ConfigBaseVariable.dbKeepConnected = "true";
// Clear the static test:
idOfTheObject = null;
startAction = null;
// Connect the dataBase...
final DBEntry entry = DBEntry.createInterface(GlobalConfiguration.dbConfig);
entry.connect();
}
@AfterAll
public static void removeDataBase() throws IOException {
LOGGER.info("Remove the test db");
DBEntry.closeAllForceMode();
ConfigBaseVariable.clearAllValue();
}
@Order(1)
@Test
public void testTableInsertAndRetrieve() throws Exception {
@ -69,14 +69,14 @@ public class TestSimpleTable {
final SimpleTable test = new SimpleTable();
test.data = TestSimpleTable.DATA_INJECTED;
final SimpleTable insertedData = DataAccess.insert(test);
Assertions.assertNotNull(insertedData);
Assertions.assertNotNull(insertedData.id);
Assertions.assertTrue(insertedData.id >= 0);
// Try to retrieve all the data:
final SimpleTable retrieve = DataAccess.get(SimpleTable.class, insertedData.id);
Assertions.assertNotNull(retrieve);
Assertions.assertNotNull(retrieve.id);
Assertions.assertEquals(insertedData.id, retrieve.id);
@ -85,13 +85,13 @@ public class TestSimpleTable {
Assertions.assertNull(retrieve.updatedAt);
TestSimpleTable.idOfTheObject = retrieve.id;
}
@Order(2)
@Test
public void testReadAllValuesUnreadable() throws Exception {
// check the full values
final SimpleTable retrieve = DataAccess.get(SimpleTable.class, TestSimpleTable.idOfTheObject, new QueryOptions(QueryOptions.SQL_NOT_READ_DISABLE, true));
Assertions.assertNotNull(retrieve);
Assertions.assertNotNull(retrieve.id);
Assertions.assertEquals(TestSimpleTable.idOfTheObject, retrieve.id);
@ -104,16 +104,11 @@ public class TestSimpleTable {
// Assertions.assertTrue(retrieve.updatedAt.after(this.startAction));
Assertions.assertEquals(retrieve.createdAt, retrieve.updatedAt);
}
@Order(3)
@Test
public void testUpdateData() throws Exception {
if ("sqlite".equalsIgnoreCase(ConfigBaseVariable.getDBType())) {
Thread.sleep(Duration.ofMillis(1100));
} else {
Thread.sleep(Duration.ofMillis(15));
}
Thread.sleep(Duration.ofMillis(15));
// Delete the entry:
final SimpleTable test = new SimpleTable();
test.data = TestSimpleTable.DATA_INJECTED_2;
@ -128,7 +123,7 @@ public class TestSimpleTable {
LOGGER.info("created @ {} updated @ {}", retrieve.createdAt, retrieve.updatedAt);
Assertions.assertTrue(retrieve.updatedAt.after(retrieve.createdAt));
}
@Order(4)
@Test
public void testDeleteTheObject() throws Exception {
@ -137,17 +132,17 @@ public class TestSimpleTable {
final SimpleTable retrieve = DataAccess.get(SimpleTable.class, TestSimpleTable.idOfTheObject);
Assertions.assertNull(retrieve);
}
@Order(5)
@Test
public void testReadDeletedObject() throws Exception {
// check if we set get deleted element
final SimpleTable retrieve = DataAccess.get(SimpleTable.class, TestSimpleTable.idOfTheObject, new QueryOptions(QueryOptions.SQL_DELETED_DISABLE, true));
Assertions.assertNull(retrieve);
}
@Order(6)
@Test
public void testReadAllValuesUnreadableOfDeletedObject() throws Exception {
@ -155,6 +150,6 @@ public class TestSimpleTable {
final SimpleTable retrieve = DataAccess.get(SimpleTable.class, TestSimpleTable.idOfTheObject,
new QueryOptions(QueryOptions.SQL_DELETED_DISABLE, true, QueryOptions.SQL_NOT_READ_DISABLE, true));
Assertions.assertNull(retrieve);
}
}

View File

@ -33,30 +33,30 @@ public class TestSimpleTableSoftDelete {
private static final String DATA_INJECTED_2 = "qsdfqsdfqsdfsqdf";
private static Long idOfTheObject = null;
private static Timestamp startAction = null;
@BeforeAll
public static void configureWebServer() throws Exception {
ConfigBaseVariable.dbType = "sqlite";
ConfigBaseVariable.dbHost = "memory";
// for test we need to connect all time the DB
ConfigBaseVariable.dbKeepConnected = "true";
// Clear the static test:
idOfTheObject = null;
startAction = null;
// Connect the dataBase...
final DBEntry entry = DBEntry.createInterface(GlobalConfiguration.dbConfig);
entry.connect();
}
@AfterAll
public static void removeDataBase() throws IOException {
LOGGER.info("Remove the test db");
DBEntry.closeAllForceMode();
ConfigBaseVariable.clearAllValue();
}
@Order(1)
@Test
public void testTableInsertAndRetrieve() throws Exception {
@ -69,14 +69,14 @@ public class TestSimpleTableSoftDelete {
final SimpleTableSoftDelete test = new SimpleTableSoftDelete();
test.data = TestSimpleTableSoftDelete.DATA_INJECTED;
final SimpleTableSoftDelete insertedData = DataAccess.insert(test);
Assertions.assertNotNull(insertedData);
Assertions.assertNotNull(insertedData.id);
Assertions.assertTrue(insertedData.id >= 0);
// Try to retrieve all the data:
final SimpleTableSoftDelete retrieve = DataAccess.get(SimpleTableSoftDelete.class, insertedData.id);
Assertions.assertNotNull(retrieve);
Assertions.assertNotNull(retrieve.id);
Assertions.assertEquals(insertedData.id, retrieve.id);
@ -86,13 +86,13 @@ public class TestSimpleTableSoftDelete {
Assertions.assertNull(retrieve.deleted);
TestSimpleTableSoftDelete.idOfTheObject = retrieve.id;
}
@Order(2)
@Test
public void testReadAllValuesUnreadable() throws Exception {
// check the full values
final SimpleTableSoftDelete retrieve = DataAccess.get(SimpleTableSoftDelete.class, TestSimpleTableSoftDelete.idOfTheObject, new QueryOptions(QueryOptions.SQL_NOT_READ_DISABLE, true));
Assertions.assertNotNull(retrieve);
Assertions.assertNotNull(retrieve.id);
Assertions.assertEquals(TestSimpleTableSoftDelete.idOfTheObject, retrieve.id);
@ -107,16 +107,12 @@ public class TestSimpleTableSoftDelete {
Assertions.assertNotNull(retrieve.deleted);
Assertions.assertEquals(false, retrieve.deleted);
}
@Order(3)
@Test
public void testUpdateData() throws Exception {
if ("sqlite".equalsIgnoreCase(ConfigBaseVariable.getDBType())) {
Thread.sleep(Duration.ofMillis(1100));
} else {
Thread.sleep(Duration.ofMillis(15));
}
Thread.sleep(Duration.ofMillis(15));
// Delete the entry:
final SimpleTableSoftDelete test = new SimpleTableSoftDelete();
test.data = TestSimpleTableSoftDelete.DATA_INJECTED_2;
@ -134,7 +130,7 @@ public class TestSimpleTableSoftDelete {
Assertions.assertNotNull(retrieve.deleted);
Assertions.assertEquals(false, retrieve.deleted);
}
@Order(4)
@Test
public void testSoftDeleteTheObject() throws Exception {
@ -148,11 +144,11 @@ public class TestSimpleTableSoftDelete {
final SimpleTableSoftDelete retrieve = DataAccess.get(SimpleTableSoftDelete.class, TestSimpleTableSoftDelete.idOfTheObject);
Assertions.assertNull(retrieve);
}
@Order(5)
@Test
public void testReadDeletedObject() throws Exception {
// check if we set get deleted element
final SimpleTableSoftDelete retrieve = DataAccess.get(SimpleTableSoftDelete.class, TestSimpleTableSoftDelete.idOfTheObject, new QueryOptions(QueryOptions.SQL_DELETED_DISABLE, true));
Assertions.assertNotNull(retrieve);
@ -162,9 +158,9 @@ public class TestSimpleTableSoftDelete {
Assertions.assertNull(retrieve.createdAt);
Assertions.assertNull(retrieve.updatedAt);
Assertions.assertNull(retrieve.deleted);
}
@Order(6)
@Test
public void testReadAllValuesUnreadableOfDeletedObject() throws Exception {
@ -181,6 +177,6 @@ public class TestSimpleTableSoftDelete {
Assertions.assertTrue(retrieve.updatedAt.after(retrieve.createdAt));
Assertions.assertNotNull(retrieve.deleted);
Assertions.assertEquals(true, retrieve.deleted);
}
}

View File

@ -5,13 +5,18 @@ import org.kar.archidata.migration.MigrationSqlStep;
import test.kar.archidata.migration.model.TypesMigrationInitialisationCurrent;
class InitializationCurrent extends MigrationSqlStep {
@Override
public String getName() {
return "Initialization";
}
public InitializationCurrent() throws Exception {
public InitializationCurrent() {
}
@Override
public void generateStep() throws Exception {
addClass(TypesMigrationInitialisationCurrent.class);
addAction("""
ALTER TABLE `TestTableMigration` AUTO_INCREMENT = 1000;

View File

@ -5,13 +5,18 @@ import org.kar.archidata.migration.MigrationSqlStep;
import test.kar.archidata.migration.model.TypesMigrationInitialisationFirst;
class InitializationFirst extends MigrationSqlStep {
@Override
public String getName() {
return "Initialization";
}
public InitializationFirst() throws Exception {
public InitializationFirst() {
}
@Override
public void generateStep() throws Exception {
addClass(TypesMigrationInitialisationFirst.class);
addAction("""
ALTER TABLE `TestTableMigration` AUTO_INCREMENT = 1000;

View File

@ -3,19 +3,24 @@ package test.kar.archidata.migration;
import org.kar.archidata.migration.MigrationSqlStep;
class Migration1 extends MigrationSqlStep {
@Override
public String getName() {
return "first migratiion";
}
public Migration1() throws Exception {
public Migration1() {
}
@Override
public void generateStep() throws Exception {
addAction("""
ALTER TABLE `TestTableMigration`
RENAME COLUMN `testData` TO `testDataMigration1`
""");
display();
}
}

View File

@ -3,19 +3,24 @@ package test.kar.archidata.migration;
import org.kar.archidata.migration.MigrationSqlStep;
class Migration2 extends MigrationSqlStep {
@Override
public String getName() {
return "Second migration";
}
public Migration2() throws Exception {
public Migration2() {
}
@Override
public void generateStep() throws Exception {
addAction("""
ALTER TABLE `TestTableMigration`
RENAME COLUMN `testDataMigration1` TO `testDataMigration2`
""");
display();
}
}

View File

@ -3,19 +3,24 @@ package test.kar.archidata.migration;
import org.kar.archidata.migration.MigrationSqlStep;
class MigrationFail extends MigrationSqlStep {
@Override
public String getName() {
return "Fail migration Test";
}
public MigrationFail() throws Exception {
public MigrationFail() {
}
@Override
public void generateStep() throws Exception {
addAction("""
ALTER TABLE `TestTableMigrationqs`
RENAME COLUMN `testDataMisqdgration1` TO `testDataMiqsdgration2`
""");
display();
}
}