From 8aa42393632e1fe444314765abe6efb8dbb1a1aa Mon Sep 17 00:00:00 2001 From: Edouard DUPIN Date: Thu, 19 Jan 2023 00:03:35 +0100 Subject: [PATCH] [DEV] correct log resource for big insert --- back/pom.xml | 4 +- .../src/org/kar/karanage/api/LogResource.java | 107 ++++++++++++++---- .../org/kar/karanage/model/StateHistory.java | 3 +- 3 files changed, 89 insertions(+), 25 deletions(-) diff --git a/back/pom.xml b/back/pom.xml index 8fd6380..3385fd4 100644 --- a/back/pom.xml +++ b/back/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.kar karanage - 0.2.2 + 0.2.3 3.1 @@ -22,7 +22,7 @@ kangaroo-and-rabbit archidata - 0.2.4 + 0.2.6 diff --git a/back/src/org/kar/karanage/api/LogResource.java b/back/src/org/kar/karanage/api/LogResource.java index 7980a1f..129725b 100644 --- a/back/src/org/kar/karanage/api/LogResource.java +++ b/back/src/org/kar/karanage/api/LogResource.java @@ -1,5 +1,6 @@ package org.kar.karanage.api; +import org.kar.archidata.GlobalConfiguration; import org.kar.archidata.SqlWrapper; import org.kar.archidata.WhereCondition; import org.kar.karanage.model.DataLog; @@ -7,16 +8,33 @@ import org.kar.karanage.model.Group; import org.kar.karanage.model.MultipleLogElement; import javax.ws.rs.core.Response; - +import org.kar.archidata.annotation.SQLComment; +import org.kar.archidata.annotation.SQLCreateTime; +import org.kar.archidata.annotation.SQLDefault; +import org.kar.archidata.annotation.SQLForeignKey; +import org.kar.archidata.annotation.SQLIfNotExists; +import org.kar.archidata.annotation.SQLLimitSize; +import org.kar.archidata.annotation.SQLNotNull; +import org.kar.archidata.annotation.SQLPrimaryKey; +import org.kar.archidata.annotation.SQLUpdateTime; +import org.kar.archidata.annotation.SQLTableLinkGeneric.ModelLink; import org.kar.archidata.annotation.security.PermitAll; +import org.kar.archidata.db.DBEntry; import org.kar.archidata.exception.FailException; import org.kar.archidata.exception.InputException; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; +import java.lang.reflect.Field; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.sql.Timestamp; +import java.sql.Types; import java.time.Instant; +import java.util.ArrayList; import java.util.List; @@ -172,6 +190,7 @@ public class LogResource { @PathParam("system") String system, @QueryParam("uuid") Long clientUuid, List listDataToInsert) throws Exception { + long start_time = System.currentTimeMillis(); Group group = SqlWrapper.getWhere(Group.class, List.of( new WhereCondition("name", "=", groupName) @@ -180,27 +199,73 @@ public class LogResource { if (group == null) { throw new InputException("group", "url: /log/{group}/... ==> Unknown group name"); } - for (MultipleLogElement dataToInsert : listDataToInsert) { - DataLog data = new DataLog(); - data.group = group.id; - data.system = system; - data.data = dataToInsert.data; - if (dataToInsert.id != null) { - data.clientId = dataToInsert.id; - } - if (clientUuid != null) { - data.clientUuid = clientUuid; - } - if (dataToInsert.time != null) { - try { - data.create_date = Timestamp.from(Instant.parse(dataToInsert.time)); - } catch (Exception ex) { - throw new InputException("time", "url: ?time=... ==> is not an iso 8601 time format"); + //System.out.println("receive to insert : " + listDataToInsert.size()); + DBEntry entry = new DBEntry(GlobalConfiguration.dbConfig); + entry.connection.setAutoCommit(false); + try { + String query = "INSERT INTO log (`create_date`, `group`, `system`, `clientUuid`, `clientId`, `data`) VALUES (?, ?, ?, ?, ?, ?)"; + // prepare the request: + PreparedStatement ps = entry.connection.prepareStatement(query);//, Statement.RETURN_GENERATED_KEYS); + + Timestamp now = Timestamp.from(Instant.now()); + int batchId = 0; + for (MultipleLogElement dataToInsert : listDataToInsert) { + Timestamp create_date = now; + if (dataToInsert.time != null) { + try { + create_date = Timestamp.from(Instant.parse(dataToInsert.time)); + } catch (Exception ex) { + throw new InputException("time", "url: ?time=... ==> is not an iso 8601 time format"); + } } - } - // TODO: Do a single insertion for multiple DATAS... - SqlWrapper.insert(data); - } + + int iii = 1; + if (create_date == null) { + ps.setNull(iii++, Types.INTEGER); + } else { + ps.setTimestamp(iii++, create_date); + } + ps.setLong(iii++, group.id); + if (system == null) { + ps.setNull(iii++, Types.VARCHAR); + } else { + ps.setString(iii++, system); + } + if (clientUuid == null) { + ps.setNull(iii++, Types.BIGINT); + } else { + ps.setLong(iii++, clientUuid); + } + if (dataToInsert.id == null) { + ps.setNull(iii++, Types.BIGINT); + } else { + ps.setLong(iii++, dataToInsert.id); + } + if (dataToInsert.data == null) { + ps.setNull(iii++, Types.VARCHAR); + } else { + ps.setString(iii++, dataToInsert.data); + } + + ps.addBatch(); + batchId++; + + if (batchId % 1000 == 0 || batchId == listDataToInsert.size()) { + //System.out.println(" insert : " + (batchId%1000) + " elements"); + //long start_time2 = System.currentTimeMillis(); + ps.executeBatch(); // Execute every 1000 items. + //long end_time2 = System.currentTimeMillis(); + //System.out.println(" inject time : " + (end_time2 - start_time2) + " ms"); + ps.clearBatch(); + } + } + entry.connection.setAutoCommit(true); + } finally { + entry.close(); + entry = null; + } + long end_time = System.currentTimeMillis(); + System.out.println("log process time=" + (end_time - start_time) + " ms insert count=" + listDataToInsert.size()); return Response.status(201).build(); } diff --git a/back/src/org/kar/karanage/model/StateHistory.java b/back/src/org/kar/karanage/model/StateHistory.java index 151e6ee..126ac52 100644 --- a/back/src/org/kar/karanage/model/StateHistory.java +++ b/back/src/org/kar/karanage/model/StateHistory.java @@ -7,7 +7,6 @@ import org.kar.archidata.annotation.SQLNotNull; import org.kar.archidata.annotation.SQLTableName; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; enum State { @@ -19,7 +18,7 @@ enum TypeData { @SQLTableName ("stateHistory") @SQLIfNotExists -@JsonInclude(Include.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_NULL) public class StateHistory extends StateInstant { // default constructor public StateHistory() {