kangaroo-and-rabbit/karanage
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

[DEV] correct log resource for big insert

Browse Source
This commit is contained in:
Edouard DUPIN 2023-01-19 00:03:35 +01:00
parent aa9331ab05
commit 8aa4239363
3 changed files with 89 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
back/pom.xml
View File

@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.kar</groupId>
<artifactId>karanage</artifactId>
<version>0.2.2</version>
<version>0.2.3</version>
<properties>
<maven.compiler.version>3.1</maven.compiler.version>
@ -22,7 +22,7 @@
<dependency>
<groupId>kangaroo-and-rabbit</groupId>
<artifactId>archidata</artifactId>
<version>0.2.4</version>
<version>0.2.6</version>
</dependency>
</dependencies>

105
back/src/org/kar/karanage/api/LogResource.java
View File

@ -1,5 +1,6 @@
package org.kar.karanage.api;
import org.kar.archidata.GlobalConfiguration;
import org.kar.archidata.SqlWrapper;
import org.kar.archidata.WhereCondition;
import org.kar.karanage.model.DataLog;
@ -7,16 +8,33 @@ import org.kar.karanage.model.Group;
import org.kar.karanage.model.MultipleLogElement;
import javax.ws.rs.core.Response;
import org.kar.archidata.annotation.SQLComment;
import org.kar.archidata.annotation.SQLCreateTime;
import org.kar.archidata.annotation.SQLDefault;
import org.kar.archidata.annotation.SQLForeignKey;
import org.kar.archidata.annotation.SQLIfNotExists;
import org.kar.archidata.annotation.SQLLimitSize;
import org.kar.archidata.annotation.SQLNotNull;
import org.kar.archidata.annotation.SQLPrimaryKey;
import org.kar.archidata.annotation.SQLUpdateTime;
import org.kar.archidata.annotation.SQLTableLinkGeneric.ModelLink;
import org.kar.archidata.annotation.security.PermitAll;
import org.kar.archidata.db.DBEntry;
import org.kar.archidata.exception.FailException;
import org.kar.archidata.exception.InputException;
import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import java.lang.reflect.Field;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
@ -172,6 +190,7 @@ public class LogResource {
@PathParam("system") String system,
@QueryParam("uuid") Long clientUuid,
List<MultipleLogElement> listDataToInsert) throws Exception {
long start_time = System.currentTimeMillis();
Group group = SqlWrapper.getWhere(Group.class,
List.of(
new WhereCondition("name", "=", groupName)
@ -180,27 +199,73 @@ public class LogResource {
if (group == null) {
throw new InputException("group", "url: /log/{group}/... ==> Unknown group name");
}
for (MultipleLogElement dataToInsert : listDataToInsert) {
DataLog data = new DataLog();
data.group = group.id;
data.system = system;
data.data = dataToInsert.data;
if (dataToInsert.id != null) {
data.clientId = dataToInsert.id;
}
if (clientUuid != null) {
data.clientUuid = clientUuid;
}
if (dataToInsert.time != null) {
try {
data.create_date = Timestamp.from(Instant.parse(dataToInsert.time));
} catch (Exception ex) {
throw new InputException("time", "url: ?time=... ==> is not an iso 8601 time format");
//System.out.println("receive to insert : " + listDataToInsert.size());
DBEntry entry = new DBEntry(GlobalConfiguration.dbConfig);
entry.connection.setAutoCommit(false);
try {
String query = "INSERT INTO log (`create_date`, `group`, `system`, `clientUuid`, `clientId`, `data`) VALUES (?, ?, ?, ?, ?, ?)";
// prepare the request:
PreparedStatement ps = entry.connection.prepareStatement(query);//, Statement.RETURN_GENERATED_KEYS);
Timestamp now = Timestamp.from(Instant.now());
int batchId = 0;
for (MultipleLogElement dataToInsert : listDataToInsert) {
Timestamp create_date = now;
if (dataToInsert.time != null) {
try {
create_date = Timestamp.from(Instant.parse(dataToInsert.time));
} catch (Exception ex) {
throw new InputException("time", "url: ?time=... ==> is not an iso 8601 time format");
}
}
int iii = 1;
if (create_date == null) {
ps.setNull(iii++, Types.INTEGER);
} else {
ps.setTimestamp(iii++, create_date);
}
ps.setLong(iii++, group.id);
if (system == null) {
ps.setNull(iii++, Types.VARCHAR);
} else {
ps.setString(iii++, system);
}
if (clientUuid == null) {
ps.setNull(iii++, Types.BIGINT);
} else {
ps.setLong(iii++, clientUuid);
}
if (dataToInsert.id == null) {
ps.setNull(iii++, Types.BIGINT);
} else {
ps.setLong(iii++, dataToInsert.id);
}
if (dataToInsert.data == null) {
ps.setNull(iii++, Types.VARCHAR);
} else {
ps.setString(iii++, dataToInsert.data);
}
ps.addBatch();
batchId++;
if (batchId % 1000 == 0 || batchId == listDataToInsert.size()) {
//System.out.println(" insert : " + (batchId%1000) + " elements");
//long start_time2 = System.currentTimeMillis();
ps.executeBatch(); // Execute every 1000 items.
//long end_time2 = System.currentTimeMillis();
//System.out.println(" inject time : " + (end_time2 - start_time2) + " ms");
ps.clearBatch();
}
}
// TODO: Do a single insertion for multiple DATAS...
SqlWrapper.insert(data);
}
entry.connection.setAutoCommit(true);
} finally {
entry.close();
entry = null;
}
long end_time = System.currentTimeMillis();
System.out.println("log process time=" + (end_time - start_time) + " ms insert count=" + listDataToInsert.size());
return Response.status(201).build();
}

3
back/src/org/kar/karanage/model/StateHistory.java
View File

@ -7,7 +7,6 @@ import org.kar.archidata.annotation.SQLNotNull;
import org.kar.archidata.annotation.SQLTableName;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
enum State {
@ -19,7 +18,7 @@ enum TypeData {
@SQLTableName ("stateHistory")
@SQLIfNotExists
@JsonInclude(Include.NON_NULL)
@JsonInclude(JsonInclude.Include.NON_NULL)
public class StateHistory extends StateInstant {
// default constructor
public StateHistory() {