[DEV] correct log resource for big insert

back/pom.xml

@@ -2,7 +2,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.kar</groupId>
   <artifactId>karanage</artifactId>
-  <version>0.2.2</version>
+  <version>0.2.3</version>
   <properties>
 
 		<maven.compiler.version>3.1</maven.compiler.version>
@@ -22,7 +22,7 @@
 		<dependency>
 			<groupId>kangaroo-and-rabbit</groupId>
 		    <artifactId>archidata</artifactId>
-		    <version>0.2.4</version>
+		    <version>0.2.6</version>
 		</dependency>
     </dependencies>
 

back/src/org/kar/karanage/api/LogResource.java

@@ -1,5 +1,6 @@
 package org.kar.karanage.api;
 
+import org.kar.archidata.GlobalConfiguration;
 import org.kar.archidata.SqlWrapper;
 import org.kar.archidata.WhereCondition;
 import org.kar.karanage.model.DataLog;
@@ -7,16 +8,33 @@ import org.kar.karanage.model.Group;
 import org.kar.karanage.model.MultipleLogElement;
 import javax.ws.rs.core.Response;
 
-
+import org.kar.archidata.annotation.SQLComment;
+import org.kar.archidata.annotation.SQLCreateTime;
+import org.kar.archidata.annotation.SQLDefault;
+import org.kar.archidata.annotation.SQLForeignKey;
+import org.kar.archidata.annotation.SQLIfNotExists;
+import org.kar.archidata.annotation.SQLLimitSize;
+import org.kar.archidata.annotation.SQLNotNull;
+import org.kar.archidata.annotation.SQLPrimaryKey;
+import org.kar.archidata.annotation.SQLUpdateTime;
+import org.kar.archidata.annotation.SQLTableLinkGeneric.ModelLink;
 import org.kar.archidata.annotation.security.PermitAll;
+import org.kar.archidata.db.DBEntry;
 import org.kar.archidata.exception.FailException;
 import org.kar.archidata.exception.InputException;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
 
+import java.lang.reflect.Field;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.sql.Timestamp;
+import java.sql.Types;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
 
 
@@ -172,6 +190,7 @@ public class LogResource {
     		@PathParam("system") String system,
     		@QueryParam("uuid") Long clientUuid,
     		List<MultipleLogElement> listDataToInsert) throws Exception {
+    	long start_time = System.currentTimeMillis();
     	Group group = SqlWrapper.getWhere(Group.class,
     			List.of(
     					new WhereCondition("name", "=", groupName)
@@ -180,27 +199,73 @@ public class LogResource {
     	if (group == null) {
     		throw new InputException("group", "url: /log/{group}/... ==> Unknown group name");
     	}
-		for (MultipleLogElement dataToInsert : listDataToInsert) {
+    	//System.out.println("receive to insert : " + listDataToInsert.size());
-    		DataLog data = new DataLog();
+        DBEntry entry = new DBEntry(GlobalConfiguration.dbConfig);
-			data.group = group.id;
+        entry.connection.setAutoCommit(false);
-			data.system = system;
+        try {
-			data.data = dataToInsert.data;
+	        String query = "INSERT INTO log (`create_date`, `group`, `system`, `clientUuid`, `clientId`, `data`) VALUES (?, ?, ?, ?, ?, ?)";
-			if (dataToInsert.id != null) {
+	        // prepare the request:
-				data.clientId = dataToInsert.id;
+	        PreparedStatement ps = entry.connection.prepareStatement(query);//, Statement.RETURN_GENERATED_KEYS);
-			}
+	
-			if (clientUuid != null) {
+	        Timestamp now = Timestamp.from(Instant.now());
-				data.clientUuid = clientUuid;
+	        int batchId = 0;
-			}
+			for (MultipleLogElement dataToInsert : listDataToInsert) {
-			if (dataToInsert.time != null) {
+				Timestamp create_date = now;
-				try {
+				if (dataToInsert.time != null) {
-					data.create_date = Timestamp.from(Instant.parse(dataToInsert.time));
+					try {
-				}  catch (Exception ex) {
+						create_date = Timestamp.from(Instant.parse(dataToInsert.time));
-					throw new InputException("time", "url: ?time=... ==> is not an iso 8601 time format");
+					}  catch (Exception ex) {
+						throw new InputException("time", "url: ?time=... ==> is not an iso 8601 time format");
+					}
 				}
-			}
+	
-			// TODO: Do a single insertion for multiple DATAS...
+		        int iii = 1;
-			SqlWrapper.insert(data);
+		        if (create_date == null) {
-		}
+		        	ps.setNull(iii++, Types.INTEGER);
+		        } else {
+		        	ps.setTimestamp(iii++, create_date);
+		        }
+		        ps.setLong(iii++, group.id);
+		        if (system == null) {
+		        	ps.setNull(iii++, Types.VARCHAR);
+		        } else {
+		        	ps.setString(iii++, system);
+		        }
+		        if (clientUuid == null) {
+		        	ps.setNull(iii++, Types.BIGINT);
+		        } else {
+		        	ps.setLong(iii++, clientUuid);
+		        }
+		        if (dataToInsert.id == null) {
+		        	ps.setNull(iii++, Types.BIGINT);
+		        } else {
+		        	ps.setLong(iii++, dataToInsert.id);
+		        }
+		        if (dataToInsert.data == null) {
+		        	ps.setNull(iii++, Types.VARCHAR);
+		        } else {
+		        	ps.setString(iii++, dataToInsert.data);
+		        }
+	
+	            ps.addBatch();
+	            batchId++;
+	
+	            if (batchId % 1000 == 0 || batchId == listDataToInsert.size()) {
+	            	//System.out.println("    insert : " + (batchId%1000) + " elements");
+	            	//long start_time2 = System.currentTimeMillis();
+	            	ps.executeBatch(); // Execute every 1000 items.
+	            	//long end_time2 = System.currentTimeMillis();
+	            	//System.out.println("    inject time : " + (end_time2 - start_time2) + " ms");
+	            	ps.clearBatch();
+	            }
+			}		
+	        entry.connection.setAutoCommit(true);
+	    } finally {
+	    	entry.close();
+	    	entry = null;
+	    }
+    	long end_time = System.currentTimeMillis();
+    	System.out.println("log process time=" + (end_time - start_time) + " ms insert count=" + listDataToInsert.size());
 		return Response.status(201).build();
     }
     

back/src/org/kar/karanage/model/StateHistory.java

@@ -7,7 +7,6 @@ import org.kar.archidata.annotation.SQLNotNull;
 import org.kar.archidata.annotation.SQLTableName;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
 
 
 enum State {
@@ -19,7 +18,7 @@ enum TypeData {
 
 @SQLTableName ("stateHistory")
 @SQLIfNotExists
-@JsonInclude(Include.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class StateHistory extends StateInstant {
 	// default constructor
 	public StateHistory() {