[DEV] Download maven dependency

src/org/atriasoft/island/Tools.java

@@ -101,6 +101,8 @@ public class Tools {
 	}
 
 	public static void fileWriteData(final Path path, final String data) {
+
+		Path tmpFile = path.getParent().resolve(path.getFileName()+"__tmp");
 		if (!Files.exists(path.getParent())) { 
 			File file = path.getParent().toFile();
 			if (!file.mkdirs()) {
@@ -108,8 +110,16 @@ public class Tools {
 			}
 		}
 		try {
-			Files.writeString(path, data);
+			Files.writeString(tmpFile, data);
 		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+        // this is the way to have atomic replacement of file and not partial downloaded file ==> move is 99.999999% atomic.
+        try {
+			Files.move(tmpFile, path);
+		} catch (IOException e) {
+			
 			// TODO Auto-generated catch block
 			e.printStackTrace();
 		}

src/org/atriasoft/island/actions/DependencySync.java

@@ -2,18 +2,25 @@ package org.atriasoft.island.actions;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Enumeration;
 import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import org.atriasoft.death.annotation.ArgCommand;
 import org.atriasoft.death.annotation.ArgDescription;
 import org.atriasoft.death.annotation.ArgExecute;
+import org.atriasoft.death.annotation.ArgName;
 import org.atriasoft.death.annotation.ArgSample;
 import org.atriasoft.exml.Exml;
 import org.atriasoft.exml.exception.ExmlException;
@@ -27,12 +34,25 @@ import org.atriasoft.island.model.manifest.Artifactory;
 import org.atriasoft.island.model.manifest.ConfigManifest;
 import org.atriasoft.island.model.manifest.Dependency;
 import org.atriasoft.island.model.maven.MavenMetadata;
+import org.atriasoft.island.model.maven.PomDependency;
+import org.atriasoft.island.model.maven.PomMaven;
 
 @ArgCommand("depend-sync")
 @ArgDescription("Syncronize all the dependencies referenced")
 @ArgSample("depend-sync")
 public class DependencySync  {
 
+	@ArgName("doc")
+	@ArgDescription("enable download of the documentation")
+	public boolean documentation = false;
+	
+	@ArgName("sources")
+	@ArgDescription("enable download of the sources")
+	public boolean sources = false;
+	
+	
+	
+	
 	private static final String MAVEN_METADATA = "maven-metadata.xml";
 
 	Artifactory findArtifactory(final List<Artifactory> artefactories, final String name) {
@@ -83,6 +103,10 @@ public class DependencySync  {
 	}
 	
 
+	// this is the maven module interface
+	public String getPomFileName(final Dependency dependency, final String version) {
+		return dependency.name() + "-" + version + ".pom";
+	}
 	public String getJavaPackageFileName(final Dependency dependency, final String version) {
 		return dependency.name() + "-" + version + ".jar";
 	}
@@ -92,10 +116,109 @@ public class DependencySync  {
 	public String getJavaSourceFileName(final Dependency dependency, final String version) {
 		return dependency.name() + "-" + version + "-sources.jar";
 	}
+	// This is the gradle module interface
+	public String getModuleFileName(final Dependency dependency, final String version) {
+		return dependency.name() + "-" + version + ".module";
+	}
+	private class UpdateDependency {
+		public Dependency dependency;
+		public List<String> download = new ArrayList<>();
+	}
 	
+	private void downloadPackage(final List<Artifactory> artefactories, final Dependency dependency, final List<UpdateDependency> alreadyDone) throws Exception {
+		Log.debug("download : " + dependency);
+		for (UpdateDependency elem : alreadyDone) {
+			if (elem.dependency.org().equals(dependency.org())
+					 && elem.dependency.name().equals(dependency.name())
+					 && elem.dependency.remote().equals(dependency.remote())) {
+				// find element ==> check version
+				for (String version : elem.download) {
+					if (version.equals(dependency.revision())) {
+						// already download ==> skip ...
+						return;
+					}
+				}
+			}
+		}
+		
+		String metadataFile = createMavenMetadataRemoteFileName(artefactories, dependency);
+		Log.print("Metadata position: " + metadataFile);
+		Path localPath = createMetadataLocalFileName(artefactories, dependency);
+		String dataAsString = readAllMatadataAndStore(metadataFile, localPath);
+		Log.print("dataAsString: " + dataAsString);
+		MavenMetadata metaData = null;
+		try {
+			metaData = Exml.parseOne(dataAsString, MavenMetadata.class, "metadata");
+		} catch (ExmlException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		Log.print("metaData=" + metaData);
+		
+		// TODO : check if the current file is newer....
+		
+		// TODO : Check if the version is download
+		
+		String lastReleaseTag = metaData.versioning().release();
+		
+		String remotePakageFileName = createBasePkgRemoteFileName(artefactories, dependency, lastReleaseTag);
+		Path localPackageFileName = createBasePkgLocalFileName(artefactories, dependency, lastReleaseTag);
+		// pom sources
+		String base = getPomFileName(dependency, lastReleaseTag);
+		readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
+		PomMaven mavenPom = null;
+		try {
+			mavenPom = Exml.parseOne(localPackageFileName.resolve(base), PomMaven.class, "project");
+		} catch (ExmlException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		// retrieve binary
+		base = getJavaPackageFileName(dependency, lastReleaseTag);
+		readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
+		// listFiles(localPackageFileName.resolve(base));
+		// retrieve javadoc
+		if (this.documentation) {
+			base = getJavaDocFileName(dependency, lastReleaseTag);
+			readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
+		}
+		// retrieve sources
+		if (this.sources) {
+			base = getJavaSourceFileName(dependency, lastReleaseTag);
+			readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
+		}
+		// module sources ==> for gradle ==> no need...
+		/*
+		base = getModuleFileName(dependency, lastReleaseTag);
+		readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
+		*/
+		boolean find = false;
+		for (UpdateDependency elem : alreadyDone) {
+			if (elem.dependency.org().equals(dependency.org())
+					 && elem.dependency.name().equals(dependency.name())
+					 && elem.dependency.remote().equals(dependency.remote())) {
+				// find element ==> check version
+				elem.download.add(dependency.revision());
+				find = true;
+				break;
+			}
+		}
+		if (!find ) {
+			UpdateDependency tmpp = new UpdateDependency();
+			tmpp.dependency = dependency;
+			tmpp.download.add(dependency.revision());
+			alreadyDone.add(tmpp);
+		}
+		if (mavenPom != null && mavenPom.dependencies() != null) {
+			for ( PomDependency value : mavenPom.dependencies()) {
+				Dependency dependencyTmp = new Dependency(value.groupId(), value.artifactId(), value.version(), dependency.remote());
+				downloadPackage(artefactories, dependencyTmp, alreadyDone);
+			}
+		}
+	}
 	@ArgExecute 
 	public void execute() throws ActionException, Exception {
-
+		List<UpdateDependency> alreadyDone = new ArrayList<>();
 		// check system is OK
 		Manifest.checkIsInit();
 		ConfigManifest configuration = Config.getUniqueConfig();
@@ -118,38 +241,8 @@ public class DependencySync  {
 			id_element++;
 			String base_display = Tools.getListBaseDisplay(id_element, dependencies.size(), elem);
 			Log.info("dep-sync : " + base_display);
-			Tools.waitForServerIfNeeded();
 			Log.debug("elem : " + elem);
-			String metadataFile = createMavenMetadataRemoteFileName(artefactories, elem);
+			downloadPackage(artefactories, elem, alreadyDone);
-			Log.print("Metadata position: " + metadataFile);
-			Path localPath = createMetadataLocalFileName(artefactories, elem);
-			String dataAsString = readAllMatadataAndStore(metadataFile, localPath);
-			MavenMetadata metaData = null;
-			try {
-				metaData = Exml.parseOne(dataAsString, MavenMetadata.class, "metadata");
-			} catch (ExmlException e) {
-				// TODO Auto-generated catch block
-				e.printStackTrace();
-			}
-			Log.print("metaData=" + metaData);
-			
-			// TODO : check if the curent file is newer....
-			
-			// TODO : Check if the version is downloaded
-			
-			String lastReleaseTag = metaData.versioning().release();
-			
-			String remotePakageFileName = createBasePkgRemoteFileName(artefactories, elem, lastReleaseTag);
-			Path localPackageFileName = createBasePkgLocalFileName(artefactories, elem, lastReleaseTag);
-			// retrieve binary
-			String base = getJavaPackageFileName(elem, lastReleaseTag);
-			readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
-			// retrieve javadoc
-			base = getJavaDocFileName(elem, lastReleaseTag);
-			readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
-			// retrieve sources
-			base = getJavaSourceFileName(elem, lastReleaseTag);
-			readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
 		}
 		Log.print("Dependency sync END");
 	}
@@ -176,7 +269,13 @@ public class DependencySync  {
 		Tools.fileWriteData(path, dataAsString);
 		return dataAsString;
 	}
+	
 	void readRemoteFileAndStore(final String urlPath, final Path path, final String baseName) throws Exception {
+		if (Files.exists(path)) {
+			Log.debug("File already download : " + baseName);
+			return;
+		}
+		try {
 			if (!Files.exists(path.getParent())) {
 				File file = path.getParent().toFile();
 				if (!file.mkdirs()) {
@@ -189,21 +288,46 @@ public class DependencySync  {
 			con.setConnectTimeout(5000);
 			con.setReadTimeout(5000);
 			con.setInstanceFollowRedirects(false);
-		FileOutputStream outputStream = new FileOutputStream(path.toFile());
 			final int BUFFER_SIZE = 1024*1024;
 			StringBuilder result = new StringBuilder();
 			InputStream inputStream = con.getInputStream();
+			Path tmpFile = path.getParent().resolve(path.getFileName()+"__tmp");
+			FileOutputStream outputStream = new FileOutputStream(tmpFile.toFile());
 	        int bytesRead = -1;
 	        int totalRead = 0;
 	        byte[] buffer = new byte[BUFFER_SIZE];
 	        while ((bytesRead = inputStream.read(buffer)) != -1) {
 	            outputStream.write(buffer, 0, bytesRead);
 	            totalRead += bytesRead;
-            System.out.print("Download: " + baseName + " =" + totalRead + " B\r");
+	            //System.out.print("Download: " + baseName + " =" + totalRead + " B\r");
 	        }
 	        System.out.print("Download: " + baseName + " =" + totalRead + " B\n");
 	        outputStream.close();
 	        inputStream.close();
+	        // this is the way to have atomic replacement of file and not partial downloaded file ==> move is 99.999999% atomic.
+	        Files.move(tmpFile, path);
+		} catch (FileNotFoundException ex) {
+			Log.warning("File not found: " + urlPath);
+		} finally {
+			
+		}
+	}
+	
+	void listFiles(final Path path) {
+		Log.info("List of file in " + path);
+		try {
+			ZipFile zipFile = new ZipFile(path.toFile());
+		
+		    Enumeration<? extends ZipEntry> entries = zipFile.entries();
+		    while(entries.hasMoreElements()){
+		        ZipEntry entry = entries.nextElement();
+		        Log.info("    - " + entry.getName() + " (" + entry.getSize() + ")");
+		        //InputStream stream = zipFile.getInputStream(entry);
+		    }
+		} catch (IOException ex) {
+			Log.error("Catch Exception : " + ex.getLocalizedMessage());
+			ex.printStackTrace();
+		}
 	}
 }
 

src/org/atriasoft/island/model/maven/MavenMetadata.java

@@ -1,9 +1,11 @@
 package org.atriasoft.island.model.maven;
 
 import org.atriasoft.exml.annotation.XmlDefaultAttibute;
+import org.atriasoft.exml.annotation.XmlIgnoreUnknow;
 import org.atriasoft.exml.annotation.XmlName;
 
 @XmlDefaultAttibute(false)
+@XmlIgnoreUnknow
 public record MavenMetadata(
 	@XmlName("groupId") String groupId,
 	@XmlName("artifactId") String artifactId,

src/org/atriasoft/island/model/maven/PomDependency.java

@@ -0,0 +1,16 @@
+package org.atriasoft.island.model.maven;
+
+import org.atriasoft.exml.annotation.XmlDefaultNullValue;
+import org.atriasoft.exml.annotation.XmlIgnoreUnknow;
+import org.atriasoft.exml.annotation.XmlName;
+import org.atriasoft.exml.annotation.XmlOptional;
+
+@XmlDefaultNullValue
+@XmlIgnoreUnknow
+public record PomDependency(
+		@XmlName("groupId") String groupId,
+		@XmlName("artifactId") String artifactId,
+		@XmlOptional @XmlName("version") String version,
+		@XmlOptional @XmlName("scope") String scope) {
+	
+}

src/org/atriasoft/island/model/maven/PomMaven.java

@@ -0,0 +1,22 @@
+package org.atriasoft.island.model.maven;
+
+import java.util.List;
+
+import org.atriasoft.exml.annotation.XmlDefaultNullValue;
+import org.atriasoft.exml.annotation.XmlIgnoreUnknow;
+import org.atriasoft.exml.annotation.XmlList;
+import org.atriasoft.exml.annotation.XmlName;
+import org.atriasoft.exml.annotation.XmlOptional;
+
+@XmlDefaultNullValue
+@XmlIgnoreUnknow
+public record PomMaven (
+	@XmlName("modelVersion") String modelVersion,
+	@XmlName("groupId") String groupId,
+	@XmlName("artifactId") String artifactId,
+	@XmlName("version") String version,
+	@XmlOptional @XmlName("name") String name,
+	@XmlOptional @XmlName("description") String description,
+	@XmlOptional @XmlName("url") String url,
+	@XmlOptional @XmlName("dependencies") @XmlList(value="dependency") List<PomDependency> dependencies) {
+}