From 1e1b4b723e1175fa0a1ec2790ceaa89fc7ad2b88 Mon Sep 17 00:00:00 2001 From: Edouard DUPIN Date: Sat, 9 Oct 2021 08:25:41 +0200 Subject: [PATCH] [DEV] Download maven dependency --- src/org/atriasoft/island/Tools.java | 12 +- .../island/actions/DependencySync.java | 242 +++++++++++++----- .../island/model/maven/MavenMetadata.java | 2 + .../island/model/maven/PomDependency.java | 16 ++ .../island/model/maven/PomMaven.java | 22 ++ 5 files changed, 234 insertions(+), 60 deletions(-) create mode 100644 src/org/atriasoft/island/model/maven/PomDependency.java create mode 100644 src/org/atriasoft/island/model/maven/PomMaven.java diff --git a/src/org/atriasoft/island/Tools.java b/src/org/atriasoft/island/Tools.java index 101262a..f19be86 100644 --- a/src/org/atriasoft/island/Tools.java +++ b/src/org/atriasoft/island/Tools.java @@ -101,6 +101,8 @@ public class Tools { } public static void fileWriteData(final Path path, final String data) { + + Path tmpFile = path.getParent().resolve(path.getFileName()+"__tmp"); if (!Files.exists(path.getParent())) { File file = path.getParent().toFile(); if (!file.mkdirs()) { @@ -108,8 +110,16 @@ public class Tools { } } try { - Files.writeString(path, data); + Files.writeString(tmpFile, data); } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + // this is the way to have atomic replacement of file and not partial downloaded file ==> move is 99.999999% atomic. + try { + Files.move(tmpFile, path); + } catch (IOException e) { + // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/src/org/atriasoft/island/actions/DependencySync.java b/src/org/atriasoft/island/actions/DependencySync.java index 8ed9621..0853b9d 100644 --- a/src/org/atriasoft/island/actions/DependencySync.java +++ b/src/org/atriasoft/island/actions/DependencySync.java @@ -2,18 +2,25 @@ package org.atriasoft.island.actions; import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Enumeration; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import org.atriasoft.death.annotation.ArgCommand; import org.atriasoft.death.annotation.ArgDescription; import org.atriasoft.death.annotation.ArgExecute; +import org.atriasoft.death.annotation.ArgName; import org.atriasoft.death.annotation.ArgSample; import org.atriasoft.exml.Exml; import org.atriasoft.exml.exception.ExmlException; @@ -27,12 +34,25 @@ import org.atriasoft.island.model.manifest.Artifactory; import org.atriasoft.island.model.manifest.ConfigManifest; import org.atriasoft.island.model.manifest.Dependency; import org.atriasoft.island.model.maven.MavenMetadata; +import org.atriasoft.island.model.maven.PomDependency; +import org.atriasoft.island.model.maven.PomMaven; @ArgCommand("depend-sync") @ArgDescription("Syncronize all the dependencies referenced") @ArgSample("depend-sync") public class DependencySync { + @ArgName("doc") + @ArgDescription("enable download of the documentation") + public boolean documentation = false; + + @ArgName("sources") + @ArgDescription("enable download of the sources") + public boolean sources = false; + + + + private static final String MAVEN_METADATA = "maven-metadata.xml"; Artifactory findArtifactory(final List artefactories, final String name) { @@ -82,7 +102,11 @@ public class DependencySync { return outPath; } - + + // this is the maven module interface + public String getPomFileName(final Dependency dependency, final String version) { + return dependency.name() + "-" + version + ".pom"; + } public String getJavaPackageFileName(final Dependency dependency, final String version) { return dependency.name() + "-" + version + ".jar"; } @@ -92,10 +116,109 @@ public class DependencySync { public String getJavaSourceFileName(final Dependency dependency, final String version) { return dependency.name() + "-" + version + "-sources.jar"; } + // This is the gradle module interface + public String getModuleFileName(final Dependency dependency, final String version) { + return dependency.name() + "-" + version + ".module"; + } + private class UpdateDependency { + public Dependency dependency; + public List download = new ArrayList<>(); + } + private void downloadPackage(final List artefactories, final Dependency dependency, final List alreadyDone) throws Exception { + Log.debug("download : " + dependency); + for (UpdateDependency elem : alreadyDone) { + if (elem.dependency.org().equals(dependency.org()) + && elem.dependency.name().equals(dependency.name()) + && elem.dependency.remote().equals(dependency.remote())) { + // find element ==> check version + for (String version : elem.download) { + if (version.equals(dependency.revision())) { + // already download ==> skip ... + return; + } + } + } + } + + String metadataFile = createMavenMetadataRemoteFileName(artefactories, dependency); + Log.print("Metadata position: " + metadataFile); + Path localPath = createMetadataLocalFileName(artefactories, dependency); + String dataAsString = readAllMatadataAndStore(metadataFile, localPath); + Log.print("dataAsString: " + dataAsString); + MavenMetadata metaData = null; + try { + metaData = Exml.parseOne(dataAsString, MavenMetadata.class, "metadata"); + } catch (ExmlException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + Log.print("metaData=" + metaData); + + // TODO : check if the current file is newer.... + + // TODO : Check if the version is download + + String lastReleaseTag = metaData.versioning().release(); + + String remotePakageFileName = createBasePkgRemoteFileName(artefactories, dependency, lastReleaseTag); + Path localPackageFileName = createBasePkgLocalFileName(artefactories, dependency, lastReleaseTag); + // pom sources + String base = getPomFileName(dependency, lastReleaseTag); + readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); + PomMaven mavenPom = null; + try { + mavenPom = Exml.parseOne(localPackageFileName.resolve(base), PomMaven.class, "project"); + } catch (ExmlException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + // retrieve binary + base = getJavaPackageFileName(dependency, lastReleaseTag); + readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); + // listFiles(localPackageFileName.resolve(base)); + // retrieve javadoc + if (this.documentation) { + base = getJavaDocFileName(dependency, lastReleaseTag); + readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); + } + // retrieve sources + if (this.sources) { + base = getJavaSourceFileName(dependency, lastReleaseTag); + readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); + } + // module sources ==> for gradle ==> no need... + /* + base = getModuleFileName(dependency, lastReleaseTag); + readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); + */ + boolean find = false; + for (UpdateDependency elem : alreadyDone) { + if (elem.dependency.org().equals(dependency.org()) + && elem.dependency.name().equals(dependency.name()) + && elem.dependency.remote().equals(dependency.remote())) { + // find element ==> check version + elem.download.add(dependency.revision()); + find = true; + break; + } + } + if (!find ) { + UpdateDependency tmpp = new UpdateDependency(); + tmpp.dependency = dependency; + tmpp.download.add(dependency.revision()); + alreadyDone.add(tmpp); + } + if (mavenPom != null && mavenPom.dependencies() != null) { + for ( PomDependency value : mavenPom.dependencies()) { + Dependency dependencyTmp = new Dependency(value.groupId(), value.artifactId(), value.version(), dependency.remote()); + downloadPackage(artefactories, dependencyTmp, alreadyDone); + } + } + } @ArgExecute public void execute() throws ActionException, Exception { - + List alreadyDone = new ArrayList<>(); // check system is OK Manifest.checkIsInit(); ConfigManifest configuration = Config.getUniqueConfig(); @@ -118,38 +241,8 @@ public class DependencySync { id_element++; String base_display = Tools.getListBaseDisplay(id_element, dependencies.size(), elem); Log.info("dep-sync : " + base_display); - Tools.waitForServerIfNeeded(); Log.debug("elem : " + elem); - String metadataFile = createMavenMetadataRemoteFileName(artefactories, elem); - Log.print("Metadata position: " + metadataFile); - Path localPath = createMetadataLocalFileName(artefactories, elem); - String dataAsString = readAllMatadataAndStore(metadataFile, localPath); - MavenMetadata metaData = null; - try { - metaData = Exml.parseOne(dataAsString, MavenMetadata.class, "metadata"); - } catch (ExmlException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - Log.print("metaData=" + metaData); - - // TODO : check if the curent file is newer.... - - // TODO : Check if the version is downloaded - - String lastReleaseTag = metaData.versioning().release(); - - String remotePakageFileName = createBasePkgRemoteFileName(artefactories, elem, lastReleaseTag); - Path localPackageFileName = createBasePkgLocalFileName(artefactories, elem, lastReleaseTag); - // retrieve binary - String base = getJavaPackageFileName(elem, lastReleaseTag); - readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); - // retrieve javadoc - base = getJavaDocFileName(elem, lastReleaseTag); - readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); - // retrieve sources - base = getJavaSourceFileName(elem, lastReleaseTag); - readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base); + downloadPackage(artefactories, elem, alreadyDone); } Log.print("Dependency sync END"); } @@ -176,34 +269,65 @@ public class DependencySync { Tools.fileWriteData(path, dataAsString); return dataAsString; } + void readRemoteFileAndStore(final String urlPath, final Path path, final String baseName) throws Exception { - if (!Files.exists(path.getParent())) { - File file = path.getParent().toFile(); - if (!file.mkdirs()) { - Log.critical("Can not create the path:" + path.getParent()); - } + if (Files.exists(path)) { + Log.debug("File already download : " + baseName); + return; + } + try { + if (!Files.exists(path.getParent())) { + File file = path.getParent().toFile(); + if (!file.mkdirs()) { + Log.critical("Can not create the path:" + path.getParent()); + } + } + URL url = new URL(urlPath); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); + con.setConnectTimeout(5000); + con.setReadTimeout(5000); + con.setInstanceFollowRedirects(false); + final int BUFFER_SIZE = 1024*1024; + StringBuilder result = new StringBuilder(); + InputStream inputStream = con.getInputStream(); + Path tmpFile = path.getParent().resolve(path.getFileName()+"__tmp"); + FileOutputStream outputStream = new FileOutputStream(tmpFile.toFile()); + int bytesRead = -1; + int totalRead = 0; + byte[] buffer = new byte[BUFFER_SIZE]; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + totalRead += bytesRead; + //System.out.print("Download: " + baseName + " =" + totalRead + " B\r"); + } + System.out.print("Download: " + baseName + " =" + totalRead + " B\n"); + outputStream.close(); + inputStream.close(); + // this is the way to have atomic replacement of file and not partial downloaded file ==> move is 99.999999% atomic. + Files.move(tmpFile, path); + } catch (FileNotFoundException ex) { + Log.warning("File not found: " + urlPath); + } finally { + + } + } + + void listFiles(final Path path) { + Log.info("List of file in " + path); + try { + ZipFile zipFile = new ZipFile(path.toFile()); + + Enumeration entries = zipFile.entries(); + while(entries.hasMoreElements()){ + ZipEntry entry = entries.nextElement(); + Log.info(" - " + entry.getName() + " (" + entry.getSize() + ")"); + //InputStream stream = zipFile.getInputStream(entry); + } + } catch (IOException ex) { + Log.error("Catch Exception : " + ex.getLocalizedMessage()); + ex.printStackTrace(); } - URL url = new URL(urlPath); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - con.setConnectTimeout(5000); - con.setReadTimeout(5000); - con.setInstanceFollowRedirects(false); - FileOutputStream outputStream = new FileOutputStream(path.toFile()); - final int BUFFER_SIZE = 1024*1024; - StringBuilder result = new StringBuilder(); - InputStream inputStream = con.getInputStream(); - int bytesRead = -1; - int totalRead = 0; - byte[] buffer = new byte[BUFFER_SIZE]; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - totalRead += bytesRead; - System.out.print("Download: " + baseName + " =" + totalRead + " B\r"); - } - System.out.print("Download: " + baseName + " =" + totalRead + " B\n"); - outputStream.close(); - inputStream.close(); } } diff --git a/src/org/atriasoft/island/model/maven/MavenMetadata.java b/src/org/atriasoft/island/model/maven/MavenMetadata.java index 4ec8862..9255ec7 100644 --- a/src/org/atriasoft/island/model/maven/MavenMetadata.java +++ b/src/org/atriasoft/island/model/maven/MavenMetadata.java @@ -1,9 +1,11 @@ package org.atriasoft.island.model.maven; import org.atriasoft.exml.annotation.XmlDefaultAttibute; +import org.atriasoft.exml.annotation.XmlIgnoreUnknow; import org.atriasoft.exml.annotation.XmlName; @XmlDefaultAttibute(false) +@XmlIgnoreUnknow public record MavenMetadata( @XmlName("groupId") String groupId, @XmlName("artifactId") String artifactId, diff --git a/src/org/atriasoft/island/model/maven/PomDependency.java b/src/org/atriasoft/island/model/maven/PomDependency.java new file mode 100644 index 0000000..2340f0d --- /dev/null +++ b/src/org/atriasoft/island/model/maven/PomDependency.java @@ -0,0 +1,16 @@ +package org.atriasoft.island.model.maven; + +import org.atriasoft.exml.annotation.XmlDefaultNullValue; +import org.atriasoft.exml.annotation.XmlIgnoreUnknow; +import org.atriasoft.exml.annotation.XmlName; +import org.atriasoft.exml.annotation.XmlOptional; + +@XmlDefaultNullValue +@XmlIgnoreUnknow +public record PomDependency( + @XmlName("groupId") String groupId, + @XmlName("artifactId") String artifactId, + @XmlOptional @XmlName("version") String version, + @XmlOptional @XmlName("scope") String scope) { + +} diff --git a/src/org/atriasoft/island/model/maven/PomMaven.java b/src/org/atriasoft/island/model/maven/PomMaven.java new file mode 100644 index 0000000..3ca3059 --- /dev/null +++ b/src/org/atriasoft/island/model/maven/PomMaven.java @@ -0,0 +1,22 @@ +package org.atriasoft.island.model.maven; + +import java.util.List; + +import org.atriasoft.exml.annotation.XmlDefaultNullValue; +import org.atriasoft.exml.annotation.XmlIgnoreUnknow; +import org.atriasoft.exml.annotation.XmlList; +import org.atriasoft.exml.annotation.XmlName; +import org.atriasoft.exml.annotation.XmlOptional; + +@XmlDefaultNullValue +@XmlIgnoreUnknow +public record PomMaven ( + @XmlName("modelVersion") String modelVersion, + @XmlName("groupId") String groupId, + @XmlName("artifactId") String artifactId, + @XmlName("version") String version, + @XmlOptional @XmlName("name") String name, + @XmlOptional @XmlName("description") String description, + @XmlOptional @XmlName("url") String url, + @XmlOptional @XmlName("dependencies") @XmlList(value="dependency") List dependencies) { +}