jatria-soft/island
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[DEV] Download maven dependency

Browse Source
This commit is contained in:
Edouard DUPIN 2021-10-09 08:25:41 +02:00
parent aefb938621
commit 1e1b4b723e
5 changed files with 234 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/org/atriasoft/island/Tools.java
View File

@ -101,6 +101,8 @@ public class Tools {
}
public static void fileWriteData(final Path path, final String data) {
Path tmpFile = path.getParent().resolve(path.getFileName()+"__tmp");
if (!Files.exists(path.getParent())) {
File file = path.getParent().toFile();
if (!file.mkdirs()) {
@ -108,8 +110,16 @@ public class Tools {
}
}
try {
Files.writeString(path, data);
Files.writeString(tmpFile, data);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// this is the way to have atomic replacement of file and not partial downloaded file ==> move is 99.999999% atomic.
try {
Files.move(tmpFile, path);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

242
src/org/atriasoft/island/actions/DependencySync.java
View File

@ -2,18 +2,25 @@ package org.atriasoft.island.actions;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.atriasoft.death.annotation.ArgCommand;
import org.atriasoft.death.annotation.ArgDescription;
import org.atriasoft.death.annotation.ArgExecute;
import org.atriasoft.death.annotation.ArgName;
import org.atriasoft.death.annotation.ArgSample;
import org.atriasoft.exml.Exml;
import org.atriasoft.exml.exception.ExmlException;
@ -27,12 +34,25 @@ import org.atriasoft.island.model.manifest.Artifactory;
import org.atriasoft.island.model.manifest.ConfigManifest;
import org.atriasoft.island.model.manifest.Dependency;
import org.atriasoft.island.model.maven.MavenMetadata;
import org.atriasoft.island.model.maven.PomDependency;
import org.atriasoft.island.model.maven.PomMaven;
@ArgCommand("depend-sync")
@ArgDescription("Syncronize all the dependencies referenced")
@ArgSample("depend-sync")
public class DependencySync {
@ArgName("doc")
@ArgDescription("enable download of the documentation")
public boolean documentation = false;
@ArgName("sources")
@ArgDescription("enable download of the sources")
public boolean sources = false;
private static final String MAVEN_METADATA = "maven-metadata.xml";
Artifactory findArtifactory(final List<Artifactory> artefactories, final String name) {
@ -82,7 +102,11 @@ public class DependencySync {
return outPath;
}
// this is the maven module interface
public String getPomFileName(final Dependency dependency, final String version) {
return dependency.name() + "-" + version + ".pom";
}
public String getJavaPackageFileName(final Dependency dependency, final String version) {
return dependency.name() + "-" + version + ".jar";
}
@ -92,10 +116,109 @@ public class DependencySync {
public String getJavaSourceFileName(final Dependency dependency, final String version) {
return dependency.name() + "-" + version + "-sources.jar";
}
// This is the gradle module interface
public String getModuleFileName(final Dependency dependency, final String version) {
return dependency.name() + "-" + version + ".module";
}
private class UpdateDependency {
public Dependency dependency;
public List<String> download = new ArrayList<>();
}
private void downloadPackage(final List<Artifactory> artefactories, final Dependency dependency, final List<UpdateDependency> alreadyDone) throws Exception {
Log.debug("download : " + dependency);
for (UpdateDependency elem : alreadyDone) {
if (elem.dependency.org().equals(dependency.org())
&& elem.dependency.name().equals(dependency.name())
&& elem.dependency.remote().equals(dependency.remote())) {
// find element ==> check version
for (String version : elem.download) {
if (version.equals(dependency.revision())) {
// already download ==> skip ...
return;
}
}
}
}
String metadataFile = createMavenMetadataRemoteFileName(artefactories, dependency);
Log.print("Metadata position: " + metadataFile);
Path localPath = createMetadataLocalFileName(artefactories, dependency);
String dataAsString = readAllMatadataAndStore(metadataFile, localPath);
Log.print("dataAsString: " + dataAsString);
MavenMetadata metaData = null;
try {
metaData = Exml.parseOne(dataAsString, MavenMetadata.class, "metadata");
} catch (ExmlException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Log.print("metaData=" + metaData);
// TODO : check if the current file is newer....
// TODO : Check if the version is download
String lastReleaseTag = metaData.versioning().release();
String remotePakageFileName = createBasePkgRemoteFileName(artefactories, dependency, lastReleaseTag);
Path localPackageFileName = createBasePkgLocalFileName(artefactories, dependency, lastReleaseTag);
// pom sources
String base = getPomFileName(dependency, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
PomMaven mavenPom = null;
try {
mavenPom = Exml.parseOne(localPackageFileName.resolve(base), PomMaven.class, "project");
} catch (ExmlException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// retrieve binary
base = getJavaPackageFileName(dependency, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
// listFiles(localPackageFileName.resolve(base));
// retrieve javadoc
if (this.documentation) {
base = getJavaDocFileName(dependency, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
}
// retrieve sources
if (this.sources) {
base = getJavaSourceFileName(dependency, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
}
// module sources ==> for gradle ==> no need...
/*
base = getModuleFileName(dependency, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
*/
boolean find = false;
for (UpdateDependency elem : alreadyDone) {
if (elem.dependency.org().equals(dependency.org())
&& elem.dependency.name().equals(dependency.name())
&& elem.dependency.remote().equals(dependency.remote())) {
// find element ==> check version
elem.download.add(dependency.revision());
find = true;
break;
}
}
if (!find ) {
UpdateDependency tmpp = new UpdateDependency();
tmpp.dependency = dependency;
tmpp.download.add(dependency.revision());
alreadyDone.add(tmpp);
}
if (mavenPom != null && mavenPom.dependencies() != null) {
for ( PomDependency value : mavenPom.dependencies()) {
Dependency dependencyTmp = new Dependency(value.groupId(), value.artifactId(), value.version(), dependency.remote());
downloadPackage(artefactories, dependencyTmp, alreadyDone);
}
}
}
@ArgExecute
public void execute() throws ActionException, Exception {
List<UpdateDependency> alreadyDone = new ArrayList<>();
// check system is OK
Manifest.checkIsInit();
ConfigManifest configuration = Config.getUniqueConfig();
@ -118,38 +241,8 @@ public class DependencySync {
id_element++;
String base_display = Tools.getListBaseDisplay(id_element, dependencies.size(), elem);
Log.info("dep-sync : " + base_display);
Tools.waitForServerIfNeeded();
Log.debug("elem : " + elem);
String metadataFile = createMavenMetadataRemoteFileName(artefactories, elem);
Log.print("Metadata position: " + metadataFile);
Path localPath = createMetadataLocalFileName(artefactories, elem);
String dataAsString = readAllMatadataAndStore(metadataFile, localPath);
MavenMetadata metaData = null;
try {
metaData = Exml.parseOne(dataAsString, MavenMetadata.class, "metadata");
} catch (ExmlException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Log.print("metaData=" + metaData);
// TODO : check if the curent file is newer....
// TODO : Check if the version is downloaded
String lastReleaseTag = metaData.versioning().release();
String remotePakageFileName = createBasePkgRemoteFileName(artefactories, elem, lastReleaseTag);
Path localPackageFileName = createBasePkgLocalFileName(artefactories, elem, lastReleaseTag);
// retrieve binary
String base = getJavaPackageFileName(elem, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
// retrieve javadoc
base = getJavaDocFileName(elem, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
// retrieve sources
base = getJavaSourceFileName(elem, lastReleaseTag);
readRemoteFileAndStore(remotePakageFileName + "/" + base, localPackageFileName.resolve(base), base);
downloadPackage(artefactories, elem, alreadyDone);
}
Log.print("Dependency sync END");
}
@ -176,34 +269,65 @@ public class DependencySync {
Tools.fileWriteData(path, dataAsString);
return dataAsString;
}
void readRemoteFileAndStore(final String urlPath, final Path path, final String baseName) throws Exception {
if (!Files.exists(path.getParent())) {
File file = path.getParent().toFile();
if (!file.mkdirs()) {
Log.critical("Can not create the path:" + path.getParent());
}
if (Files.exists(path)) {
Log.debug("File already download : " + baseName);
return;
}
try {
if (!Files.exists(path.getParent())) {
File file = path.getParent().toFile();
if (!file.mkdirs()) {
Log.critical("Can not create the path:" + path.getParent());
}
}
URL url = new URL(urlPath);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("GET");
con.setConnectTimeout(5000);
con.setReadTimeout(5000);
con.setInstanceFollowRedirects(false);
final int BUFFER_SIZE = 1024*1024;
StringBuilder result = new StringBuilder();
InputStream inputStream = con.getInputStream();
Path tmpFile = path.getParent().resolve(path.getFileName()+"__tmp");
FileOutputStream outputStream = new FileOutputStream(tmpFile.toFile());
int bytesRead = -1;
int totalRead = 0;
byte[] buffer = new byte[BUFFER_SIZE];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
totalRead += bytesRead;
//System.out.print("Download: " + baseName + " =" + totalRead + " B\r");
}
System.out.print("Download: " + baseName + " =" + totalRead + " B\n");
outputStream.close();
inputStream.close();
// this is the way to have atomic replacement of file and not partial downloaded file ==> move is 99.999999% atomic.
Files.move(tmpFile, path);
} catch (FileNotFoundException ex) {
Log.warning("File not found: " + urlPath);
} finally {
}
}
void listFiles(final Path path) {
Log.info("List of file in " + path);
try {
ZipFile zipFile = new ZipFile(path.toFile());
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while(entries.hasMoreElements()){
ZipEntry entry = entries.nextElement();
Log.info(" - " + entry.getName() + " (" + entry.getSize() + ")");
//InputStream stream = zipFile.getInputStream(entry);
}
} catch (IOException ex) {
Log.error("Catch Exception : " + ex.getLocalizedMessage());
ex.printStackTrace();
}
URL url = new URL(urlPath);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("GET");
con.setConnectTimeout(5000);
con.setReadTimeout(5000);
con.setInstanceFollowRedirects(false);
FileOutputStream outputStream = new FileOutputStream(path.toFile());
final int BUFFER_SIZE = 1024*1024;
StringBuilder result = new StringBuilder();
InputStream inputStream = con.getInputStream();
int bytesRead = -1;
int totalRead = 0;
byte[] buffer = new byte[BUFFER_SIZE];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
totalRead += bytesRead;
System.out.print("Download: " + baseName + " =" + totalRead + " B\r");
}
System.out.print("Download: " + baseName + " =" + totalRead + " B\n");
outputStream.close();
inputStream.close();
}
}

2
src/org/atriasoft/island/model/maven/MavenMetadata.java
View File

@ -1,9 +1,11 @@
package org.atriasoft.island.model.maven;
import org.atriasoft.exml.annotation.XmlDefaultAttibute;
import org.atriasoft.exml.annotation.XmlIgnoreUnknow;
import org.atriasoft.exml.annotation.XmlName;
@XmlDefaultAttibute(false)
@XmlIgnoreUnknow
public record MavenMetadata(
@XmlName("groupId") String groupId,
@XmlName("artifactId") String artifactId,

16
src/org/atriasoft/island/model/maven/PomDependency.java Normal file
View File

@ -0,0 +1,16 @@
package org.atriasoft.island.model.maven;
import org.atriasoft.exml.annotation.XmlDefaultNullValue;
import org.atriasoft.exml.annotation.XmlIgnoreUnknow;
import org.atriasoft.exml.annotation.XmlName;
import org.atriasoft.exml.annotation.XmlOptional;
@XmlDefaultNullValue
@XmlIgnoreUnknow
public record PomDependency(
@XmlName("groupId") String groupId,
@XmlName("artifactId") String artifactId,
@XmlOptional @XmlName("version") String version,
@XmlOptional @XmlName("scope") String scope) {
}

22
src/org/atriasoft/island/model/maven/PomMaven.java Normal file
View File

@ -0,0 +1,22 @@
package org.atriasoft.island.model.maven;
import java.util.List;
import org.atriasoft.exml.annotation.XmlDefaultNullValue;
import org.atriasoft.exml.annotation.XmlIgnoreUnknow;
import org.atriasoft.exml.annotation.XmlList;
import org.atriasoft.exml.annotation.XmlName;
import org.atriasoft.exml.annotation.XmlOptional;
@XmlDefaultNullValue
@XmlIgnoreUnknow
public record PomMaven (
@XmlName("modelVersion") String modelVersion,
@XmlName("groupId") String groupId,
@XmlName("artifactId") String artifactId,
@XmlName("version") String version,
@XmlOptional @XmlName("name") String name,
@XmlOptional @XmlName("description") String description,
@XmlOptional @XmlName("url") String url,
@XmlOptional @XmlName("dependencies") @XmlList(value="dependency") List<PomDependency> dependencies) {
}