Author: tchemit Date: 2014-03-17 11:10:45 +0100 (Mon, 17 Mar 2014) New Revision: 975 Url: http://forge.codelutin.com/projects/echobase/repository/revisions/975 Log: refs-60 #2931 Added: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/CommandLineUtils.java trunk/echobase-services/src/main/resources/rscript/ trunk/echobase-services/src/main/resources/rscript/extractMap.r Modified: trunk/echobase-domain/pom.xml trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfigurationOption.java trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java trunk/echobase-domain/src/main/resources/i18n/echobase-domain_en_GB.properties trunk/echobase-domain/src/main/resources/i18n/echobase-domain_fr_FR.properties trunk/echobase-services/pom.xml trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/embeddedapplication/EmbeddedApplicationService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportMap/ExportMapService.java trunk/echobase-ui/pom.xml trunk/pom.xml Modified: trunk/echobase-domain/pom.xml =================================================================== --- trunk/echobase-domain/pom.xml 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-domain/pom.xml 2014-03-17 10:10:45 UTC (rev 975) @@ -160,6 +160,11 @@ </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </dependency> + + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java 2014-03-17 10:10:45 UTC (rev 975) @@ -180,6 +180,13 @@ return file; } + public File getRscriptExecutablePath() { + File file = applicationConfig.getOptionAsFile( + EchoBaseConfigurationOption.RSCRIPT_EXECTUABLE_PATH.key); + Preconditions.checkNotNull(file); + return file; + } + public File getLogConfigFile() { File file = applicationConfig.getOptionAsFile( EchoBaseConfigurationOption.LOG_CONFIG_FILE.key); Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfigurationOption.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfigurationOption.java 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfigurationOption.java 2014-03-17 10:10:45 UTC (rev 975) @@ -49,6 +49,10 @@ "echobase.internal.db.directory", n("echobase.config.internal.db.directory.description"), "${echobase.data.directory}/internaldb", File.class), + RSCRIPT_EXECTUABLE_PATH( + "echobase.Rscript.executable.path", + n("echobase.config.Rscript.executable.path.description"), + "/usr/bin/Rscript", File.class), EMBEDDED("echobase.embedded", n("echobase.config.embedded.description"), "false", boolean.class), Added: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/CommandLineUtils.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/CommandLineUtils.java (rev 0) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/CommandLineUtils.java 2014-03-17 10:10:45 UTC (rev 975) @@ -0,0 +1,107 @@ +package fr.ifremer.echobase.io; + +/* + * #%L + * EchoBase :: Domain + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import fr.ifremer.echobase.EchoBaseTechnicalException; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.LogOutputStream; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.logging.Log; + +import java.io.File; +import java.io.IOException; + +/** + * Utils method around a command line execution. + * + * @author tchemit <chemit@codelutin.com> + * @since 1.8 + */ +public class CommandLineUtils { + + public static CommandLine newCommand(File exec, Object... args) { + return newCommand(exec.getAbsolutePath(), args); + } + + public static CommandLine newCommand(String exec, Object... args) { + CommandLine result = new CommandLine(exec); + + if (args != null) { + for (Object arg : args) { + String argument; + if (arg instanceof File) { + argument = ((File) arg).getAbsolutePath(); + } else { + argument = arg.toString(); + } + result.addArgument(argument); + } + } + return result; + } + + public static void invokeCommandeLine(Log logger, + CommandLine commandLine, + String errorMsg, + File workDir) { + DefaultExecutor executor = new DefaultExecutor(); + + ExecutorLog executorLog = new ExecutorLog(); + executor.setStreamHandler(new PumpStreamHandler(executorLog)); + + // set resource directory + executor.setWorkingDirectory(workDir); + + // invoke + int exitValue; + try { + if (logger.isInfoEnabled()) { + logger.info("Command line '" + commandLine.toString() + "' will be execute"); + } + exitValue = executor.execute(commandLine); + } catch (IOException eee) { + throw new EchoBaseTechnicalException(errorMsg + "\n" + executorLog.getLog(), eee); + } + + if (logger.isInfoEnabled()) { + logger.info("Command line end with exit with value : " + exitValue); + } + if (exitValue != 0) { + throw new EchoBaseTechnicalException(errorMsg + "\n" + executorLog.getLog()); + } + } + + protected static class ExecutorLog extends LogOutputStream { + + protected StringBuilder executorLog = new StringBuilder(); + + @Override + protected void processLine(String line, int level) { + executorLog.append(line); + } + + public String getLog() { + return executorLog.toString(); + } + } +} Property changes on: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/CommandLineUtils.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java 2014-03-17 10:10:45 UTC (rev 975) @@ -35,11 +35,13 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; +import java.io.OutputStreamWriter; import java.util.Collection; import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; @@ -239,6 +241,44 @@ return sql; } + public static boolean isExecutableFile(File file) { + boolean isExecutableFile = (file != null && file.exists() && file.canExecute() && !file.isDirectory()); + return isExecutableFile; + } + + public static void forceMkdir(File directory) { + try { + FileUtils.forceMkdir(directory); + } catch (IOException e) { + throw new EchoBaseTechnicalException("Could not create directory: " + directory, e); + } + } + + public static void copyResource(String resourcePath, + File outputFile, + boolean executable) throws IOException { + InputStream inputStream = EchoBaseIOUtil.class.getResourceAsStream(resourcePath); + Preconditions.checkNotNull(inputStream, + "could not find resource " + resourcePath); + try { + if (log.isInfoEnabled()) { + log.info("Copy configuration to " + resourcePath + " to " + outputFile); + } + OutputStreamWriter outputStream = new OutputStreamWriter( + new FileOutputStream(outputFile), Charsets.UTF_8); + try { + IOUtils.copy(inputStream, outputStream); + } finally { + outputStream.close(); + } + if (executable) { + outputFile.setExecutable(true); + } + } finally { + inputStream.close(); + } + } + protected EchoBaseIOUtil() { // no instance of helper class } Modified: trunk/echobase-domain/src/main/resources/i18n/echobase-domain_en_GB.properties =================================================================== --- trunk/echobase-domain/src/main/resources/i18n/echobase-domain_en_GB.properties 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-domain/src/main/resources/i18n/echobase-domain_en_GB.properties 2014-03-17 10:10:45 UTC (rev 975) @@ -26,6 +26,7 @@ echobase.common.importType.resultsVoyage.short=R Voyage echobase.common.importType.voyage=Voyage import echobase.common.importType.voyage.short=Complete voyage +echobase.config.Rscript.executable.path.description=Location of the R command echobase.config.csv.separator.description=Csv separator character echobase.config.data.directory.description=Path to application data echobase.config.documentationUrl.description=Where to find EchoBase online documentation Modified: trunk/echobase-domain/src/main/resources/i18n/echobase-domain_fr_FR.properties =================================================================== --- trunk/echobase-domain/src/main/resources/i18n/echobase-domain_fr_FR.properties 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-domain/src/main/resources/i18n/echobase-domain_fr_FR.properties 2014-03-17 10:10:45 UTC (rev 975) @@ -26,6 +26,7 @@ echobase.common.importType.resultsVoyage.short=R Voyage echobase.common.importType.voyage=Import Campagne echobase.common.importType.voyage.short=Campagne complête +echobase.config.Rscript.executable.path.description=Chemin vers l'exécutable R echobase.config.csv.separator.description=Caractère séparateur pour les fichiers csv echobase.config.data.directory.description=Répertoire des données de l'application echobase.config.documentationUrl.description=Où trouver la documentation en ligne d'EchoBase Modified: trunk/echobase-services/pom.xml =================================================================== --- trunk/echobase-services/pom.xml 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-services/pom.xml 2014-03-17 10:10:45 UTC (rev 975) @@ -148,6 +148,11 @@ </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </dependency> + + <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/embeddedapplication/EmbeddedApplicationService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/embeddedapplication/EmbeddedApplicationService.java 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/embeddedapplication/EmbeddedApplicationService.java 2014-03-17 10:10:45 UTC (rev 975) @@ -20,7 +20,6 @@ */ package fr.ifremer.echobase.services.service.embeddedapplication; -import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import fr.ifremer.echobase.EchoBaseTechnicalException; import fr.ifremer.echobase.entities.EchoBaseInternalPersistenceContext; @@ -62,7 +61,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.util.List; /** @@ -336,27 +334,29 @@ File targetDirectory, boolean executable) throws IOException { String resourcePath = EMBEDDED_PATH + resourceName; - InputStream inputStream = getClass().getResourceAsStream(resourcePath); - Preconditions.checkNotNull(inputStream, - "could not find resource " + resourcePath); - try { - File outputFile = new File(targetDirectory, resourceName); - if (log.isInfoEnabled()) { - log.info("Copy configuration to " + resourceName + " to " + outputFile); - } - OutputStreamWriter outputStream = new OutputStreamWriter( - new FileOutputStream(outputFile), Charsets.UTF_8); - try { - IOUtils.copy(inputStream, outputStream); - } finally { - outputStream.close(); - } - if (executable) { - outputFile.setExecutable(true); - } - } finally { - inputStream.close(); - } + File outputFile = new File(targetDirectory, resourceName); + EchoBaseIOUtil.copyResource(resourcePath, outputFile, executable); +// InputStream inputStream = getClass().getResourceAsStream(resourcePath); +// Preconditions.checkNotNull(inputStream, +// "could not find resource " + resourcePath); +// try { +// File outputFile = new File(targetDirectory, resourceName); +// if (log.isInfoEnabled()) { +// log.info("Copy configuration to " + resourceName + " to " + outputFile); +// } +// OutputStreamWriter outputStream = new OutputStreamWriter( +// new FileOutputStream(outputFile), Charsets.UTF_8); +// try { +// IOUtils.copy(inputStream, outputStream); +// } finally { +// outputStream.close(); +// } +// if (executable) { +// outputFile.setExecutable(true); +// } +// } finally { +// inputStream.close(); +// } } public static void copyEmbeddedBinaryFile(String resourceName, Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportMap/ExportMapService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportMap/ExportMapService.java 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportMap/ExportMapService.java 2014-03-17 10:10:45 UTC (rev 975) @@ -22,24 +22,28 @@ */ import com.google.common.base.Preconditions; +import fr.ifremer.coser.CoserTechnicalException; import fr.ifremer.coser.bean.EchoBaseProject; import fr.ifremer.echobase.EchoBaseTechnicalException; import fr.ifremer.echobase.entities.references.Mission; +import fr.ifremer.echobase.io.CommandLineUtils; import fr.ifremer.echobase.io.EchoBaseIOUtil; import fr.ifremer.echobase.persistence.JdbcConfiguration; import fr.ifremer.echobase.services.EchoBaseServiceSupport; import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import org.apache.commons.exec.CommandLine; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Export; -import org.nuiton.util.FileUtil; import org.nuiton.util.TimeLog; import javax.inject.Inject; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; +import java.util.Map; /** * Created on 3/1/14. @@ -64,8 +68,8 @@ Preconditions.checkNotNull(model.getDbConfiguration()); Preconditions.checkNotNull(model.getMissionId()); - // compute nb steps (invoke EchoR + create archive tree + - int nbSteps = 3; + // compute nb steps (invoke EchoR + extract souces + create archive tree + create archive) + int nbSteps = 4; if (log.isInfoEnabled()) { log.info("NB steps: " + nbSteps); @@ -78,17 +82,22 @@ String missionName = mission.getName(); File tempDirectory = model.getWorkingDirectory(); - File echoRBaseDir = new File(tempDirectory, "EchoR"); - FileUtil.createDirectoryIfNecessary(echoRBaseDir); + // 1 - invoke EchoR - invokeEchoR(echoRBaseDir, model.getDbConfiguration(), missionName); + File echoRBaseDir = invokeEchoR(missionName, + tempDirectory, + model.getDbConfiguration()); model.incrementsProgress(); - // 2 - create archive tree + // 2 - Extract source files + + model.incrementsProgress(); + + // 3 - create archive tree File archiveBaseDir = new File(tempDirectory, "echobase"); - FileUtil.createDirectoryIfNecessary(archiveBaseDir); + EchoBaseIOUtil.forceMkdir(archiveBaseDir); if (log.isInfoEnabled()) { log.info("Will store maps in " + archiveBaseDir); } @@ -97,7 +106,7 @@ model.incrementsProgress(); - // 3 - create archive + // 4 - create archive String fileName = model.getFileName(); File zipFile = new File(tempDirectory, fileName + ".zip"); @@ -109,18 +118,95 @@ EchoBaseIOUtil.compressZipFile(zipFile, archiveBaseDir); } - protected void invokeEchoR(File dir, JdbcConfiguration dbConfiguration, String missionName) { + protected File invokeEchoR(String missionName, + File tempDirectory, + JdbcConfiguration dbConfiguration) throws IOException { if (log.isInfoEnabled()) { log.info("Will invoke EchoR on mission: " + missionName); log.info("Will connect to db: " + dbConfiguration.getUrl()); } + File workingDirectory = new File(tempDirectory, "r"); + EchoBaseIOUtil.forceMkdir(workingDirectory); + String scriptPrefixPath = new File(workingDirectory, "EchoR").getAbsolutePath(); + + File rExecutableFile = getConfiguration().getRscriptExecutablePath(); + if (!EchoBaseIOUtil.isExecutableFile(rExecutableFile)) { + throw new EchoBaseTechnicalException("Could not find R executable at: " + rExecutableFile); + } + + // copy script to temporary directory + File scriptFile = new File(tempDirectory, "extractMaps.r"); + if (log.isInfoEnabled()) { + log.info("Copy R script to " + scriptFile); + } + EchoBaseIOUtil.copyResource("rscript/extractMaps.r", scriptFile, true); + + // Extract params from jdbcurl + //jdbc:postgresql://demo.codelutin.com:5433/echobase-latest-2011-2.6 + String url = StringUtils.substringAfter(dbConfiguration.getUrl(), "jdbc:postgresql://"); + if (!url.contains("/")) { + throw new CoserTechnicalException("Invalid jdbc url: " + url + " (should contains a /"); + } + String dbHost = StringUtils.substringBefore(url, "/"); + String dbName = StringUtils.substringAfter(url, "/"); + String dbPort = "5432"; + if (dbHost.contains(":")) { + // a port is given + dbHost = StringUtils.substringBefore(dbHost, ":"); + dbPort = StringUtils.substringAfter(dbHost, ":"); + } + + if (log.isInfoEnabled()) { + log.info("mission: " + missionName); + log.info("extractPath: " + scriptPrefixPath); + log.info("dbHost: " + dbHost); + log.info("dbPort: " + dbPort); + log.info("dbName: " + dbName); + } + + CommandLine commandLine = CommandLineUtils.newCommand( + rExecutableFile, + missionName, + scriptPrefixPath, + dbHost, + dbName, + dbPort, + dbConfiguration.getLogin(), + dbConfiguration.getPassword()); + + CommandLineUtils.invokeCommandeLine(log, commandLine, "Failed to launch R script: " + scriptFile, workingDirectory); + + File result = new File(scriptPrefixPath + "meanMaps"); + return result; } - protected void createArchiveContent(File archiveBaseDir, File echoRBaseDir, String missionName) throws IOException { + /** + * Execute R script passed in param and add args to call script + * + * @param workingDirectory working directory + * @param scriptFile script to execute + * @param args arguments to pass to script + */ + public void executeRScript(File workingDirectory, + File scriptFile, + Map<String, String> args) { + // launch R command + File rExecutableFile = getConfiguration().getRscriptExecutablePath(); + if (!EchoBaseIOUtil.isExecutableFile(rExecutableFile)) { + throw new EchoBaseTechnicalException("Could not find R executable at: " + rExecutableFile); + } + CommandLine commandLine = CommandLineUtils.newCommand(rExecutableFile, args); + + CommandLineUtils.invokeCommandeLine(log, commandLine, "Failed to launch R script: " + scriptFile, workingDirectory); + } + + protected void createArchiveContent(File archiveBaseDir, File mapsDirectory, String missionName) throws IOException { + EchoBaseProject coserProject = new EchoBaseProject(archiveBaseDir); - coserProject.setFacadeName("Atlantique"); + coserProject.setSurveyName(missionName); + coserProject.setFacadeName("atlantique"); coserProject.setZoneName("gdgciem8"); coserProject.setAuthor("EchoBase"); coserProject.setComment("Created by Echobase application (db: " + getUserDbUrl() + ")"); @@ -129,15 +215,16 @@ File mapDir = coserProject.getMapsDirectory(); - FileUtil.createDirectoryIfNecessary(mapDir); + EchoBaseIOUtil.forceMkdir(mapDir); FilenameFilter filter = EchoBaseProject.newMapSpeciesFilenameFilter(missionName); - // files are in echoRBaseDir-meanMaps // copy all of them in mapDir - File[] files = new File(echoRBaseDir, "invoke-meanMaps").listFiles(filter); - for (File file : files) { - FileUtils.copyFileToDirectory(file, mapDir); + File[] files = mapsDirectory.listFiles(filter); + if (files != null) { + for (File file : files) { + FileUtils.copyFileToDirectory(file, mapDir); + } } // create species.csv Added: trunk/echobase-services/src/main/resources/rscript/extractMap.r =================================================================== --- trunk/echobase-services/src/main/resources/rscript/extractMap.r (rev 0) +++ trunk/echobase-services/src/main/resources/rscript/extractMap.r 2014-03-17 10:10:45 UTC (rev 975) @@ -0,0 +1,28 @@ +## To extract maps from EchoR +## Args are +## missionName +## exportPrefixPath +## dbHost +## dbName +## dbUser +## dbPwd +## dbPort + +args <- commandArgs(TRUE) +missionName <- args[1] +exportPrefixPath <- args[2] +dbHost <- args[3] +dbName <- args[4] +dbUser <- args[5] +dbPwd <- args[6] +dbPort <- args[7] + +library('EchoR'); +GridMaps4Echobase(mission=missionName, + voyage='%', + path.export=exportPrefixPath, + host=dbHost, + dbname=dbName, + user=dbUser, + password=dbPwd, + port=dbPort); \ No newline at end of file Modified: trunk/echobase-ui/pom.xml =================================================================== --- trunk/echobase-ui/pom.xml 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/echobase-ui/pom.xml 2014-03-17 10:10:45 UTC (rev 975) @@ -33,10 +33,6 @@ ${project.build.directory}/${embeddedWarName}.zip </redmine.releaseFiles> - <deployFiles> - ${project.build.directory}/${fullWarName}.war - </deployFiles> - <!-- Post Release configuration --> <skipPostRelease>false</skipPostRelease> Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2014-03-07 23:23:32 UTC (rev 974) +++ trunk/pom.xml 2014-03-17 10:10:45 UTC (rev 975) @@ -6,7 +6,7 @@ <parent> <groupId>org.nuiton</groupId> <artifactId>mavenpom4redmine</artifactId> - <version>5.0.1-SNAPSHOT</version> + <version>5.0.1</version> </parent> <groupId>fr.ifremer</groupId> @@ -267,6 +267,13 @@ <version>4.2.10.Final</version> </dependency> + <!-- Commons --> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + <version>1.1</version> + </dependency> + <!-- base postgres --> <dependency> <groupId>org.postgresql</groupId>