Author: bpoussin Date: 2014-06-29 20:44:00 +0200 (Sun, 29 Jun 2014) New Revision: 4026 Url: http://forge.codelutin.com/projects/isis-fish/repository/revisions/4026 Log: - Add new default ResultStorage (store in pseudo csv compressed file) - old simulation with mapped storage are automaticaly detected - Add configuration for matrix type thresold (dense/sparse: default 1000 value) - guava cache support sparse matrix - factorisation of ResultStorage (ResultMappedStorage, ResultStorageInMemory, ResultStorageCSV) in ResultStorageAbstract - add Trace for equation (but not very useful, because equation name is generated, perhaps need entity to string in name ?) Added: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java trunk/src/main/java/fr/ifremer/isisfish/util/SimpleParser.java Modified: trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java trunk/src/main/java/fr/ifremer/isisfish/aspect/TraceAspect.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationStorage.java trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java Modified: trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -63,6 +63,7 @@ import fr.ifremer.isisfish.util.IsisCacheBackend; import fr.ifremer.isisfish.util.IsisCacheBackendOnGuava; import fr.ifremer.isisfish.vcs.VCS; +import org.nuiton.math.matrix.DoubleVector; /** * Isis fish configuration. @@ -948,10 +949,10 @@ /** * L'implementation de la class gérant les implémentations de vecteur de - * matrice pour les simulations + * matrice pleine pour les simulations * * @return - * @since 4.2.1.1 + * @since 4.2.0.2 */ public Class getSimulationMatrixVectorClass() { Class result = getOptionAsClass(Option.SIMULATION_MATRIX_VECTOR_CLASS.key); @@ -959,6 +960,29 @@ } /** + * L'implementation de la class gérant les implémentations de vecteur de + * matrice creuse pour les simulations + * + * @return + * @since 4.2.0.2 + */ + public Class getSimulationMatrixVectorSparseClass() { + Class result = getOptionAsClass(Option.SIMULATION_MATRIX_VECTOR_SPARSE_CLASS.key); + return result; + } + + /** + * Seuil d'uitilisation des matrices creuse + * + * @return + * @since 4.2.0.2 + */ + public int getSimulationMatrixThresholdUseSparse() { + int result = getOptionAsInt(Option.SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS.key); + return result; + } + + /** * Retourne la factory a utilise pour le cache * * @return factory a utilise pour le backend de cache @@ -1004,6 +1028,8 @@ // pour les simulations SIMULATION_MATRIX_VECTOR_CLASS("simulation.matrix.vector.class", n("isisfish.config.simulation.matrix.vector.class.description"), DoubleBigVector.class.getName()), + SIMULATION_MATRIX_VECTOR_SPARSE_CLASS("simulation.matrix.vector.sparse.class", n("isisfish.config.simulation.matrix.vector.sparse.class.description"), DoubleVector.class.getName()), + SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS("simulation.matrix.threshold.use.sparse.class", n("isisfish.config.simulation.matrix.threshold.use.sparse.class.description"), "1000"), /** Nombre maximum de thread de simulation in process. */ SIMULATOR_IN_MAXTHREADS("simulation.in.max.threads", n("isisfish.config.main.simulation.in.max.threads.description"), "1"), Modified: trunk/src/main/java/fr/ifremer/isisfish/aspect/TraceAspect.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/aspect/TraceAspect.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/aspect/TraceAspect.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -76,6 +76,8 @@ } @Before("execution(* scripts..*(..))" + + " || execution(* fr.ifremer.isisfish.entities..*(..))" + + " || execution(* org.nuiton.math.matrix..*(..))" + " || execution(* simulators..*(..))" + " || execution(* rules..*(..)) " + " || execution(* simulationplans..*(..)) " @@ -86,6 +88,8 @@ @AfterThrowing(throwing = "ex", pointcut = "execution(* scripts..*(..))" + + " || execution(* fr.ifremer.isisfish.entities..*(..))" + + " || execution(* org.nuiton.math.matrix..*(..))" + " || execution(* simulators..*(..))" + " || execution(* rules..*(..)) " + " || execution(* simulationplans..*(..)) " @@ -96,6 +100,8 @@ } @After("execution(* scripts..*(..))" + + " || execution(* fr.ifremer.isisfish.entities..*(..))" + + " || execution(* org.nuiton.math.matrix..*(..))" + " || execution(* simulators..*(..))" + " || execution(* rules..*(..)) " + " || execution(* simulationplans..*(..)) " Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -25,61 +25,32 @@ package fr.ifremer.isisfish.datastore; -import static org.nuiton.i18n.I18n.t; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; -import org.apache.commons.collections4.BidiMap; -import org.apache.commons.collections4.bidimap.DualHashBidiMap; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.math.matrix.DoubleBigMappedVector; -import org.nuiton.math.matrix.DoubleBigVector; import org.nuiton.math.matrix.MatrixFactory; import org.nuiton.math.matrix.MatrixHelper; -import org.nuiton.math.matrix.MatrixIterator; import org.nuiton.math.matrix.MatrixND; import org.nuiton.math.matrix.MatrixSemanticsDecorator; import org.nuiton.math.matrix.SemanticsDecorator; import org.nuiton.topia.TopiaContext; -import org.nuiton.topia.TopiaException; -import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.util.ArrayUtil; -import org.nuiton.util.HashList; import fr.ifremer.isisfish.IsisFish; -import fr.ifremer.isisfish.IsisFishDAOHelper; -import fr.ifremer.isisfish.IsisFishException; import fr.ifremer.isisfish.IsisFishRuntimeException; -import fr.ifremer.isisfish.entities.ActiveRule; -import fr.ifremer.isisfish.entities.ActiveRuleDAO; -import fr.ifremer.isisfish.entities.Population; -import fr.ifremer.isisfish.export.Export; -import fr.ifremer.isisfish.export.SensitivityExport; -import fr.ifremer.isisfish.rule.Rule; -import fr.ifremer.isisfish.simulator.Objective; -import fr.ifremer.isisfish.simulator.Optimization; -import fr.ifremer.isisfish.simulator.SimulationContext; -import fr.ifremer.isisfish.simulator.SimulationException; -import fr.ifremer.isisfish.simulator.SimulationPlan; -import fr.ifremer.isisfish.simulator.SimulationResultGetter; -import fr.ifremer.isisfish.simulator.SimulationResultListener; -import fr.ifremer.isisfish.types.Month; import fr.ifremer.isisfish.types.TimeStep; import fr.ifremer.isisfish.util.BitUtil; +import java.util.HashMap; /** * Cette classe permet de conserver des résultats de simulation. Elle permet @@ -93,15 +64,13 @@ * Mise a jour: $Date$ * par : $Author$ */ -public class ResultMappedStorage implements SimulationResultListener, - SimulationResultGetter, ResultStorage { // ResultStorage +public class ResultMappedStorage extends ResultStorageAbstract { // ResultStorage /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(ResultMappedStorage.class); - protected MatrixFactory matrixFactory = MatrixFactory.getInstance(DoubleBigVector.class); - - protected SimulationStorage simulation = null; + protected MatrixFactory matrixFactory; + protected RandomAccessFile raf; protected long offset; protected Map<TimeStep, Map<String, ResultMapped>> stepNameResults = @@ -109,91 +78,7 @@ protected Map<String, Map<TimeStep, ResultMapped>> nameStepResults = new TreeMap<String, Map<TimeStep, ResultMapped>>(); - /** result enabled */ - transient protected Set<String> enabledResult = null; - /** - * Convertie une entite, month, timestep en string et inversement. - * Entity = "TopiaId:Entity.toString" - */ - static protected class EntitySemanticsDecorator implements SemanticsDecorator { - static final private String SEP = ":"; - - protected TopiaContext tx; - /** en cle les representation interne (get) en valeur les valeurs decoree (getKey) */ - protected BidiMap<Object, Object> cache = new DualHashBidiMap<>(); - - public EntitySemanticsDecorator() { - } - - public EntitySemanticsDecorator(TopiaContext tx) { - this.tx = tx; - } - - public Object decorate(Object internalValue) { - Object result = cache.get(internalValue); - if (result == null && internalValue != null) { - result = internalValue; - if (internalValue instanceof String) { - // on retrouve souvent les memes semantiques dans les matrices - // pour minimiser les chaines en memoire, on prend la representation - // interne avant de l'utiliser comme cle. - internalValue = ((String)internalValue).intern(); - result = internalValue; - if (StringUtils.startsWith((String)internalValue, Month.class.getName())) { - String val = StringUtils.substringAfter((String)internalValue, SEP); - int monthNumber = Integer.parseInt(val); - result = new Month(monthNumber); - } else if (StringUtils.startsWith((String)internalValue, TimeStep.class.getName())) { - String val = StringUtils.substringAfter((String)internalValue, SEP); - int stepNumber = Integer.parseInt(val); - result = new TimeStep(stepNumber); - } else if (StringUtils.startsWith((String)internalValue, "fr.ifremer.isisfish.entities.")) { - if (tx == null) { - result = StringUtils.substringAfter((String)internalValue, SEP); - } else { - try { - String id = StringUtils.substringBefore((String)internalValue, SEP); - result = tx.findByTopiaId(id); - } catch (TopiaException eee) { - log.info("Fallback use string representation because" - + " i can't decorate (String->Entity): " - + internalValue, eee); - // si on arrive pas a convertir cette fois-ci, on - // renvoi internalValue pour que le undecorate est toutes les infos - // et donc que la prochaine fois on y arrive peut etre - } - } - } - } - cache.put(internalValue, result); - } - return result; - } - - public Object undecorate(Object decoratedValue) { - Object result = cache.getKey(decoratedValue); - if (result == null && decoratedValue != null) { - if (decoratedValue instanceof Month) { - result = Month.class.getName() + SEP + ((Month)decoratedValue).getMonthNumber(); - } else if (decoratedValue instanceof TimeStep) { - result = TimeStep.class.getName() + SEP + ((TimeStep)decoratedValue).getStep(); - } else if (decoratedValue instanceof TopiaEntity) { - result = ((TopiaEntity)decoratedValue).getTopiaId() + SEP + decoratedValue; - } else { - result = String.valueOf(decoratedValue); - } - // on retrouve souvent les memes semantiques dans les matrices - // pour minimiser les chaines en memoire, on prend la representation - // interne avant de l'utiliser comme cle. - cache.put(((String)result).intern(), decoratedValue); - } - return result; - } - - } - - /** * Represente un resultat * Lors de la construction, si on ne passe que le RandomAccessFile et l'offset, on relie un resultat * si on passe toutes les infos, on ecrit le resultat. @@ -418,13 +303,8 @@ * @param simulation storage to get result */ public ResultMappedStorage(SimulationStorage simulation) throws IOException { - this.simulation = simulation; + super(simulation); - // instanciation de la factory avec l'implementation - // choisie dans la config - Class vectorClass = IsisFish.config.getMappedResultMatrixVectorClass(); - matrixFactory = MatrixFactory.getInstance(vectorClass); - File file = SimulationStorage.getResultFile(simulation.getDirectory()); // il faut toujours ouvrir en 'rw' car sinon les matrices n'arrive pas @@ -434,13 +314,24 @@ // on lit les donnees deja presente offset = 0; while (offset < raf.length()) { - ResultMapped r = new ResultMapped(matrixFactory, raf, offset); + ResultMapped r = new ResultMapped(getMatrixFactory(), raf, offset); storeResult(r); offset += r.size(); } } @Override + protected MatrixFactory getMatrixFactory() { + if (matrixFactory == null) { + // instanciation de la factory avec l'implementation + // choisie dans la config + Class vectorClass = IsisFish.config.getMappedResultMatrixVectorClass(); + matrixFactory = MatrixFactory.getInstance(vectorClass); + } + return matrixFactory; + } + + @Override protected void finalize() throws Throwable { try { raf.close(); @@ -451,7 +342,7 @@ @Override public void delete() { - close(); + super.delete(); File file = SimulationStorage.getResultFile(simulation.getDirectory()); file.delete(); } @@ -474,12 +365,7 @@ String name = r.getName(); getResult(step).put(name, r); - Map<TimeStep, ResultMapped> stepResult = nameStepResults.get(name); - if (stepResult == null) { - stepResult = new TreeMap<TimeStep, ResultMapped>(); - nameStepResults.put(name, stepResult); - } - stepResult.put(step, r); + getResult(name).put(step, r); } protected Map<String, ResultMapped> getResult(TimeStep step) { @@ -491,80 +377,42 @@ return result; } - protected ResultMapped getResult(TimeStep step, String name) { - ResultMapped result = getResult(step).get(name); - return result; - } - - protected ResultMapped getResult(TimeStep step, String name, Population pop) { - name += " " + pop; - ResultMapped result = getResult(step, name); - return result; - } - - protected Map<TimeStep, ResultMapped> getResult(String name, Population pop) { - name += " " + pop; - Map<TimeStep, ResultMapped> result = getResult(name); - return result; - } - protected Map<TimeStep, ResultMapped> getResult(String name) { Map<TimeStep, ResultMapped> result = nameStepResults.get(name); if (result == null) { - // on retourne une map vide, pour ne jamais retourne null - result = Collections.EMPTY_MAP; + result = new TreeMap<TimeStep, ResultMapped>(); + nameStepResults.put(name, result); } return result; } + @Override + protected MatrixND readResult(TimeStep step, String name) { + MatrixND result = null; - public void addResult(TimeStep step, MatrixND mat) throws IsisFishException { - addResult(false, step, mat.getName(), mat); - } - - public void addResult(TimeStep step, Population pop, MatrixND mat) throws IsisFishException { - addResult(false, step, mat.getName(), pop, mat); - } - - public void addResult(boolean force, TimeStep step, MatrixND mat) throws IsisFishException { - addResult(force, step, mat.getName(), mat); - } - - public void addResult(boolean force, TimeStep step, Population pop, MatrixND mat) throws IsisFishException { - addResult(force, step, mat.getName(), pop, mat); - } - - public void addResult(TimeStep step, String name, Population pop, MatrixND mat) throws IsisFishException { - addResult(false, step, name, pop, mat); - } - - public void addResult(TimeStep step, String name, MatrixND mat) throws IsisFishException { - addResult(false, step, name, mat); - } - - public void addResult(boolean force, TimeStep step, String name, Population pop, MatrixND mat) throws IsisFishException { - if (force || isEnabled(name)) { - doAddResult(step, name + " " + pop, mat); + ResultMapped rm = getResult(step).get(name); + if (rm != null) { + result = rm.getMatrix(); } - } - public void addResult(boolean force, TimeStep step, String name, MatrixND mat) throws IsisFishException { - if (force || isEnabled(name)) { - doAddResult(step, name, mat); - } + return result; } - protected void doAddResult(TimeStep step, String name, MatrixND mat) throws IsisFishException { - // si la matrice n'a pas de semantique on refuse - for (int i = 0; i < mat.getDimCount(); i++) { - // la semantique n'est pas bonne des qu'il y a un null dedans - if (mat.getSemantic(i).contains(null)) { - throw new SimulationException( - "Erreur le résultat que vous souhaitez enregistrer n'a pas d'information convenable pour la dimension: " - + i + " " + mat.getDimensionName(i)); + @Override + protected Map<TimeStep, MatrixND> readResult(String name) { + Map<TimeStep, MatrixND> result = Collections.EMPTY_MAP; + Map<TimeStep, ResultMapped> rms = nameStepResults.get(name); + if (rms != null) { + result = new HashMap<TimeStep, MatrixND>(); + for (Map.Entry<TimeStep, ResultMapped> e : rms.entrySet()) { + result.put(e.getKey(), e.getValue().getMatrix()); } } + return result; + } + @Override + protected void writeResult(TimeStep step, String name, MatrixND mat) { try { ResultMapped r = new ResultMapped(raf, offset, step, name, mat); storeResult(r); @@ -575,144 +423,15 @@ // temps de simulation, si on ne force pas le sync. // raf.getFD().sync(); } catch (IOException eee) { - log.warn("Can't add result '" + name + "' at step " + step, eee); + throw new IsisFishRuntimeException("Can't write result", eee); } } - - - - - - - /** - * Permet de savoir si lorsque l'on ajoutera ce resultat, il sera - * sauvé ou non. - * - * Check for result name returned by : - * <ul> - * <li>{@link Export#getNecessaryResult()}</li> - * <li>{@link SensitivityExport#getNecessaryResult()}</li> - * <li>{@link Rule#getNecessaryResult()}</li> - * <li>{@link SimulationPlan#getNecessaryResult()}</li> - * </ul> - * - * @param name result name - * @return {@code true} if result is enabled - */ - public boolean isEnabled(String name) { - name = name.trim(); - if (enabledResult == null && simulation.getSimulationParametersFile().exists()) { - enabledResult = new HashSet<String>(); - - Collection<String> resultEnabled = simulation.getParameter() - .getResultEnabled(); - enabledResult.addAll(resultEnabled); - - // test on export - List<String> exportNames = simulation.getParameter().getExportNames(); - if (exportNames != null) { - for (String exportName : exportNames) { - ExportStorage storage = ExportStorage.getExport(exportName); - try { - Export export = storage.getNewInstance(); - for (String resultName : export.getNecessaryResult()) { - enabledResult.add(resultName); - } - } catch (IsisFishException eee) { - if (log.isWarnEnabled()) { - log.warn(t("isisfish.error.instanciate.export", - exportName), eee); - } - } - } - } - - // test on sensitivity export - List<SensitivityExport> sensitivityExports = simulation - .getParameter().getSensitivityExport(); - if (sensitivityExports != null) { - for (SensitivityExport sensitivityExport : sensitivityExports) { - for (String resultName : sensitivityExport.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - } - - // test on rules - List<Rule> rules = simulation.getParameter().getRules(); - if (rules != null) { - for (Rule rule : rules) { - for (String resultName : rule.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - } - - // test on plans - List<SimulationPlan> plans = simulation.getParameter().getSimulationPlans(); - if (plans != null) { - for (SimulationPlan plan : plans) { - for (String resultName : plan.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - } - - // on objective and optimization - Objective objective = simulation.getParameter().getObjective(); - if (objective != null) { - for (String resultName : objective.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - - Optimization optimization = simulation.getParameter().getOptimization(); - if (optimization != null) { - for (String resultName : optimization.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - - log.info("Enabled result: " + enabledResult); - } - // par defaut on dit qu'on conserve le resultat - boolean result = true; - if (enabledResult != null) { - result = enabledResult.contains(name); - } - return result; + @Override + protected void writeActiveRule(TimeStep step, String name, String params) { + // TODO } - - public void addActiveRule(TimeStep step, Rule rule) throws IsisFishException { - try { - TopiaContext tx = null; - boolean mustClose = false; - - if (simulation == SimulationContext.get().getSimulationStorage()) { - tx = SimulationContext.get().getDbResult(); - } - if (tx == null) { - // not in simulation, create transaction - tx = simulation.getStorage().beginTransaction(); - mustClose = true; - } - ActiveRuleDAO ps = IsisFishDAOHelper.getActiveRuleDAO(tx); - ActiveRule result = ps.create(); - result.setActiveRuleStep(step); - result.setName(RuleStorage.getName(rule)); - result.setParam(RuleStorage.getParamAsString(rule)); - ps.update(result); - if (mustClose) { - tx.commitTransaction(); - tx.closeContext(); - } - } catch (TopiaException eee) { - throw new IsisFishException("Can't add result", eee); - } - } - /** * Retourne la liste de tous les résultats. Si le résultat est categorisé * par une population alors le nom de la population est automatiquement @@ -722,279 +441,5 @@ List<String> result = new ArrayList<String>(nameStepResults.keySet()); return result; } - - /** - * Retourne la matrice stocke pour un pas de temps - * @param step le pas de temps que l'on souhaite - * @param pop la population pour lequelle on souhaite le resultat - * @param name le nom des resultats dont on veut la matrice - * @return La matrice demandée ou null si aucune matrice ne correspond a - * la demande. - */ - public MatrixND getMatrix(TimeStep step, Population pop, String name) { - String newName = name + " " + pop; - return getMatrix(step, newName, null); - } - - /** - * Retourne la matrice stocke pour un pas de temps - * @param step le pas de temps que l'on souhaite - * @param pop la population pour lequelle on souhaite le resultat - * @param name le nom des resultats dont on veut la matrice - * @return La matrice demandée ou null si aucune matrice ne correspond a - * la demande. - */ - public MatrixND getMatrix(TimeStep step, Population pop, String name, TopiaContext tx) { - String newName = name + " " + pop; - return getMatrix(step, newName, tx); - } - - public MatrixND getMatrix(TimeStep step, String name) { - return getMatrix(step, name, null); - } - - /** - * Retourne une matrice contenant tous les pas de temps. - * - * @param name le nom des resultats dont on veut une matrice globale. - */ - public MatrixND getMatrix(TimeStep step, String name, TopiaContext tx) { - ResultMapped r = getResult(step, name); - MatrixND mat = null; - if (r != null) { - tx = getTx(tx); - mat = r.getMatrix(tx); - } - return mat; - } - - /** - * Retourne une matrice contenant tous les pas de temps. - * @param pop la population pour lequel on souhaite la matrice - * @param name le nom des resultats dont on veut une matrice globale. - */ - public MatrixND getMatrix(Population pop, String name) { - String newName = name + " " + pop; - return getMatrix(newName, null); - } - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(fr.ifremer.isisfish.entities.Population, java.lang.String, org.nuiton.topia.TopiaContext) - */ - @Override - public MatrixND getMatrix(Population pop, String name, TopiaContext tx) { - String newName = name + " " + pop; - return getMatrix(newName, tx); - } - - /** - * Retourne une matrice contenant tous les pas de temps. - * - * @param name le nom des resultats dont on veut une matrice globale. - */ - public MatrixND getMatrix(String name) { - return getMatrix(name, null); - } - - /** - * Retourne une matrice contenant tous les pas de temps. - * @param name le nom des resultats dont on veut une matrice globale. - */ - public MatrixND getMatrix(String name, TopiaContext tx) { - log.debug("Get result: " + name); - - MatrixND resultMat = null; - Map<TimeStep, ResultMapped> results = getResult(name); - // recuperation des resultats qui nous interesse - if (!results.isEmpty()) { - // creation de la liste de date - TimeStep lastStep = getLastStep(); - List<TimeStep> steps = new ArrayList<TimeStep>(); - TimeStep step = new TimeStep(0); - steps.add(step); - while (step.before(lastStep)) { - step = step.next(); - steps.add(step); - } - - if (log.isTraceEnabled()) { - log.trace("Steps list : " + steps); - } - - - MatrixND mat = results.values().iterator().next().getMatrix(); - - // recuperation des noms des dimensions - String[] dimNames = new String[1 + mat.getDimCount()]; - dimNames[0] = t("isisfish.common.date"); - for (int i = 1; i < dimNames.length; i++) { - dimNames[i] = mat.getDimensionName(i - 1); - } - - // creation de la semantique pour la matrice resultat. +1 pour les dates - List[] sem = new List[1 + mat.getDimCount()]; - sem[0] = steps; - - for (int i = 1; i < sem.length; i++) { - sem[i] = new HashList(); - } - - for (ResultMapped result : results.values()) { - MatrixND mattmp = result.getMatrix(); - if (log.isTraceEnabled()) { - log.trace("Ajout de la semantics: " - + Arrays.asList(mattmp.getSemantics())); - } - - for (int s = 0; s < mattmp.getDimCount(); s++) { - sem[s + 1].addAll(mattmp.getSemantic(s)); - } - } - - if (log.isTraceEnabled()) { - log.trace("La semantique final est: " + Arrays.asList(sem)); - } - - // creation de la matrice resultat - resultMat = matrixFactory.create(name, sem, dimNames); - - // recuperation du resultat pour chaque date de la simulation, de Date(0) à lastDate - for (ResultMapped result : results.values()) { - TimeStep d = result.getStep(); - mat = result.getMatrix(); - // on met ce resultat dans la matrice result si besoin - if (mat != null) { - // on recupere dans la matrice resultat l'endroit on il faut - // mettre la matrice - MatrixND submat = resultMat.getSubMatrix(0, d, 1); - // on met les valeur de mat dans la sous matrice extraite - for (MatrixIterator mi = mat.iterator(); mi.next();) { - submat.setValue( - ArrayUtil.concat(new Object[] { d }, - mi.getSemanticsCoordinates()), mi.getValue()); - } - } - } - } - if (resultMat != null) { - // on decore la matrice resultat au dernier moment, tous les calcules - // ce font avec les strings - tx = getTx(tx); - resultMat = new MatrixSemanticsDecorator(resultMat, new EntitySemanticsDecorator(tx)); - } - return resultMat; - } - - /** - * Get last simulation date. - * - * @return last simulation date - */ - public TimeStep getLastStep() { - int monthNumber = simulation.getParameter().getNumberOfYear() - * Month.NUMBER_OF_MONTH; - TimeStep result = new TimeStep(monthNumber - 1); // -1 because date begin at 0 - return result; - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationResultListener#addResult(fr.ifremer.isisfish.simulator.SimulationContext, fr.ifremer.isisfish.types.TimeStep, java.lang.String, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(SimulationContext context, TimeStep step, String name, - MatrixND mat) throws IsisFishException { - doAddResult(step, name, mat); - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationResultGetter#getMatrix(fr.ifremer.isisfish.simulator.SimulationContext, fr.ifremer.isisfish.types.TimeStep, java.lang.String) - */ - @Override - public MatrixND getMatrix(SimulationContext context, TimeStep step, String name) { - MatrixND result = null; - try { - result = getMatrix(step, name, context.getDB()); - } catch (TopiaException eee) { - if (log.isWarnEnabled()) { - log.warn(String.format("Can't get result: %1$s", name), eee); - } - } - return result; - - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationResultGetter#getMatrix(fr.ifremer.isisfish.simulator.SimulationContext, java.lang.String) - */ - @Override - public MatrixND getMatrix(SimulationContext context, String name) { - MatrixND result = null; - try { - result = getMatrix(name, context.getDB()); - } catch (TopiaException eee) { - if (log.isWarnEnabled()) { - log.warn(String.format("Can't get result: %1$s", name), eee); - } - } - return result; - } - - /** - * Try to find better tx. If argument is not null, return it. otherwize - * try to get tx in SimulationContext. - * - * @param tx - * @return - */ - protected TopiaContext getTx(TopiaContext tx) { - TopiaContext result = tx; - if (tx == null) { - // si on a pas de tx, on recherche si on est dans une simulation - // pour recuperer la tx de la simulation - if (simulation == SimulationContext.get().getSimulationStorage()) { - try { - result = SimulationContext.get().getDB(); - } catch (TopiaException eee) { - throw new IsisFishRuntimeException("Can't get database from SimulationContext", eee); - } - } - } - return result; - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationListener#afterSimulation(fr.ifremer.isisfish.simulator.SimulationContext) - */ - @Override - public void afterSimulation(SimulationContext context) { - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationListener#beforeSimulation(fr.ifremer.isisfish.simulator.SimulationContext) - */ - @Override - public void beforeSimulation(SimulationContext context) { - } - - // public void addActivatedRule(ResultStorage self, Date date, RegleParam rule){ - // List rules = (List)activatedRules.get(date); - // if(rules == null){ - // activatedRules.put(date, rules = new LinkedList()); - // } - // rules.add(rule); - // } - - // /** - // * Retourne pour une date données tous les RegleParam qui ont été activé - // * a la date demandé. - // * @return une list de {@link fr.ifremer.nodb.RegleParam} - // */ - // public List getActivatedRule(ResultStorage self, Date date){ - // List rules = (List)activatedRules.get(date); - // if(rules == null){ - // activatedRules.put(date, rules = new LinkedList()); - // } - // return rules; - // } - } Copied: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java (from rev 4023, trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java) =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java (rev 0) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -0,0 +1,681 @@ +/* + * #%L + * IsisFish + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2012 Ifremer, Code Lutin, Benjamin Poussin, Chatellier Eric + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +package fr.ifremer.isisfish.datastore; + +import static org.nuiton.i18n.I18n.t; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections4.BidiMap; +import org.apache.commons.collections4.bidimap.DualHashBidiMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.math.matrix.MatrixIterator; +import org.nuiton.math.matrix.MatrixND; +import org.nuiton.math.matrix.MatrixSemanticsDecorator; +import org.nuiton.math.matrix.SemanticsDecorator; +import org.nuiton.topia.TopiaContext; +import org.nuiton.topia.TopiaException; +import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.ArrayUtil; +import org.nuiton.util.HashList; + +import fr.ifremer.isisfish.IsisFishException; +import fr.ifremer.isisfish.IsisFishRuntimeException; +import fr.ifremer.isisfish.entities.Population; +import fr.ifremer.isisfish.export.Export; +import fr.ifremer.isisfish.export.SensitivityExport; +import fr.ifremer.isisfish.rule.Rule; +import fr.ifremer.isisfish.simulator.Objective; +import fr.ifremer.isisfish.simulator.Optimization; +import fr.ifremer.isisfish.simulator.SimulationContext; +import fr.ifremer.isisfish.simulator.SimulationException; +import fr.ifremer.isisfish.simulator.SimulationPlan; +import fr.ifremer.isisfish.simulator.SimulationResultGetter; +import fr.ifremer.isisfish.simulator.SimulationResultListener; +import fr.ifremer.isisfish.types.Month; +import fr.ifremer.isisfish.types.TimeStep; +import org.nuiton.math.matrix.MatrixFactory; + +/** + * Classe abstraite servant a factoriser tous les traitements commun au differente + * implantation. Lorsqu'on herite de cette classe il reste a definir la lecture + * et l'ecriture reel des resultats (en memoire, en base, binary file, ...) + * + * Created: 24 juin 2014 + * + * @author Benjamin Poussin : poussin@codelutin.com + * @version $Revision$ + * + * Mise a jour: $Date$ + * par : $Author$ + */ +public abstract class ResultStorageAbstract implements SimulationResultListener, + SimulationResultGetter, ResultStorage { // ResultStorage + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(ResultStorageAbstract.class); + + protected SimulationStorage simulation = null; + + /** result enabled */ + transient protected Set<String> enabledResult = null; + + /** + * Convertie une entite, month, timestep en string et inversement. + * Entity = "TopiaId:Entity.toString" + */ + static protected class EntitySemanticsDecorator implements SemanticsDecorator { + static final private String SEP = ":"; + + protected TopiaContext tx; + /** en cle les representation interne (get) en valeur les valeurs decoree (getKey) */ + protected BidiMap<Object, Object> cache = new DualHashBidiMap<>(); + + public EntitySemanticsDecorator() { + } + + public EntitySemanticsDecorator(TopiaContext tx) { + this.tx = tx; + } + + @Override + public Object decorate(Object internalValue) { + Object result = cache.get(internalValue); + if (result == null && internalValue != null) { + result = internalValue; + if (internalValue instanceof String) { + // on retrouve souvent les memes semantiques dans les matrices + // pour minimiser les chaines en memoire, on prend la representation + // interne avant de l'utiliser comme cle. + internalValue = ((String)internalValue).intern(); + result = internalValue; + if (StringUtils.startsWith((String)internalValue, Month.class.getName())) { + String val = StringUtils.substringAfter((String)internalValue, SEP); + int monthNumber = Integer.parseInt(val); + result = new Month(monthNumber); + } else if (StringUtils.startsWith((String)internalValue, TimeStep.class.getName())) { + String val = StringUtils.substringAfter((String)internalValue, SEP); + int stepNumber = Integer.parseInt(val); + result = new TimeStep(stepNumber); + } else if (StringUtils.startsWith((String)internalValue, "fr.ifremer.isisfish.entities.")) { + if (tx == null) { + result = StringUtils.substringAfter((String)internalValue, SEP); + } else { + try { + String id = StringUtils.substringBefore((String)internalValue, SEP); + result = tx.findByTopiaId(id); + } catch (TopiaException eee) { + log.info("Fallback use string representation because" + + " i can't decorate (String->Entity): " + + internalValue, eee); + // si on arrive pas a convertir cette fois-ci, on + // renvoi internalValue pour que le undecorate est toutes les infos + // et donc que la prochaine fois on y arrive peut etre + } + } + } + } + cache.put(internalValue, result); + } + return result; + } + + @Override + public Object undecorate(Object decoratedValue) { + Object result = cache.getKey(decoratedValue); + if (result == null && decoratedValue != null) { + if (decoratedValue instanceof Month) { + result = Month.class.getName() + SEP + ((Month)decoratedValue).getMonthNumber(); + } else if (decoratedValue instanceof TimeStep) { + result = TimeStep.class.getName() + SEP + ((TimeStep)decoratedValue).getStep(); + } else if (decoratedValue instanceof TopiaEntity) { + result = ((TopiaEntity)decoratedValue).getTopiaId() + SEP + decoratedValue; + } else { + result = String.valueOf(decoratedValue); + } + // on retrouve souvent les memes semantiques dans les matrices + // pour minimiser les chaines en memoire, on prend la representation + // interne avant de l'utiliser comme cle. + cache.put(((String)result).intern(), decoratedValue); + } + return result; + } + + } + + /** + * Return one result for step and name + * + * @param step + * @param name + * @return + */ + abstract protected MatrixND readResult(TimeStep step, String name); + /** + * Return all available result for name in argument + * @param name + * @return + */ + abstract protected Map<TimeStep, MatrixND> readResult(String name); + /** + * write result + * @param step + * @param name + * @param mat + */ + abstract protected void writeResult(TimeStep step, String name, MatrixND mat); + /** + * Write active rule + * @param step + * @param name + * @param params + */ + abstract protected void writeActiveRule(TimeStep step, String name, String params); + /** + * return list of all result name available + * @return + */ + abstract public List<String> getResultName(); + + /** + * sub class can overwrite this method to return specifique factory + * @return + */ + protected MatrixFactory getMatrixFactory() { + return MatrixFactory.getInstance(); + } + + protected MatrixND decorate(MatrixND mat, TopiaContext tx) { + MatrixND result = null; + if (mat != null) { + // on decore la matrice resultat au dernier moment, tous les calcules + // ce font avec les strings + tx = getTx(tx); + result = new MatrixSemanticsDecorator(mat, new EntitySemanticsDecorator(tx)); + } + return result; + } + + /** + * Les ResultStorage ne doivent pas etre instancier directement, mais + * recuperer a partir d'un + * {@link fr.ifremer.isisfish.datastore.SimulationStorage#getResultStorage()} + * + * @param simulation storage to get result + */ + public ResultStorageAbstract(SimulationStorage simulation) { + this.simulation = simulation; + } + + @Override + public void delete() { + close(); + } + + @Override + public void close() { + } + + @Override + public void addResult(TimeStep step, MatrixND mat) throws IsisFishException { + addResult(false, step, mat.getName(), mat); + } + + @Override + public void addResult(TimeStep step, Population pop, MatrixND mat) throws IsisFishException { + addResult(false, step, mat.getName(), pop, mat); + } + + @Override + public void addResult(boolean force, TimeStep step, MatrixND mat) throws IsisFishException { + addResult(force, step, mat.getName(), mat); + } + + @Override + public void addResult(boolean force, TimeStep step, Population pop, MatrixND mat) throws IsisFishException { + addResult(force, step, mat.getName(), pop, mat); + } + + @Override + public void addResult(TimeStep step, String name, Population pop, MatrixND mat) throws IsisFishException { + addResult(false, step, name, pop, mat); + } + + @Override + public void addResult(TimeStep step, String name, MatrixND mat) throws IsisFishException { + addResult(false, step, name, mat); + } + + @Override + public void addResult(boolean force, TimeStep step, String name, Population pop, MatrixND mat) throws IsisFishException { + if (force || isEnabled(name)) { + doAddResult(step, name + " " + pop, mat); + } + } + + @Override + public void addResult(boolean force, TimeStep step, String name, MatrixND mat) throws IsisFishException { + if (force || isEnabled(name)) { + doAddResult(step, name, mat); + } + } + + protected void doAddResult(TimeStep step, String name, MatrixND mat) throws IsisFishException { + // si la matrice n'a pas de semantique on refuse + for (int i = 0; i < mat.getDimCount(); i++) { + // la semantique n'est pas bonne des qu'il y a un null dedans + if (mat.getSemantic(i).contains(null)) { + throw new SimulationException( + "Erreur le résultat que vous souhaitez enregistrer n'a pas d'information convenable pour la dimension: " + + i + " " + mat.getDimensionName(i)); + } + } + + try { + writeResult(step, name, mat); + } catch (Exception eee) { + log.warn("Can't add result '" + name + "' at step " + step, eee); + } + } + + /** + * Permet de savoir si lorsque l'on ajoutera ce resultat, il sera + * sauvé ou non. + * + * Check for result name returned by : + * <ul> + * <li>{@link Export#getNecessaryResult()}</li> + * <li>{@link SensitivityExport#getNecessaryResult()}</li> + * <li>{@link Rule#getNecessaryResult()}</li> + * <li>{@link SimulationPlan#getNecessaryResult()}</li> + * </ul> + * + * @param name result name + * @return {@code true} if result is enabled + */ + @Override + public boolean isEnabled(String name) { + name = name.trim(); + if (enabledResult == null && simulation.getSimulationParametersFile().exists()) { + enabledResult = new HashSet<String>(); + + Collection<String> resultEnabled = simulation.getParameter() + .getResultEnabled(); + enabledResult.addAll(resultEnabled); + + // test on export + List<String> exportNames = simulation.getParameter().getExportNames(); + if (exportNames != null) { + for (String exportName : exportNames) { + ExportStorage storage = ExportStorage.getExport(exportName); + try { + Export export = storage.getNewInstance(); + Collections.addAll(enabledResult, export.getNecessaryResult()); + } catch (IsisFishException eee) { + if (log.isWarnEnabled()) { + log.warn(t("isisfish.error.instanciate.export", + exportName), eee); + } + } + } + } + + // test on sensitivity export + List<SensitivityExport> sensitivityExports = simulation + .getParameter().getSensitivityExport(); + if (sensitivityExports != null) { + for (SensitivityExport sensitivityExport : sensitivityExports) { + Collections.addAll(enabledResult, sensitivityExport.getNecessaryResult()); + } + } + + // test on rules + List<Rule> rules = simulation.getParameter().getRules(); + if (rules != null) { + for (Rule rule : rules) { + Collections.addAll(enabledResult, rule.getNecessaryResult()); + } + } + + // test on plans + List<SimulationPlan> plans = simulation.getParameter().getSimulationPlans(); + if (plans != null) { + for (SimulationPlan plan : plans) { + Collections.addAll(enabledResult, plan.getNecessaryResult()); + } + } + + // on objective and optimization + Objective objective = simulation.getParameter().getObjective(); + if (objective != null) { + Collections.addAll(enabledResult, objective.getNecessaryResult()); + } + + Optimization optimization = simulation.getParameter().getOptimization(); + if (optimization != null) { + Collections.addAll(enabledResult, optimization.getNecessaryResult()); + } + + log.info("Enabled result: " + enabledResult); + } + // par defaut on dit qu'on conserve le resultat + boolean result = true; + if (enabledResult != null) { + result = enabledResult.contains(name); + } + return result; + } + + + @Override + public void addActiveRule(TimeStep step, Rule rule) throws IsisFishException { + String name = RuleStorage.getName(rule); + String params = RuleStorage.getParamAsString(rule); + writeActiveRule(step, name, params); + } + + /** + * Retourne la matrice stocke pour un pas de temps + * @param step le pas de temps que l'on souhaite + * @param pop la population pour lequelle on souhaite le resultat + * @param name le nom des resultats dont on veut la matrice + * @return La matrice demandée ou null si aucune matrice ne correspond a + * la demande. + */ + @Override + public MatrixND getMatrix(TimeStep step, Population pop, String name) { + String newName = name + " " + pop; + return getMatrix(step, newName, null); + } + + /** + * Retourne la matrice stocke pour un pas de temps + * @param step le pas de temps que l'on souhaite + * @param pop la population pour lequelle on souhaite le resultat + * @param name le nom des resultats dont on veut la matrice + * @return La matrice demandée ou null si aucune matrice ne correspond a + * la demande. + */ + public MatrixND getMatrix(TimeStep step, Population pop, String name, TopiaContext tx) { + String newName = name + " " + pop; + return getMatrix(step, newName, tx); + } + + @Override + public MatrixND getMatrix(TimeStep step, String name) { + return getMatrix(step, name, null); + } + + /** + * Retourne une matrice contenant tous les pas de temps. + * + * @param name le nom des resultats dont on veut une matrice globale. + */ + @Override + public MatrixND getMatrix(TimeStep step, String name, TopiaContext tx) { + MatrixND mat = readResult(step, name); + mat = decorate(mat, tx); + return mat; + } + + /** + * Retourne une matrice contenant tous les pas de temps. + * @param pop la population pour lequel on souhaite la matrice + * @param name le nom des resultats dont on veut une matrice globale. + */ + @Override + public MatrixND getMatrix(Population pop, String name) { + String newName = name + " " + pop; + return getMatrix(newName, null); + } + + /* (non-Javadoc) + * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(fr.ifremer.isisfish.entities.Population, java.lang.String, org.nuiton.topia.TopiaContext) + */ + @Override + public MatrixND getMatrix(Population pop, String name, TopiaContext tx) { + String newName = name + " " + pop; + return getMatrix(newName, tx); + } + + /** + * Retourne une matrice contenant tous les pas de temps. + * + * @param name le nom des resultats dont on veut une matrice globale. + */ + @Override + public MatrixND getMatrix(String name) { + return getMatrix(name, null); + } + + /** + * Retourne une matrice contenant tous les pas de temps. + * @param name le nom des resultats dont on veut une matrice globale. + */ + @Override + public MatrixND getMatrix(String name, TopiaContext tx) { + log.debug("Get result: " + name); + + MatrixND resultMat = null; + Map<TimeStep, MatrixND> results = readResult(name); + // recuperation des resultats qui nous interesse + if (!results.isEmpty()) { + // creation de la liste de date + TimeStep lastStep = getLastStep(); + List<TimeStep> steps = new ArrayList<TimeStep>(); + TimeStep step = new TimeStep(0); + steps.add(step); + while (step.before(lastStep)) { + step = step.next(); + steps.add(step); + } + + if (log.isTraceEnabled()) { + log.trace("Steps list : " + steps); + } + + + MatrixND mat = results.values().iterator().next(); + + // recuperation des noms des dimensions + String[] dimNames = new String[1 + mat.getDimCount()]; + dimNames[0] = t("isisfish.common.date"); + for (int i = 1; i < dimNames.length; i++) { + dimNames[i] = mat.getDimensionName(i - 1); + } + + // creation de la semantique pour la matrice resultat. +1 pour les dates + List[] sem = new List[1 + mat.getDimCount()]; + sem[0] = steps; + + for (int i = 1; i < sem.length; i++) { + sem[i] = new HashList(); + } + + for (MatrixND mattmp : results.values()) { + if (log.isTraceEnabled()) { + log.trace("Ajout de la semantics: " + + Arrays.asList(mattmp.getSemantics())); + } + + for (int s = 0; s < mattmp.getDimCount(); s++) { + sem[s + 1].addAll(mattmp.getSemantic(s)); + } + } + + if (log.isTraceEnabled()) { + log.trace("La semantique final est: " + Arrays.asList(sem)); + } + + // creation de la matrice resultat + resultMat = getMatrixFactory().create(name, sem, dimNames); + + // recuperation du resultat pour chaque date de la simulation, de Date(0) à lastDate + for (Map.Entry<TimeStep, MatrixND> result : results.entrySet()) { + TimeStep d = result.getKey(); + mat = result.getValue(); + // on met ce resultat dans la matrice result si besoin + if (mat != null) { + // on recupere dans la matrice resultat l'endroit on il faut + // mettre la matrice + MatrixND submat = resultMat.getSubMatrix(0, d, 1); + // on met les valeur de mat dans la sous matrice extraite + for (MatrixIterator mi = mat.iterator(); mi.next();) { + submat.setValue( + ArrayUtil.concat(new Object[] { d }, + mi.getSemanticsCoordinates()), mi.getValue()); + } + } + } + } + + resultMat = decorate(resultMat, tx); + return resultMat; + } + + /** + * Get last simulation date. + * + * @return last simulation date + */ + @Override + public TimeStep getLastStep() { + int monthNumber = simulation.getParameter().getNumberOfYear() + * Month.NUMBER_OF_MONTH; + TimeStep result = new TimeStep(monthNumber - 1); // -1 because date begin at 0 + return result; + } + + /* + * @see fr.ifremer.isisfish.simulator.SimulationResultListener#addResult(fr.ifremer.isisfish.simulator.SimulationContext, fr.ifremer.isisfish.types.TimeStep, java.lang.String, org.nuiton.math.matrix.MatrixND) + */ + @Override + public void addResult(SimulationContext context, TimeStep step, String name, + MatrixND mat) throws IsisFishException { + doAddResult(step, name, mat); + } + + /* + * @see fr.ifremer.isisfish.simulator.SimulationResultGetter#getMatrix(fr.ifremer.isisfish.simulator.SimulationContext, fr.ifremer.isisfish.types.TimeStep, java.lang.String) + */ + @Override + public MatrixND getMatrix(SimulationContext context, TimeStep step, String name) { + MatrixND result = null; + try { + result = getMatrix(step, name, context.getDB()); + } catch (TopiaException eee) { + if (log.isWarnEnabled()) { + log.warn(String.format("Can't get result: %1$s", name), eee); + } + } + return result; + + } + + /* + * @see fr.ifremer.isisfish.simulator.SimulationResultGetter#getMatrix(fr.ifremer.isisfish.simulator.SimulationContext, java.lang.String) + */ + @Override + public MatrixND getMatrix(SimulationContext context, String name) { + MatrixND result = null; + try { + result = getMatrix(name, context.getDB()); + } catch (TopiaException eee) { + if (log.isWarnEnabled()) { + log.warn(String.format("Can't get result: %1$s", name), eee); + } + } + return result; + } + + /** + * Try to find better tx. If argument is not null, return it. otherwize + * try to get tx in SimulationContext. + * + * @param tx + * @return + */ + protected TopiaContext getTx(TopiaContext tx) { + TopiaContext result = tx; + if (tx == null) { + // si on a pas de tx, on recherche si on est dans une simulation + // pour recuperer la tx de la simulation + if (simulation == SimulationContext.get().getSimulationStorage()) { + try { + result = SimulationContext.get().getDB(); + } catch (TopiaException eee) { + throw new IsisFishRuntimeException("Can't get database from SimulationContext", eee); + } + } + } + return result; + } + + /* + * @see fr.ifremer.isisfish.simulator.SimulationListener#afterSimulation(fr.ifremer.isisfish.simulator.SimulationContext) + */ + @Override + public void afterSimulation(SimulationContext context) { + } + + /* + * @see fr.ifremer.isisfish.simulator.SimulationListener#beforeSimulation(fr.ifremer.isisfish.simulator.SimulationContext) + */ + @Override + public void beforeSimulation(SimulationContext context) { + } + + // public void addActivatedRule(ResultStorage self, Date date, RegleParam rule){ + // List rules = (List)activatedRules.get(date); + // if(rules == null){ + // activatedRules.put(date, rules = new LinkedList()); + // } + // rules.add(rule); + // } + + // /** + // * Retourne pour une date données tous les RegleParam qui ont été activé + // * a la date demandé. + // * @return une list de {@link fr.ifremer.nodb.RegleParam} + // */ + // public List getActivatedRule(ResultStorage self, Date date){ + // List rules = (List)activatedRules.get(date); + // if(rules == null){ + // activatedRules.put(date, rules = new LinkedList()); + // } + // return rules; + // } + +} Added: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java (rev 0) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -0,0 +1,303 @@ +package fr.ifremer.isisfish.datastore; + + +import fr.ifremer.isisfish.IsisFishRuntimeException; +import fr.ifremer.isisfish.types.TimeStep; +import fr.ifremer.isisfish.util.SimpleParser; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.math.matrix.MatrixIterator; +import org.nuiton.math.matrix.MatrixND; + +/** + * Ecrit les resultats dans des fichiers au format pseudo csv compresser (gz) + * Seule les valeurs differentes de 0 sont ecrite dans le fichier + * + * Format d'une matrice: + * <pre> + * # commentaire + * [nom] + * [nom dimension1]:[semantique1];[semantique2];... + * [nom dimension2]:[semantique1];[semantique2];... + * ... + * [ligne blanche] + * [coordonnee1];[coordonnee2];...;[valeur] + * [coordonnee1];[coordonnee2];...;[valeur] + * [coordonnee1];[coordonnee2];...;[valeur] + * ... + * </pre> + * + * Exemple + * <pre> + * MaMatrice + * Mois:Janvier;Fevrier;Mars + * Ville:Nantes;Paris;Nice + * + * 1;1;2;13.5 + * 0;2;1;4.2 + * </pre> + * + * TODO: + * stat du nombre de demande de resultat or pas de temps courant + * stat sur max ecart entre pas de temps courant et pas de temps demande + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class ResultStorageCSV extends ResultStorageAbstract { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(ResultStorageCSV.class); + + static final private Charset charset = Charset.forName("UTF-8"); + + protected EntitySemanticsDecorator decorator; + + public ResultStorageCSV(SimulationStorage simulation) { + super(simulation); + decorator = new EntitySemanticsDecorator(); + } + + /** + * donne le repertoire de stockage des resultats + * @return + */ + protected File getDirectory() { + File file = SimulationStorage.getResultDirectory(simulation.getDirectory()); + return file; + } + + /** + * Donne le repertoire de stockage pour un certain type de resultat matriciel + * @param name + * @return + */ + protected File getMatrixDirectory(String name) { + File file = new File(getDirectory(), "matrix" + File.separator + name); + return file; + } + + protected File getMatrixFile(TimeStep step, String name) { + File file = new File(getMatrixDirectory(name), step.getStep() + "-" + name + ".csv.gz"); + return file; + } + + protected MatrixND readMatrix(File file) throws IOException { + return readMatrix(file.getPath()); + } + + protected MatrixND readMatrix(String file) throws IOException { + MatrixND result = null; + LineNumberReader in = null; + try { + in = new LineNumberReader(new InputStreamReader( + new GZIPInputStream(new FileInputStream(file)), charset)); + + SimpleParser sp = new SimpleParser(in, true); + + // lecture du nom de la matrice + String name = sp.readString('\n'); + + + // lecture du nom des dimensions et des semantics + List<String> dimNames = new ArrayList<String>(); + List<List<String>> semantics = new ArrayList<List<String>>(); + + String dimName = sp.readString(':'); + while (!sp.isEOL() || StringUtils.isNotBlank(dimName)) { + dimNames.add(dimName); + List<String> sems = new ArrayList<String>(); + semantics.add(sems); + while (!sp.isEOL()) { + String sem = sp.readString(';'); + sems.add(sem); + } + dimName = sp.readString(':'); + } + + // creation de la matrice resultat avec les infos collectes + result = getMatrixFactory().create(name, + semantics.toArray(new List[semantics.size()]), + dimNames.toArray(new String[dimNames.size()])); + + + // lecture des data + int nbDim = dimNames.size(); + int[] coord = new int[nbDim]; + + while (!sp.isEOF()) { + for (int i=0; i<nbDim; i++) { + coord[i] = sp.readInt(';'); + } + double v = sp.readDouble(';'); + result.setValue(coord, v); + } + + } finally { + IOUtils.closeQuietly(in); + } + + return result; + } + + /** + * Extract and create TimeStep from file name. If file name doesn't contains + * TimeStep null is returned + * @param file + * @return + */ + protected TimeStep getTimeStep(String file) { + TimeStep result = null; + try { + String num = StringUtils.substringBefore(file, "-"); + int step = Integer.parseInt(num); + result = new TimeStep(step); + } catch(NumberFormatException eee) { + log.error("File doesn't contains step information: " + file, eee); + } + return result; + } + + @Override + public void delete() { + super.delete(); + try { + File file = getDirectory(); + FileUtils.deleteDirectory(file); + } catch (IOException eee) { + throw new IsisFishRuntimeException("Can't delete results", eee); + } + } + + @Override + protected MatrixND readResult(TimeStep step, String name) { + File file = getMatrixFile(step, name); + MatrixND result = null; + if (file.exists()) { + try { + result = readMatrix(file); + } catch (IOException eee) { + log.error("Can't read result file: " + file, eee); + } + } + return result; + } + + @Override + protected Map<TimeStep, MatrixND> readResult(String name) { + File dir = getMatrixDirectory(name); + String[] files = dir.list(); + + Map<TimeStep, MatrixND> result = new HashMap<TimeStep, MatrixND>(); + for(String file : files) { + try { + TimeStep step = getTimeStep(file); + if (step != null) { + MatrixND mat = readMatrix(dir + File.separator + file); + if (mat != null) { + result.put(step, mat); + } + } + } catch (Exception eee) { + log.error("Can't read result file: " + file, eee); + } + } + return result; + } + + @Override + protected void writeResult(TimeStep step, String name, MatrixND mat) { + File file = getMatrixFile(step, name); + file.getParentFile().mkdirs(); + PrintWriter out = null; + try { + out = new PrintWriter(new BufferedWriter(new OutputStreamWriter( + new GZIPOutputStream(new FileOutputStream(file)), + charset + ))); + + out.print(name); + out.println(); + + List[] sems = mat.getSemantics(); + for (int i=0, maxi=sems.length; i<maxi; i++) { + String dimName = mat.getDimensionName(i); + out.print(dimName); + + List l = sems[i]; + char sep = ':'; + for (Object o : l) { + out.print(sep); + out.print(decorator.undecorate(o)); + sep = ';'; + } + out.println(); + } + + // blank line to separate header and data + out.println(); + + for (MatrixIterator i = mat.iteratorNotZero(); i.hasNext();) { + i.next(); + int[] pos = i.getCoordinates(); + double value = i.getValue(); + for (int p : pos) { + out.print(p); + out.print(';'); + } + out.print(value); + out.println(); + } + + } catch (Exception eee) { + throw new IsisFishRuntimeException("Can't write result: " + file, eee); + } finally { + IOUtils.closeQuietly(out); + } + + } + + @Override + protected void writeActiveRule(TimeStep step, String name, String params) { + // TODO + } + + @Override + public List<String> getResultName() { + File dir = getMatrixDirectory(""); + String[] files = dir.list(); + List<String> result; + if (files != null) { + Arrays.sort(files); + result = Arrays.asList(files); + } else { + result = Collections.EMPTY_LIST; + } + return result; + } + +} Property changes on: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -25,37 +25,17 @@ package fr.ifremer.isisfish.datastore; -import static org.nuiton.i18n.I18n.t; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.math.matrix.MatrixFactory; -import org.nuiton.math.matrix.MatrixIterator; import org.nuiton.math.matrix.MatrixND; import org.nuiton.topia.TopiaContext; -import org.nuiton.util.ArrayUtil; -import org.nuiton.util.HashList; - -import fr.ifremer.isisfish.IsisFishException; -import fr.ifremer.isisfish.entities.Population; -import fr.ifremer.isisfish.export.Export; -import fr.ifremer.isisfish.export.SensitivityExport; -import fr.ifremer.isisfish.rule.Rule; -import fr.ifremer.isisfish.simulator.Objective; -import fr.ifremer.isisfish.simulator.Optimization; -import fr.ifremer.isisfish.simulator.SimulationContext; -import fr.ifremer.isisfish.simulator.SimulationException; -import fr.ifremer.isisfish.simulator.SimulationPlan; -import fr.ifremer.isisfish.simulator.SimulationResultGetter; import fr.ifremer.isisfish.types.TimeStep; -import fr.ifremer.isisfish.types.Month; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -73,14 +53,13 @@ * Mise a jour: $Date$ * par : $Author$ */ -public class ResultStorageInMemory implements SimulationResultGetter, ResultStorage { +public class ResultStorageInMemory extends ResultStorageAbstract { /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(ResultStorageInMemory.class); final static public String MAX_TIME_STEP = "ResultStorageInMemory.maxTimeStep"; protected int maxTimeStep = 12; // default to 1 year - protected SimulationStorage simulation = null; LinkedHashMap<TimeStep, Map<String, MatrixND>> data = new LinkedHashMap<TimeStep, Map<String, MatrixND>>() { private static final long serialVersionUID = 1L; @@ -90,7 +69,7 @@ } }; - LinkedHashMap<TimeStep, List<Rule>> rules = new LinkedHashMap<TimeStep, List<Rule>>() { + LinkedHashMap<TimeStep, List<String>> rules = new LinkedHashMap<TimeStep, List<String>>() { private static final long serialVersionUID = 1L; @Override protected boolean removeEldestEntry(Map.Entry eldest) { @@ -98,10 +77,6 @@ } }; - - /** result enabled */ - transient protected Set<String> enabledResult = null; - /** * Les ResultStorage ne doivent pas etre instancier directement, mais * recuperer a partir d'un @@ -110,187 +85,24 @@ * @param simulation storage to get result */ public ResultStorageInMemory(SimulationStorage simulation) { - this.simulation = simulation; + super(simulation); maxTimeStep = Integer.parseInt(simulation.getParameter().getTagValue().get(MAX_TIME_STEP)); } @Override public void delete() { + super.delete(); data.clear(); } @Override - public void close() { - // do nothing, closed by simulation storage closing + protected MatrixND decorate(MatrixND mat, TopiaContext tx) { + // for in memory do nothing, matrice has already semantics (not string) + return mat; } - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#isEnabled(java.lang.String) - */ @Override - public boolean isEnabled(String name) { - name = name.trim(); - if (enabledResult == null) { - enabledResult = new HashSet<String>(); - - Collection<String> resultEnabled = simulation.getParameter() - .getResultEnabled(); - enabledResult.addAll(resultEnabled); - - // test on export - List<String> exportNames = simulation.getParameter().getExportNames(); - if (exportNames != null) { - for (String exportName : exportNames) { - ExportStorage storage = ExportStorage.getExport(exportName); - try { - Export export = storage.getNewInstance(); - for (String resultName : export.getNecessaryResult()) { - enabledResult.add(resultName); - } - } catch (IsisFishException eee) { - if (log.isWarnEnabled()) { - log.warn(t("isisfish.error.instanciate.export", - exportName), eee); - } - } - } - } - - // test on sensitivity export - List<SensitivityExport> sensitivityExports = simulation - .getParameter().getSensitivityExport(); - if (sensitivityExports != null) { - for (SensitivityExport sensitivityExport : sensitivityExports) { - for (String resultName : sensitivityExport.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - } - - // test on rules - List<Rule> rules = simulation.getParameter().getRules(); - if (rules != null) { - for (Rule rule : rules) { - for (String resultName : rule.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - } - - // test on plans - List<SimulationPlan> plans = simulation.getParameter().getSimulationPlans(); - if (plans != null) { - for (SimulationPlan plan : plans) { - for (String resultName : plan.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - } - - // on objective and optimization - Objective objective = simulation.getParameter().getObjective(); - if (objective != null) { - for (String resultName : objective.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - - Optimization optimization = simulation.getParameter().getOptimization(); - if (optimization != null) { - for (String resultName : optimization.getNecessaryResult()) { - enabledResult.add(resultName); - } - } - - log.info("Enabled result: " + enabledResult); - } - boolean result = enabledResult.contains(name); - return result; - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(fr.ifremer.isisfish.types.TimeStep, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(TimeStep step, MatrixND mat) throws IsisFishException { - addResult(false, step, mat.getName(), mat); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(fr.ifremer.isisfish.types.TimeStep, fr.ifremer.isisfish.entities.Population, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(TimeStep step, Population pop, MatrixND mat) throws IsisFishException { - addResult(false, step, mat.getName(), pop, mat); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(boolean, fr.ifremer.isisfish.types.TimeStep, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(boolean force, TimeStep step, MatrixND mat) throws IsisFishException { - addResult(force, step, mat.getName(), mat); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(boolean, fr.ifremer.isisfish.types.TimeStep, fr.ifremer.isisfish.entities.Population, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(boolean force, TimeStep step, Population pop, MatrixND mat) throws IsisFishException { - addResult(force, step, mat.getName(), pop, mat); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(fr.ifremer.isisfish.types.TimeStep, java.lang.String, fr.ifremer.isisfish.entities.Population, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(TimeStep step, String name, Population pop, MatrixND mat) throws IsisFishException { - addResult(false, step, name, pop, mat); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(fr.ifremer.isisfish.types.TimeStep, java.lang.String, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(TimeStep step, String name, MatrixND mat) throws IsisFishException { - addResult(false, step, name, mat); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(boolean, fr.ifremer.isisfish.types.TimeStep, java.lang.String, fr.ifremer.isisfish.entities.Population, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(boolean force, TimeStep step, String name, Population pop, MatrixND mat) throws IsisFishException { - if (force || isEnabled(name)) { - doAddResult(step, name + " " + pop, mat); - } - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addResult(boolean, fr.ifremer.isisfish.types.TimeStep, java.lang.String, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(boolean force, TimeStep step, String name, MatrixND mat) throws IsisFishException { - if (force || isEnabled(name)) { - doAddResult(step, name, mat); - } - } - - protected void doAddResult(TimeStep step, String name, MatrixND mat) throws IsisFishException { - // si la matrice n'a pas de semantique on refuse - for (int i = 0; i < mat.getDimCount(); i++) { - // la semantique n'est pas bonne des qu'il y a un null dedans - if (mat.getSemantic(i).contains(null)) { - throw new SimulationException( - "Erreur le résultat que vous souhaitez enregistrer n'a pas d'information convenable pour la dimension: " - + i + " " + mat.getDimensionName(i)); - } - } - - // on fait une copie pour avoir reellement des resultats independant - // FIXME echatellier 20120829 : faire une copie optimiser - // suivant l'implementation du vector plutot qu'un parcourt - // via un iterateur de semantiques (plus couteux) + protected void writeResult(TimeStep step, String name, MatrixND mat) { MatrixND newMat = mat.copy(); Map<String, MatrixND> mats = data.get(step); if (mats == null) { @@ -299,24 +111,17 @@ } mats.put(name, newMat); - } + } - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#addActiveRule(fr.ifremer.isisfish.types.TimeStep, fr.ifremer.isisfish.rule.Rule) - */ @Override - public void addActiveRule(TimeStep step, Rule rule) throws IsisFishException { - List<Rule> list = rules.get(step); + protected void writeActiveRule(TimeStep step, String name, String params) { + List<String> list = rules.get(step); if (list == null) { - list = new LinkedList<Rule>(); + list = new LinkedList<String>(); rules.put(step, list); } - list.add(rule); - -// result.setActiveRuleStep(step); -// result.setName(RuleStorage.getName(rule)); -// result.setParam(RuleStorage.getParamAsString(rule)); + list.add(name); } /* (non-Javadoc) @@ -335,19 +140,10 @@ } /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(fr.ifremer.isisfish.types.TimeStep, fr.ifremer.isisfish.entities.Population, java.lang.String) - */ - @Override - public MatrixND getMatrix(TimeStep step, Population pop, String name) { - String newName = name + " " + pop; - return getMatrix(step, newName); - } - - /* (non-Javadoc) * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(fr.ifremer.isisfish.types.TimeStep, java.lang.String) */ @Override - public MatrixND getMatrix(TimeStep step, String name) { + public MatrixND readResult(TimeStep step, String name) { MatrixND result = null; Map<String, MatrixND> mats = data.get(step); if (mats != null) { @@ -356,160 +152,21 @@ return result; } - @Override - public MatrixND getMatrix(TimeStep step, String name, TopiaContext tx) { - return getMatrix(step, name); - } - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(fr.ifremer.isisfish.entities.Population, java.lang.String) - */ - @Override - public MatrixND getMatrix(Population pop, String name) { - String newName = name + " " + pop; - return getMatrix(newName); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(fr.ifremer.isisfish.entities.Population, java.lang.String, org.nuiton.topia.TopiaContext) - */ - @Override - public MatrixND getMatrix(Population pop, String name, TopiaContext tx) { - String newName = name + " " + pop; - return getMatrix(newName, tx); - } - - /* (non-Javadoc) * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(java.lang.String) */ @Override - public MatrixND getMatrix(String name) { - MatrixND result = null; - - List<TimeStep> steps = new ArrayList<TimeStep>(); - Map<TimeStep, MatrixND> mats = new LinkedHashMap<TimeStep, MatrixND>(); - MatrixND matExample = null; + public Map<TimeStep, MatrixND> readResult(String name) { + Map<TimeStep, MatrixND> result = new LinkedHashMap<TimeStep, MatrixND>(); for (Map.Entry<TimeStep, Map<String, MatrixND>> e : data.entrySet()) { TimeStep ts = e.getKey(); MatrixND mat = e.getValue().get(name); if (mat != null) { - steps.add(ts); - mats.put(ts, mat); - matExample = mat; + result.put(ts, mat); } } - if (mats.size() > 0) { - // recuperation des noms des dimensions - String[] dimNames = new String[1 + matExample.getDimCount()]; - dimNames[0] = t("isisfish.common.date"); - for (int i = 1; i < dimNames.length; i++) { - dimNames[i] = matExample.getDimensionName(i - 1); - } - - // creation de la semantique pour la matrice resultat. +1 pour les dates - List[] sem = new List[1 + matExample.getDimCount()]; - sem[0] = steps; - - for (int i = 1; i < sem.length; i++) { - sem[i] = new HashList(); - } - - for (MatrixND mattmp : mats.values()) { - if (log.isTraceEnabled()) { - log.trace("Ajout de la semantics: " - + Arrays.asList(mattmp.getSemantics())); - } - - for (int s = 0; s < mattmp.getDimCount(); s++) { - sem[s + 1].addAll(mattmp.getSemantic(s)); - } - } - - if (log.isTraceEnabled()) { - log.trace("La semantique final est: " + Arrays.asList(sem)); - } - - // creation de la matrice resultat - result = MatrixFactory.getInstance().create(name, sem, dimNames); - - // recuperation du resultat pour chaque date de la simulation, de Date(0) à lastDate - for (Map.Entry<TimeStep, MatrixND> e : mats.entrySet()) { - TimeStep d = e.getKey(); - MatrixND mat = e.getValue(); - // on recupere dans la matrice resultat l'endroit on il faut - // mettre la matrice - MatrixND submat = result.getSubMatrix(0, d, 1); - // on met les valeur de mat dans la sous matrice extraite - for (MatrixIterator mi = mat.iterator(); mi.next();) { - submat.setValue( - ArrayUtil.concat(new Object[] { d }, - mi.getSemanticsCoordinates()), mi.getValue()); - } - } - } - return result; } - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(java.lang.String, org.nuiton.topia.TopiaContext) - */ - @Override - public MatrixND getMatrix(String name, TopiaContext tx) { - return getMatrix(name); - } - - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#getLastStep() - */ - @Override - public TimeStep getLastStep() { - int monthNumber = simulation.getParameter().getNumberOfYear() - * Month.NUMBER_OF_MONTH; - TimeStep result = new TimeStep(monthNumber - 1); // -1 because date begin at 0 - return result; - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationResultListener#addResult(fr.ifremer.isisfish.simulator.SimulationContext, fr.ifremer.isisfish.types.TimeStep, java.lang.String, org.nuiton.math.matrix.MatrixND) - */ - @Override - public void addResult(SimulationContext context, TimeStep step, String name, - MatrixND mat) throws IsisFishException { - doAddResult(step, name, mat); - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationResultGetter#getMatrix(fr.ifremer.isisfish.simulator.SimulationContext, fr.ifremer.isisfish.types.TimeStep, java.lang.String) - */ - @Override - public MatrixND getMatrix(SimulationContext context, TimeStep step, String name) { - MatrixND result = getMatrix(step, name); - return result; - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationResultGetter#getMatrix(fr.ifremer.isisfish.simulator.SimulationContext, java.lang.String) - */ - @Override - public MatrixND getMatrix(SimulationContext context, String name) { - MatrixND result = getMatrix(name); - return result; - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationListener#afterSimulation(fr.ifremer.isisfish.simulator.SimulationContext) - */ - @Override - public void afterSimulation(SimulationContext context) { - } - - /* - * @see fr.ifremer.isisfish.simulator.SimulationListener#beforeSimulation(fr.ifremer.isisfish.simulator.SimulationContext) - */ - @Override - public void beforeSimulation(SimulationContext context) { - } - } Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationStorage.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationStorage.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationStorage.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -78,6 +78,7 @@ public static final String PARAMETERS_FILENAME = "parameters.properties"; public static final String RESULT_XML_FILENAME = "isis-mexico-output.xml"; public static final String RESULT_FILENAME = "results.mapped"; + public static final String RESULT_DIRECTORY = "results"; public static final String RESULT_EXPORT_DIRECTORY = "resultExports"; public static final String DESIGN_PLAN_FILENAME = "isis-mexico-designplan.xml"; public static final String SENSITIVITY_RESULTS = "sensitivityresults"; @@ -260,6 +261,21 @@ } /** + * Retourne le fichier de stockage des resultats + * de la simulation. + * + * @param root le repertoire de stockage de la simulation + * @return + */ + public static File getResultDirectory(File root) { + File result = new File(root, RESULT_DIRECTORY); + if (!result.exists()) { + result.mkdirs(); + } + return result; + } + + /** * Retourne le fichier de stockage des parametres de la simulation courante. * * @return parameter file @@ -523,7 +539,14 @@ resultStorage = new ResultStorageInMemory(this); } else { try { - resultStorage = new ResultMappedStorage(this); + if (getResultFile(getDirectory()).exists()) { + // if result.mapped exist, use ResultMappedStorage + // need to read old simulation + resultStorage = new ResultMappedStorage(this); + } else { + // else use ResultStorageCSV, the new default result storage + resultStorage = new ResultStorageCSV(this); + } } catch (IOException ex) { throw new StorageException("Can't get storage", ex); } Modified: trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -25,7 +25,6 @@ package fr.ifremer.isisfish.simulator; -import fr.ifremer.isisfish.IsisConfig; import fr.ifremer.isisfish.IsisFish; import java.io.File; import java.util.HashMap; @@ -105,7 +104,10 @@ protected SimulationContext() { // force matrix backend for this simulation - MatrixFactory.initMatrixFactoryThreadLocal(IsisFish.config.getSimulationMatrixVectorClass()); + MatrixFactory.initMatrixFactoryThreadLocal( + IsisFish.config.getSimulationMatrixVectorClass(), + IsisFish.config.getSimulationMatrixVectorSparseClass(), + IsisFish.config.getSimulationMatrixThresholdUseSparse()); } /** Modified: trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -423,8 +423,10 @@ // context.fireBeforeSimulation(); String matrixBackend = MatrixFactory.getInstance().getVectorClass().getName(); + String matrixSparseBackend = MatrixFactory.getInstance().getSparseVectorClass().getName(); + int threshold = MatrixFactory.getInstance().getThresholdSparse(); simulation.getInformation().addInformation("Matrix backend: " - + matrixBackend); + + matrixBackend + " and " + matrixSparseBackend + " threshold: " + threshold); log.info("Matrix backend: " + matrixBackend); simulatorObject.simulate(context); Modified: trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -226,6 +226,8 @@ modelBuilder.addOption(IsisConfig.Option.SIMULATOR_SUB_MAXPROCESS); modelBuilder.addOption(IsisConfig.Option.SIMULATOR_SUB_MAXMEMORY); modelBuilder.addOption(IsisConfig.Option.SIMULATION_MATRIX_VECTOR_CLASS); + modelBuilder.addOption(IsisConfig.Option.SIMULATION_MATRIX_VECTOR_SPARSE_CLASS); + modelBuilder.addOption(IsisConfig.Option.SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS); modelBuilder.addOption(IsisConfig.Option.MAPPED_RESULT_MATRIX_VECTOR_CLASS); // category database vcs Modified: trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java 2014-06-17 16:17:51 UTC (rev 4025) +++ trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -85,12 +85,9 @@ // on prend le nombre de case de la matrice pour sa taille // on ne tient pas compte des dimensions qui sont des objets // utilises ailleurs - int[] dim = ((MatrixND)value).getDim(); - int size = 1; - for (int i=0; i<dim.length; i++) { - size *= dim[i]; - } - result += size * DOUBLE_SIZE; // size is wrong, because depend on matrix type (float or double) + int size = ((MatrixND)value).getNumberOfAssignedValue(); + // size is wrong, because depend on matrix type (float or double), but approximation is acceptable + result += size * DOUBLE_SIZE; } else if (value instanceof Collection) { // List<Zone|Pop|Str|Metier|Cell> // on prend la taille de la collection, car la donnees Added: trunk/src/main/java/fr/ifremer/isisfish/util/SimpleParser.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/util/SimpleParser.java (rev 0) +++ trunk/src/main/java/fr/ifremer/isisfish/util/SimpleParser.java 2014-06-29 18:44:00 UTC (rev 4026) @@ -0,0 +1,131 @@ +package fr.ifremer.isisfish.util; + + +import java.io.BufferedReader; +import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Very simple parse write for efficiency. Input stream must be correct no check + * is done on data. + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class SimpleParser { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(SimpleParser.class); + + protected boolean intern; + protected BufferedReader in; + protected StringBuilder sb = new StringBuilder(); + + protected boolean eof = false; + protected boolean eol = false; + /** + * + * @param in BufferedReader because Reader must support mark and reset + */ + public SimpleParser(BufferedReader in, boolean intern) { + this.in = in; + this.intern = intern; + } + + /** + * + * @return true if end of file + */ + public boolean isEOF() { + return eof; + } + + /** + * + * @return true if end of line + */ + public boolean isEOL() { + return eol; + } + + protected void read(char sep) throws IOException { + eol = false; + sb.setLength(0); + + int i; + while ((i = in.read()) != -1) { + char c = (char)i; + if (c == '\r') { + // try to eat \n after \r + in.mark(1); + c = (char)in.read(); + if (c != '\n') { + in.reset(); + } + eol = true; + break; + } else if (c == '\n') { + eol = true; + break; + } else if (c == sep) { + break; + } else { + sb.append(c); + } + } + + if (i != -1) { + // test if not eof + in.mark(1); + i = in.read(); + in.reset(); + } + eof = i == -1; + } + + public String readString(char sep) throws IOException { + read(sep); + String result = sb.toString(); + if (intern) { + result = result.intern(); + } + return result; + } + + public int readInt(char sep) throws IOException { + read(sep); + int result = parseInt(sb); + return result; + } + + public double readDouble(char sep) throws IOException { + read(sep); + double result = parseDouble(sb); + return result; + } + + protected double parseDouble(StringBuilder s) throws IOException { + return Double.parseDouble(s.toString()); + } + + /** + * custom parseInt: 4 fois plus performant que Integer.parseInt + * ne supporte que les chiffres positif, et la chaine doit forcement etre un nombre valide + * @param sb la chaine representant l'entier + * @return + */ + protected int parseInt(StringBuilder sb) { + int result = 0; + for (int i=0, maxi=sb.length(); i<maxi; i++) { + char c = sb.charAt(i); + result = result * 10 + c - '0'; + } + return result; + } + + +} Property changes on: trunk/src/main/java/fr/ifremer/isisfish/util/SimpleParser.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL