Author: tchemit Date: 2014-04-05 13:00:41 +0200 (Sat, 05 Apr 2014) New Revision: 1002 Url: http://forge.codelutin.com/projects/echobase/repository/revisions/1002 Log: refs-60 extract Raw data Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportRow.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportRow.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportRow.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/service/exportCoser/ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserServiceTest.java trunk/echobase-services/src/test/resources/echobase-services-test.properties Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportRow.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/AcousticImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/BiometrySampleImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/TotalSampleImportModel.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FakeEchoBaseServiceContext.java Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/config/EchoBaseConfiguration.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -65,11 +65,15 @@ } public EchoBaseConfiguration(Properties propos) { + this("echobase.properties", propos); + } + public EchoBaseConfiguration(String configFileName, Properties propos) { + EchoBaseDateConverter.initDateConverter(); applicationConfig = new ApplicationConfig(); - applicationConfig.setConfigFileName("echobase.properties"); + applicationConfig.setConfigFileName(configFileName); if (log.isInfoEnabled()) { log.info(this + " is initializing..."); Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -25,10 +25,10 @@ import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.DataMetadata; import fr.ifremer.echobase.entities.references.DataQuality; +import org.nuiton.csv.ValueParser; import org.nuiton.csv.ValueParserFormatter; +import org.nuiton.csv.ValueSetter; import org.nuiton.topia.service.csv.TopiaCsvCommons; -import org.nuiton.csv.ValueParser; -import org.nuiton.csv.ValueSetter; import java.util.Date; import java.util.List; @@ -45,6 +45,7 @@ public static final String CELLULE_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSS"; public static final ValueParser<Date> IMPORT_DAY_TIME_ECHOBASE = new DateValue(CELLULE_DATE_FORMAT); + public static final ValueParser<Date> IMPORT_DAY_TIME_ECHOBASE2 = new DateValue("dd/MM/yyyy HH:mm:ss.SSSS"); @@ -56,9 +57,14 @@ } }; - public static final ValueParser<Float> NA_TO_FLOAT_PARSER = new FloatParserFormatter(null, true) { + public static final ValueParserFormatter<Float> NA_TO_FLOAT_PARSER_FORMATTER = new FloatParserFormatter(null, true) { @Override + public String format(Float value) { + return value == null ? "NA" : super.format(value); + } + + @Override protected Float parseNoneEmptyValue(String value) { Float result = null; if (!"NA".equals(value)) { @@ -84,9 +90,14 @@ }; - public static final ValueParser<Integer> NA_TO_INTEGER_PARSER = new IntegerParserFormatter(null, true) { + public static final ValueParserFormatter<Integer> NA_TO_INTEGER_PARSER_FORMATTER = new IntegerParserFormatter(null, true) { @Override + public String format(Integer value) { + return value == null ? "NA" : super.format(value); + } + + @Override protected Integer parseNoneEmptyValue(String value) { Integer result = null; if (!"NA".equals(value)) { Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -708,6 +708,10 @@ return persistenceContext.getVoyageDao().findByTopiaId(id); } + public List<Voyage> getVoyagesForMission(Mission mission) { + return persistenceContext.getVoyageDao().forMissionEquals(mission).findAll(); + } + public long countVoyage() { return persistenceContext.getVoyageDao().count(); } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserService.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserService.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -29,8 +29,10 @@ import fr.ifremer.coser.bean.EchoBaseProject; import fr.ifremer.echobase.EchoBaseTechnicalException; import fr.ifremer.echobase.entities.data.Result; +import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.DataMetadata; import fr.ifremer.echobase.entities.references.Mission; +import fr.ifremer.echobase.entities.references.SampleDataType; import fr.ifremer.echobase.entities.references.Species; import fr.ifremer.echobase.io.CommandLineUtils; import fr.ifremer.echobase.io.EchoBaseIOUtil; @@ -81,7 +83,8 @@ Preconditions.checkNotNull(model.getPopulationIndicator()); Preconditions.checkNotNull(model.getCommunityIndicator()); - model.setNbSteps(5 + + model.setNbSteps(4 + + 4 + // raw data model.getPopulationIndicator().size() + model.getCommunityIndicator().size()); @@ -103,8 +106,7 @@ model.incrementsProgress(); // 3 - Extract source files - extractRawData(project, model); - model.incrementsProgress(); + extractRawData(project, mission, model); // 4 - Extract population indicators extractPopulationIndicators(model, project, mission); @@ -113,7 +115,7 @@ extractCommunityIndicators(model, project, mission); // 6 - Export species file - exportSpeciesFile(project); + extractSpeciesFile(project); model.incrementsProgress(); // 7 - create archive @@ -190,6 +192,9 @@ Mission mission) { File exportFile = echoBaseProject.getPopulationIndicatorsFile(); + if (log.isInfoEnabled()) { + log.info("Generate " + exportFile); + } PopulationIndicatorExportModel exportModel = new PopulationIndicatorExportModel(getCsvSeparator()); @@ -228,7 +233,6 @@ PopulationIndicatorExportRow row = new PopulationIndicatorExportRow(); row.setMission(mission); row.setIndicator(dataMetadata); - row.setList("L1"); //FIXME FAKE! Should remove it Species species = result.getCategory().getSpeciesCategory().getSpecies(); row.setSpecies(species); row.setStratum("Total"); //FIXME How to find this ? @@ -269,6 +273,9 @@ File exportFile = echoBaseProject.getCommunityIndicatorsFile(); + if (log.isInfoEnabled()) { + log.info("Generate " + exportFile); + } CommunityIndicatorExportModel exportModel = new CommunityIndicatorExportModel(getCsvSeparator()); List<CommunityIndicatorExportRow> rows = Lists.newArrayList(); @@ -374,7 +381,7 @@ } String dbHost = StringUtils.substringBefore(url, "/"); String dbName = StringUtils.substringAfter(url, "/"); - String dbPort = "5432"; + String dbPort = "5432"; // default pg port if (dbHost.contains(":")) { // a port is given dbPort = StringUtils.substringAfter(dbHost, ":"); @@ -428,21 +435,84 @@ //------------------------------------------------------------------------// protected void extractRawData(EchoBaseProject project, + Mission mission, ExportCoserConfiguration model) throws IOException { - //TODO Do this + File rawDataDirectory = project.getRawDataDirectory(); + EchoBaseIOUtil.forceMkdir(rawDataDirectory); + if (log.isInfoEnabled()) { log.info("Extract raw data for mission: " + model.getMissionId() + - " to : " + project.getRawDataDirectory()); + " to : " + rawDataDirectory); } - FileUtils.write(new File(project.getRawDataDirectory(), "TODO.txt"), "TODO"); + List<Voyage> voyages = persistenceService.getVoyagesForMission(mission); + + { // create catch file + File file = new File(rawDataDirectory, "Captures.csv"); + if (log.isInfoEnabled()) { + log.info("generate " + file); + } + RawDataCatchExportModel exportModel = new RawDataCatchExportModel(getCsvSeparator()); + + List<RawDataCatchExportRow> rows = exportModel.prepareRows(voyages); + try { + Export.exportToFile(exportModel, rows, file); + } catch (Exception e) { + throw new EchoBaseTechnicalException("Could not export catches to " + file, e); + } + model.incrementsProgress(); + } + + { // create size file + + SampleDataType weightSampleDataType = persistenceService.getSampleDataTypeByName("LTcm1"); + SampleDataType lengthSampleDataType = persistenceService.getSampleDataTypeByName("WeightAtLengthkg"); + + File file = new File(rawDataDirectory, "Tailles.csv"); + if (log.isInfoEnabled()) { + log.info("generate " + file); + } + RawDataSizeExportModel exportModel = new RawDataSizeExportModel(getCsvSeparator()); + + List<RawDataSizeExportRow> rows = exportModel.prepareRows(voyages, + weightSampleDataType, + lengthSampleDataType); + try { + Export.exportToFile(exportModel, rows, file); + } catch (Exception e) { + throw new EchoBaseTechnicalException("Could not export size to " + file, e); + } + model.incrementsProgress(); + } + { // create strate file + + DataMetadata surfaceMetdata = persistenceService.getDataMetadataByName("Surface"); + + File file = new File(rawDataDirectory, "Strates.csv"); + if (log.isInfoEnabled()) { + log.info("generate " + file); + } + RawDataStrataExportModel exportModel = new RawDataStrataExportModel(getCsvSeparator()); + + List<RawDataStrataExportRow> rows = exportModel.prepareRows(voyages, surfaceMetdata); + try { + Export.exportToFile(exportModel, rows, file); + } catch (Exception e) { + throw new EchoBaseTechnicalException("Could not export strata to " + file, e); + } + model.incrementsProgress(); + } + + // create fishingOperation file + // TODO + model.incrementsProgress(); } //------------------------------------------------------------------------// //--- Extract Species File -----------------------------------------------// //------------------------------------------------------------------------// - protected void exportSpeciesFile(EchoBaseProject project) { + protected void extractSpeciesFile(EchoBaseProject project) { File speciesFile = project.getSpeciesDefinitionFile(); SpeciesExportModel speciesExportModel = new SpeciesExportModel(getCsvSeparator()); try { Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportModel.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -36,7 +36,6 @@ super(separator); newColumnForExport("Campagne", EchoBaseCsvUtil.<PopulationIndicatorExportRow, String>newBeanProperty(PopulationIndicatorExportRow.PROPERTY_MISSION_NAME)); newColumnForExport("Indicateur", EchoBaseCsvUtil.<PopulationIndicatorExportRow, String>newBeanProperty(PopulationIndicatorExportRow.PROPERTY_INDICATOR_NAME)); - newColumnForExport("Liste", PopulationIndicatorExportRow.PROPERTY_LIST); newColumnForExport("Espece", EchoBaseCsvUtil.<PopulationIndicatorExportRow, String>newBeanProperty(PopulationIndicatorExportRow.PROPERTY_SPECIES_NAME)); newColumnForExport("Strate", PopulationIndicatorExportRow.PROPERTY_STRATUM); newColumnForExport("Annee", PopulationIndicatorExportRow.PROPERTY_DATE, EchoBaseCsvUtil.YEAR); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportRow.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/PopulationIndicatorExportRow.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -39,8 +39,6 @@ public static final String PROPERTY_INDICATOR_NAME = "indicator.name"; - public static final String PROPERTY_LIST = "list"; - public static final String PROPERTY_SPECIES_NAME = "species.codeMemo"; public static final String PROPERTY_STRATUM = "stratum"; @@ -57,8 +55,6 @@ protected DataMetadata indicator; - protected String list; - protected Species species; protected String stratum; @@ -77,11 +73,6 @@ return indicator; } - public String getList() { - return list; - } - - public Species getSpecies() { return species; } @@ -114,10 +105,6 @@ this.indicator = indicator; } - public void setList(String list) { - this.list = list; - } - public void setSpecies(Species species) { this.species = species; } Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportModel.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,156 @@ +package fr.ifremer.echobase.services.service.exportCoser; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import fr.ifremer.echobase.csv.EchoBaseCsvUtil; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.Transect; +import fr.ifremer.echobase.entities.data.Transit; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import org.apache.commons.lang3.mutable.MutableFloat; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.tuple.Pair; +import org.nuiton.csv.ext.AbstractExportModel; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * Created on 4/5/14. + * + * @author Tony Chemit <chemit@codelutin.com> + * @since 2.6 + */ +public class RawDataCatchExportModel extends AbstractExportModel<RawDataCatchExportRow> { + + /* + SELECT + voyage.name AS Campagne, + species.baracoudacode AS Espece, + operation.id AS Trait, + sample.numbersampled AS Nombre, + sample.sampleweight AS Poids + FROM + transit AS transit, + voyage AS voyage, + transect AS transect, + operation AS operation, + sample AS sample, + speciescategory AS speciescategory, + species AS species + WHERE + transit.voyage = voyage.topiaid AND + transect.transit = transit.topiaid AND + operation.transect = transect.topiaid AND + sample.operation = operation.topiaid AND + sample.speciescategory = speciescategory.topiaid AND + speciescategory.species = species.topiaid + +dans cette requête, il faut en plus : +- enlever le '_' dans le baracoudacode +- créer un champ "Annee" avec l'année du voyage +- faire une somme des 'Nombre" et "Poids" en groupant sur les champs +"Campagne","trait", "Espece" et "Annee", mais j'y arrive pas... + */ + + public RawDataCatchExportModel(char separator) { + super(separator); + newColumnForExport("Campagne", EchoBaseCsvUtil.<RawDataCatchExportRow, String>newBeanProperty(RawDataCatchExportRow.PROPERTY_VOYAGE_NAME)); + newColumnForExport("Annee", EchoBaseCsvUtil.<RawDataCatchExportRow, Date>newBeanProperty(RawDataCatchExportRow.PROPERTY_VOYAGE_YEAR), EchoBaseCsvUtil.YEAR); + newColumnForExport("Trait", EchoBaseCsvUtil.<RawDataCatchExportRow, String>newBeanProperty(RawDataCatchExportRow.PROPERTY_OPERATION_ID)); + newColumnForExport("Espece", EchoBaseCsvUtil.<RawDataCatchExportRow, String>newBeanProperty(RawDataCatchExportRow.PROPERTY_SPECIES_CODE)); + newColumnForExport("Nombre", RawDataCatchExportRow.PROPERTY_OPERATION_NUMBER, EchoBaseCsvUtil.PRIMITIVE_INTEGER); + newColumnForExport("Poids", RawDataCatchExportRow.PROPERTY_OPERATION_WEIGHT, EchoBaseCsvUtil.PRIMITIVE_FLOAT); + } + + public List<RawDataCatchExportRow> prepareRows(List<Voyage> voyages) { + List<RawDataCatchExportRow> rows = Lists.newArrayList(); + for (Voyage voyage : voyages) { + prepareRows(voyage, rows); + } + return rows; + } + + protected void prepareRows(Voyage voyage, List<RawDataCatchExportRow> rows) { + if (voyage.isTransitNotEmpty()) { + for (Transit transit : voyage.getTransit()) { + + if (transit.isTransectNotEmpty()) { + for (Transect transect : transit.getTransect()) { + + if (transect.isOperationNotEmpty()) { + for (Operation operation : transect.getOperation()) { + + if (operation.isSampleNotEmpty()) { + + // compute sum of number and weight per species + Map<Species, Pair<MutableInt, MutableFloat>> speciesMap = Maps.newHashMap(); + for (Sample sample : operation.getSample()) { + + SpeciesCategory speciesCategory = sample.getSpeciesCategory(); + if (speciesCategory != null) { + Species species = speciesCategory.getSpecies(); + Pair<MutableInt, MutableFloat> numberWeight = speciesMap.get(species); + if (numberWeight == null) { + numberWeight = Pair.of(new MutableInt(), new MutableFloat()); + speciesMap.put(species, numberWeight); + } + Integer numberSampled = sample.getNumberSampled(); + if (numberSampled != null) { + numberWeight.getLeft().add(numberSampled); + } + Float sampleWeight = sample.getSampleWeight(); + if (sampleWeight != null) { + numberWeight.getRight().add(sampleWeight); + } + } + } + + // create rows + for (Map.Entry<Species, Pair<MutableInt, MutableFloat>> entry : speciesMap.entrySet()) { + Species species = entry.getKey(); + Pair<MutableInt, MutableFloat> numberWeight = entry.getValue(); + + RawDataCatchExportRow row = new RawDataCatchExportRow(); + row.setVoyage(voyage); + row.setOperation(operation); + row.setSpecies(species); + row.setNumber(numberWeight.getLeft().intValue()); + row.setWeight(numberWeight.getRight().floatValue()); + rows.add(row); + } + } + } + } + } + } + } + } + } + +} Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportModel.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportRow.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportRow.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,99 @@ +package fr.ifremer.echobase.services.service.exportCoser; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.Species; + +/** + * Created on 4/5/14. + * + * @author Tony Chemit <chemit@codelutin.com> + * @since 2.6 + */ +public class RawDataCatchExportRow { + + public static final String PROPERTY_VOYAGE = "voyage"; + + public static final String PROPERTY_SPECIES_CODE = "species." + Species.PROPERTY_CODE_MEMO; + + public static final String PROPERTY_OPERATION_ID = "operation." + Operation.PROPERTY_ID; + + public static final String PROPERTY_VOYAGE_NAME = PROPERTY_VOYAGE + "." + Voyage.PROPERTY_NAME; + + public static final String PROPERTY_VOYAGE_YEAR = PROPERTY_VOYAGE + "." + Voyage.PROPERTY_START_DATE; + + public static final String PROPERTY_OPERATION_NUMBER = "number"; + + public static final String PROPERTY_OPERATION_WEIGHT = "weight"; + + protected Voyage voyage; + + protected Species species; + + protected Operation operation; + + protected int number; + + protected float weight; + + public Voyage getVoyage() { + return voyage; + } + + public void setVoyage(Voyage voyage) { + this.voyage = voyage; + } + + public Species getSpecies() { + return species; + } + + public void setSpecies(Species species) { + this.species = species; + } + + public Operation getOperation() { + return operation; + } + + public void setOperation(Operation operation) { + this.operation = operation; + } + + public int getNumber() { + return number; + } + + public void setNumber(int number) { + this.number = number; + } + + public float getWeight() { + return weight; + } + + public void setWeight(float weight) { + this.weight = weight; + } +} Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataCatchExportRow.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportModel.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,249 @@ +package fr.ifremer.echobase.services.service.exportCoser; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.google.common.collect.Lists; +import fr.ifremer.echobase.csv.EchoBaseCsvUtil; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.entities.data.Transect; +import fr.ifremer.echobase.entities.data.Transit; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.SampleDataType; +import fr.ifremer.echobase.entities.references.SexCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import org.apache.commons.collections4.keyvalue.MultiKey; +import org.apache.commons.collections4.map.MultiKeyMap; +import org.nuiton.csv.ValueFormatter; +import org.nuiton.csv.ext.AbstractExportModel; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * Created on 4/5/14. + * + * @author Tony Chemit <chemit@codelutin.com> + * @since 2.6 + */ +public class RawDataSizeExportModel extends AbstractExportModel<RawDataSizeExportRow> { + + /* + SELECT + voyage.name AS Campagne, + operation.id AS Trait, + species.baracoudacode AS Espece, + sexcategory.name AS Sexe, + sampledata.datalabel, + sampledata.datavalue, + sampledatatype.name + FROM + voyage AS voyage, + mission AS mission, + transit AS transit, + transect AS transect, + operation AS operation, + sample AS sample, + sampledata AS sampledata, + sampledatatype AS sampledatatype, + sampletype AS sampletype, + speciescategory AS speciescategory, + species AS species, + sizecategory AS sizecategory, + sexcategory AS sexcategory + WHERE + voyage.mission = mission.topiaid AND + transit.voyage = voyage.topiaid AND + transect.transit = transit.topiaid AND + operation.transect = transect.topiaid AND + sample.operation = operation.topiaid AND + sampledata.sample = sample.topiaid AND + sampledata.sampledatatype = sampledatatype.topiaid AND + sample.sampletype = sampletype.topiaid AND + sample.speciescategory = speciescategory.topiaid AND + speciescategory.species = species.topiaid AND + speciescategory.sizecategory = sizecategory.topiaid AND + speciescategory.sexcategory = sexcategory.topiaid AND + sampletype.name = 'Subsample' + +dans cette requête, il faut en plus : +- enlever le '_' dans le baracoudacode +- créer un champ Annee avec l'année du voyage +- mettre en colonne les champs LTcm1 et WeightAtLengthkg +- faire une somme des datavalues des 'LTcm1 et WeightAtLengthkg +respectivement en groupant sur les champs Campagne,trait, Espece +et "Annee", mais j'y arrive pas... +- ajouter une colonne "maturite" avec des NA +- convertir les "N" de la colonne "sexe" en NA + */ + + public RawDataSizeExportModel(char separator) { + super(separator); + newColumnForExport("Campagne", EchoBaseCsvUtil.<RawDataSizeExportRow, String>newBeanProperty(RawDataSizeExportRow.PROPERTY_VOYAGE_NAME)); + newColumnForExport("Annee", EchoBaseCsvUtil.<RawDataSizeExportRow, Date>newBeanProperty(RawDataSizeExportRow.PROPERTY_VOYAGE_YEAR), EchoBaseCsvUtil.YEAR); + newColumnForExport("Trait", EchoBaseCsvUtil.<RawDataSizeExportRow, String>newBeanProperty(RawDataSizeExportRow.PROPERTY_OPERATION_ID)); + newColumnForExport("Espece", EchoBaseCsvUtil.<RawDataSizeExportRow, String>newBeanProperty(RawDataSizeExportRow.PROPERTY_SPECIES_CODE)); + newColumnForExport("Sexe", RawDataSizeExportRow.PROPERTY_SEX_CATEGORY, new ValueFormatter<SexCategory>() { + @Override + public String format(SexCategory value) { + String result; + if (value == null || "N".equals(value.getName())) { + result = "NA"; + } else { + result = value.getName(); + } + return result; + } + }); + newColumnForExport("Maturite", RawDataSizeExportRow.PROPERTY_MATURITE); + newColumnForExport("Longueur", RawDataSizeExportRow.PROPERTY_LENGTH_STEP); + newColumnForExport("Nombre", RawDataSizeExportRow.PROPERTY_NUMBER, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER); + newColumnForExport("Poids", RawDataSizeExportRow.PROPERTY_WEIGHT, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); + newColumnForExport("Age", RawDataSizeExportRow.PROPERTY_AGE, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER); + } + + public List<RawDataSizeExportRow> prepareRows(List<Voyage> voyages, + SampleDataType weightSampleDataType, + SampleDataType lenghtSampleDataType) { + List<RawDataSizeExportRow> rows = Lists.newArrayList(); + for (Voyage voyage : voyages) { + prepareRows(voyage, rows, weightSampleDataType, lenghtSampleDataType); + } + return rows; + } + + protected void prepareRows(Voyage voyage, + List<RawDataSizeExportRow> rows, + SampleDataType weightSampleDataType, + SampleDataType lenghtSampleDataType) { + if (voyage.isTransitNotEmpty()) { + for (Transit transit : voyage.getTransit()) { + + if (transit.isTransectNotEmpty()) { + for (Transect transect : transit.getTransect()) { + + if (transect.isOperationNotEmpty()) { + for (Operation operation : transect.getOperation()) { + + if (operation.isSampleNotEmpty()) { + + // compute sum of number and weight per species and lengthStep + MultiKeyMap<Object, LengthStepContext> map = new MultiKeyMap<Object, LengthStepContext>(); + + for (Sample sample : operation.getSample()) { + + SpeciesCategory speciesCategory = sample.getSpeciesCategory(); + if (speciesCategory != null) { + Species species = speciesCategory.getSpecies(); + SexCategory sexCategory = speciesCategory.getSexCategory(); + + if (sample.isSampleDataNotEmpty()) { + for (SampleData sampleData : sample.getSampleData()) { + if (weightSampleDataType.equals(sampleData.getSampleDataType())) { + + // register a weight + addWeight(species, sexCategory, map, sampleData); + } else if (lenghtSampleDataType.equals(sampleData.getSampleDataType())) { + // register a number + addNumber(species, sexCategory, map, sampleData); + } + } + } + + + } + } + + // create rows + for (Map.Entry<MultiKey<?>, LengthStepContext> entry : map.entrySet()) { + MultiKey<?> key = entry.getKey(); + Species species = (Species) key.getKey(0); + String lengthStep = (String) key.getKey(1); + + LengthStepContext numberWeight = entry.getValue(); + + RawDataSizeExportRow row = new RawDataSizeExportRow(); + row.setVoyage(voyage); + row.setOperation(operation); + row.setSpecies(species); + row.setSexCategory(numberWeight.sexCategory); + row.setMaturite("NA"); + row.setLengthStep(lengthStep); + row.setNumber((int) numberWeight.number); + row.setWeight(numberWeight.weight); + rows.add(row); + } + } + } + } + } + } + } + } + } + + protected void addWeight(Species species, SexCategory sexCategory, MultiKeyMap<Object, LengthStepContext> map, SampleData sampleData) { + String lengthStep = sampleData.getDataLabel(); + LengthStepContext lengthStepContext = map.get(species, lengthStep); + if (lengthStepContext == null) { + lengthStepContext = new LengthStepContext(sexCategory); + map.put(species, lengthStep, lengthStepContext); + } + Float weight = sampleData.getDataValue(); + lengthStepContext.addWeight(weight); + } + + protected void addNumber(Species species, SexCategory sexCategory, MultiKeyMap<Object, LengthStepContext> map, SampleData sampleData) { + String lengthStep = sampleData.getDataLabel(); + LengthStepContext lengthStepContext = map.get(species, lengthStep); + if (lengthStepContext == null) { + lengthStepContext = new LengthStepContext(sexCategory); + map.put(species, lengthStep, lengthStepContext); + } + Float number = sampleData.getDataValue(); + lengthStepContext.addNumber(number); + } + + static class LengthStepContext { + + protected final SexCategory sexCategory; + + protected float number; + + protected float weight; + + LengthStepContext(SexCategory sexCategory) { + this.sexCategory = sexCategory; + } + + void addNumber(float number) { + this.number += number; + } + + void addWeight(float weight) { + this.weight += weight; + } + } +} \ No newline at end of file Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportModel.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportRow.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportRow.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,167 @@ +package fr.ifremer.echobase.services.service.exportCoser; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.SexCategory; +import fr.ifremer.echobase.entities.references.Species; + +/** + * Created on 4/5/14. + * + * @author Tony Chemit <chemit@codelutin.com> + * @since 2.6 + */ +public class RawDataSizeExportRow { + + public static final String PROPERTY_VOYAGE = "voyage"; + + public static final String PROPERTY_SPECIES_CODE = "species." + Species.PROPERTY_CODE_MEMO; + + public static final String PROPERTY_OPERATION_ID = "operation." + Operation.PROPERTY_ID; + + public static final String PROPERTY_VOYAGE_NAME = PROPERTY_VOYAGE + "." + Voyage.PROPERTY_NAME; + + public static final String PROPERTY_VOYAGE_YEAR = PROPERTY_VOYAGE + "." + Voyage.PROPERTY_START_DATE; + + public static final String PROPERTY_SEX_CATEGORY = "sexCategory"; + + public static final String PROPERTY_MATURITE = "maturite"; + + public static final String PROPERTY_LENGTH_STEP = "lengthStep"; + + public static final String PROPERTY_NUMBER = "number"; + + public static final String PROPERTY_WEIGHT = "weight"; + + public static final String PROPERTY_AGE = "age"; + + protected Voyage voyage; + + protected Species species; + + protected Operation operation; + + protected SexCategory sexCategory; + + protected SampleData sampleData; + + protected String lengthStep; + + protected Integer number; + + protected Float weight; + + protected Integer age; + + protected String maturite = "NA"; + + public Voyage getVoyage() { + return voyage; + } + + public void setVoyage(Voyage voyage) { + this.voyage = voyage; + } + + public Species getSpecies() { + return species; + } + + public void setSpecies(Species species) { + this.species = species; + } + + public Operation getOperation() { + return operation; + } + + public void setOperation(Operation operation) { + this.operation = operation; + } + + public int getNumber() { + return number; + } + + public void setNumber(int number) { + this.number = number; + } + + public float getWeight() { + return weight; + } + + public void setWeight(float weight) { + this.weight = weight; + } + + public SexCategory getSexCategory() { + return sexCategory; + } + + public void setSexCategory(SexCategory sexCategory) { + this.sexCategory = sexCategory; + } + + public SampleData getSampleData() { + return sampleData; + } + + public void setSampleData(SampleData sampleData) { + this.sampleData = sampleData; + } + + public String getLengthStep() { + return lengthStep; + } + + public void setLengthStep(String lengthStep) { + this.lengthStep = lengthStep; + } + + public void setNumber(Integer number) { + this.number = number; + } + + public void setWeight(Float weight) { + this.weight = weight; + } + + public Integer getAge() { + return age; + } + + public void setAge(Integer age) { + this.age = age; + } + + public String getMaturite() { + return maturite; + } + + public void setMaturite(String maturite) { + this.maturite = maturite; + } +} Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataSizeExportRow.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportModel.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,101 @@ +package fr.ifremer.echobase.services.service.exportCoser; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.google.common.collect.Lists; +import fr.ifremer.echobase.csv.EchoBaseCsvUtil; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.DataMetadata; +import org.nuiton.csv.ext.AbstractExportModel; + +import java.util.List; + +/** + * Created on 4/5/14. + * + * @author Tony Chemit <chemit@codelutin.com> + * @since 2.6 + */ +public class RawDataStrataExportModel extends AbstractExportModel<RawDataStrataExportRow> { + + /* + SELECT + voyage.name AS Campagne, + cell.name AS Strate, + datametadata.name, + data.datavalue AS Surface + FROM + voyage AS voyage, + mission AS mission, + cell AS cell, + celltype AS celltype, + data AS data, + datametadata AS datametadata + WHERE + voyage.mission = mission.topiaid AND + cell.voyage = voyage.topiaid AND + cell.celltype = celltype.topiaid AND + data.cell = cell.topiaid AND + data.datametadata = datametadata.topiaid AND + celltype.id = 'Region' AND + datametadata.name = 'Surface' + */ + + public RawDataStrataExportModel(char separator) { + super(separator); + newColumnForExport("Campagne", EchoBaseCsvUtil.<RawDataStrataExportRow, String>newBeanProperty(RawDataStrataExportRow.PROPERTY_VOYAGE_NAME)); + newColumnForExport("Strate", EchoBaseCsvUtil.<RawDataStrataExportRow, String>newBeanProperty(RawDataStrataExportRow.PROPERTY_CELL_NAME)); + newColumnForExport("Surface", EchoBaseCsvUtil.<RawDataStrataExportRow, String>newBeanProperty(RawDataStrataExportRow.PROPERTY_DATA_VALUE)); + } + + public List<RawDataStrataExportRow> prepareRows(List<Voyage> voyages, DataMetadata surfaceMetadata) { + List<RawDataStrataExportRow> rows = Lists.newArrayList(); + for (Voyage voyage : voyages) { + prepareRows(voyage, surfaceMetadata, rows); + } + return rows; + } + + protected void prepareRows(Voyage voyage, DataMetadata surfaceMetadata, List<RawDataStrataExportRow> rows) { + + for (Cell cell : voyage.getRegionCells()) { + + if (cell.isDataNotEmpty()) { + + for (Data data : cell.getData()) { + + if (surfaceMetadata.equals(data.getDataMetadata())) { + RawDataStrataExportRow row = new RawDataStrataExportRow(); + row.setVoyage(voyage); + row.setCell(cell); + row.setData(data); + rows.add(row); + } + } + } + } + + } + +} \ No newline at end of file Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportModel.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportRow.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportRow.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,71 @@ +package fr.ifremer.echobase.services.service.exportCoser; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.Voyage; + +/** + * Created on 4/5/14. + * + * @author Tony Chemit <chemit@codelutin.com> + * @since 2.6 + */ +public class RawDataStrataExportRow { + + public static final String PROPERTY_VOYAGE_NAME = "voyage." + Voyage.PROPERTY_NAME; + + public static final String PROPERTY_CELL_NAME = "cell." + Cell.PROPERTY_NAME; + + public static final String PROPERTY_DATA_VALUE = "data." + Data.PROPERTY_DATA_VALUE; + + protected Voyage voyage; + + protected Cell cell; + + protected Data data; + + public Voyage getVoyage() { + return voyage; + } + + public void setVoyage(Voyage voyage) { + this.voyage = voyage; + } + + public Cell getCell() { + return cell; + } + + public void setCell(Cell cell) { + this.cell = cell; + } + + public Data getData() { + return data; + } + + public void setData(Data data) { + this.data = data; + } +} Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/RawDataStrataExportRow.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/AcousticImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/AcousticImportModel.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/AcousticImportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -107,17 +107,17 @@ newMandatoryColumn("MOVIES_EILayer\\cellset\\lat", AcousticImportRow.PROPERTY_CELL_LATITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BF newMandatoryColumn("MOVIES_EILayer\\cellset\\long", AcousticImportRow.PROPERTY_CELL_LONGITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BG - newMandatoryColumn("MOVIES_EILayer\\cellset\\volume", AcousticImportRow.PROPERTY_CELL_VOLUME, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BH + newMandatoryColumn("MOVIES_EILayer\\cellset\\volume", AcousticImportRow.PROPERTY_CELL_VOLUME, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER);//BH newMandatoryColumn("MOVIES_EILayer\\cellset\\area", AcousticImportRow.PROPERTY_CELL_SURFACE, EchoBaseCsvUtil.PRIMITIVE_INTEGER);//BI newIgnoredColumn("MOVIES_EILayer\\cellset\\diststart");//BJ newIgnoredColumn("MOVIES_EILayer\\cellset\\distend");//BK newMandatoryColumn("MOVIES_EILayer\\cellset\\thresholdup", AcousticImportRow.PROPERTY_E_ITHRESHOLD_HIGH, EchoBaseCsvUtil.PRIMITIVE_INTEGER);//BL newMandatoryColumn("MOVIES_EILayer\\cellset\\thresholdlow", AcousticImportRow.PROPERTY_E_ITHRESHOLD_LOW, EchoBaseCsvUtil.PRIMITIVE_INTEGER);//BM newIgnoredColumn("MOVIES_EILayer\\eilayer");//BN - newMandatoryColumn("MOVIES_EILayer\\eilayer\\sa", AcousticImportRow.PROPERTY_CELL_NASC, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER);//BO + newMandatoryColumn("MOVIES_EILayer\\eilayer\\sa", AcousticImportRow.PROPERTY_CELL_NASC, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER);//BO newIgnoredColumn("MOVIES_EILayer\\eilayer\\sv");//BP - newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ - newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR + newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER);//BQ + newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER);//BR newIgnoredColumn("MOVIES_EILayer\\boterr");//BR newIgnoredColumn("MOVIES_EILayer\\boterr\\sa");//BT newIgnoredColumn("MOVIES_EILayer\\boterr\\ni");//BU Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/BiometrySampleImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/BiometrySampleImportModel.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/BiometrySampleImportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -50,7 +50,7 @@ newForeignKeyColumn("name", SampleData.PROPERTY_SAMPLE_DATA_TYPE, SampleDataType.class, SampleDataType.PROPERTY_NAME, sampleDataTypeMap); newMandatoryColumn(SampleData.PROPERTY_DATA_LABEL); - newMandatoryColumn(SampleData.PROPERTY_DATA_VALUE, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER); + newMandatoryColumn(SampleData.PROPERTY_DATA_VALUE, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); } @Override Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportModel.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -57,10 +57,10 @@ newMandatoryColumn(Sample.PROPERTY_NUMBER_SAMPLED, EchoBaseCsvUtil.PRIMITIVE_INTEGER); newMandatoryColumn(SubSampleImportRow.PROPERTY_NUMBER_AT_LENGTH, EchoBaseCsvUtil.PRIMITIVE_FLOAT); - newMandatoryColumn(SubSampleImportRow.PROPERTY_WEIGHT_AT_LENGTH, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER); + newMandatoryColumn(SubSampleImportRow.PROPERTY_WEIGHT_AT_LENGTH, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); newMandatoryColumn(SubSampleImportRow.PROPERTY_LENGTH_CLASS); newMandatoryColumn(SubSampleImportRow.PROPERTY_UNITS); - newMandatoryColumn(SubSampleImportRow.PROPERTY_ROUND, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER); + newMandatoryColumn(SubSampleImportRow.PROPERTY_ROUND, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER); } @Override Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/TotalSampleImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/TotalSampleImportModel.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/TotalSampleImportModel.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -48,11 +48,11 @@ newForeignKeyColumn(TotalSampleImportRow.PROPERTY_SIZE_CATEGORY, SizeCategory.class, SizeCategory.PROPERTY_NAME, sizeCategoryMap); newMandatoryColumn(Sample.PROPERTY_SAMPLE_WEIGHT, EchoBaseCsvUtil.PRIMITIVE_FLOAT); - newMandatoryColumn(Sample.PROPERTY_NUMBER_SAMPLED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER); + newMandatoryColumn(Sample.PROPERTY_NUMBER_SAMPLED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER); - newMandatoryColumn(TotalSampleImportRow.PROPERTY_MEAN_LENGTH, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER); - newMandatoryColumn(TotalSampleImportRow.PROPERTY_MEAN_WEIGHT, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER); - newMandatoryColumn(TotalSampleImportRow.PROPERTY_NO_PER_KG, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER); + newMandatoryColumn(TotalSampleImportRow.PROPERTY_MEAN_LENGTH, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); + newMandatoryColumn(TotalSampleImportRow.PROPERTY_MEAN_WEIGHT, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); + newMandatoryColumn(TotalSampleImportRow.PROPERTY_NO_PER_KG, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); newMandatoryColumn(TotalSampleImportRow.PROPERTY_SORTED_WEIGHT, EchoBaseCsvUtil.PRIMITIVE_FLOAT); } Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -21,6 +21,10 @@ * #L% */ +import com.google.common.collect.Sets; + +import java.util.Set; + /** * Obtain some fixtures for service tests. * @@ -176,4 +180,62 @@ public String TRANSECT_LICENSE() { return "transectLicence"; } + + public Set<String> COSER_INDICATORS() { + return Sets.newHashSet( + "EA", + "Abundance", + "Shannonmod", + "propLW20", + "L50", + "ycg", + "propLW25", + "propL30", + "Imin", + "Imax", + "Conserv", + "meanWbar", + "Btot", + "l0.25", + "Biomass", + "microS", + "Lvar", + "Iso", + "Lbar", + "propLW15", + "xcg", + "propL20", + "Ntot", + "propL25", + "biomSmall", + "Gtot", + "l0.75", + "PropMat", + "xaxe1.2", + "Delta", + "xaxe1.1", + "yaxe1.2", + "I", + "yaxe2.2", + "l0.95", + "yaxe2.1", + "Wtot", + "xaxe2.1", + "biomBig", + "Wbcomm", + "propL15", + "lnN", + "Npatch", + "xaxe2.2", + "sexRatio", + "meanQuant0.95", + "Dbar", + "Lbcomm", + "l0.05", + "propLW30", + "yaxe1.1", + "Wbar", + "PA", + "SA"); + } } Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FakeEchoBaseServiceContext.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FakeEchoBaseServiceContext.java 2014-04-02 08:06:04 UTC (rev 1001) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FakeEchoBaseServiceContext.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -44,7 +44,6 @@ import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; -import java.net.MalformedURLException; import java.net.URL; import java.util.Date; import java.util.Locale; @@ -148,7 +147,7 @@ Properties defaultProps = new Properties(); defaultProps.put(EchoBaseConfigurationOption.DATA_DIRECTORY.getKey(), testDir); - configuration = new EchoBaseConfiguration(defaultProps); + configuration = new EchoBaseConfiguration("echobase-services-test.properties", defaultProps); JdbcConfiguration dbConf = JdbcConfiguration.newEmbeddedConfig(testDir); echoBaseTopiaApplicationContext = EchobaseTopiaApplicationContexts.newWorkingDb(dbConf); @@ -158,7 +157,7 @@ @Override protected void finished(Description description) { super.finished(description); - if (echoBaseTopiaApplicationContext!=null && !echoBaseTopiaApplicationContext.isClosed()) { + if (echoBaseTopiaApplicationContext != null && !echoBaseTopiaApplicationContext.isClosed()) { echoBaseTopiaApplicationContext.closeContext(); } } @@ -219,13 +218,18 @@ @Override public URL getCoserApiURL() { - try { - return new URL("http://localhost:8080/coser-web/json"); - } catch (MalformedURLException e) { - throw new EchoBaseTechnicalException(e); - } + return getConfiguration().getCoserApiURL(); } +// @Override +// public URL getCoserApiURL() { +// try { +// return new URL("http://localhost:8080/coser-web/json"); +// } catch (MalformedURLException e) { +// throw new EchoBaseTechnicalException(e); +// } +// } + @Override public <S extends EchoBaseService> S newService(Class<S> serviceClass) { // instantiate service using empty constructor Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserServiceTest.java (rev 0) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserServiceTest.java 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,154 @@ +package fr.ifremer.echobase.services.service.exportCoser; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2014 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import fr.ifremer.coser.bean.EchoBaseProject; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.Mission; +import fr.ifremer.echobase.services.EchoBaseTestServiceSupport; +import fr.ifremer.echobase.services.FakeEchoBaseServiceContext; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import org.junit.Assert; +import org.junit.Test; +import org.nuiton.topia.persistence.TopiaEntities; +import org.nuiton.util.FileUtil; + +import javax.inject.Inject; +import java.io.File; +import java.util.List; +import java.util.Set; + +/** + * Created on 4/5/14. + * + * @author Tony Chemit <chemit@codelutin.com> + * @since 2.6 + */ +public class ExportCoserServiceTest extends EchoBaseTestServiceSupport { + + protected FakeEchoBaseServiceContext initContext() { + return new FakeEchoBaseServiceContext( + fixtures.IMPORT_DATA_ECHOBASE_CATCHES_AND_VOYAGE_RESULT()); + } + + @Inject + private ExportCoserService exportService; + + @Inject + private UserDbPersistenceService persistenceService; + + @Test + public void testExtractRawData() throws Exception { + + File workingDirectory = new File(getTestDir(), "testExtractRawData"); + FileUtil.createDirectoryIfNecessary(workingDirectory); + + EchoBaseProject echobaseProject = new EchoBaseProject(workingDirectory); + + ExportCoserConfiguration model = new ExportCoserConfiguration(); + model.setMissionId(fixtures.MISSION_ID()); + + Mission mission = persistenceService.getMission(model.getMissionId()); + + Assert.assertFalse(echobaseProject.getRawDataDirectory().exists()); + Assert.assertFalse(new File(echobaseProject.getRawDataDirectory(), "Captures.csv").exists()); + Assert.assertFalse(new File(echobaseProject.getRawDataDirectory(), "Tailles.csv").exists()); + Assert.assertFalse(new File(echobaseProject.getRawDataDirectory(), "Strates.csv").exists()); + + exportService.extractRawData(echobaseProject, mission, model); + + Assert.assertTrue(echobaseProject.getRawDataDirectory().exists()); + Assert.assertTrue(new File(echobaseProject.getRawDataDirectory(), "Captures.csv").exists()); + Assert.assertTrue(new File(echobaseProject.getRawDataDirectory(), "Tailles.csv").exists()); + Assert.assertTrue(new File(echobaseProject.getRawDataDirectory(), "Strates.csv").exists()); + } + + @Test + public void testExtractSpeciesFile() throws Exception { + + File workingDirectory = new File(getTestDir(), "testExtractSpeciesFile"); + FileUtil.createDirectoryIfNecessary(workingDirectory); + + EchoBaseProject echobaseProject = new EchoBaseProject(workingDirectory); + + ExportCoserConfiguration model = new ExportCoserConfiguration(); + model.setMissionId(fixtures.MISSION_ID()); + + Assert.assertFalse(echobaseProject.getSpeciesDefinitionFile().exists()); + + exportService.extractSpeciesFile(echobaseProject); + + Assert.assertTrue(echobaseProject.getSpeciesDefinitionFile().exists()); + } + + @Test + public void testExtractPopulationIndicators() throws Exception { + + Set<String> indicatorNames = fixtures.COSER_INDICATORS(); + List<DataMetadata> dataMetadatas = persistenceService.getDataMetadatasInName(indicatorNames); + Set<String> indicatorIds = Sets.newHashSet(Iterables.transform(dataMetadatas, TopiaEntities.getTopiaIdFunction())); + + File workingDirectory = new File(getTestDir(), "testExtractCommunityIndicators"); + FileUtil.createDirectoryIfNecessary(workingDirectory); + + ExportCoserConfiguration model = new ExportCoserConfiguration(); + model.setMissionId(fixtures.MISSION_ID()); + model.setPopulationIndicator(indicatorIds); + + EchoBaseProject echobaseProject = new EchoBaseProject(workingDirectory); + + Mission mission = persistenceService.getMission(model.getMissionId()); + + Assert.assertFalse(echobaseProject.getPopulationIndicatorsFile().exists()); + + exportService.extractPopulationIndicators(model, echobaseProject, mission); + + Assert.assertTrue(echobaseProject.getPopulationIndicatorsFile().exists()); + } + + @Test + public void testExtractCommunityIndicators() throws Exception { + + Set<String> indicatorNames = fixtures.COSER_INDICATORS(); + List<DataMetadata> dataMetadatas = persistenceService.getDataMetadatasInName(indicatorNames); + Set<String> indicatorIds = Sets.newHashSet(Iterables.transform(dataMetadatas, TopiaEntities.getTopiaIdFunction())); + + File workingDirectory = new File(getTestDir(), "testExtractCommunityIndicators"); + FileUtil.createDirectoryIfNecessary(workingDirectory); + + ExportCoserConfiguration model = new ExportCoserConfiguration(); + model.setMissionId(fixtures.MISSION_ID()); + model.setCommunityIndicator(indicatorIds); + + EchoBaseProject echobaseProject = new EchoBaseProject(workingDirectory); + + Mission mission = persistenceService.getMission(model.getMissionId()); + + Assert.assertFalse(echobaseProject.getCommunityIndicatorsFile().exists()); + + exportService.extractCommunityIndicators(model, echobaseProject, mission); + + Assert.assertTrue(echobaseProject.getCommunityIndicatorsFile().exists()); + } +} Property changes on: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/service/exportCoser/ExportCoserServiceTest.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native Added: trunk/echobase-services/src/test/resources/echobase-services-test.properties =================================================================== --- trunk/echobase-services/src/test/resources/echobase-services-test.properties (rev 0) +++ trunk/echobase-services/src/test/resources/echobase-services-test.properties 2014-04-05 11:00:41 UTC (rev 1002) @@ -0,0 +1,21 @@ +### +# #%L +# EchoBase :: Services +# %% +# Copyright (C) 2011 - 2014 Ifremer, Codelutin +# %% +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# #L% +### +echobase.coser.api.url=http://localhost:8090/coser-web/json/ \ No newline at end of file Property changes on: trunk/echobase-services/src/test/resources/echobase-services-test.properties ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native