This is an automated email from the git hooks/post-receive script. New commit to branch feature/8170 in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit a65f2e791ef3d8b9cd6d5b82c813db2f3f2f03d8 Author: Tony CHEMIT <chemit@codelutin.com> Date: Sat Mar 26 14:00:13 2016 +0100 Mise en place d'une nouvelle API d'action d'import (pour traiter chaque import de fichier unitairement et non plus via des méthodes de services), du coup on ne va plus utiliser qu'un seul service d'import (grosse simplification) --- .../importdata/DuplicatedSampleException.java | 47 ++ .../service/importdata/ImportDataFileResult.java | 149 +++++ .../service/importdata/ImportDataResult.java | 38 ++ .../service/importdata/ImportDataService.java | 510 +++++++++++++++++ .../service/importdata/MissionService.java | 61 ++ .../NoTransitFoundBetweenDateException.java | 45 ++ .../importdata/actions/AbstractImportAction.java | 352 ++++++++++++ .../importdata/actions/AcousticImportAction.java | 616 +++++++++++++++++++++ .../actions/BiometrySampleImportAction.java | 136 +++++ .../actions/CommonTransectImportAction.java | 121 ++++ .../actions/CommonTransitImportAction.java | 100 ++++ .../actions/CommonVoyageImportAction.java | 100 ++++ .../actions/GearMetadataImportAction.java | 100 ++++ .../importdata/actions/OperationImportAction.java | 104 ++++ .../actions/OperationMetadataImportAction.java | 94 ++++ .../actions/RegionAssociationImportAction.java | 84 +++ .../importdata/actions/RegionImportAction.java | 118 ++++ ...EchotypeAndSpeciesCategoryCellImportAction.java | 134 +++++ .../ResultsEsduByEchotypeCellImportAction.java | 118 ++++ ...sEsduSpeciesAndAgeCategoryCellImportAction.java | 138 +++++ ...EsduSpeciesAndSizeCategoryCellImportAction.java | 140 +++++ .../actions/ResultsMapFishCellImportAction.java | 194 +++++++ .../actions/ResultsMapOtherCellImportAction.java | 168 ++++++ .../actions/ResultsRegionImportAction.java | 126 +++++ .../actions/ResultsVoyageEchotypeImportAction.java | 128 +++++ .../ResultsVoyageLengthAgeKeyImportAction.java | 95 ++++ .../ResultsVoyageLengthWeightKeyImportAction.java | 109 ++++ .../importdata/actions/SubSampleImportAction.java | 151 +++++ .../actions/TotalSampleImportAction.java | 162 ++++++ .../importdata/AbstractImportDataServiceIT.java | 28 +- .../importdata/AcousticImportServiceIT.java | 2 +- .../service/importdata/CatchesImportServiceIT.java | 8 +- .../importdata/CommonAllImportServiceIT.java | 4 +- .../importdata/CommonTransectImportServiceIT.java | 4 +- .../importdata/CommonTransitImportServiceIT.java | 4 +- .../importdata/CommonVoyageImportServiceIT.java | 4 +- .../importdata/OperationImportServiceIT.java | 2 +- .../importdata/ResultsEsduCellImportServiceIT.java | 8 +- .../ResultsMapFishCellImportServiceIT.java | 2 +- .../ResultsMapOtherCellImportServiceIT.java | 2 +- .../ResultsRegionCellImportServiceIT.java | 2 +- .../importdata/ResultsVoyageImportServiceIT.java | 8 +- .../removedata/AbstractRemoveDataServiceTest.java | 42 +- 43 files changed, 4490 insertions(+), 68 deletions(-) diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/DuplicatedSampleException.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/DuplicatedSampleException.java new file mode 100644 index 0000000..a5f69bf --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/DuplicatedSampleException.java @@ -0,0 +1,47 @@ +package fr.ifremer.echobase.services.service.importdata; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2015 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.references.SampleType; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import org.nuiton.csv.ImportRuntimeException; + +import java.util.Locale; + +import static org.nuiton.i18n.I18n.l; + +/** + * Created on 1/21/15. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 2.9 + */ +public class DuplicatedSampleException extends ImportRuntimeException { + + private static final long serialVersionUID = 1L; + + public DuplicatedSampleException(Locale locale, Operation operation, SampleType sampleType, Species species, SizeCategory sizeCategory) { + super(l(locale, "echobase.importError.duplicate.sample", operation.getId(), sampleType.getName(), species.getBaracoudaCode(), sizeCategory.getName())); + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataFileResult.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataFileResult.java new file mode 100644 index 0000000..40c93a7 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataFileResult.java @@ -0,0 +1,149 @@ +package fr.ifremer.echobase.services.service.importdata; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.ImportLog; +import fr.ifremer.echobase.io.InputFile; +import org.apache.commons.lang3.mutable.MutableInt; +import org.nuiton.topia.persistence.TopiaEntity; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * Created on 26/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportDataFileResult { + + /** + * Import file. + */ + private final InputFile importFile; + + /** + * Processed Import file (used to compare with result file). + * + * This file may not be exactly the same than the one given in the import configuration, we for example may + * remove ignored columns. + */ + private final InputFile processedImportile; + + /** + * Export file. + * + * Used to check that import is really ok and stored then in the {@link ImportLog}. + */ + private final InputFile exportFile; + + /** Entity types to import csv datas. */ + protected final Set<EchoBaseUserEntityEnum> entityTypes; + + /** Count of created entities. */ + protected final Map<EchoBaseUserEntityEnum, MutableInt> numberCreated; + + /** Count of updated entities. */ + protected final Map<EchoBaseUserEntityEnum, MutableInt> numberUpdated; + + /** Count of updated entities. */ + protected final Map<EchoBaseUserEntityEnum, MutableInt> numberNotImported; + + /** Ids of created entities. */ + protected final LinkedHashSet<String> ids; + + /** Ids of not imported entities. */ + protected final LinkedHashSet<String> notImportedIds; + + public ImportDataFileResult(InputFile importFile) { + this.importFile = importFile; + this.processedImportile = InputFile.newFile(importFile.getLabel() + " - " + importFile.getFileName() + ".processed"); + this.exportFile = InputFile.newFile(importFile.getLabel() + " - " + importFile.getFileName() + ".checked"); + this.entityTypes = Collections.unmodifiableSet(Sets.newHashSet(EchoBaseUserEntityEnum.values())); + this.numberCreated = new TreeMap<>(); + this.numberUpdated = new TreeMap<>(); + this.numberNotImported = new TreeMap<>(); + for (EchoBaseUserEntityEnum entityType : entityTypes) { + numberCreated.put(entityType, new MutableInt()); + numberUpdated.put(entityType, new MutableInt()); + numberNotImported.put(entityType, new MutableInt()); + } + this.ids = new LinkedHashSet<>(); + this.notImportedIds = new LinkedHashSet<>(); + } + + public InputFile getImportFile() { + return importFile; + } + + public InputFile getProcessedImportile() { + return processedImportile; + } + + public InputFile getExportFile() { + return exportFile; + } + + public <E extends TopiaEntity> void addId(EchoBaseUserEntityEnum entityEnum, E entity) { + this.ids.add(entity.getTopiaId()); + incrementsNumberCreated(entityEnum); + } + + public <E extends TopiaEntity> void addNotImportedId(EchoBaseUserEntityEnum entityEnum, E entity) { + addNotImportedId(entityEnum, entity.getTopiaId()); + } + + public void addNotImportedId(EchoBaseUserEntityEnum entityEnum, String id) { + this.notImportedIds.add(id); + incrementsNumberNotImported(entityEnum); + } + + public LinkedHashSet<String> getIds() { + return ids; + } + + public LinkedHashSet<String> getNotImportedIds() { + return notImportedIds; + } + + public Set<EchoBaseUserEntityEnum> getEntityTypes() { + return ImmutableSet.copyOf(entityTypes); + } + + public int getNumberCreated(EchoBaseUserEntityEnum entityType) { + return getInteger(numberCreated, entityType); + } + + public int getNumberUpdated(EchoBaseUserEntityEnum entityType) { + return getInteger(numberUpdated, entityType); + } + + public int getNumberNotImported(EchoBaseUserEntityEnum entityType) { + return getInteger(numberNotImported, entityType); + } + + public void incrementsNumberCreated(EchoBaseUserEntityEnum entityType) { + increments(numberCreated, entityType); + } + + public void incrementsNumberNotImported(EchoBaseUserEntityEnum entityType) { + increments(numberNotImported, entityType); + } + + public void incrementsNumberUpdated(EchoBaseUserEntityEnum entityType) { + increments(numberUpdated, entityType); + } + + protected int getInteger(Map<EchoBaseUserEntityEnum, MutableInt> map, EchoBaseUserEntityEnum entityType) { + return map.get(entityType).intValue(); + } + + protected void increments(Map<EchoBaseUserEntityEnum, MutableInt> map, EchoBaseUserEntityEnum entityType) { + MutableInt result = map.get(entityType); + result.increment(); + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataResult.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataResult.java new file mode 100644 index 0000000..43365aa --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataResult.java @@ -0,0 +1,38 @@ +package fr.ifremer.echobase.services.service.importdata; + +import fr.ifremer.echobase.entities.EchoBaseUser; + +import java.util.Collections; +import java.util.Set; + +/** + * Created on 26/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportDataResult<C extends AbstractImportConfiguration> { + + private final C configuration; + + private final EchoBaseUser user; + + private final Set<ImportDataFileResult> importResults; + + public ImportDataResult(C configuration, EchoBaseUser user, Set<ImportDataFileResult> importResults) { + this.configuration = configuration; + this.user = user; + this.importResults = Collections.unmodifiableSet(importResults); + } + + public C getConfiguration() { + return configuration; + } + + public EchoBaseUser getUser() { + return user; + } + + public Set<ImportDataFileResult> getImportResults() { + return importResults; + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataService.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataService.java new file mode 100644 index 0000000..003ef6f --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportDataService.java @@ -0,0 +1,510 @@ +package fr.ifremer.echobase.services.service.importdata; + +import com.google.common.collect.Lists; +import fr.ifremer.echobase.EchoBaseTechnicalException; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.ImportFile; +import fr.ifremer.echobase.entities.ImportLog; +import fr.ifremer.echobase.entities.ImportType; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.EchoBaseServiceSupport; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.actions.AbstractImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.AcousticImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.BiometrySampleImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.CommonTransectImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.CommonTransitImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.CommonVoyageImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.GearMetadataImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.OperationImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.OperationMetadataImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.RegionAssociationImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.RegionImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsEsduByEchotypeCellImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsEsduSpeciesAndAgeCategoryCellImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsEsduSpeciesAndSizeCategoryCellImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsMapFishCellImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsMapOtherCellImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsRegionImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsVoyageEchotypeImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsVoyageLengthAgeKeyImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.ResultsVoyageLengthWeightKeyImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.SubSampleImportAction; +import fr.ifremer.echobase.services.service.importdata.actions.TotalSampleImportAction; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.util.TimeLog; + +import javax.inject.Inject; +import java.sql.SQLException; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import static org.nuiton.i18n.I18n.l; + +/** + * Created on 26/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportDataService extends EchoBaseServiceSupport { + +// public static final Pattern REMOVE_DOUBLE_QUOTES_PATTERN = Pattern.compile("\"(.+)\""); + + /** Logger. */ + private static final Log log = LogFactory.getLog(AbstractImportDataService.class); + + public static final TimeLog TIME_LOG = new TimeLog(AbstractImportDataService.class); + + @Inject + protected UserDbPersistenceService persistenceService; + + + public String doImportCommons(CommonImportConfiguration configuration, EchoBaseUser user) throws ImportException { + + Collection<AbstractImportAction> importActions = new LinkedHashSet<>(); + ImportType importMode = configuration.getImportType(); + switch (importMode) { + + case COMMON_ALL: { + CommonVoyageImportAction importAction = new CommonVoyageImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + { + CommonTransitImportAction importAction = new CommonTransitImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user, false); + importActions.add(importAction); + } + { + CommonTransectImportAction importAction = new CommonTransectImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user, false); + importActions.add(importAction); + } + break; + + case COMMON_VOYAGE: { + CommonVoyageImportAction importAction = new CommonVoyageImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + break; + + case COMMON_TRANSIT: { + CommonTransitImportAction importAction = new CommonTransitImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user, true); + importActions.add(importAction); + } + break; + + case COMMON_TRANSECT: { + CommonTransectImportAction importAction = new CommonTransectImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user, true); + importActions.add(importAction); + } + break; + + default: + throw new EchoBaseTechnicalException("Can not treate import result of type " + importMode); + + } + + return doImport(configuration, user, importActions); + + } + + public String doImportOperations(OperationImportConfiguration configuration, EchoBaseUser user) throws ImportException { + + Collection<AbstractImportAction> importActions = new LinkedHashSet<>(); + OperationImportAction operationImportAction = new OperationImportAction(getLocale(), getCsvSeparator(), persistenceService, configuration, user); + importActions.add(operationImportAction); + + OperationMetadataImportAction operationMetadataImportAction = new OperationMetadataImportAction(getLocale(), getCsvSeparator(), persistenceService, configuration, user); + importActions.add(operationMetadataImportAction); + + GearMetadataImportAction gearMetadataImportAction = new GearMetadataImportAction(getLocale(), getCsvSeparator(), persistenceService, configuration, user); + importActions.add(gearMetadataImportAction); + + return doImport(configuration, user, importActions); + + } + + public String doImportCatches(CatchesImportConfiguration configuration, EchoBaseUser user) throws ImportException { + + Collection<AbstractImportAction> importActions = new LinkedHashSet<>(); + + InputFile totalSampleFile = configuration.getTotalSampleFile(); + if (totalSampleFile.hasFile()) { + + TotalSampleImportAction importAction = new TotalSampleImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + + InputFile subSampleFile = configuration.getSubSampleFile(); + if (subSampleFile.hasFile()) { + + SubSampleImportAction importAction = new SubSampleImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + + InputFile biometrySampleFile = configuration.getBiometrySampleFile(); + if (biometrySampleFile.hasFile()) { + + BiometrySampleImportAction importAction = new BiometrySampleImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + + return doImport(configuration, user, importActions); + + } + + public String doImportAcoustics(AcousticImportConfiguration configuration, EchoBaseUser user) throws ImportException { + + AcousticImportAction acousticImportAction = new AcousticImportAction(getLocale(), getCsvSeparator(), persistenceService, configuration, user); + Set<AcousticImportAction> importActions = Collections.singleton(acousticImportAction); + return doImport(configuration, user, importActions); + + } + + public String doImportResults(ResultsImportConfiguration configuration, EchoBaseUser user) throws ImportException { + + Collection<AbstractImportAction> importActions = new LinkedHashSet<>(); + + ImportType importMode = configuration.getImportType(); + + switch (importMode) { + + case RESULT_VOYAGE: { + // check if there is a such same import on this voyage + boolean importExists = persistenceService.isImportLogForVoyageExists(configuration.getVoyageId()); + if (importExists) { + if (log.isWarnEnabled()) { + log.warn("there is already a voyage result import for this voyage, won't import."); + } + throw new ResultsVoyageImportAlreadyExistException(getLocale()); + } + + if (configuration.getLengthAgeKeyFile().hasFile()) { + + ResultsVoyageLengthAgeKeyImportAction importAction = new ResultsVoyageLengthAgeKeyImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + + } + + if (configuration.getLengthWeightKeyFile().hasFile()) { + + ResultsVoyageLengthWeightKeyImportAction importAction = new ResultsVoyageLengthWeightKeyImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + + } + + if (configuration.getEchotypeFile().hasFile()) { + + ResultsVoyageEchotypeImportAction importAction = new ResultsVoyageEchotypeImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + + } + + } + break; + + case RESULT_ESDU: + if (configuration.getEsduByEchotypeFile().hasFile()) { + + ResultsEsduByEchotypeCellImportAction importAction = new ResultsEsduByEchotypeCellImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + + } + + if (configuration.getEsduByEchotypeAndSpeciesCategoryFile().hasFile()) { + + ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction importAction = new ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + + } + + if (configuration.getEsduBySpeciesAndSizeCategoryFile().hasFile()) { + + ResultsEsduSpeciesAndSizeCategoryCellImportAction importAction = new ResultsEsduSpeciesAndSizeCategoryCellImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + + } + + if (configuration.getEsduBySpeciesAndAgeCategoryFile().hasFile()) { + + ResultsEsduSpeciesAndAgeCategoryCellImportAction importAction = new ResultsEsduSpeciesAndAgeCategoryCellImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + + } + + break; + + case RESULT_REGION: { + RegionImportAction importAction = new RegionImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + { + RegionAssociationImportAction importAction = new RegionAssociationImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + { + ResultsRegionImportAction importAction = new ResultsRegionImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + break; + + case RESULT_MAP_FISH: { + ResultsMapFishCellImportAction importAction = new ResultsMapFishCellImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + break; + + case RESULT_MAP_OTHER: { + ResultsMapOtherCellImportAction importAction = new ResultsMapOtherCellImportAction(persistenceService, getLocale(), getCsvSeparator(), configuration, user); + importActions.add(importAction); + } + break; + default: + throw new EchoBaseTechnicalException( + "Can not treate import result of type " + importMode); + } + + return doImport(configuration, user, importActions); + + } + + + protected <M extends AbstractImportConfiguration, A extends AbstractImportAction> String doImport(M configuration, EchoBaseUser user, Collection<A> importActions) throws ImportException { + + long s0 = TimeLog.getTime(); + + long nbSteps = configuration.computeNbSteps(); + + if (log.isInfoEnabled()) { + log.info("Nb lines to import " + nbSteps); + } + + s0 = TIME_LOG.log(s0, "computeNbSteps"); + + try { + + Set<ImportDataFileResult> importDataFileResults = new LinkedHashSet<>(); + + for (A importAction : importActions) { + ImportDataFileResult importDataFileResult = importAction.run(); + importDataFileResults.add(importDataFileResult); + + } + ImportDataResult<M> importDataResult = new ImportDataResult<>(configuration, user, importDataFileResults); + + s0 = TIME_LOG.log(s0, "importDone"); + + // add result in log book and compute resume to show in result + String result = computeResultAndLogBookEntry(importDataResult); + + // do commit + persistenceService.commit(); + + TIME_LOG.log(s0, "importCommited"); + + return result; + + } catch (EchoBaseTechnicalException e) { + throw new ImportException(e.getMessage(), e); + } + } + + public <M extends AbstractImportConfiguration> ImportLog computeImportLogEntry(ImportDataResult<M> result) { + + EchoBaseUser user = result.getUser(); + M configuration = result.getConfiguration(); + Date importDate = newDate(); + String importUser = user.getEmail(); + + StringBuilder buffer = new StringBuilder(); + + if (StringUtils.isNotEmpty(configuration.getImportNotes())) { + buffer.append("Notes :").append(configuration.getImportNotes()).append('\n'); + } + + List<String> importIds = Lists.newArrayList(); + + for (ImportDataFileResult importResult : result.getImportResults()) { + + InputFile importFile = importResult.getImportFile(); + + importIds.addAll(importResult.getIds()); + + StringBuilder description = new StringBuilder(); + description.append("Depuis Fichier ").append(importFile.getFileName()); + + for (EchoBaseUserEntityEnum entityType : importResult.getEntityTypes()) { + int numberCreated = importResult.getNumberCreated(entityType); + int numberUpdated = importResult.getNumberUpdated(entityType); + int numberNotImported = importResult.getNumberNotImported(entityType); + description.append("\n\tEntité ").append(entityType); + if (numberCreated > 0) { + description.append(" [nombre de création : "); + description.append(numberCreated); + description.append("]"); + } + if (numberUpdated > 0) { + description.append(" [nombre de mises à jour : "); + description.append(numberUpdated); + description.append("]"); + } + if (numberUpdated > 0) { + description.append(" [nombre de non importées : "); + description.append(numberNotImported); + description.append("]"); + } + + } + buffer.append(description.toString()).append('\n'); + } + String importText = buffer.toString().trim(); + + ImportLog importLog = persistenceService.createImportLog(configuration.getVoyageId(), + configuration.getImportType(), + importUser, + importDate, + importText, + importIds); + + for (ImportDataFileResult importDataFileResult : result.getImportResults()) { + + ImportFile importFile = createImportFile(importDataFileResult.getImportFile(), importDataFileResult.getExportFile()); + importLog.addImportFile(importFile); + + + } + + return importLog; + + } + + public <M extends AbstractImportConfiguration> ImportLog computeImportLogEntry(M configuration, EchoBaseUser user) { + + Date importDate = newDate(); + String importUser = user.getEmail(); + + StringBuilder buffer = new StringBuilder(); + + if (StringUtils.isNotEmpty(configuration.getImportNotes())) { + buffer.append("Notes :").append(configuration.getImportNotes()).append('\n'); + } + + List<String> importIds = Lists.newArrayList(); + + String importText = buffer.toString().trim(); + + return persistenceService.createImportLog(configuration.getVoyageId(), + configuration.getImportType(), + importUser, + importDate, + importText, + importIds); + + } + + protected ImportFile createImportFile(InputFile importFile, InputFile checkFile) { + + ImportFile result= persistenceService.createImportFile(importFile, checkFile); + + if (log.isInfoEnabled()) { + try { + log.info("Add ImportFile: " + result.getName() + " - size: " + result.getFile().length()); + } catch (SQLException e) { + throw new EchoBaseTechnicalException("Can't get length of importFile " + result.getName()); + } + try { + log.info("Add checkFile: " + result.getName() + " - size: " + result.getCheckFile().length()); + } catch (SQLException e) { + throw new EchoBaseTechnicalException("Can't get length of checkFile " + result.getName()); + } + } + return result; + + } + +// protected void addImportFiles(ImportLog importLog, InputFile... inputFiles) { +// +// for (InputFile inputFile : inputFiles) { +// +// ImportFile importFile = persistenceService.createImportFile(inputFile); +// +// if (log.isInfoEnabled()) { +// try { +// log.info("Add ImportFile: " + importFile.getName() + " - size: " + importFile.getFile().length()); +// } catch (SQLException e) { +// throw new EchoBaseTechnicalException("Can't get length of importFile " + importFile.getName()); +// } +// } +// importLog.addImportFile(importFile); +// +// } +// +// } + + protected <M extends AbstractImportConfiguration> String computeResultAndLogBookEntry(ImportDataResult<M> importDataResult) { + + Date importDate = newDate(); + String importUser = importDataResult.getUser().getEmail(); + + ImportLog importLog = computeImportLogEntry(importDataResult); + + String importType = getImportLabel(importDataResult.getConfiguration()); + + String result = importLog.getImportText(); + + persistenceService.createEntityModificationLog("Import", importType, importUser, importDate, result); + return result; + + } + + protected <M extends AbstractImportConfiguration> String getImportLabel(M configuration) { + return l(getLocale(), configuration.getImportType().getI18nKey()); + } + + public interface ImportDataAction<M extends AbstractImportConfiguration> { + + String doImport(ImportDataService service, M configuration, EchoBaseUser user) throws ImportException; + + } + + public static class AcousticImportDataAction implements ImportDataAction<AcousticImportConfiguration> { + @Override + public String doImport(ImportDataService service, AcousticImportConfiguration configuration, EchoBaseUser user) throws ImportException { + return service.doImportAcoustics(configuration, user); + } + } + + public static class CommonImportDataAction implements ImportDataAction<CommonImportConfiguration> { + @Override + public String doImport(ImportDataService service, CommonImportConfiguration configuration, EchoBaseUser user) throws ImportException { + return service.doImportCommons(configuration, user); + } + } + + public static class OperationsImportDataAction implements ImportDataAction<OperationImportConfiguration> { + @Override + public String doImport(ImportDataService service, OperationImportConfiguration configuration, EchoBaseUser user) throws ImportException { + return service.doImportOperations(configuration, user); + } + } + + public static class ResultsImportDataAction implements ImportDataAction<ResultsImportConfiguration> { + @Override + public String doImport(ImportDataService service, ResultsImportConfiguration configuration, EchoBaseUser user) throws ImportException { + return service.doImportResults(configuration, user); + } + } + + public static class CatchesImportDataAction implements ImportDataAction<CatchesImportConfiguration> { + @Override + public String doImport(ImportDataService service, CatchesImportConfiguration configuration, EchoBaseUser user) throws ImportException { + return service.doImportCatches(configuration, user); + } + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/MissionService.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/MissionService.java new file mode 100644 index 0000000..f3b7a0f --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/MissionService.java @@ -0,0 +1,61 @@ +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services.service.importdata; + +import com.google.common.base.Preconditions; +import fr.ifremer.echobase.EchoBaseTechnicalException; +import fr.ifremer.echobase.entities.references.Mission; +import fr.ifremer.echobase.services.EchoBaseServiceSupport; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import org.nuiton.topia.persistence.TopiaException; + +import javax.inject.Inject; + +/** + * Service to launch a "common data" import. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 0.3 + */ +public class MissionService extends EchoBaseServiceSupport { + + @Inject + protected UserDbPersistenceService persistenceService; + + public Mission createMission(Mission mission) throws MissionNameAlreadyExistException { + + Preconditions.checkNotNull(mission); + + try { + // check mission name is unique + boolean exists = persistenceService.isMissionExistByName(mission.getName()); + + if (exists) { + throw new MissionNameAlreadyExistException(); + } + Mission result = persistenceService.createMission(mission); + persistenceService.commit(); + return result; + } catch (TopiaException eee) { + throw new EchoBaseTechnicalException(eee); + } + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/NoTransitFoundBetweenDateException.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/NoTransitFoundBetweenDateException.java new file mode 100644 index 0000000..e773918 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/NoTransitFoundBetweenDateException.java @@ -0,0 +1,45 @@ +package fr.ifremer.echobase.services.service.importdata; + +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2015 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import fr.ifremer.echobase.entities.data.Voyage; +import org.nuiton.csv.ImportRuntimeException; + +import java.util.Date; +import java.util.Locale; + +import static org.nuiton.i18n.I18n.l; + +/** + * Created on 1/21/15. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 2.9 + */ +public class NoTransitFoundBetweenDateException extends ImportRuntimeException { + + private static final long serialVersionUID = 1L; + + public NoTransitFoundBetweenDateException(Locale locale, Voyage voyage, Date timeCoverageStart, Date timeCoverageEnd) { + super(l(locale, "echobase.importError.no.transit.between.date", voyage.getName(), timeCoverageStart, timeCoverageEnd)); + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/AbstractImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/AbstractImportAction.java new file mode 100644 index 0000000..1f390b1 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/AbstractImportAction.java @@ -0,0 +1,352 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import fr.ifremer.echobase.EchoBaseTechnicalException; +import fr.ifremer.echobase.csv.EchoBaseCsvUtil; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.Echotype; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Result; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.entities.references.AgeCategory; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.entities.references.SampleDataType; +import fr.ifremer.echobase.entities.references.SexCategory; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.AbstractImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.DataMetadataNotFoundException; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ImportException; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ImportModel; +import org.nuiton.csv.ImportRuntimeException; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; + +import static fr.ifremer.echobase.services.service.importdata.AbstractImportDataService.REMOVE_DOUBLE_QUOTES_PATTERN; +import static org.nuiton.i18n.I18n.l; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public abstract class AbstractImportAction<M extends AbstractImportConfiguration, E, C extends ImportModel<E>> implements Closeable { + + /** Logger. */ + private static final Log log = LogFactory.getLog(AbstractImportAction.class); + protected final UserDbPersistenceService persistenceService; + private final Locale locale; + private final M configuration; + private final EchoBaseUser user; + private final InputFile inputFile; + private final C importModel; + //FIXME Use a real cache + protected Map<String, SpeciesCategory> speciesCategories = Maps.newTreeMap(); + //FIXME Use a real cache + protected Map<String, Category> categories = Maps.newTreeMap(); + private Reader reader; + private Import<E> csvImport; + + protected AbstractImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + M configuration, + EchoBaseUser user, + InputFile inputFile) { + this.persistenceService = persistenceService; + this.locale = locale; + this.configuration = configuration; + this.user = user; + this.inputFile = inputFile; + this.importModel = createCsvImportModel(csvSeparator, configuration); + } + + public ImportDataFileResult run() throws ImportException { + + ImportDataFileResult result = new ImportDataFileResult(inputFile); + try { + performImport(getConfiguration(), inputFile, result); + } catch (ImportRuntimeException e) { + throw new ImportException(locale, inputFile, e); + } + return result; + + } + + protected abstract void performImport(M configuration, InputFile inputFile, ImportDataFileResult result); + + protected abstract C createCsvImportModel(char csvSeparator, M configuration); + + public final Locale getLocale() { + return locale; + } + + public final M getConfiguration() { + return configuration; + } + + public final EchoBaseUser getUser() { + return user; + } + + public InputFile getInputFile() { + return inputFile; + } + + protected final Import<E> open() { + if (reader != null) { + throw new IllegalStateException("Reader was already opened"); + } + if (csvImport != null) { + throw new IllegalStateException("csvImport was already opened"); + } + try { + reader = new BufferedReader(new InputStreamReader(new FileInputStream(inputFile.getFile()), Charsets.UTF_8)); + } catch (FileNotFoundException e) { + throw new EchoBaseTechnicalException("Could not find import file " + inputFile.getFile(), e); + } + return csvImport = Import.newImport(importModel, reader); + } + + @Override + public void close() throws IOException { + if (reader != null) { + IOUtils.closeQuietly(reader); + } + if (csvImport != null) { + IOUtils.closeQuietly(csvImport); + } + } + + public final String getImportLabel() { + return l(getLocale(), configuration.getImportType().getI18nKey()); + } + + public final String getImportMessage() { + return l(getLocale(), "echobase.importLabel.withFile", getImportLabel(), inputFile.getFileName()); + } + + protected void incrementsProgress() { + configuration.incrementsProgress(); + } + + protected final void doFlushTransaction(int rowNumber) { + configuration.incrementsProgress(); + if (rowNumber % 1000 == 0) { + // flush each 1000 imported rows + String message = getImportMessage(); + if (log.isDebugEnabled()) { + log.debug(message); + } + persistenceService.flush(); + } + } + + protected SpeciesCategory getSpeciesCategory(Species species, + SizeCategory sizeCategory, + AgeCategory ageCategory, + SexCategory sexCategory, + ImportDataFileResult importResult) { + + String key = species == null ? "" : species.getBaracoudaCode(); + key += "#" + (ageCategory == null ? "" : ageCategory.getName()); + key += "#" + (sizeCategory == null ? "" : sizeCategory.getName()); + key += "#" + (sexCategory == null ? "" : sexCategory.getName()); + + SpeciesCategory category = speciesCategories.get(key); + + if (category == null) { + + // try to find it in db + + category = persistenceService.getSpeciesCategory(species, sizeCategory, ageCategory, sexCategory); + + if (category == null) { + + // not found in db, create it + + category = persistenceService.createSpeciesCategory(species, sizeCategory, ageCategory, sexCategory); + importResult.incrementsNumberCreated(EchoBaseUserEntityEnum.SpeciesCategory); + } + + speciesCategories.put(key, category); + } + return category; + } + + protected Category getResultCategory(Echotype echotype, + Species species, + SizeCategory sizeCategory, + AgeCategory ageCategory, + ImportDataFileResult importResult) { + + // get species categorie first + + SpeciesCategory speciesCategory = getSpeciesCategory(species, sizeCategory, ageCategory, null, importResult); + + return getResultCategory(echotype, speciesCategory, importResult); + } + + protected Category getResultCategory(Echotype echotype, SpeciesCategory speciesCategory, ImportDataFileResult importResult) { + + String key = speciesCategory == null ? "" : speciesCategory.getTopiaId(); + key += "#" + (echotype == null ? "" : echotype.getName()); + + Category category = categories.get(key); + + if (category == null) { + + // try to find it in db + + category = persistenceService.getCategoryByEchotypeAndSpeciesCategory(echotype, speciesCategory); + + if (category == null) { + + // not found in db, create it + category = persistenceService.createCategory(echotype, speciesCategory); + importResult.incrementsNumberCreated(EchoBaseUserEntityEnum.Category); + } + + categories.put(key, category); + } + + return category; + } + + protected void createCellData(Cell cell, + DataMetadata dataMetaData, + String dataValue, + DataQuality dataQuality, + ImportDataFileResult importResult) { + + Data data = persistenceService.createData(dataMetaData, dataValue); + cell.addData(data); + cell.setDataQuality(dataQuality); + + importResult.incrementsNumberCreated(EchoBaseUserEntityEnum.Data); + + } + + protected List<DataMetadata> getMetas(String[] columnNamesToExclude, Map<String, DataMetadata> dataMetadataMap, String... headers) { + + List<String> metadataNames = Lists.newArrayList(headers); + for (String columnToExclude : columnNamesToExclude) { + metadataNames.remove(columnToExclude); + metadataNames.remove("\"" + columnToExclude + "\""); + } + + List<DataMetadata> result = Lists.newArrayList(); + + Locale locale = getLocale(); + + for (String metadataName : metadataNames) { + + Matcher matcher = REMOVE_DOUBLE_QUOTES_PATTERN.matcher(metadataName); + if (matcher.matches()) { + metadataName = matcher.group(1); + } + DataMetadata dataMetadata = dataMetadataMap.get(metadataName); + if (dataMetadata == null) { + throw new DataMetadataNotFoundException(locale, metadataName, dataMetadataMap.keySet()); + } + result.add(dataMetadata); + } + return result; + + } + + protected void addResults(EchoBaseCsvUtil.ResultAble row, + Cell cell, + Category category, + String resultLabel, + ImportDataFileResult importResult, + boolean collecIds, + boolean importNAResults) { + + List<Result> results = row.getResult(); + + for (Result result : results) { + + if (!importNAResults && EchoBaseCsvUtil.NA.equals(result.getResultValue())) { + + // Do not import NA results + continue; + } + result.setDataQuality(row.getDataQuality()); + result.setCategory(category); + result.setResultLabel(resultLabel); + + Result resultCreated = persistenceService.createResult(result); + cell.addResult(resultCreated); + + if (collecIds) { + + importResult.addId(EchoBaseUserEntityEnum.Result, resultCreated); + + } else { + + importResult.incrementsNumberCreated(EchoBaseUserEntityEnum.Result); + } + + } + + } + + protected Sample addSample(Operation operation, Sample sample, ImportDataFileResult importResult) { + + Preconditions.checkNotNull(operation); + Preconditions.checkNotNull(sample); + Sample result = persistenceService.createSample(sample); + operation.addSample(result); + importResult.addId(EchoBaseUserEntityEnum.Sample, sample); + return result; + + } + + protected SampleData addSampleData(SampleDataType sampleDataType, String label, float value, Sample sample, ImportDataFileResult importResult) { + + SampleData sampleData = persistenceService.createSampleData(sampleDataType, label, value); + sample.addSampleData(sampleData); + importResult.incrementsNumberCreated(EchoBaseUserEntityEnum.SampleData); + return sampleData; + + } + + //FIXME Define a real exception + protected void checkOperationWithTotalOrUnsortedSample(int rowNumber, Collection<Operation> operationWithTotalOrUnsortedSample, Operation operation) { + if (!operationWithTotalOrUnsortedSample.contains(operation)) { + + // can not accept this import + throw new ImportRuntimeException("At line " + rowNumber + ", wants to add a subsample sample but operation " + operation.getId() + " has no sample of type *Total*, nor *Unsorted*"); + } + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/AcousticImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/AcousticImportAction.java new file mode 100644 index 0000000..eadc34e --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/AcousticImportAction.java @@ -0,0 +1,616 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import fr.ifremer.echobase.csv.EchoBaseCsvUtil; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.DataAcquisition; +import fr.ifremer.echobase.entities.data.DataProcessing; +import fr.ifremer.echobase.entities.data.Transect; +import fr.ifremer.echobase.entities.data.Transit; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.AcousticInstrument; +import fr.ifremer.echobase.entities.references.AcousticInstruments; +import fr.ifremer.echobase.entities.references.CellType; +import fr.ifremer.echobase.entities.references.CellTypeImpl; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.entities.references.Vessel; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.AcousticImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.EsduCellNotFoundException; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.TransectNotFoundException; +import fr.ifremer.echobase.services.service.importdata.TransitNotFoundException; +import fr.ifremer.echobase.services.service.importdata.csv.AcousticImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.AcousticImportRow; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class AcousticImportAction extends AbstractImportAction<AcousticImportConfiguration, AcousticImportRow, AcousticImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(AcousticImportAction.class); + + private final DateFormat cellDateFormat; + private final Map<String, DataMetadata> dataMetadatasByName; + private String lastEsduId; + + public AcousticImportAction(Locale locale, + char csvSeparator, + UserDbPersistenceService persistenceService, + AcousticImportConfiguration configuration, + EchoBaseUser user) { + + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getMoviesFile()); + + dataMetadatasByName = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + + cellDateFormat = new SimpleDateFormat(EchoBaseCsvUtil.CELLULE_DATE_FORMAT); + + } + + @Override + protected AcousticImportModel createCsvImportModel(char csvSeparator, AcousticImportConfiguration configuration) { + Map<String, AcousticInstrument> instrumentsById = persistenceService.getEntitiesMap(AcousticInstrument.class, AcousticInstruments.ACOUSTIC_INSTRUMENT_ID); + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + return new AcousticImportModel(csvSeparator, instrumentsById, dataQualityMap); + } + + @Override + protected void performImport(AcousticImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts " + getImportMessage()); + } + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get selected vessel + Vessel vessel = persistenceService.getVessel(configuration.getVesselId()); + + CellType esduCellType = persistenceService.getCellTypeById(CellTypeImpl.ESDU); + CellType elementaryCellType = persistenceService.getCellTypeById(CellTypeImpl.ELEMENTARY); + + boolean addDataAcquisition = getConfiguration().isAddDataAcquisition(); + + String suffix = getConfiguration().getCellPositionReference().getMetadataNameSuffix(); + + List<Cell> elementaryCells = Lists.newArrayList(); + + Locale locale = getLocale(); + String dataProcessingId = null; + + + String processingTemplate = getConfiguration().getProcessingTemplate(); + + try (Import<AcousticImportRow> importer = open()) { + DataAcquisition dataAcquisition = null; + DataProcessing dataProcessing = null; + + Cell elementaryCell; + int rowNumber = 0; + + incrementsProgress(); + + Set<String> createdDataAcquisitions = Sets.newHashSet(); + + for (AcousticImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + if (dataProcessingId == null) { + + // compute once for all the common dataprocessing id used + // for hole the movies file + + dataProcessingId = row.getEiLayer() + processingTemplate; + } + + Date startDate = row.getCellDateStart(); + Date endDate = row.getCellDateEnd(); + + // get transect to use + Transit transit = voyage.getTransit(startDate); + if (transit == null) { + + // can not find correct transit + throw new TransitNotFoundException(locale, voyage, startDate, rowNumber); + } + + Transect transect = transit.getTransect(vessel); + if (transect == null) { + + // can not find correct transect + throw new TransectNotFoundException(locale, voyage, vessel, startDate, rowNumber); + } + + AcousticInstrument instrument = row.getAcousticInstrument(); + boolean isME70 = AcousticInstruments.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); + + if (dataAcquisition == null || !instrument.equals(dataAcquisition.getAcousticInstrument())) { + + // need to use another data acquisition + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + "] New instrument to use (" + instrument.getId() + ")"); + } + + if (CollectionUtils.isNotEmpty(elementaryCells)) { + + // means miss a esdu cell + // can not find correct transect + throw new EsduCellNotFoundException(locale, voyage, vessel, instrument, rowNumber); + } + + String softwareVersion = getSoftwareVersion(isME70); + + String soundSpeedCalculations = getSoundSpeedCalculations(isME70); + + // try to get existing data acquisition + dataAcquisition = getDataAcquisition(transect, instrument, addDataAcquisition); + + if (dataAcquisition == null) { + + // need to create the data acquisition + dataAcquisition = createDataAcquisition(instrument, softwareVersion, soundSpeedCalculations, row); + + // collect id of the import + result.addId(EchoBaseUserEntityEnum.DataAcquisition, dataAcquisition); + createdDataAcquisitions.add(dataAcquisition.getTopiaId()); + + // add dataAcquisition to transect + transect.addDataAcquisition(dataAcquisition); + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + "] New dataAquisition to use (number: " + result.getNumberCreated(EchoBaseUserEntityEnum.DataAcquisition) + ")"); + } + + // create data processing + dataProcessing = createDataProcessing(dataProcessingId, softwareVersion, soundSpeedCalculations, row); + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + "] New dataProcessing to use (" + dataProcessing.getId() + ")"); + } + + result.incrementsNumberCreated(EchoBaseUserEntityEnum.DataProcessing); + + // add it to data acquisition + dataAcquisition.addDataProcessing(dataProcessing); + } + } + + int cellType = row.getCellType(); + + String esduCellId = cellDateFormat.format(endDate); + + String previousLastEsduId = lastEsduId; + + if (lastEsduId == null || !lastEsduId.equals(esduCellId)) { + + lastEsduId = esduCellId; + // find a new esdu cell id + if (log.isDebugEnabled()) { + log.debug("row: " + rowNumber + " - detects new esduCellId:" + lastEsduId); + } + } + boolean isEsduCell = cellType == 4; + + if (isEsduCell) { + + if (!esduCellId.equals(previousLastEsduId)) { + + // This should not happen ? + if (log.isWarnEnabled()) { + log.warn("row: " + rowNumber + " A previous esduCellId was detected (" + previousLastEsduId + "); but the esdu id was never found."); + } + } + if (log.isDebugEnabled()) { + log.debug("row: " + rowNumber + " - found esdu cell:" + lastEsduId); + } + } + + if (!row.isCellToAdd()) { + + // dead cell not to be imported + + String id = rowNumber + " - " + cellType + " : " + esduCellId; + if (isEsduCell) { + + // keep this fact into import result + if (log.isWarnEnabled()) { + log.warn("Will not import esdu cell of row " + id); + } + result.addNotImportedId(EchoBaseUserEntityEnum.Cell, id); + } else { + + // just a debug log + if (log.isTraceEnabled()) { + log.trace("Will not import cell of row " + id); + } + } + + continue; + } + + DataQuality dataQuality = row.getDataQuality(); + + // collect Esdu cellIds only if dataAcquisitions was not created here + boolean collectCellIds = !createdDataAcquisitions.contains(dataAcquisition.getTopiaId()); + + + if (isEsduCell) { + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + "] Esdu cell (" + esduCellId + ")"); + } + + // this is a esdu cell row + + // create esdu cell + Cell esduCell = persistenceService.createCell(esduCellType, esduCellId); + + esduCell.setDataQuality(dataQuality); + + // add all found elementary cells + esduCell.addAllChilds(elementaryCells); + + // clear elementary cells + elementaryCells.clear(); + + if (collectCellIds) { + + // collect ids (dataProcessing already exists) + result.addId(EchoBaseUserEntityEnum.Cell, esduCell); + } else { + + result.incrementsNumberCreated(EchoBaseUserEntityEnum.Cell); + } + + Preconditions.checkNotNull(dataProcessing); + + // add it to data processing + dataProcessing.addCell(esduCell); + + // create esdu cell data + createEsduCellData(suffix, esduCell, row, dataQuality, result); + } else { + + // this is a elementary cell row + + int cellNum = row.getCellNum(); + + boolean surface = cellType == 0; + + String elementaryCellId = cellNum + (surface ? "S" : "B"); + + if (log.isTraceEnabled()) { + log.trace("[row " + rowNumber + "] elementary cell (" + elementaryCellId + ")"); + } + + // create the elementary cell + elementaryCell = persistenceService.createCell(elementaryCellType, elementaryCellId); + elementaryCell.setDataQuality(dataQuality); + + // keep (to attach them to esdu cell) + elementaryCells.add(elementaryCell); + + result.incrementsNumberCreated(EchoBaseUserEntityEnum.Cell); + + // create datas of the elementary cell + createElementaryCellData(suffix, elementaryCell, surface, row, dataQuality, result); + } + } + + } + + } + + private DataAcquisition getDataAcquisition(Transect transect, AcousticInstrument instrument, boolean addDataAcquisition) { + + DataAcquisition result = null; + if (!addDataAcquisition && !transect.isDataAcquisitionEmpty()) { + + // try to obtain an existing data acquisiton + + for (DataAcquisition acquisition : transect.getDataAcquisition()) { + + if (instrument.equals(acquisition.getAcousticInstrument())) { + + // found a matching dataAcquisition from his acoustic instrument + result = acquisition; + break; + } + } + } + return result; + } + + private void createEsduCellData(String suffix, + Cell cell, + AcousticImportRow row, + DataQuality dataQuality, + ImportDataFileResult importResult) { + + String dataValue; + + // create Latitude data + dataValue = String.valueOf(row.getCellLatitude()); + createCellData(cell, "Latitude" + suffix, dataValue, dataQuality, importResult); + + // create Longitude data + dataValue = String.valueOf(row.getCellLongitude()); + createCellData(cell, "Longitude" + suffix, dataValue, dataQuality, importResult); + + // create Depth data + dataValue = row.getEsduCellDataDepth(); + createCellData(cell, "DepthRefSurface" + suffix, dataValue, dataQuality, importResult); + + // create Time Start data + dataValue = cellDateFormat.format(row.getCellDateStart()); + createCellData(cell, "TimeStart", dataValue, dataQuality, importResult); + + // create Time end data + dataValue = cellDateFormat.format(row.getCellDateEnd()); + createCellData(cell, "TimeEnd", dataValue, dataQuality, importResult); + + // create NASC data + if (row.getCellNasc() != null) { + dataValue = String.valueOf(row.getCellNasc()); + createCellData(cell, "NASC", dataValue, dataQuality, importResult); + } + + // create Volume data + if (row.getCellVolume() != null) { + dataValue = String.valueOf(row.getCellVolume()); + createCellData(cell, "Volume", dataValue, dataQuality, importResult); + } + + // create Surface data + dataValue = String.valueOf(row.getCellSurface()); + createCellData(cell, "Surface", dataValue, dataQuality, importResult); + + // create NumberOfSamplesRecorded data + if (row.getCellNumberOfSamplesRecorded() != null) { + dataValue = String.valueOf(row.getCellNumberOfSamplesRecorded()); + createCellData(cell, "NumberOfSamplesRecorded", dataValue, dataQuality, importResult); + } + // create NumberOfSamplesEchoIntegrated data + if (row.getCellNumberOfSamplesEchoIntegrated() != null) { + dataValue = String.valueOf(row.getCellNumberOfSamplesEchoIntegrated()); + createCellData(cell, "NumberOfSamplesEchoIntegrated", dataValue, dataQuality, importResult); + } + } + + private void createElementaryCellData(String suffix, + Cell cell, + boolean surface, + AcousticImportRow row, + DataQuality dataQuality, + ImportDataFileResult importResult) { + + String dataValue; + String startMeta; + String endMeta; + + if (surface) { + + // surface + + startMeta = "DepthRefSurfaceStart"; + endMeta = "DepthRefSurfaceEnd"; + + } else { + + // bottom + + startMeta = "DepthRefBottomStart"; + endMeta = "DepthRefBottomEnd"; + } + + // create Latitude data + dataValue = String.valueOf(row.getCellLatitude()); + createCellData(cell, "Latitude" + suffix, dataValue, dataQuality, importResult); + + // create Longitude data + dataValue = String.valueOf(row.getCellLongitude()); + createCellData(cell, "Longitude" + suffix, dataValue, dataQuality, importResult); + + // create depth start data + dataValue = String.valueOf(row.getCellDepthStart()); + createCellData(cell, startMeta, dataValue, dataQuality, importResult); + + // create depth end data + dataValue = String.valueOf(row.getCellDepthEnd()); + createCellData(cell, endMeta, dataValue, dataQuality, importResult); + + + // create Time Start data + dataValue = cellDateFormat.format(row.getCellDateStart()); + createCellData(cell, "TimeStart", dataValue, dataQuality, importResult); + + // create Time end data + dataValue = cellDateFormat.format(row.getCellDateEnd()); + createCellData(cell, "TimeEnd", dataValue, dataQuality, importResult); + + // create acoustic density data + if (row.getCellNasc() != null) { + dataValue = String.valueOf(row.getCellNasc()); + createCellData(cell, "NASC", dataValue, dataQuality, importResult); + } + // create Volume data + if (row.getCellVolume() != null) { + dataValue = String.valueOf(row.getCellVolume()); + createCellData(cell, "Volume", dataValue, dataQuality, importResult); + } + // create Surface data + dataValue = String.valueOf(row.getCellSurface()); + createCellData(cell, "Surface", dataValue, dataQuality, importResult); + + // create NumberOfSamplesRecorded data + if (row.getCellNumberOfSamplesRecorded() != null) { + dataValue = String.valueOf(row.getCellNumberOfSamplesRecorded()); + createCellData(cell, "NumberOfSamplesRecorded", dataValue, dataQuality, importResult); + } + // create NumberOfSamplesEchoIntegrated data + if (row.getCellNumberOfSamplesEchoIntegrated() != null) { + dataValue = String.valueOf(row.getCellNumberOfSamplesEchoIntegrated()); + createCellData(cell, "NumberOfSamplesEchoIntegrated", dataValue, dataQuality, importResult); + } + } + + private DataAcquisition createDataAcquisition(AcousticInstrument instrument, + String softwareVersion, + String soundSpeedCalculations, + AcousticImportRow row) { + + AcousticImportConfiguration configuration = getConfiguration(); + + String transceiverAcquisitionAbsorptionDescription = + configuration.getTransceiverAcquisitionAbsorptionDescription(); + String loggedDataFormat = configuration.getLoggedDataFormat(); + String loggedDataDatatype = configuration.getLoggedDataDatatype(); + String pingDutyCycle = configuration.getPingDutyCycle(); + + DataAcquisition dataAcquisition = persistenceService.createDataAcquisition(instrument); + + // fill from manual configuration + dataAcquisition.setTransceiverAcquisitionAbsorptionDescription(transceiverAcquisitionAbsorptionDescription); + dataAcquisition.setAcquisitionSoftwareVersion(softwareVersion); + dataAcquisition.setLoggedDataFormat(loggedDataFormat); + dataAcquisition.setLoggedDataDatatype(loggedDataDatatype); + dataAcquisition.setPingDutyCycle(pingDutyCycle); + dataAcquisition.setEchosounderSoundSpeed(row.getSoundCelerity()); + dataAcquisition.setSoundSpeedCalculations(soundSpeedCalculations); + + // fill from csv file + Float transceiverAcquisitionAbsorption = row.getTransceiverAcquisitionAbsorption(); + Float transducerAcquisitionBeamAngleAthwartship = row.getTransducerAcquisitionBeamAngleAthwartship(); + Float transducerAcquisitionBeamAngleAlongship = row.getTransducerAcquisitionBeamAngleAlongship(); + Float transducerAcquisitionPsi = row.getTransducerAcquisitionPsi(); + Float transceiverAcquisitionPower = row.getTransceiverAcquisitionPower(); + Float transceiverAcquisitionPulseLength = row.getTransceiverAcquisitionPulseLength(); + Float transceiverAcquisitionGain = row.getTransceiverAcquisitionGain(); + Float transceiverAcquisitionSacorrection = row.getTransceiverAcquisitionSacorrection(); + + dataAcquisition.setTransceiverAcquisitionAbsorption(transceiverAcquisitionAbsorption); + dataAcquisition.setTransducerAcquisitionBeamAngleAthwartship(transducerAcquisitionBeamAngleAthwartship); + dataAcquisition.setTransducerAcquisitionBeamAngleAlongship(transducerAcquisitionBeamAngleAlongship); + dataAcquisition.setTransducerAcquisitionPsi(transducerAcquisitionPsi); + dataAcquisition.setTransceiverAcquisitionPower(transceiverAcquisitionPower); + dataAcquisition.setTransceiverAcquisitionPulseLength(transceiverAcquisitionPulseLength); + dataAcquisition.setTransceiverAcquisitionGain(transceiverAcquisitionGain); + dataAcquisition.setTransceiverAcquisitionSacorrection(transceiverAcquisitionSacorrection); + + return dataAcquisition; + } + + private DataProcessing createDataProcessing(String id, + String softwareVersion, + String soundSpeedCalculations, + AcousticImportRow row) { + + AcousticImportConfiguration configuration = getConfiguration(); + + String transceiverAcquisitionAbsorptionDescription = + configuration.getTransceiverAcquisitionAbsorptionDescription(); + String processingTemplate = configuration.getProcessingTemplate(); + String processingDescription = configuration.getProcessingDescription(); + String sounderConstant = configuration.getSounderConstant(); + float digitThreshold = configuration.getDigitThreshold(); + String acousticDensityUnit = configuration.getAcousticDensityUnit(); + String notes = configuration.getNotes(); + + DataProcessing dataProcessing = persistenceService.createDataProcessing(id, processingTemplate); + + // fill from manual configuration + dataProcessing.setProcessingDescription(processingDescription); + dataProcessing.setSounderConstant(sounderConstant); + dataProcessing.setDigitThreshold(digitThreshold); + dataProcessing.setAcousticDensityUnit(acousticDensityUnit); + dataProcessing.setNotes(notes); + dataProcessing.setProcessingSoftwareVersion(softwareVersion); + dataProcessing.setTransceiverProcessingAbsorptionDescription(transceiverAcquisitionAbsorptionDescription); + dataProcessing.setEchosounderSoundSpeed(row.getSoundCelerity()); + dataProcessing.setSoundSpeedCalculations(soundSpeedCalculations); + + // fill from csv file + + Float transceiverAcquisitionAbsorption = row.getTransceiverAcquisitionAbsorption(); + Float transducerAcquisitionBeamAngleAthwartship = row.getTransducerAcquisitionBeamAngleAthwartship(); + Float transducerAcquisitionBeamAngleAlongship = row.getTransducerAcquisitionBeamAngleAlongship(); + Float transducerAcquisitionPsi = row.getTransducerAcquisitionPsi(); + Float transceiverAcquisitionGain = row.getTransceiverAcquisitionGain(); + Float transceiverAcquisitionSacorrection = row.getTransceiverAcquisitionSacorrection(); + int eIThresholdLow = row.geteIThresholdLow(); + int eIThresholdHigh = row.geteIThresholdHigh(); + + dataProcessing.seteIThresholdLow(eIThresholdLow); + dataProcessing.seteIThresholdHigh(eIThresholdHigh); + dataProcessing.setTransceiverProcessingSacorrection(transceiverAcquisitionSacorrection); + dataProcessing.setTransceiverProcessingAbsorption(transceiverAcquisitionAbsorption); + dataProcessing.setTransceiverProcessingGain(transceiverAcquisitionGain); + dataProcessing.setTransducerProcessingPsi(transducerAcquisitionPsi); + dataProcessing.setTransducerProcessingBeamAngleAthwartship(transducerAcquisitionBeamAngleAthwartship); + dataProcessing.setTransducerProcessingBeamAngleAlongship(transducerAcquisitionBeamAngleAlongship); + + return dataProcessing; + } + + private void createCellData(Cell cell, +// Map<String, DataMetadata> dataMetadatasByName, + String metadataName, + String dataValue, + DataQuality dataQuality, + ImportDataFileResult importResult) { + createCellData(cell, + dataMetadatasByName.get(metadataName), + dataValue, + dataQuality, + importResult); + } + + private String getSoftwareVersion(boolean isME70) { + String result; + if (isME70) { + result = getConfiguration().getAcquisitionSoftwareVersionME70(); + } else { + result = getConfiguration().getAcquisitionSoftwareVersionER60(); + } + return result; + } + + private String getSoundSpeedCalculations(boolean isME70) { + String result; + if (isME70) { + result = getConfiguration().getSoundSpeedCalculationsME70(); + } else { + result = getConfiguration().getSoundSpeedCalculationsER60(); + } + return result; + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/BiometrySampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/BiometrySampleImportAction.java new file mode 100644 index 0000000..6b6b850 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/BiometrySampleImportAction.java @@ -0,0 +1,136 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Collections2; +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Operations; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.SampleDataType; +import fr.ifremer.echobase.entities.references.SampleDataTypes; +import fr.ifremer.echobase.entities.references.SampleType; +import fr.ifremer.echobase.entities.references.SampleTypeImpl; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CatchesImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.csv.BiometrySampleImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.BiometrySampleImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collection; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class BiometrySampleImportAction extends AbstractImportAction<CatchesImportConfiguration, BiometrySampleImportRow, BiometrySampleImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(BiometrySampleImportAction.class); + + public BiometrySampleImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + CatchesImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getSubSampleFile()); + } + + @Override + protected BiometrySampleImportModel createCsvImportModel(char csvSeparator, CatchesImportConfiguration configuration) { + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get all operation for this voyage and this vessel + Collection<Operation> operations = voyage.getAllOperations(); + + Map<String, Operation> operationMap = Maps.uniqueIndex(operations, Operations.OPERATION_ID); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + Map<String, SampleDataType> sampleDataTypeMap = persistenceService.getEntitiesMap(SampleDataType.class, SampleDataTypes.SAMPLE_DATA_TYPE_NAME); + + return new BiometrySampleImportModel(csvSeparator, operationMap, speciesMap, sampleDataTypeMap); + + } + + @Override + protected void performImport(CatchesImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of subSample from file " + inputFile.getFileName()); + } + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get all operation for this voyage and this vessel + Collection<Operation> operations = voyage.getAllOperations(); + + Collection<Operation> operationWithTotalOrUnsortedSample = Collections2.filter(operations, Operations.OPERATION_WITH_TOTAL_OR_UNSORTED_SAMPLE); + + SampleType sampleTypeIndividual = persistenceService.getSampleTypeByName(SampleTypeImpl.INDIVIDUAL_SAMPLE_TYPE); + Map<String, Sample> samples = Maps.newTreeMap(); + + try (Import<BiometrySampleImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (BiometrySampleImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Operation operation = row.getOperation(); + + checkOperationWithTotalOrUnsortedSample(rowNumber, operationWithTotalOrUnsortedSample, operation); + + Species species = row.getSpecies(); + + int numFish = row.getNumFish(); + + String sampleKey = operation.getId() + "_" + species.getBaracoudaCode() + "_" + numFish; + + Sample sample = samples.get(sampleKey); + + if (sample == null) { + + // create a new sample + sample = persistenceService.newSample(); + + sample.setSampleType(sampleTypeIndividual); + + SpeciesCategory category = getSpeciesCategory(species, null, null, null, result); + + sample.setSpeciesCategory(category); + + sample = addSample(operation, sample, result); + + samples.put(sampleKey, sample); + } + + SampleData sampleData = row.getSampleData(); + + SampleData sampleDataCreated = persistenceService.createSampleData(sampleData); + sample.addSampleData(sampleDataCreated); + result.incrementsNumberCreated(EchoBaseUserEntityEnum.SampleData); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonTransectImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonTransectImportAction.java new file mode 100644 index 0000000..08e4022 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonTransectImportAction.java @@ -0,0 +1,121 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Transect; +import fr.ifremer.echobase.entities.data.Transit; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.Vessel; +import fr.ifremer.echobase.entities.references.Vessels; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CommonImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.NoTransitFoundBetweenDateException; +import fr.ifremer.echobase.services.service.importdata.csv.TransectImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.TransectImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collections; +import java.util.Date; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class CommonTransectImportAction extends AbstractImportAction<CommonImportConfiguration, TransectImportRow, TransectImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(CommonTransectImportAction.class); + + private final boolean collectIds; + + public CommonTransectImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + CommonImportConfiguration configuration, + EchoBaseUser user, + boolean collectIds) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getTransectFile()); + this.collectIds = collectIds; + } + + @Override + protected TransectImportModel createCsvImportModel(char csvSeparator, CommonImportConfiguration configuration) { + + // get usable vessels + Map<String, Vessel> vesselMap = persistenceService.getEntitiesMap(Vessel.class, Vessels.VESSEL_NAME); + + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // restrict voyage to use to this voyage + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + return new TransectImportModel(csvSeparator, voyageMap, vesselMap); + + } + + @Override + protected void performImport(CommonImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of transects from file " + inputFile.getFileName()); + } + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + String datum = getConfiguration().getDatum(); + String license = getConfiguration().getTransectLicence(); + String geospatialVerticalPositive = getConfiguration().getTransectGeospatialVerticalPositive(); + String binUnitsPingAxis = getConfiguration().getTransectBinUnitsPingAxis(); + + try (Import<TransectImportRow> importer = open()) { + + incrementsProgress(); + + for (TransectImportRow row : importer) { + + incrementsProgress(); + Transect transect = row.getTransect(); + + Date timeCoverageStart = transect.getTimeCoverageStart(); + Date timeCoverageEnd = transect.getTimeCoverageEnd(); + + Transit transit = voyage.getTransit(timeCoverageStart, timeCoverageEnd); + if (transit == null) { + throw new NoTransitFoundBetweenDateException(getLocale(), voyage, timeCoverageStart, timeCoverageEnd); + } + + transect.setDatum(datum); + transect.setLicence(license); + transect.setGeospatialVerticalPositive(geospatialVerticalPositive); + transect.setBinUnitsPingAxis(binUnitsPingAxis); + + Transect createdTransect = persistenceService.createTransect(transect); + + transit.addTransect(createdTransect); + + if (collectIds) { + + // collect id of the import + result.addId(EchoBaseUserEntityEnum.Transect, createdTransect); + } else { + + result.incrementsNumberCreated(EchoBaseUserEntityEnum.Transect); + } + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonTransitImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonTransitImportAction.java new file mode 100644 index 0000000..2d2d800 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonTransitImportAction.java @@ -0,0 +1,100 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Transit; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CommonImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.csv.TransitImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.TransitImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collections; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class CommonTransitImportAction extends AbstractImportAction<CommonImportConfiguration, TransitImportRow, TransitImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(CommonTransitImportAction.class); + + private final boolean collectIds; + + public CommonTransitImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + CommonImportConfiguration configuration, + EchoBaseUser user, + boolean collectIds) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getTransitFile()); + this.collectIds = collectIds; + } + + @Override + protected TransitImportModel createCsvImportModel(char csvSeparator, CommonImportConfiguration configuration) { + + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // restrict voyage to use to this voyage + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + return new TransitImportModel(csvSeparator, voyageMap); + + } + + @Override + protected void performImport(CommonImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of transits from file " + inputFile.getFileName()); + } + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + String relatedActivity = getConfiguration().getTransitRelatedActivity(); + + try (Import<TransitImportRow> importer = open()) { + + incrementsProgress(); + + for (TransitImportRow row : importer) { + + incrementsProgress(); + Transit transit = row.getTransit(); + + transit.setRelatedActivity(relatedActivity); + + Transit createdTransit = persistenceService.createTransit(transit); + + voyage.addTransit(createdTransit); + + if (collectIds) { + + // collect id of the import + result.addId(EchoBaseUserEntityEnum.Transit, createdTransit); + + } else { + + result.incrementsNumberCreated(EchoBaseUserEntityEnum.Transit); + } + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonVoyageImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonVoyageImportAction.java new file mode 100644 index 0000000..a5328fb --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/CommonVoyageImportAction.java @@ -0,0 +1,100 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.AreaOfOperation; +import fr.ifremer.echobase.entities.references.Mission; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CommonImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.MoreThanOnceVoyageToImportException; +import fr.ifremer.echobase.services.service.importdata.NoVoyageToImportException; +import fr.ifremer.echobase.services.service.importdata.csv.VoyageImportModel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Locale; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class CommonVoyageImportAction extends AbstractImportAction<CommonImportConfiguration, Voyage, VoyageImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(CommonVoyageImportAction.class); + + public CommonVoyageImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + CommonImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getVoyageFile()); + } + + @Override + protected VoyageImportModel createCsvImportModel(char csvSeparator, CommonImportConfiguration configuration) { + return new VoyageImportModel(csvSeparator); + } + + @Override + public void performImport(CommonImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + + if (log.isInfoEnabled()) { + log.info("Starts import of voyages from file " + inputFile.getFileName()); + } + + Mission mission = persistenceService.getMission(configuration.getMissionId()); + AreaOfOperation areaOfOperation = persistenceService.getAreaOfOperation(configuration.getAreaOfOperationId()); + + String voyageDescription = getConfiguration().getVoyageDescription(); + String datum = getConfiguration().getDatum(); + Locale locale = getLocale(); + + Voyage newVoyage = null; + + try (Import<Voyage> importer = open()) { + + incrementsProgress(); + + for (Voyage voyage : importer) { + + incrementsProgress(); + + voyage.setMission(mission); + voyage.setAreaOfOperation(areaOfOperation); + voyage.setDescription(voyageDescription); + voyage.setDatum(datum); + + Voyage createdVoyage = persistenceService.createVoyage(voyage); + if (newVoyage == null) { + newVoyage = createdVoyage; + } else { + // this means a voyage file with more than one row not possible... + + throw new MoreThanOnceVoyageToImportException(locale); + } + + // collect id of the import + result.addId(EchoBaseUserEntityEnum.Voyage, voyage); + + } + + } + + if (newVoyage == null) { + throw new NoVoyageToImportException(locale); + } + + // push back to id of the voyage in configuration for next imports + getConfiguration().setVoyageId(newVoyage.getTopiaId()); + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/GearMetadataImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/GearMetadataImportAction.java new file mode 100644 index 0000000..10391be --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/GearMetadataImportAction.java @@ -0,0 +1,100 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.GearMetadataValue; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Operations; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.Gear; +import fr.ifremer.echobase.entities.references.GearMetadata; +import fr.ifremer.echobase.entities.references.GearMetadatas; +import fr.ifremer.echobase.entities.references.Gears; +import fr.ifremer.echobase.entities.references.Vessel; +import fr.ifremer.echobase.entities.references.Vessels; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.OperationImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.GearMetadataValueImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.GearMetadataValueImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collection; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class GearMetadataImportAction extends AbstractImportAction<OperationImportConfiguration, GearMetadataValueImportRow, GearMetadataValueImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(GearMetadataImportAction.class); + + public GearMetadataImportAction(Locale locale, + char csvSeparator, + UserDbPersistenceService persistenceService, + OperationImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getOperationMetadataFile()); + + } + + @Override + protected GearMetadataValueImportModel createCsvImportModel(char csvSeparator, OperationImportConfiguration configuration) { + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get usable vessels from the voyage by their name + Map<String, Vessel> vesselMap = Maps.uniqueIndex(voyage.getAllVessels(), Vessels.VESSEL_NAME); + + // get all operation for this voyage and this vessel + Collection<Operation> operations = voyage.getAllOperations(); + + // get usable gears by their casino gear name + Map<String, Gear> gearMap = persistenceService.getEntitiesMap(Gear.class, Gears.GEAR_CASINO_GEAR_NAME); + + // split them by operation Id + Map<String, Operation> operationMap = Maps.uniqueIndex(operations, Operations.OPERATION_ID); + + Map<String, GearMetadata> gearMetadatasMap = persistenceService.getEntitiesMap(GearMetadata.class, GearMetadatas.GEAR_METADATA_NAME); + return new GearMetadataValueImportModel(csvSeparator, vesselMap, gearMetadatasMap, gearMap, operationMap); + } + + @Override + protected void performImport(OperationImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of gear metadata values from file " + inputFile.getFileName()); + } + + try (Import<GearMetadataValueImportRow> importer = open()) { + + int rowNumber = 0; + incrementsProgress(); + for (GearMetadataValueImportRow row : importer) { + + doFlushTransaction(++rowNumber); + Operation operation = row.getOperation(); + + GearMetadataValue gearMetadataValuetoCreate = row.getGearMetadataValue(); + GearMetadataValue gearMetadataValue = persistenceService.createGearMetadataValue(gearMetadataValuetoCreate); + + operation.addGearMetadataValue(gearMetadataValue); + result.incrementsNumberCreated(EchoBaseUserEntityEnum.GearMetadataValue); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/OperationImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/OperationImportAction.java new file mode 100644 index 0000000..62e9770 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/OperationImportAction.java @@ -0,0 +1,104 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Transect; +import fr.ifremer.echobase.entities.data.Transit; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.DepthStratum; +import fr.ifremer.echobase.entities.references.DepthStratums; +import fr.ifremer.echobase.entities.references.Gear; +import fr.ifremer.echobase.entities.references.Gears; +import fr.ifremer.echobase.entities.references.Vessel; +import fr.ifremer.echobase.entities.references.Vessels; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.OperationImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.OperationImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.OperationImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Date; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class OperationImportAction extends AbstractImportAction<OperationImportConfiguration, OperationImportRow, OperationImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(OperationImportAction.class); + + public OperationImportAction(Locale locale, + char csvSeparator, + UserDbPersistenceService persistenceService, + OperationImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getOperationFile()); + } + + @Override + protected OperationImportModel createCsvImportModel(char csvSeparator, OperationImportConfiguration configuration) { + + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get usable vessels from the voyage by their name + Map<String, Vessel> vesselMap = Maps.uniqueIndex(voyage.getAllVessels(), Vessels.VESSEL_NAME); + + // get usable gears by their casino gear name + Map<String, Gear> gearMap = persistenceService.getEntitiesMap(Gear.class, Gears.GEAR_CASINO_GEAR_NAME); + + Map<String, DepthStratum> depthStratumMap = persistenceService.getEntitiesMap(DepthStratum.class, DepthStratums.DEPTH_STRATUM_ID); + return new OperationImportModel(csvSeparator, vesselMap, gearMap, depthStratumMap); + } + + @Override + protected void performImport(OperationImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of operation from file " + inputFile.getFileName()); + } + + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + try (Import<OperationImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (OperationImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Vessel vessel = row.getVessel(); + + Operation operation = row.getOperation(); + + Date startTime = operation.getGearShootingStartTime(); + Date endTime = operation.getGearShootingEndTime(); + + Transit transit = voyage.getTransit(startTime, endTime); + + Transect transect = transit.getTransect(vessel); + + Operation createdOperation = persistenceService.createOperation(operation); + + // collect ids + result.addId(EchoBaseUserEntityEnum.Operation, createdOperation); + + transect.addOperation(createdOperation); + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/OperationMetadataImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/OperationMetadataImportAction.java new file mode 100644 index 0000000..24e63eb --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/OperationMetadataImportAction.java @@ -0,0 +1,94 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.OperationMetadataValue; +import fr.ifremer.echobase.entities.data.Operations; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.OperationMetadata; +import fr.ifremer.echobase.entities.references.OperationMetadatas; +import fr.ifremer.echobase.entities.references.Vessel; +import fr.ifremer.echobase.entities.references.Vessels; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.OperationImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.OperationMetadataValueImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.OperationMetadataValueImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collection; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class OperationMetadataImportAction extends AbstractImportAction<OperationImportConfiguration, OperationMetadataValueImportRow, OperationMetadataValueImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(OperationMetadataImportAction.class); + + public OperationMetadataImportAction(Locale locale, + char csvSeparator, + UserDbPersistenceService persistenceService, + OperationImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getOperationMetadataFile()); + } + + @Override + protected OperationMetadataValueImportModel createCsvImportModel(char csvSeparator, OperationImportConfiguration configuration) { + + // get voyage + Voyage voyage = persistenceService.getVoyage(getConfiguration().getVoyageId()); + + // get usable vessels from the voyage by their name + Map<String, Vessel> vesselMap = Maps.uniqueIndex(voyage.getAllVessels(), Vessels.VESSEL_NAME); + + // get all operation for this voyage and this vessel + Collection<Operation> operations = voyage.getAllOperations(); + + // split them by operation Id + Map<String, Operation> operationMap = Maps.uniqueIndex(operations, Operations.OPERATION_ID); + + Map<String, OperationMetadata> operationMetadatasMap = persistenceService.getEntitiesMap(OperationMetadata.class, OperationMetadatas.OPERATION_METADATA_NAME); + return new OperationMetadataValueImportModel(csvSeparator, vesselMap, operationMetadatasMap, operationMap); + } + + @Override + protected void performImport(OperationImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of operation metadata values from file " + inputFile.getFileName()); + } + + try (Import<OperationMetadataValueImportRow> importer = open()) { + + int rowNumber = 0; + incrementsProgress(); + for (OperationMetadataValueImportRow row : importer) { + + doFlushTransaction(++rowNumber); + Operation operation = row.getOperation(); + + OperationMetadataValue operationMetadataValueToCreate = row.getOperationMetadataValue(); + OperationMetadataValue operationMetadataValue = persistenceService.createOperationMetadataValue(operationMetadataValueToCreate); + + operation.addOperationMetadataValue(operationMetadataValue); + result.incrementsNumberCreated(EchoBaseUserEntityEnum.OperationMetadataValue); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/RegionAssociationImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/RegionAssociationImportAction.java new file mode 100644 index 0000000..9485b33 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/RegionAssociationImportAction.java @@ -0,0 +1,84 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Cells; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.RegionCellAssociationImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.RegionCellAssociationImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collections; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class RegionAssociationImportAction extends AbstractImportAction<ResultsImportConfiguration, RegionCellAssociationImportRow, RegionCellAssociationImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(RegionAssociationImportAction.class); + + public RegionAssociationImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getRegionAssociationFile()); + } + + @Override + protected RegionCellAssociationImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // authorize only the selected voyage to be imported + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, Cell> regionsMap = Maps.uniqueIndex(voyage.getRegionCells(), Cells.CELL_BY_NAME); + return new RegionCellAssociationImportModel(csvSeparator, + voyageMap, + regionsMap, + persistenceService.newCellValueParser(voyage)); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of Region cells association from file " + inputFile.getFileName()); + } + + try (Import<RegionCellAssociationImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (RegionCellAssociationImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell regionCell = row.getRegionCell(); + Cell esduCell = row.getEsduCell(); + regionCell.addChilds(esduCell); + result.incrementsNumberUpdated(EchoBaseUserEntityEnum.Cell); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/RegionImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/RegionImportAction.java new file mode 100644 index 0000000..87ba05e --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/RegionImportAction.java @@ -0,0 +1,118 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.CellType; +import fr.ifremer.echobase.entities.references.CellTypes; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadataImpl; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.RegionCellImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.RegionCellImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class RegionImportAction extends AbstractImportAction<ResultsImportConfiguration, RegionCellImportRow, RegionCellImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(RegionImportAction.class); + + public RegionImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getRegionsFile()); + } + + @Override + protected RegionCellImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // authorize only the selected voyage to be imported + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Set<CellType> allCellTypes = persistenceService.getRegionCellTypes(); + + // authorize only to use region* cell types + Map<String, CellType> cellTypeMap = Maps.uniqueIndex(allCellTypes, CellTypes.CELL_TYPE_ID); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + return new RegionCellImportModel(csvSeparator, voyageMap, cellTypeMap, dataQualityMap); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + DataMetadata dataCoordinateMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.REGION_ENV_COORDINATES); + + DataMetadata dataSurfaceMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.SURFACE); + + if (log.isInfoEnabled()) { + log.info("Starts import of Region cells from file " + inputFile.getFileName()); + } + + try (Import<RegionCellImportRow> importer = open()) { + + Cell cell = null; + + incrementsProgress(); + int rowNumber = 0; + for (RegionCellImportRow row : importer) { + + doFlushTransaction(++rowNumber); + if (cell == null || !row.getName().equals(cell.getName())) { + + cell = persistenceService.createCell(row.getCellType(), row.getName()); + voyage.addPostCell(cell); + + // collect ids + result.addId(EchoBaseUserEntityEnum.Cell, cell); + + // add surface data + createCellData(cell, + dataSurfaceMeta, + String.valueOf(row.getDataSurface()), + row.getDataQuality(), + result); + } + + // add coordinate data + createCellData(cell, + dataCoordinateMeta, + row.getDataCoordinate(), + row.getDataQuality(), + result); + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction.java new file mode 100644 index 0000000..0430e84 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction.java @@ -0,0 +1,134 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Cells; +import fr.ifremer.echobase.entities.data.DataProcessing; +import fr.ifremer.echobase.entities.data.Echotype; +import fr.ifremer.echobase.entities.data.Echotypes; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.entities.references.SizeCategories; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultByEchotypeAndSpeciesCategoryImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultByEchotypeAndSpeciesCategoryImportRow; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportModel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ext.CsvReaders; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction extends AbstractImportAction<ResultsImportConfiguration, EsduResultByEchotypeAndSpeciesCategoryImportRow, EsduResultByEchotypeAndSpeciesCategoryImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction.class); + + public ResultsEsduByEchotypeAndSpeciesCategoryCellImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getEsduByEchotypeAndSpeciesCategoryFile()); + + } + + @Override + protected EsduResultByEchotypeAndSpeciesCategoryImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // first get header of file to detect which results to import + String[] header = CsvReaders.getHeader(getInputFile().getFile(), csvSeparator); + + Map<String, DataMetadata> dataMetadataMap = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + List<DataMetadata> metas = getMetas(MapCellImportModel.COLUMN_NAMES_TO_EXCLUDE, dataMetadataMap, header); + + // get selected dataProcessing + DataProcessing dataProcessing = persistenceService.getDataProcessing(configuration.getDataProcessingId()); + + // get esdu cells usables + Map<String, Cell> esduCellMap = Maps.uniqueIndex(dataProcessing.getCell(), Cells.CELL_BY_NAME); + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + // index all echotypes of this voyage + Map<String, Echotype> echotypeMap = Maps.uniqueIndex(voyage.getEchotype(), Echotypes.ECHOTYPE_NAME); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + Map<String, SizeCategory> sizeCategoryMap = Maps.newTreeMap(); + sizeCategoryMap.putAll(persistenceService.getEntitiesMap(SizeCategory.class, SizeCategories.SIZE_CATEGORY_NAME)); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + return new EsduResultByEchotypeAndSpeciesCategoryImportModel(csvSeparator, + voyageMap, + echotypeMap, + speciesMap, + sizeCategoryMap, + voyage, + esduCellMap, + metas, + dataQualityMap); + + } + + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of acoustic result by echotype and species category from file " + inputFile.getFileName()); + } + + String resultLabel = getConfiguration().getResultLabel(); + + try (Import<EsduResultByEchotypeAndSpeciesCategoryImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (EsduResultByEchotypeAndSpeciesCategoryImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell cell = row.getCell(); + + Category category = getResultCategory(row.getEchotype(), + row.getSpecies(), + row.getSizeCategory(), + null, + result); + + addResults(row, cell, category, resultLabel, result, true, true); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduByEchotypeCellImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduByEchotypeCellImportAction.java new file mode 100644 index 0000000..9d4e2e2 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduByEchotypeCellImportAction.java @@ -0,0 +1,118 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Cells; +import fr.ifremer.echobase.entities.data.DataProcessing; +import fr.ifremer.echobase.entities.data.Echotype; +import fr.ifremer.echobase.entities.data.Echotypes; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultByEchotypeImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultByEchotypeImportRow; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportModel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ext.CsvReaders; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsEsduByEchotypeCellImportAction extends AbstractImportAction<ResultsImportConfiguration, EsduResultByEchotypeImportRow, EsduResultByEchotypeImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsEsduByEchotypeCellImportAction.class); + + public ResultsEsduByEchotypeCellImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getEsduByEchotypeFile()); + } + + @Override + protected EsduResultByEchotypeImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // first get header of file to detect which results to import + String[] header = CsvReaders.getHeader(getInputFile().getFile(), csvSeparator); + + Map<String, DataMetadata> dataMetadataMap = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + List<DataMetadata> metas = getMetas(MapCellImportModel.COLUMN_NAMES_TO_EXCLUDE, dataMetadataMap, header); + + // get selected dataProcessing + DataProcessing dataProcessing = persistenceService.getDataProcessing(configuration.getDataProcessingId()); + + // get esdu cells usables + Map<String, Cell> esduCellMap = Maps.uniqueIndex(dataProcessing.getCell(), Cells.CELL_BY_NAME); + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + // index all echotypes of this voyage + Map<String, Echotype> echotypeMap = Maps.uniqueIndex(voyage.getEchotype(), Echotypes.ECHOTYPE_NAME); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + return new EsduResultByEchotypeImportModel(csvSeparator, + voyageMap, + echotypeMap, + voyage, + esduCellMap, + metas, + dataQualityMap); + } + + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of esdu result by echotype from file " + inputFile.getFileName()); + } + + String resultLabel = getConfiguration().getResultLabel(); + + try (Import<EsduResultByEchotypeImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + + for (EsduResultByEchotypeImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell cell = row.getCell(); + + Category category = getResultCategory(row.getEchotype(), null, result); + + addResults(row, cell, category, resultLabel, result, true, true); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduSpeciesAndAgeCategoryCellImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduSpeciesAndAgeCategoryCellImportAction.java new file mode 100644 index 0000000..1fee2a7 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduSpeciesAndAgeCategoryCellImportAction.java @@ -0,0 +1,138 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Cells; +import fr.ifremer.echobase.entities.data.DataProcessing; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.AgeCategories; +import fr.ifremer.echobase.entities.references.AgeCategory; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultBySpeciesAndAgeCategoryImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultBySpeciesAndAgeCategoryImportRow; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportModel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ext.CsvReaders; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsEsduSpeciesAndAgeCategoryCellImportAction extends AbstractImportAction<ResultsImportConfiguration, EsduResultBySpeciesAndAgeCategoryImportRow, EsduResultBySpeciesAndAgeCategoryImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsEsduSpeciesAndAgeCategoryCellImportAction.class); + + public ResultsEsduSpeciesAndAgeCategoryCellImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getEsduBySpeciesAndAgeCategoryFile()); + } + + @Override + protected EsduResultBySpeciesAndAgeCategoryImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // first get header of file to detect which results to import + String[] header = CsvReaders.getHeader(getInputFile().getFile(), csvSeparator); + + Map<String, DataMetadata> dataMetadataMap = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + List<DataMetadata> metas = getMetas(MapCellImportModel.COLUMN_NAMES_TO_EXCLUDE, dataMetadataMap, header); + + // get selected dataProcessing + DataProcessing dataProcessing = persistenceService.getDataProcessing(configuration.getDataProcessingId()); + + // get esdu cells usables + Map<String, Cell> esduCellMap = Maps.uniqueIndex(dataProcessing.getCell(), Cells.CELL_BY_NAME); + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + return new EsduResultBySpeciesAndAgeCategoryImportModel(csvSeparator, + voyageMap, + speciesMap, + voyage, + esduCellMap, + metas, + dataQualityMap); + + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of acoustic result by speices and size category from file " + inputFile.getFileName()); + } + + Map<String, AgeCategory> ageCategoryMap = Maps.newTreeMap(); + ageCategoryMap.putAll(persistenceService.getEntitiesMap(AgeCategory.class, AgeCategories.AGE_CATEGORY_NAME)); + + String resultLabel = getConfiguration().getResultLabel(); + + try (Import<EsduResultBySpeciesAndAgeCategoryImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + + for (EsduResultBySpeciesAndAgeCategoryImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell cell = row.getCell(); + + String ageCategoryName = row.getAgeCategory(); + + AgeCategory ageCategory = ageCategoryMap.get(ageCategoryName); + + if (ageCategory == null) { + + // create a new sizeCategory + ageCategory = persistenceService.createAgeCategory(ageCategoryName, row.getAgeCategoryMeaning()); + result.incrementsNumberCreated(EchoBaseUserEntityEnum.AgeCategory); + if (log.isInfoEnabled()) { + log.info("Creates a new AgeCategory " + ageCategoryName); + } + ageCategoryMap.put(ageCategoryName, ageCategory); + } + + Category category = getResultCategory(null, row.getSpecies(), null, ageCategory, result); + + addResults(row, cell, category, resultLabel, result, true, true); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduSpeciesAndSizeCategoryCellImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduSpeciesAndSizeCategoryCellImportAction.java new file mode 100644 index 0000000..2a4f307 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsEsduSpeciesAndSizeCategoryCellImportAction.java @@ -0,0 +1,140 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Cells; +import fr.ifremer.echobase.entities.data.DataProcessing; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.entities.references.SizeCategories; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultBySpeciesAndSizeCategoryImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.EsduResultBySpeciesAndSizeCategoryImportRow; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportModel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ext.CsvReaders; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsEsduSpeciesAndSizeCategoryCellImportAction extends AbstractImportAction<ResultsImportConfiguration, EsduResultBySpeciesAndSizeCategoryImportRow, EsduResultBySpeciesAndSizeCategoryImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsEsduSpeciesAndSizeCategoryCellImportAction.class); + + public ResultsEsduSpeciesAndSizeCategoryCellImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getEsduBySpeciesAndSizeCategoryFile()); + } + + @Override + protected EsduResultBySpeciesAndSizeCategoryImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // first get header of file to detect which results to import + String[] header = CsvReaders.getHeader(getInputFile().getFile(), csvSeparator); + + Map<String, DataMetadata> dataMetadataMap = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + List<DataMetadata> metas = getMetas(MapCellImportModel.COLUMN_NAMES_TO_EXCLUDE, dataMetadataMap, header); + + // get selected dataProcessing + DataProcessing dataProcessing = persistenceService.getDataProcessing(configuration.getDataProcessingId()); + + // get esdu cells usables + Map<String, Cell> esduCellMap = Maps.uniqueIndex(dataProcessing.getCell(), Cells.CELL_BY_NAME); + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + return new EsduResultBySpeciesAndSizeCategoryImportModel(csvSeparator, + voyageMap, + speciesMap, + voyage, + esduCellMap, + metas, + dataQualityMap); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of acoustic result by speices and size category from file " + inputFile.getFileName()); + } + + Map<String, SizeCategory> sizeCategoryMap = Maps.newTreeMap(); + sizeCategoryMap.putAll(persistenceService.getEntitiesMap(SizeCategory.class, SizeCategories.SIZE_CATEGORY_NAME)); + + String resultLabel = getConfiguration().getResultLabel(); + + try (Import<EsduResultBySpeciesAndSizeCategoryImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (EsduResultBySpeciesAndSizeCategoryImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell cell = row.getCell(); + + String sizeCategoryName = row.getSizeCategory(); + + SizeCategory sizeCategory = sizeCategoryMap.get(sizeCategoryName); + + if (sizeCategory == null) { + + // create a new sizeCategory + sizeCategory = persistenceService.createSizeCategory(sizeCategoryName, row.getSizeCategoryMeaning()); + result.incrementsNumberCreated(EchoBaseUserEntityEnum.SizeCategory); + if (log.isInfoEnabled()) { + log.info("Creates a new SizeCategory " + sizeCategoryName); + } + sizeCategoryMap.put(sizeCategoryName, sizeCategory); + } + + Category category = getResultCategory(null, + row.getSpecies(), + sizeCategory, + null, + result); + + addResults(row, cell, category, resultLabel, result, true, true); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsMapFishCellImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsMapFishCellImportAction.java new file mode 100644 index 0000000..0652502 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsMapFishCellImportAction.java @@ -0,0 +1,194 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.AgeCategories; +import fr.ifremer.echobase.entities.references.AgeCategory; +import fr.ifremer.echobase.entities.references.CellType; +import fr.ifremer.echobase.entities.references.CellTypeImpl; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadataImpl; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.entities.references.SizeCategories; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ext.CsvReaders; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsMapFishCellImportAction extends AbstractImportAction<ResultsImportConfiguration, MapCellImportRow, MapCellImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsMapFishCellImportAction.class); + + public ResultsMapFishCellImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getMapsFile()); + } + + @Override + protected MapCellImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, DataMetadata> dataMetadataMap = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + // first get header of file to detect which results to import + String[] header = CsvReaders.getHeader(getInputFile().getFile(), csvSeparator); + + List<DataMetadata> metas = getMetas(MapCellImportModel.COLUMN_NAMES_TO_EXCLUDE, dataMetadataMap, header); + + CellType cellType = persistenceService.getCellTypeById(CellTypeImpl.MAP); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + Map<String, SizeCategory> sizeCategoryMap = Maps.newTreeMap(); + sizeCategoryMap.putAll(persistenceService.getEntitiesMap(SizeCategory.class, SizeCategories.SIZE_CATEGORY_NAME)); + + Map<String, AgeCategory> ageCategoryMap = Maps.newTreeMap(); + ageCategoryMap.putAll(persistenceService.getEntitiesMap(AgeCategory.class, AgeCategories.AGE_CATEGORY_NAME)); + + return new MapCellImportModel(csvSeparator, + cellType, + voyageMap, + speciesMap, + sizeCategoryMap, + ageCategoryMap, + metas, + dataQualityMap); + + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of Map cells from file " + inputFile.getFileName()); + } + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + String resultLabel = getConfiguration().getResultLabel(); + + DataMetadata dataLongitudeMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_CELL_LONGITUDE); + + DataMetadata dataLatitudeMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_CELL_LATITUDE); + + DataMetadata dataDepthMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_CELL_DEPTH); + + DataMetadata dataLongitudeLagMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_LONGITUDE_LAG); + + DataMetadata dataLatitudeLagMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_LATITUDE_LAG); + + DataMetadata dataDepthLagMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_DEPTH_LAG); + + + try (Import<MapCellImportRow> importer = open()) { + + incrementsProgress(); + + int rowNumber = 0; + for (MapCellImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell cell = persistenceService.createCell(row.getCell()); + voyage.addPostCell(cell); + + // collect ids + result.addId(EchoBaseUserEntityEnum.Cell, cell); + + DataQuality dataQuality = row.getDataQuality(); + + // add gridCellLongitude data + createCellData(cell, + dataLongitudeMeta, + String.valueOf(row.getGridCellLongitude()), + dataQuality, + result); + + // add gridCellLatitude data + createCellData(cell, + dataLatitudeMeta, + String.valueOf(row.getGridCellLatitude()), + dataQuality, + result); + + // add gridCellDepth data + createCellData(cell, + dataDepthMeta, + String.valueOf(row.getGridCellDepth()), + dataQuality, + result); + + // add gridLongitudeLag data + createCellData(cell, + dataLongitudeLagMeta, + String.valueOf(row.getGridLongitudeLag()), + dataQuality, + result); + + // add gridLatitudeLag data + createCellData(cell, + dataLatitudeLagMeta, + String.valueOf(row.getGridLatitudeLag()), + dataQuality, + result); + + // add gridDepthLag data + createCellData(cell, + dataDepthLagMeta, + String.valueOf(row.getGridDepthLag()), + dataQuality, + result); + + Category category = getResultCategory(null, + row.getSpecies(), + row.getSizeCategory(), + row.getAgeCategory(), + result); + + addResults(row, cell, category, resultLabel, result, false, true); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsMapOtherCellImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsMapOtherCellImportAction.java new file mode 100644 index 0000000..9409251 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsMapOtherCellImportAction.java @@ -0,0 +1,168 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.CellType; +import fr.ifremer.echobase.entities.references.CellTypeImpl; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadataImpl; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.MapCellImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ext.CsvReaders; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsMapOtherCellImportAction extends AbstractImportAction<ResultsImportConfiguration, MapCellImportRow, MapCellImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsMapOtherCellImportAction.class); + + public ResultsMapOtherCellImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getMapsFile()); + } + + @Override + protected MapCellImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, DataMetadata> dataMetadataMap = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + // first get header of file to detect which results to import + String[] header = CsvReaders.getHeader(getInputFile().getFile(), csvSeparator); + + List<DataMetadata> metas = getMetas(MapCellImportModel.COLUMN_NAMES_TO_EXCLUDE, dataMetadataMap, header); + + CellType cellType = persistenceService.getCellTypeById(CellTypeImpl.MAP); + + return new MapCellImportModel(csvSeparator, + cellType, + voyageMap, + metas, + dataQualityMap); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of Map Other cells from file " + inputFile.getFileName()); + } + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + String resultLabel = getConfiguration().getResultLabel(); + + DataMetadata dataLongitudeMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_CELL_LONGITUDE); + + DataMetadata dataLatitudeMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_CELL_LATITUDE); + + DataMetadata dataDepthMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_CELL_DEPTH); + + DataMetadata dataLongitudeLagMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_LONGITUDE_LAG); + + DataMetadata dataLatitudeLagMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_LATITUDE_LAG); + + DataMetadata dataDepthLagMeta = persistenceService.getDataMetadataByName(DataMetadataImpl.GRID_DEPTH_LAG); + + try (Import<MapCellImportRow> importer = open()) { + + incrementsProgress(); + + int rowNumber = 0; + for (MapCellImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell cell = persistenceService.createCell(row.getCell()); + voyage.addPostCell(cell); + + // collect ids + result.addId(EchoBaseUserEntityEnum.Cell, cell); + + DataQuality dataQuality = row.getDataQuality(); + + // add gridCellLongitude data + createCellData(cell, + dataLongitudeMeta, + String.valueOf(row.getGridCellLongitude()), + dataQuality, + result); + + // add gridCellLatitude data + createCellData(cell, + dataLatitudeMeta, + String.valueOf(row.getGridCellLatitude()), + dataQuality, + result); + + // add gridCellDepth data + createCellData(cell, + dataDepthMeta, + String.valueOf(row.getGridCellDepth()), + dataQuality, + result); + + // add gridLongitudeLag data + createCellData(cell, + dataLongitudeLagMeta, + String.valueOf(row.getGridLongitudeLag()), + dataQuality, + result); + + // add gridLatitudeLag data + createCellData(cell, + dataLatitudeLagMeta, + String.valueOf(row.getGridLatitudeLag()), + dataQuality, + result); + + // add gridDepthLag data + createCellData(cell, + dataDepthLagMeta, + String.valueOf(row.getGridDepthLag()), + dataQuality, + result); + + addResults(row, cell, null, resultLabel, result, false, true); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsRegionImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsRegionImportAction.java new file mode 100644 index 0000000..276faf8 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsRegionImportAction.java @@ -0,0 +1,126 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Cells; +import fr.ifremer.echobase.entities.data.Echotype; +import fr.ifremer.echobase.entities.data.Echotypes; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.DataMetadata; +import fr.ifremer.echobase.entities.references.DataMetadatas; +import fr.ifremer.echobase.entities.references.DataQualities; +import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.entities.references.SizeCategories; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.RegionCellResultImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.RegionCellResultImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; +import org.nuiton.csv.ext.CsvReaders; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsRegionImportAction extends AbstractImportAction<ResultsImportConfiguration, RegionCellResultImportRow, RegionCellResultImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsRegionImportAction.class); + + public ResultsRegionImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getRegionResultFile()); + } + + @Override + protected RegionCellResultImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + + // first get header of file to detect which results to import + String[] header = CsvReaders.getHeader(getInputFile().getFile(), csvSeparator); + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // authorize only the selected voyage to be imported + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, Cell> regionsMap = Maps.uniqueIndex(voyage.getRegionCells(), Cells.CELL_BY_NAME); + + Map<String, DataQuality> dataQualityMap = persistenceService.getEntitiesMap(DataQuality.class, DataQualities.DATA_QUALITY_NAME); + + // index all echotypes of thi voyage + Map<String, Echotype> echotypeMap = Maps.uniqueIndex(voyage.getEchotype(), Echotypes.ECHOTYPE_NAME); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + Map<String, SizeCategory> sizeCategoryMap = Maps.newTreeMap(); + sizeCategoryMap.putAll(persistenceService.getEntitiesMap(SizeCategory.class, SizeCategories.SIZE_CATEGORY_NAME)); + + Map<String, DataMetadata> dataMetadataMap = persistenceService.getEntitiesMap(DataMetadata.class, DataMetadatas.DATA_METADATA_NAME); + + List<DataMetadata> metas = getMetas(RegionCellResultImportModel.COLUMN_NAMES_TO_EXCLUDE, dataMetadataMap, header); + + return new RegionCellResultImportModel(csvSeparator, + voyageMap, + regionsMap, + echotypeMap, + speciesMap, + sizeCategoryMap, + metas, + dataQualityMap); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of Region cell results from file " + inputFile.getFileName()); + } + + String resultLabel = getConfiguration().getResultLabel(); + + try (Import<RegionCellResultImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (RegionCellResultImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Cell cell = row.getCell(); + + Category category = getResultCategory(row.getEchotype(), + row.getSpecies(), + row.getSizeCategory(), + null, + result); + + addResults(row, cell, category, resultLabel, result, false, false); + + } + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageEchotypeImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageEchotypeImportAction.java new file mode 100644 index 0000000..c4b1b01 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageEchotypeImportAction.java @@ -0,0 +1,128 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.Echotype; +import fr.ifremer.echobase.entities.data.Echotypes; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.DepthStratum; +import fr.ifremer.echobase.entities.references.DepthStratums; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.EchotypeImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.EchotypeImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collection; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsVoyageEchotypeImportAction extends AbstractImportAction<ResultsImportConfiguration, EchotypeImportRow, EchotypeImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsVoyageEchotypeImportAction.class); + + public ResultsVoyageEchotypeImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getEchotypeFile()); + } + + @Override + protected EchotypeImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + // authorize only the selected voyage to be imported + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, DepthStratum> depthStratumMap = persistenceService.getEntitiesMap(DepthStratum.class, DepthStratums.DEPTH_STRATUM_ID); + return new EchotypeImportModel(csvSeparator, voyageMap, depthStratumMap, speciesMap); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of echotype from file " + inputFile.getFileName()); + } + + try (Import<EchotypeImportRow> importer = open()) { + + incrementsProgress(); + + int rowNumber = 0; + for (EchotypeImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Voyage voyage = row.getVoyage(); + + Echotype rowEchotype = row.getEchotype(); + + String echotypeName = rowEchotype.getName(); + + // check if there is a echotype for the voyage and this name + Echotype echotype = null; + + if (!voyage.isEchotypeEmpty()) { + + Collection<Echotype> echotypes = voyage.getEchotype(); + Predicate<Echotype> predicate = Echotypes.newEchotypeByNamePredicate(echotypeName); + echotype = Iterables.find(echotypes, predicate, null); + + } + + if (echotype == null) { + + // creates it + echotype = persistenceService.createEchotype(rowEchotype); + + // attach it to voyage + voyage.addEchotype(echotype); + + // collect ids + result.addId(EchoBaseUserEntityEnum.Echotype, echotype); + + } else { + + Species species = rowEchotype.getSpecies().iterator().next(); + Species existingSpecie = echotype.getSpeciesByTopiaId(species.getTopiaId()); + + if (existingSpecie == null) { + + // add this species + echotype.addSpecies(species); + + } + + } + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageLengthAgeKeyImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageLengthAgeKeyImportAction.java new file mode 100644 index 0000000..853b973 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageLengthAgeKeyImportAction.java @@ -0,0 +1,95 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.LengthAgeKey; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.entities.references.Strata; +import fr.ifremer.echobase.entities.references.Stratas; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.LengthAgeKeyImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.LengthAgeKeyImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collections; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsVoyageLengthAgeKeyImportAction extends AbstractImportAction<ResultsImportConfiguration, LengthAgeKeyImportRow, LengthAgeKeyImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsVoyageLengthAgeKeyImportAction.class); + + public ResultsVoyageLengthAgeKeyImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getLengthAgeKeyFile()); + } + + @Override + protected LengthAgeKeyImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + Map<String, Strata> strataMap = persistenceService.getEntitiesMap(Strata.class, Stratas.STRATA_BY_NAME); + + // authorize only the selected voyage to be imported + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + return new LengthAgeKeyImportModel(csvSeparator, voyageMap, strataMap, speciesMap); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of lenthAgeKey from file " + inputFile.getFileName()); + } + + try (Import<LengthAgeKeyImportRow> importer = open()) { + + incrementsProgress(); + + int rowNumber = 0; + for (LengthAgeKeyImportRow row : importer) { + + doFlushTransaction(++rowNumber); + + Voyage voyage = row.getVoyage(); + + LengthAgeKey lengthAgeKeyToCreate = row.getLengthAgeKey(); + LengthAgeKey lengthAgeKey = persistenceService.createLengthAgeKey(lengthAgeKeyToCreate); + + // attach it to voyage + voyage.addLengthAgeKey(lengthAgeKey); + + // collect ids + result.addId(EchoBaseUserEntityEnum.LengthAgeKey, lengthAgeKey); + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageLengthWeightKeyImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageLengthWeightKeyImportAction.java new file mode 100644 index 0000000..456efa8 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ResultsVoyageLengthWeightKeyImportAction.java @@ -0,0 +1,109 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.data.LengthWeightKey; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.data.Voyages; +import fr.ifremer.echobase.entities.references.SizeCategories; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.entities.references.Strata; +import fr.ifremer.echobase.entities.references.Stratas; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.csv.LengthWeightKeyImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.LengthWeightKeyImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collections; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class ResultsVoyageLengthWeightKeyImportAction extends AbstractImportAction<ResultsImportConfiguration, LengthWeightKeyImportRow, LengthWeightKeyImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ResultsVoyageLengthWeightKeyImportAction.class); + + public ResultsVoyageLengthWeightKeyImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + ResultsImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getLengthWeightKeyFile()); + } + + @Override + protected LengthWeightKeyImportModel createCsvImportModel(char csvSeparator, ResultsImportConfiguration configuration) { + + // get selected voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + Map<String, Strata> strataMap = persistenceService.getEntitiesMap(Strata.class, Stratas.STRATA_BY_NAME); + + // authorize only the selected voyage to be imported + Map<String, Voyage> voyageMap = Maps.uniqueIndex(Collections.singletonList(voyage), Voyages.VOYAGE_NAME); + + Map<String, SizeCategory> sizeCategoryMap = persistenceService.getEntitiesMap(SizeCategory.class, SizeCategories.SIZE_CATEGORY_NAME); + + return new LengthWeightKeyImportModel(csvSeparator, + voyageMap, + strataMap, + sizeCategoryMap, + speciesMap); + } + + @Override + protected void performImport(ResultsImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of lenthWeightKey from file " + inputFile.getFileName()); + } + + try (Import<LengthWeightKeyImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (LengthWeightKeyImportRow row : importer) { + + doFlushTransaction(++rowNumber); + Voyage voyage = row.getVoyage(); + + // find speciesCategory + SpeciesCategory speciesCategory = getSpeciesCategory(row.getSpecies(), row.getSizeCategory(), null, null, result); + + LengthWeightKey toCreate = row.getLengthWeightKey(); + toCreate.setSpeciesCategory(speciesCategory); + + LengthWeightKey lengthWeightKey = persistenceService.createLengthWeightKey(toCreate); + + //TODO should we import it ? + //lengthWeightKey.setMetadata(row.getMetadata()); + + // attach it to voyage + voyage.addLengthWeightKey(lengthWeightKey); + + // collect ids + result.addId(EchoBaseUserEntityEnum.LengthWeightKey, lengthWeightKey); + + } + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/SubSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/SubSampleImportAction.java new file mode 100644 index 0000000..bca73d2 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/SubSampleImportAction.java @@ -0,0 +1,151 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Collections2; +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Operations; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.SampleDataType; +import fr.ifremer.echobase.entities.references.SampleDataTypeImpl; +import fr.ifremer.echobase.entities.references.SampleType; +import fr.ifremer.echobase.entities.references.SampleTypeImpl; +import fr.ifremer.echobase.entities.references.SexCategories; +import fr.ifremer.echobase.entities.references.SexCategory; +import fr.ifremer.echobase.entities.references.SizeCategories; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CatchesImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.csv.SubSampleImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.SubSampleImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collection; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class SubSampleImportAction extends AbstractImportAction<CatchesImportConfiguration, SubSampleImportRow, SubSampleImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(SubSampleImportAction.class); + + public SubSampleImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + CatchesImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getSubSampleFile()); + } + + @Override + protected SubSampleImportModel createCsvImportModel(char csvSeparator, CatchesImportConfiguration configuration) { + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get all operation for this voyage and this vessel + Collection<Operation> operations = voyage.getAllOperations(); + + Map<String, Operation> operationMap = Maps.uniqueIndex(operations, Operations.OPERATION_ID); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + Map<String, SizeCategory> sizeCategoryMap = persistenceService.getEntitiesMap(SizeCategory.class, SizeCategories.SIZE_CATEGORY_NAME); + + Map<String, SexCategory> sexCategoryMap = persistenceService.getEntitiesMap(SexCategory.class, SexCategories.SEX_CATEGORY_NAME); + return new SubSampleImportModel(csvSeparator, operationMap, speciesMap, sizeCategoryMap, sexCategoryMap); + } + + @Override + protected void performImport(CatchesImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of subSample from file " + inputFile.getFileName()); + } + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get all operation for this voyage and this vessel + Collection<Operation> operations = voyage.getAllOperations(); + + Collection<Operation> operationWithTotalOrUnsortedSample = Collections2.filter(operations, Operations.OPERATION_WITH_TOTAL_OR_UNSORTED_SAMPLE); + + SampleType sampleTypeSubsample = persistenceService.getSampleTypeByName(SampleTypeImpl.SUB_SAMPLE_TYPE); + SampleDataType sampleDataTypeNumberAtLength = persistenceService.getSampleDataTypeByName(SampleDataTypeImpl.NUMBER_AT_LENGTH); + SampleDataType sampleDataTypeNumberAtLength05cm = persistenceService.getSampleDataTypeByName(SampleDataTypeImpl.LTCM0_5); + SampleDataType sampleDataTypeNumberAtLength1cm = persistenceService.getSampleDataTypeByName(SampleDataTypeImpl.L_TCM_1); + SampleDataType sampleDataTypeWeightAtLength = persistenceService.getSampleDataTypeByName(SampleDataTypeImpl.WEIGHT_AT_LENGTHKG); + + try (Import<SubSampleImportRow> importer = open()) { + + incrementsProgress(); + int rowNumber = 0; + for (SubSampleImportRow row : importer) { + + doFlushTransaction(++rowNumber); + Operation operation = row.getOperation(); + + checkOperationWithTotalOrUnsortedSample(rowNumber, operationWithTotalOrUnsortedSample, operation); + + Species species = row.getSpecies(); + SizeCategory sizeCategory = row.getSizeCategory(); + SexCategory sexCategory = row.getSexCategory(); + + SpeciesCategory category = getSpeciesCategory(species, sizeCategory, null, sexCategory, result); + + // find the sample with this category + Sample sample = operation.getSample(category, sampleTypeSubsample); + + if (sample == null) { + + // must create it + sample = row.getSample(); + + sample.setSpeciesCategory(category); + sample.setSampleType(sampleTypeSubsample); + + sample = addSample(operation, sample, result); + } + + Integer round = row.getRound(); + SampleDataType dataType = sampleDataTypeNumberAtLength; + + if (round == null) { + dataType = sampleDataTypeNumberAtLength; + } else if (round == 5) { + dataType = sampleDataTypeNumberAtLength05cm; + } else if (round == 1) { + dataType = sampleDataTypeNumberAtLength1cm; + } + + //create numberAtLength data + addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result); + + if (row.getWeightAtLength() != null) { + + //create weightAtLength data + addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result); + } + + } + + } + + } + +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/TotalSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/TotalSampleImportAction.java new file mode 100644 index 0000000..f38d7c3 --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/TotalSampleImportAction.java @@ -0,0 +1,162 @@ +package fr.ifremer.echobase.services.service.importdata.actions; + +import com.google.common.collect.Maps; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Operations; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.SampleDataType; +import fr.ifremer.echobase.entities.references.SampleDataTypeImpl; +import fr.ifremer.echobase.entities.references.SampleType; +import fr.ifremer.echobase.entities.references.SampleTypeImpl; +import fr.ifremer.echobase.entities.references.SizeCategories; +import fr.ifremer.echobase.entities.references.SizeCategory; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.Species2; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.io.InputFile; +import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CatchesImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.DuplicatedSampleException; +import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; +import fr.ifremer.echobase.services.service.importdata.csv.TotalSampleImportModel; +import fr.ifremer.echobase.services.service.importdata.csv.TotalSampleImportRow; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.Import; + +import java.util.Collection; +import java.util.Locale; +import java.util.Map; + +/** + * Created on 25/03/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 4.0 + */ +public class TotalSampleImportAction extends AbstractImportAction<CatchesImportConfiguration, TotalSampleImportRow, TotalSampleImportModel> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(TotalSampleImportAction.class); + + public TotalSampleImportAction(UserDbPersistenceService persistenceService, + Locale locale, + char csvSeparator, + CatchesImportConfiguration configuration, + EchoBaseUser user) { + super(persistenceService, locale, csvSeparator, configuration, user, configuration.getTotalSampleFile()); + } + + @Override + protected TotalSampleImportModel createCsvImportModel(char csvSeparator, CatchesImportConfiguration configuration) { + + // get voyage + Voyage voyage = persistenceService.getVoyage(configuration.getVoyageId()); + + // get all operation for this voyage and this vessel + Collection<Operation> operations = voyage.getAllOperations(); + + // split them by operation Id + Map<String, Operation> operationMap = Maps.uniqueIndex(operations, Operations.OPERATION_ID); + + Map<String, Species> speciesMap = persistenceService.getEntitiesMap(Species.class, Species2.SPECIES_BARACOUDA_CODE); + + Map<String, SizeCategory> sizeCategoryMap = persistenceService.getEntitiesMap(SizeCategory.class, SizeCategories.SIZE_CATEGORY_NAME); + + return new TotalSampleImportModel(csvSeparator, operationMap, speciesMap, sizeCategoryMap); + } + + + @Override + protected void performImport(CatchesImportConfiguration configuration, InputFile inputFile, ImportDataFileResult result) { + + if (log.isInfoEnabled()) { + log.info("Starts import of totalSample from file " + inputFile.getFileName()); + } + + + SampleType sampleTypeTotal = persistenceService.getSampleTypeByName(SampleTypeImpl.TOTAL_SAMPLE_TYPE); + SampleType sampleTypeUnsorted = persistenceService.getSampleTypeByName(SampleTypeImpl.UNSORTED_SAMPLE_TYPE); + SampleType sampleTypeSorted = persistenceService.getSampleTypeByName(SampleTypeImpl.SORTED_SAMPLE_TYPE); + + SampleDataType sampleDataTypeMeanLength = persistenceService.getSampleDataTypeByName(SampleDataTypeImpl.MEAN_LENGTHCM); + SampleDataType sampleDataTypeMeanWeight = persistenceService.getSampleDataTypeByName(SampleDataTypeImpl.MEAN_WEIGHTG); + SampleDataType sampleDataTypeNoPerKg = persistenceService.getSampleDataTypeByName(SampleDataTypeImpl.NO_PER_KG); + + try (Import<TotalSampleImportRow> importer = open()) { + + int rowNumber = 0; + incrementsProgress(); + for (TotalSampleImportRow row : importer) { + + doFlushTransaction(++rowNumber); + Operation operation = row.getOperation(); + + Species species = row.getSpecies(); + SizeCategory sizeCategory = row.getSizeCategory(); + + SpeciesCategory category = getSpeciesCategory(species, sizeCategory, null, null, result); + + SampleType sampleType; + + if (SizeCategories.IS_HORS_VRAC.apply(sizeCategory)) { + + // hors vrac case + sampleType = sampleTypeUnsorted; + + } else { + // none hors vrac case + sampleType = sampleTypeTotal; + } + + Sample sample = operation.getSample(category, sampleType); + + if (sample != null) { + + // can not have twice same sample + throw new DuplicatedSampleException(getLocale(), operation, sampleType, species, sizeCategory); + + } + + + // must create it + + sample = row.getSample(); + sample.setSpeciesCategory(category); + sample.setSampleType(sampleType); + sample = addSample(operation, sample, result); + + // create datas + + if (row.getMeanLength() != null) { + + //create meanLength data + addSampleData(sampleDataTypeMeanLength, null, row.getMeanLength(), sample, result); + } + + if (row.getMeanWeight() != null) { + + //create meanWeight data + addSampleData(sampleDataTypeMeanWeight, null, row.getMeanWeight(), sample, result); + } + + if (row.getNoPerKg() != null) { + + //create noPerKg data + addSampleData(sampleDataTypeNoPerKg, null, row.getNoPerKg(), sample, result); + } + + Sample createdSortedSample = persistenceService.newSample(); + createdSortedSample.setSampleType(sampleTypeSorted); + createdSortedSample.setSampleWeight(row.getSortedWeight()); + + // create sorted sample + createdSortedSample = addSample(operation, createdSortedSample, result); + } + } + + } + +} diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AbstractImportDataServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AbstractImportDataServiceIT.java index 0fe2af2..e0f6d49 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AbstractImportDataServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AbstractImportDataServiceIT.java @@ -20,8 +20,10 @@ */ package fr.ifremer.echobase.services.service.importdata; +import com.google.common.base.Charsets; import com.google.common.collect.Collections2; import com.google.common.collect.Sets; +import fr.ifremer.echobase.EchoBaseTechnicalException; import fr.ifremer.echobase.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.entities.EchoBaseUser; import fr.ifremer.echobase.entities.data.Cell; @@ -55,8 +57,12 @@ import org.nuiton.topia.persistence.TopiaException; import org.nuiton.topia.service.csv.in.AbstractImportModel; import org.nuiton.util.TimeLog; +import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStreamReader; import java.io.Reader; import java.util.List; import java.util.Set; @@ -71,8 +77,7 @@ import java.util.regex.Matcher; public abstract class AbstractImportDataServiceIT extends EchoBaseTestServiceSupport { /** Logger. */ - private static final Log log = - LogFactory.getLog(AbstractImportDataServiceIT.class); + private static final Log log = LogFactory.getLog(AbstractImportDataServiceIT.class); public static final TimeLog TIME_LOG = new TimeLog(AbstractImportDataServiceIT.class); @@ -106,18 +111,15 @@ public abstract class AbstractImportDataServiceIT extends EchoBaseTestServiceSup assertNbEntities(Echotype.class, fixtures.NB_ECHOTYPE()); } - protected <M extends AbstractImportConfiguration, S extends AbstractImportDataService<M>> - List<EchoBaseCsvFileImportResult> doImport(M conf, - Class<S> serviceType, - int nbResults) throws ImportException { + protected <M extends AbstractImportConfiguration> List<EchoBaseCsvFileImportResult> doImport(M conf, ImportDataService.ImportDataAction<M> importDataAction, int nbResults) throws ImportException { - S service = newService(serviceType); + ImportDataService service = newService(ImportDataService.class); EchoBaseUser fakeUser = createFakeUser(); long s0 = TimeLog.getTime(); - String resume = service.doImport(conf, fakeUser); + String resume = importDataAction.doImport(service, conf, fakeUser); TIME_LOG.log(s0, "doImport"); @@ -222,7 +224,7 @@ public abstract class AbstractImportDataServiceIT extends EchoBaseTestServiceSup CellType esduCellType = cellTypeDAO.forIdEquals("Esdu").findUnique(); // Preconditions.checkNotNull(esduCellType); - try (Reader reader = service.getInputFileReader(inputFile)) { + try (Reader reader = getInputFileReader(inputFile)) { try (Import<ResultEdsuImportMockImportModelRow> importer = Import.newImport(csvModel, reader)) { @@ -250,6 +252,14 @@ public abstract class AbstractImportDataServiceIT extends EchoBaseTestServiceSup } } + protected Reader getInputFileReader(InputFile inputFile) throws EchoBaseTechnicalException { + try { + return new BufferedReader(new InputStreamReader(new FileInputStream(inputFile.getFile()), Charsets.UTF_8)); + } catch (FileNotFoundException e) { + throw new EchoBaseTechnicalException("Could not find import file " + inputFile.getFile(), e); + } + } + public static class ResultEdsuImportMockImportModel extends AbstractImportModel<ResultEdsuImportMockImportModelRow> { public ResultEdsuImportMockImportModel(char separator, diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AcousticImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AcousticImportServiceIT.java index 5c21fa5..c79f0bc 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AcousticImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/AcousticImportServiceIT.java @@ -80,7 +80,7 @@ public class AcousticImportServiceIT extends AbstractImportDataServiceIT { prepareInputFile(conf.getMoviesFile(), getImportPath("movies.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, AcousticImportService.class, 1); + result = doImport(conf, new ImportDataService.AcousticImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_CELL()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CatchesImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CatchesImportServiceIT.java index a2b39c7..102993b 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CatchesImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CatchesImportServiceIT.java @@ -65,7 +65,7 @@ public class CatchesImportServiceIT extends AbstractImportDataServiceIT { getImportPath("biometrysample.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CatchesImportService.class, 3); + result = doImport(conf, new ImportDataService.CatchesImportDataAction(), 3); assertNbIDs(result, 0, fixtures.NB_SAMPLE_TOTAL()); @@ -102,7 +102,7 @@ public class CatchesImportServiceIT extends AbstractImportDataServiceIT { getImportPath("totalsample.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CatchesImportService.class, 1); + result = doImport(conf, new ImportDataService.CatchesImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_SAMPLE_TOTAL()); assertCsvImportResult(result, 0, Sample.class, fixtures.NB_SAMPLE_TOTAL()); @@ -126,7 +126,7 @@ public class CatchesImportServiceIT extends AbstractImportDataServiceIT { getImportPath("subsample.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CatchesImportService.class, 1); + result = doImport(conf, new ImportDataService.CatchesImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_SAMPLE_UNSORTED()); assertCsvImportResult(result, 0, Sample.class, fixtures.NB_SAMPLE_UNSORTED()); @@ -150,7 +150,7 @@ public class CatchesImportServiceIT extends AbstractImportDataServiceIT { getImportPath("biometrysample.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CatchesImportService.class, 1); + result = doImport(conf, new ImportDataService.CatchesImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_SAMPLE_BIOMETRY()); assertCsvImportResult(result, 0, Sample.class, fixtures.NB_SAMPLE_BIOMETRY()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonAllImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonAllImportServiceIT.java index e2c1af3..fca3f96 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonAllImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonAllImportServiceIT.java @@ -30,7 +30,7 @@ import org.junit.Test; import java.util.List; /** - * Tests {@link CommonImportService} with {@link ImportType#COMMON_ALL} mode. + * Tests common imports with {@link ImportType#COMMON_ALL} mode. * * @author Tony Chemit - chemit@codelutin.com * @since 0.3 @@ -75,7 +75,7 @@ public class CommonAllImportServiceIT extends AbstractImportDataServiceIT { conf.setImportType(ImportType.COMMON_ALL); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CommonAllImportService.class, 3); + result = doImport(conf, new ImportDataService.CommonImportDataAction(), 3); assertNbIDs(result, 0, fixtures.NB_VOYAGE()); assertCsvImportResult(result, 0, Voyage.class, fixtures.NB_VOYAGE()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransectImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransectImportServiceIT.java index eb47256..bbb88b9 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransectImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransectImportServiceIT.java @@ -29,7 +29,7 @@ import org.junit.Test; import java.util.List; /** - * Tests {@link CommonImportService} with {@link ImportType#COMMON_TRANSECT} mode. + * Tests common imports with {@link ImportType#COMMON_TRANSECT} mode. * * @author Tony Chemit - chemit@codelutin.com * @since 0.3 @@ -71,7 +71,7 @@ public class CommonTransectImportServiceIT extends AbstractImportDataServiceIT { } List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CommonTransectImportService.class, 1); + result = doImport(conf, new ImportDataService.CommonImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_TRANSECT()); assertCsvImportResult(result, 0, Transect.class, fixtures.NB_TRANSECT()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransitImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransitImportServiceIT.java index f4d8da7..9c0aa4f 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransitImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonTransitImportServiceIT.java @@ -31,7 +31,7 @@ import org.junit.Test; import java.util.List; /** - * Tests {@link CommonImportService} with {@link ImportType#COMMON_TRANSIT} mode. + * Tests common imports with {@link ImportType#COMMON_TRANSIT} mode. * * @author Tony Chemit - chemit@codelutin.com * @since 2.2 @@ -73,7 +73,7 @@ public class CommonTransitImportServiceIT extends AbstractImportDataServiceIT { conf.setImportType(ImportType.COMMON_TRANSIT); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CommonTransitImportService.class, 1); + result = doImport(conf, new ImportDataService.CommonImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_TRANSIT()); assertCsvImportResult(result, 0, Transit.class, fixtures.NB_TRANSIT()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonVoyageImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonVoyageImportServiceIT.java index bf19b06..4adc0cc 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonVoyageImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/CommonVoyageImportServiceIT.java @@ -31,7 +31,7 @@ import org.junit.Test; import java.util.List; /** - * Tests {@link CommonImportService} with {@link ImportType#COMMON_VOYAGE} mode. + * Tests common imports with {@link ImportType#COMMON_VOYAGE} mode. * * @author Tony Chemit - chemit@codelutin.com * @since 2.2 @@ -70,7 +70,7 @@ public class CommonVoyageImportServiceIT extends AbstractImportDataServiceIT { conf.setImportType(ImportType.COMMON_VOYAGE); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, CommonVoyageImportService.class, 1); + result = doImport(conf, new ImportDataService.CommonImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_VOYAGE()); assertCsvImportResult(result, 0, Voyage.class, fixtures.NB_VOYAGE()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/OperationImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/OperationImportServiceIT.java index a4109a1..520126b 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/OperationImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/OperationImportServiceIT.java @@ -67,7 +67,7 @@ public class OperationImportServiceIT extends AbstractImportDataServiceIT { getImportPath("gearmetadatavalue.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, OperationImportService.class, 3); + result = doImport(conf, new ImportDataService.OperationsImportDataAction(), 3); assertNbIDs(result, 0, fixtures.NB_OPERATION()); assertCsvImportResult(result, 0, Operation.class, fixtures.NB_OPERATION()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsEsduCellImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsEsduCellImportServiceIT.java index b50e712..bc36133 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsEsduCellImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsEsduCellImportServiceIT.java @@ -75,7 +75,7 @@ public class ResultsEsduCellImportServiceIT extends AbstractImportDataServiceIT conf.setDataProcessingId(getDataProcessingId()); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsEsduCellImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, 43533); @@ -109,7 +109,7 @@ public class ResultsEsduCellImportServiceIT extends AbstractImportDataServiceIT conf.setDataProcessingId(getDataProcessingId()); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsEsduCellImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, 357291); @@ -144,7 +144,7 @@ public class ResultsEsduCellImportServiceIT extends AbstractImportDataServiceIT conf.setDataProcessingId(getDataProcessingId()); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsEsduCellImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, 68108); assertCsvImportResult(result, 0, Result.class, 68108); @@ -179,7 +179,7 @@ public class ResultsEsduCellImportServiceIT extends AbstractImportDataServiceIT conf.setDataProcessingId(getDataProcessingId()); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsEsduCellImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, 10021); assertCsvImportResult(result, 0, Category.class, 14); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapFishCellImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapFishCellImportServiceIT.java index 7b817e4..965f984 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapFishCellImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapFishCellImportServiceIT.java @@ -67,7 +67,7 @@ public class ResultsMapFishCellImportServiceIT extends AbstractImportDataService prepareInputFile(conf.getMapsFile(), getImportPath("mapsFish.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsMapFishCellImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, 380); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapOtherCellImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapOtherCellImportServiceIT.java index cc6e1a6..46b52c2 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapOtherCellImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsMapOtherCellImportServiceIT.java @@ -67,7 +67,7 @@ public class ResultsMapOtherCellImportServiceIT extends AbstractImportDataServic prepareInputFile(conf.getMapsFile(), getImportPath("mapsOther.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsMapOtherCellImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, 380); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsRegionCellImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsRegionCellImportServiceIT.java index 68f4ce7..216e921 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsRegionCellImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsRegionCellImportServiceIT.java @@ -77,7 +77,7 @@ public class ResultsRegionCellImportServiceIT extends AbstractImportDataServiceI conf.getRegionAssociationFile()); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsRegionCellImportService.class, 3); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 3); assertNbIDs(result, 0, 10); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsVoyageImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsVoyageImportServiceIT.java index e0158e0..1d0758b 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsVoyageImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/ResultsVoyageImportServiceIT.java @@ -70,7 +70,7 @@ public class ResultsVoyageImportServiceIT extends AbstractImportDataServiceIT { prepareInputFile(conf.getEchotypeFile(), getImportPath("echotype.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsVoyageImportService.class, 3); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 3); assertNbIDs(result, 0, fixtures.NB_LENGTH_AGE_KEY()); assertCsvImportResult(result, 0, LengthAgeKey.class, fixtures.NB_LENGTH_AGE_KEY()); @@ -102,7 +102,7 @@ public class ResultsVoyageImportServiceIT extends AbstractImportDataServiceIT { getImportPath("lengthAgeKey.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsVoyageImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_LENGTH_AGE_KEY()); assertCsvImportResult(result, 0, LengthAgeKey.class, fixtures.NB_LENGTH_AGE_KEY()); @@ -128,7 +128,7 @@ public class ResultsVoyageImportServiceIT extends AbstractImportDataServiceIT { getImportPath("lengthWeightKey.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsVoyageImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_LENGTH_WEIGHT_KEY()); assertCsvImportResult(result, 0, LengthWeightKey.class, fixtures.NB_LENGTH_WEIGHT_KEY()); @@ -153,7 +153,7 @@ public class ResultsVoyageImportServiceIT extends AbstractImportDataServiceIT { prepareInputFile(conf.getEchotypeFile(), getImportPath("echotype.csv.gz")); List<EchoBaseCsvFileImportResult> result; - result = doImport(conf, ResultsVoyageImportService.class, 1); + result = doImport(conf, new ImportDataService.ResultsImportDataAction(), 1); assertNbIDs(result, 0, fixtures.NB_ECHOTYPE()); assertCsvImportResult(result, 0, Echotype.class, fixtures.NB_ECHOTYPE()); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/ui/actions/removedata/AbstractRemoveDataServiceTest.java b/echobase-services/src/test/java/fr/ifremer/echobase/ui/actions/removedata/AbstractRemoveDataServiceTest.java index bf26bbc..f35112c 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/ui/actions/removedata/AbstractRemoveDataServiceTest.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/ui/actions/removedata/AbstractRemoveDataServiceTest.java @@ -76,21 +76,12 @@ import fr.ifremer.echobase.entities.references.Vessel; import fr.ifremer.echobase.services.EchoBaseTestServiceSupport; import fr.ifremer.echobase.services.FakeEchoBaseServiceContext; import fr.ifremer.echobase.services.service.importdata.AbstractImportConfiguration; -import fr.ifremer.echobase.services.service.importdata.AbstractImportDataService; import fr.ifremer.echobase.services.service.importdata.AcousticImportConfiguration; -import fr.ifremer.echobase.services.service.importdata.AcousticImportService; import fr.ifremer.echobase.services.service.importdata.CatchesImportConfiguration; -import fr.ifremer.echobase.services.service.importdata.CatchesImportService; -import fr.ifremer.echobase.services.service.importdata.CommonAllImportService; import fr.ifremer.echobase.services.service.importdata.CommonImportConfiguration; +import fr.ifremer.echobase.services.service.importdata.ImportDataService; import fr.ifremer.echobase.services.service.importdata.OperationImportConfiguration; -import fr.ifremer.echobase.services.service.importdata.OperationImportService; -import fr.ifremer.echobase.services.service.importdata.ResultsEsduCellImportService; import fr.ifremer.echobase.services.service.importdata.ResultsImportConfiguration; -import fr.ifremer.echobase.services.service.importdata.ResultsMapFishCellImportService; -import fr.ifremer.echobase.services.service.importdata.ResultsMapOtherCellImportService; -import fr.ifremer.echobase.services.service.importdata.ResultsRegionCellImportService; -import fr.ifremer.echobase.services.service.importdata.ResultsVoyageImportService; import fr.ifremer.echobase.services.service.removedata.RemoveDataConfiguration; import fr.ifremer.echobase.services.service.removedata.RemoveDataService; import org.apache.commons.collections4.CollectionUtils; @@ -392,7 +383,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS persistenceContext.commit(); importCommonVoyage1Id = createImport( - CommonAllImportService.class, new CommonImportConfiguration(getLocale()), ImportType.COMMON_VOYAGE, "Common/Voyage1", @@ -418,7 +408,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS persistenceContext.commit(); importCommonVoyage2Id = createImport( - CommonAllImportService.class, new CommonImportConfiguration(getLocale()), ImportType.COMMON_VOYAGE, "Common/Voyage2", @@ -446,7 +435,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS persistenceContext.commit(); importCommonTransit1Id = createImport( - CommonAllImportService.class, new CommonImportConfiguration(getLocale()), ImportType.COMMON_TRANSIT, "Common/Transit1", @@ -474,7 +462,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS persistenceContext.commit(); importCommonTransit2Id = createImport( - CommonAllImportService.class, new CommonImportConfiguration(getLocale()), ImportType.COMMON_TRANSIT, "Common/Transit2", @@ -510,7 +497,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS persistenceContext.commit(); importCommonTransect1Id = createImport( - CommonAllImportService.class, new CommonImportConfiguration(getLocale()), ImportType.COMMON_TRANSECT, "Common/Transect1", @@ -545,7 +531,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS persistenceContext.commit(); importCommonTransect2Id = createImport( - CommonAllImportService.class, new CommonImportConfiguration(getLocale()), ImportType.COMMON_TRANSECT, "Common/Transect2", @@ -584,7 +569,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS operation.addGearMetadataValue(gearMetadataValue); importOperation1Id = createImport( - OperationImportService.class, new OperationImportConfiguration(getLocale()), ImportType.OPERATION, "Operation1", @@ -645,7 +629,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS biometrySample.addSampleData(biometrySampleData); importCatches1Id = createImport( - CatchesImportService.class, new CatchesImportConfiguration(getLocale()), ImportType.CATCHES, "Catches1", @@ -690,7 +673,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS elementaryCell.addData(elementaryData); importAcoustic1Id = createImport( - AcousticImportService.class, new AcousticImportConfiguration(getLocale()), ImportType.ACOUSTIC, "Acoustic1", @@ -729,7 +711,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS voyage.addLengthWeightKey(lengthWeightKey); importResultVoyage1Id = createImport( - ResultsVoyageImportService.class, new ResultsImportConfiguration(getLocale()), ImportType.RESULT_VOYAGE, "ResultsVoyage1", @@ -754,7 +735,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS cell.addResult(result); importResultEsdu1Id = createImport( - ResultsEsduCellImportService.class, new ResultsImportConfiguration(getLocale()), ImportType.RESULT_ESDU, "ResultsEsdu1", @@ -787,7 +767,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS region.addResult(result); importResultRegion1Id = createImport( - ResultsRegionCellImportService.class, new ResultsImportConfiguration(getLocale()), ImportType.RESULT_REGION, "ResultsRegion1", @@ -816,7 +795,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS map.addResult(result); importResultMapFish1Id = createImport( - ResultsMapFishCellImportService.class, new ResultsImportConfiguration(getLocale()), ImportType.RESULT_MAP_FISH, "ResultsMapFish1", @@ -844,7 +822,6 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS map.addResult(result); importResultMapOther1Id = createImport( - ResultsMapOtherCellImportService.class, new ResultsImportConfiguration(getLocale()), ImportType.RESULT_MAP_OTHER, "ResultsMapOther1", @@ -852,21 +829,18 @@ public abstract class AbstractRemoveDataServiceTest extends EchoBaseTestServiceS map); } - private <C extends AbstractImportConfiguration, S extends AbstractImportDataService<C>> String createImport( - Class<S> serviceType, - C importConf, - ImportType importType, - String importNote, - String voyageId, - TopiaEntity... importIds) { + private <C extends AbstractImportConfiguration> String createImport(C importConf, + ImportType importType, + String importNote, + String voyageId, + TopiaEntity... importIds) { importConf.setImportType(importType); importConf.setImportNotes(importNote); importConf.setVoyageId(voyageId); - S importService = newService(serviceType); + ImportDataService importService = newService(ImportDataService.class); - ImportLog importLog = importService.computeImportLogEntry( - importConf, createFakeUser()); + ImportLog importLog = importService.computeImportLogEntry(importConf, createFakeUser()); for (TopiaEntity importId : importIds) { importLog.addImportId(importId.getTopiaId()); -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.