branch develop updated (4c7c2a1 -> e9095b6)
This is an automated email from the git hooks/post-receive script. New change to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git from 4c7c2a1 Fixes #8168 (Mise à jour des librairies) new eeef6aa Ajout bidirection sur le modèle (A reverter quand import branche Atlantos) new 42961ba Début d'optimisation des check d'import new 952bddb Ajout bidirection sur le modèle (A reverter quand import branche Atlantos) new bc5e510 Fin de l'optimisation des export (See #8170) new 4500852 Utilisation ToPIA 3.1 new 3d92141 On doit rajouter un order by à la main + correction de la fenetre à calculer new 5fc7551 Afficher les fichiers résultats d'import dans le tableau de bord new 3495e8a Bien vérifier que les fichiers importés sont correctes (See #8170) new e9095b6 Livrable #8170 Merge branch 'feature/8170' into develop The 9 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit e9095b622ed3bad89d9b5dc5a9854d242b237f4c Merge: 4c7c2a1 3495e8a Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 9 12:30:16 2016 +0200 Livrable #8170 Merge branch 'feature/8170' into develop commit 3495e8a1d06126c5c68c3966252a35cd97d96e50 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 9 12:27:59 2016 +0200 Bien vérifier que les fichiers importés sont correctes (See #8170) commit 5fc7551f8266a512b1b823e99adc98a5d4e7cd1e Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed May 4 12:26:41 2016 +0200 Afficher les fichiers résultats d'import dans le tableau de bord commit 3d921412e28327e8c60d497b939c8c1fff2b4723 Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed May 4 11:34:02 2016 +0200 On doit rajouter un order by à la main + correction de la fenetre à calculer commit 4500852eae783c33d02c3a18472d7ab3455bcacc Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed May 4 11:31:27 2016 +0200 Utilisation ToPIA 3.1 commit bc5e510b7dac443a765e966334eca1feb8d4aac4 Author: Tony CHEMIT <chemit@codelutin.com> Date: Tue May 3 10:16:20 2016 +0200 Fin de l'optimisation des export (See #8170) commit 952bddb60fbf609052fea6bca7ba244923c3d5d2 Author: Tony CHEMIT <chemit@codelutin.com> Date: Tue May 3 10:09:14 2016 +0200 Ajout bidirection sur le modèle (A reverter quand import branche Atlantos) commit 42961ba52814b9ab11781ba84c88e9231bff3df1 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 2 11:26:34 2016 +0200 Début d'optimisation des check d'import commit eeef6aaec5fc57a5de6efd6f96c7bdf5214a2e47 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 2 11:19:42 2016 +0200 Ajout bidirection sur le modèle (A reverter quand import branche Atlantos) Summary of changes: .../echobase/entities/ImportFileIdTopiaDao.java | 46 +++++++++ .../fr/ifremer/echobase/entities/ImportedCell.java | 41 ++++++++ .../echobase/entities/ImportedCellResult.java | 41 ++++++++ .../entities/ImportedSampleDataResult.java | 48 +++++++++ .../fr/ifremer/echobase/io/EchoBaseIOUtil.java | 82 ++++++++++++++++ echobase-domain/src/main/xmi/echobase.zargo | Bin 90472 -> 90458 bytes .../echobase/services/service/DbEditorService.java | 17 ++-- .../services/service/UserDbPersistenceService.java | 30 +++++- .../service/importdata/CheckFileException.java | 32 ++++++ .../service/importdata/ImportException.java | 26 +++++ .../actions/ImportDataActionSupport.java | 46 +++++++-- .../actions/VoyageAcousticsImportAction.java | 83 ++++------------ .../VoyageCatchesBiometrySampleImportAction.java | 34 ++----- .../VoyageCatchesSubSampleImportAction.java | 107 ++++++--------------- .../VoyageCatchesTotalSampleImportAction.java | 92 +++++++----------- .../actions/VoyageCommonsTransectImportAction.java | 15 +-- .../actions/VoyageCommonsTransitImportAction.java | 17 +--- .../actions/VoyageCommonsVoyageImportAction.java | 8 +- .../VoyageOperationsGearMetadataImportAction.java | 36 +------ .../VoyageOperationsOperationImportAction.java | 25 +---- ...ageOperationsOperationMetadataImportAction.java | 37 +------ .../VoyageResultsCellImportDataActionSupport.java | 75 ++++++--------- ...VoyageResultsRegionAssociationImportAction.java | 31 +++--- .../actions/VoyageResultsRegionImportAction.java | 41 +++----- ...oyageResultsVoyageLengthAgeKeyImportAction.java | 8 +- ...geResultsVoyageLengthWeightKeyImportAction.java | 8 +- .../csv/VoyageCatchesBiometrySampleImportRow.java | 5 +- .../csv/VoyageCatchesTotalSampleImportRow.java | 9 +- ...VoyageOperationsGearMetadataValueImportRow.java | 5 +- .../csv/VoyageOperationsOperationImportRow.java | 3 +- ...eOperationsOperationMetadataValueImportRow.java | 5 +- .../csv/VoyageResultsRegionCellImportRow.java | 3 +- .../echobase/services/EchoBaseServiceFixtures.java | 8 ++ .../VoyageCatchesAllImportServiceIT.java | 16 +-- .../VoyageResultsRegionCellImportServiceIT.java | 7 +- ...oyageResultsRegionCell_AllImportServiceIT.java} | 15 ++- .../ui/actions/importData/DownloadInputFile.java | 73 ++++++++++++++ .../ui/actions/workingDb/DownloadExportFile.java | 30 +++--- ...oadImportFile.java => DownloadFileSupport.java} | 24 ++--- .../ui/actions/workingDb/DownloadImportFile.java | 80 ++------------- .../main/resources/config/struts-importData.xml | 11 +++ .../src/main/resources/config/struts-workingDb.xml | 11 +++ .../resources/i18n/echobase-ui_en_GB.properties | 5 + .../resources/i18n/echobase-ui_fr_FR.properties | 5 + .../jsp/importData/importDataActionResult.jsp | 62 ++++++++++++ .../WEB-INF/jsp/importData/resultCommonImport.jsp | 2 +- .../webapp/WEB-INF/jsp/workingDb/dashboard.jsp | 70 ++++++++++---- pom.xml | 2 +- 48 files changed, 859 insertions(+), 618 deletions(-) create mode 100644 echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCell.java create mode 100644 echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCellResult.java create mode 100644 echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedSampleDataResult.java create mode 100644 echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/CheckFileException.java copy echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/{VoyageResultsRegionCellImportServiceIT.java => VoyageResultsRegionCell_AllImportServiceIT.java} (87%) create mode 100644 echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importData/DownloadInputFile.java copy echobase-services/src/main/java/fr/ifremer/echobase/services/service/exportCoser/GenerateCoserMapException.java => echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadExportFile.java (65%) copy echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/{DownloadImportFile.java => DownloadFileSupport.java} (83%) create mode 100644 echobase-ui/src/main/webapp/WEB-INF/jsp/importData/importDataActionResult.jsp -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit eeef6aaec5fc57a5de6efd6f96c7bdf5214a2e47 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 2 11:19:42 2016 +0200 Ajout bidirection sur le modèle (A reverter quand import branche Atlantos) --- echobase-domain/src/main/xmi/echobase.zargo | Bin 90472 -> 90457 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/echobase-domain/src/main/xmi/echobase.zargo b/echobase-domain/src/main/xmi/echobase.zargo index f979600..43367ab 100644 Binary files a/echobase-domain/src/main/xmi/echobase.zargo and b/echobase-domain/src/main/xmi/echobase.zargo differ -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 42961ba52814b9ab11781ba84c88e9231bff3df1 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 2 11:26:34 2016 +0200 Début d'optimisation des check d'import --- .../echobase/entities/ImportFileIdTopiaDao.java | 19 ++ .../services/service/UserDbPersistenceService.java | 7 + .../actions/ImportDataActionSupport.java | 6 + .../VoyageCatchesBiometrySampleImportAction.java | 31 +-- .../VoyageCatchesSubSampleImportAction.java | 257 +++++++++++++++------ .../VoyageCatchesTotalSampleImportAction.java | 19 +- .../actions/VoyageCommonsTransectImportAction.java | 9 - .../actions/VoyageCommonsTransitImportAction.java | 11 - .../VoyageOperationsGearMetadataImportAction.java | 36 +-- .../VoyageOperationsOperationImportAction.java | 25 +- ...ageOperationsOperationMetadataImportAction.java | 37 +-- .../VoyageResultsCellImportDataActionSupport.java | 70 +++--- ...VoyageResultsRegionAssociationImportAction.java | 31 ++- .../actions/VoyageResultsRegionImportAction.java | 41 +--- ...oyageResultsVoyageLengthAgeKeyImportAction.java | 8 +- ...geResultsVoyageLengthWeightKeyImportAction.java | 8 +- .../csv/VoyageCatchesBiometrySampleImportRow.java | 5 +- .../csv/VoyageCatchesTotalSampleImportRow.java | 4 +- ...VoyageOperationsGearMetadataValueImportRow.java | 5 +- .../csv/VoyageOperationsOperationImportRow.java | 3 +- ...eOperationsOperationMetadataValueImportRow.java | 5 +- .../csv/VoyageResultsRegionCellImportRow.java | 3 +- .../echobase/services/EchoBaseServiceFixtures.java | 8 + .../VoyageCatchesAllImportServiceIT.java | 16 +- .../VoyageResultsRegionCellImportServiceIT.java | 7 +- 25 files changed, 332 insertions(+), 339 deletions(-) diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java index 3df60bf..5555d79 100644 --- a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java @@ -3,6 +3,9 @@ package fr.ifremer.echobase.entities; import com.google.common.collect.ImmutableMap; import org.nuiton.topia.persistence.HqlAndParametersBuilder; import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.pagination.PaginationResult; + +import java.util.Map; public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFileId> { @@ -24,6 +27,22 @@ public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFil } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { + + HqlAndParametersBuilder<ImportFileId> builder = newHqlAndParametersBuilder(); + builder.setSelectClause(" SELECT entityId"); + builder.addEquals(ImportFileId.PROPERTY_IMPORT_FILE, importFile); + builder.addWhereClause(ImportFileId.PROPERTY_ENTITY_ID + " LIKE :" + ImportFileId.PROPERTY_ENTITY_ID, ImmutableMap.<String, Object>builder().put(ImportFileId.PROPERTY_ENTITY_ID, entityType.getName() + "#%").build()); + builder.setOrderByArguments(ImportFileId.PROPERTY_IMPORT_ORDER); + + String idHql = builder.getHql(); + Map<String, Object> hqlParameters = builder.getHqlParameters(); + PaginationResult<ImportFileId> pager = initPagination(idHql, hqlParameters, batchSize); + String hql = "From " + entityType.getName() + " Where topiaId IN (" + idHql + ")"; + return findAllLazy(hql, hqlParameters, pager); + + } + public long getImportFileIdsCountForImportFile(ImportFile importFile) { return forImportFileEquals(importFile).count(); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java index bd06702..7f49194 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java @@ -1036,6 +1036,13 @@ public class UserDbPersistenceService extends EchoBaseServiceSupport { } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { + + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedEntities(importFile, entityType); + + } + public long getImportFileIdsCount(ImportFile importFile) { ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java index 6f4a0b1..9c225db 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java @@ -391,6 +391,12 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS } + protected <EE extends TopiaEntity> Iterable<EE> getImportedEntities(Class<EE> type, ImportDataFileResult result) { + + return persistenceService.getImportedEntities(result.getImportFile(), type); + + } + protected Iterable<ImportFileId> getImportFileIds(ImportDataFileResult result) { return persistenceService.getImportFileIdsForImportFile(result.getImportFile()); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java index be86dbd..bcd88eb 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java @@ -2,7 +2,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -34,8 +33,6 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesBiometrySampleImportAction.class); - private final Map<String, String> sampleIdToOperationId = new TreeMap<>(); - private final Map<String, String> sampleDataIdToSampleId = new TreeMap<>(); public VoyageCatchesBiometrySampleImportAction(VoyageCatchesImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getBiometrySampleFile()); @@ -102,20 +99,14 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport addProcessedRow(result, row); - // add to sample ids cache (need to compute imported export later) - sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); - SampleData sampleDataFishIndex = persistenceService.createSampleData(sampleDataTypeFishIndex, null, (float) numFish); sample.addSampleData(sampleDataFishIndex); addId(result, EchoBaseUserEntityEnum.SampleData, sampleDataFishIndex, rowNumber); - SampleData sampleData = row.getSampleData(); SampleData sampleDataCreated = persistenceService.createSampleData(row.getSampleData()); sample.addSampleData(sampleDataCreated); addId(result, EchoBaseUserEntityEnum.SampleData, sampleDataCreated, rowNumber); - sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); - } } @@ -129,31 +120,15 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport SampleData sampleDataFishIndex = null; - for (ImportFileId importFileId : filterIds(SampleData.class, result)) { + for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { - String sampleDataId = importFileId.getEntityId(); - SampleData sampleData = persistenceService.getSampleData(sampleDataId); if (sampleDataTypeFishIndex.equals(sampleData.getSampleDataType())) { sampleDataFishIndex = sampleData; continue; } - String sampleId = sampleDataIdToSampleId.get(sampleDataId); - Preconditions.checkNotNull(sampleId); - - String operationId = sampleIdToOperationId.get(sampleId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - Sample sample = persistenceService.getSample(sampleId); - Preconditions.checkNotNull(sample); - - Preconditions.checkNotNull(sample.getSampleDataByTopiaId(sampleDataId)); - if (log.isInfoEnabled()) { - log.info("Adding sampleData: " + sampleId + " to imported export."); + log.info("Adding sampleData: " + sampleData.getTopiaId() + " to imported export."); } Preconditions.checkNotNull(sampleDataFishIndex); @@ -161,7 +136,7 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport Float numFishSampleDataValue = sampleDataFishIndex.getDataValue(); Preconditions.checkNotNull(numFishSampleDataValue); - VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(operation, sample, sampleData, numFishSampleDataValue.intValue()); + VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(sampleData, numFishSampleDataValue.intValue()); addImportedRow(result, importedRow); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java index 2eaacbd..6852f5f 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -23,10 +22,9 @@ import org.nuiton.csv.Import; import java.util.Collection; import java.util.LinkedHashSet; -import java.util.Map; +import java.util.LinkedList; +import java.util.List; import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; /** * Created on 25/03/16. @@ -38,8 +36,6 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesSubSampleImportAction.class); - private final Map<String, String> sampleIdToOperationId = new TreeMap<>(); - private final Map<String, String> sampleDataIdToSampleId = new TreeMap<>(); public VoyageCatchesSubSampleImportAction(VoyageCatchesImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getSubSampleFile()); @@ -71,7 +67,7 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA SampleDataType sampleDataTypeNumberAtLength1cm = importDataContext.getSampleDataTypeNumberAtLength1cm(); SampleDataType sampleDataTypeWeightAtLength = importDataContext.getSampleDataTypeWeightAtLength(); - Set<String> createdSampleIds = new TreeSet<>(); +// Set<String> createdSampleIds = new TreeSet<>(); try (Import<VoyageCatchesSubSampleImportRow> importer = open()) { @@ -102,16 +98,16 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA sample.setSampleType(sampleTypeSubsample); sample = addSample(operation, sample, result, rowNumber); - createdSampleIds.add(sample.getTopiaId()); +// createdSampleIds.add(sample.getTopiaId()); } - // add to sample ids cache (need to compute imported export later) - sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); +// // add to sample ids cache (need to compute imported export later) +// sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); addProcessedRow(result, row); - boolean sampleWasCreated = createdSampleIds.contains(sample.getTopiaId()); +// boolean sampleWasCreated = createdSampleIds.contains(sample.getTopiaId()); Integer round = row.getRound(); SampleDataType dataType; @@ -129,15 +125,17 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA { //create numberAtLength data - SampleData sampleData = addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result, true, rowNumber); - sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); +// SampleData sampleData = + addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result, true, rowNumber); +// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } if (row.getWeightAtLength() != null) { //create weightAtLength data - SampleData sampleData = addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result, true, rowNumber); - sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); +// SampleData sampleData = + addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result, true, rowNumber); +// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } @@ -150,91 +148,53 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA @Override protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { - String currentSampleId = null; +// String currentSampleId = null; - Set<String> sampleDataIds = new LinkedHashSet<>(); + List<SampleData> sampleDataIds = new LinkedList<>(); - for (ImportFileId importedFileId : getImportFileIds(result)) { + Sample sample = null; + + for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { - String importedId = importedFileId.getEntityId(); if (log.isInfoEnabled()) { - log.info("Test imported id: " + importedId); + log.info("Test imported id: " + sampleData.getTopiaId()); } - if (importedId.startsWith(Sample.class.getName() + "#")) { - - // new sample to import + if (sample == null) { - if (currentSampleId == null) { - - // first sample to collect - currentSampleId = importedId; + // first row + sample = sampleData.getSample(); - } else { + } else { - // flush previous sample - addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); + if (!sample.equals(sampleData.getSample())) { - // starts to collect new sample - currentSampleId = importedId; + // flush sample + flushSample(importDataContext, sample, result, sampleDataIds); sampleDataIds.clear(); - } - - } else if (importedId.startsWith(SampleData.class.getName() + "#")) { - - // coming from an existing sample - String sampleId = sampleDataIdToSampleId.get(importedId); - Preconditions.checkNotNull(sampleId); - - if (currentSampleId == null) { - - // first sample to treat - currentSampleId = sampleId; - - } else { - - if (!currentSampleId.equals(sampleId)) { - - // flush previous sample - addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); - - // starts to collect new sample - currentSampleId = sampleId; - sampleDataIds.clear(); - - } + sample = sampleData.getSample(); } - // add this sample data to current sample to flush - sampleDataIds.add(importedId); - - } else { - throw new IllegalStateException("Cant deal with imported id: " + importedId); } + sampleDataIds.add(sampleData); + } if (!sampleDataIds.isEmpty()) { // flush last sample - addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); + flushSample(importDataContext, sample, result, sampleDataIds); } } - protected void addImportedSample(VoyageCatchesImportDataContext importDataContext, String sampleId, ImportDataFileResult result, Set<String> sampleDataIds) { - - String operationId = sampleIdToOperationId.get(sampleId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); + protected void flushSample(VoyageCatchesImportDataContext importDataContext, Sample sample, ImportDataFileResult result, List<SampleData> sampleDataIds) { - Sample sample = persistenceService.getSample(sampleId); Preconditions.checkNotNull(sample); if (log.isInfoEnabled()) { @@ -250,11 +210,11 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA VoyageCatchesSubSampleImportRow importedRow = null; + Operation operation = sample.getOperation(); + String lastLengthClass = null; boolean lengthDataTypeFound = false; - for (String sampleDataId : sampleDataIds) { - - SampleData aSampleData = sample.getSampleDataByTopiaId(sampleDataId); + for (SampleData aSampleData : sampleDataIds) { String lengthClass = aSampleData.getDataLabel(); @@ -296,6 +256,155 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA } +// @Override +// protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { +// +// String currentSampleId = null; +// +// Set<String> sampleDataIds = new LinkedHashSet<>(); +// +// for (ImportFileId importedFileId : getImportFileIds(result)) { +// +// String importedId = importedFileId.getEntityId(); +// if (log.isInfoEnabled()) { +// log.info("Test imported id: " + importedId); +// } +// +// if (importedId.startsWith(Sample.class.getName() + "#")) { +// +// // new sample to import +// +// if (currentSampleId == null) { +// +// // first sample to collect +// currentSampleId = importedId; +// +// } else { +// +// // flush previous sample +// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); +// +// // starts to collect new sample +// currentSampleId = importedId; +// sampleDataIds.clear(); +// +// } +// +// } else if (importedId.startsWith(SampleData.class.getName() + "#")) { +// +// // coming from an existing sample +// String sampleId = sampleDataIdToSampleId.get(importedId); +// Preconditions.checkNotNull(sampleId); +// +// if (currentSampleId == null) { +// +// // first sample to treat +// currentSampleId = sampleId; +// +// } else { +// +// if (!currentSampleId.equals(sampleId)) { +// +// // flush previous sample +// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); +// +// // starts to collect new sample +// currentSampleId = sampleId; +// sampleDataIds.clear(); +// +// } +// +// } +// +// // add this sample data to current sample to flush +// sampleDataIds.add(importedId); +// +// } else { +// throw new IllegalStateException("Cant deal with imported id: " + importedId); +// } +// +// +// } +// +// if (!sampleDataIds.isEmpty()) { +// +// // flush last sample +// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); +// +// } +// +// } +// +// protected void addImportedSample(VoyageCatchesImportDataContext importDataContext, String sampleId, ImportDataFileResult result, Set<String> sampleDataIds) { +// +// String operationId = sampleIdToOperationId.get(sampleId); +// Preconditions.checkNotNull(operationId); +// +// Operation operation = persistenceService.getOperation(operationId); +// Preconditions.checkNotNull(operation); +// +// Sample sample = persistenceService.getSample(sampleId); +// Preconditions.checkNotNull(sample); +// +// if (log.isInfoEnabled()) { +// log.info("Adding sample: " + sample + " to imported export."); +// } +// +// Set<SampleDataType> lengthSampleDataTypes = new LinkedHashSet<>(); +// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength()); +// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength05cm()); +// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength1cm()); +// +// Preconditions.checkState(sample.isSampleDataNotEmpty()); +// +// VoyageCatchesSubSampleImportRow importedRow = null; +// +// String lastLengthClass = null; +// boolean lengthDataTypeFound = false; +// for (String sampleDataId : sampleDataIds) { +// +// SampleData aSampleData = sample.getSampleDataByTopiaId(sampleDataId); +// +// String lengthClass = aSampleData.getDataLabel(); +// +// boolean isLengthDataType = lengthSampleDataTypes.contains(aSampleData.getSampleDataType()); +// +// if (lastLengthClass == null) { +// +// // first row +// lastLengthClass = lengthClass; +// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); +// lengthDataTypeFound = false; +// +// } else { +// +// if ((isLengthDataType && lengthDataTypeFound) || !lengthClass.equals(lastLengthClass)) { +// +// // changing length class or length data type already found +// addImportedRow(result, importedRow); +// +// lastLengthClass = lengthClass; +// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); +// lengthDataTypeFound = false; +// } +// } +// +// if (log.isInfoEnabled()) { +// log.info("Treated sample data: " + aSampleData); +// } +// +// addImportedSampleData(importDataContext, importedRow, aSampleData); +// +// if (isLengthDataType) { +// lengthDataTypeFound = true; +// } +// +// } +// +// addImportedRow(result, importedRow); +// +// } + protected void addImportedSampleData(VoyageCatchesImportDataContext importDataContext, VoyageCatchesSubSampleImportRow importedRow, SampleData sampleData) { SampleDataType sampleDataTypeNumberAtLength = importDataContext.getSampleDataTypeNumberAtLength(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java index 7d0cfba..49d8098 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java @@ -3,7 +3,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -39,7 +38,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesTotalSampleImportAction.class); - private final Map<String, String> sampleIdToOperationId = new TreeMap<>(); private final Map<String, String> sampleIdToSortedSampleId = new TreeMap<>(); public VoyageCatchesTotalSampleImportAction(VoyageCatchesImportDataContext importDataContext) { @@ -63,7 +61,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat log.info("Starts import of totalSample from file " + inputFile.getFileName()); } - sampleIdToOperationId.clear(); sampleIdToSortedSampleId.clear(); SpeciesCategoryCache speciesCategoryCache = importDataContext.getSpeciesCategoryCache(); @@ -147,7 +144,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat Sample sortedSample = addSample(operation, createdSortedSample, result, rowNumber); // add to sample ids cache (need to compte imported export later) - sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); sampleIdToSortedSampleId.put(sample.getTopiaId(), sortedSample.getTopiaId()); } @@ -164,9 +160,9 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat SampleDataType sampleDataTypeNoPerKg = importDataContext.getSampleDataTypeNoPerKg(); Collection<String> sortedSampleIds = sampleIdToSortedSampleId.values(); - for (ImportFileId importFileId : filterIds(Sample.class, result)) { + for (Sample sample : getImportedEntities(Sample.class, result)) { - String sampleId = importFileId.getEntityId(); + String sampleId = sample.getTopiaId(); if (sortedSampleIds.contains(sampleId)) { // skip sorted sample @@ -177,22 +173,13 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat log.info("Adding sample: " + sampleId + " to imported export."); } - String operationId = sampleIdToOperationId.get(sampleId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - Sample sample = persistenceService.getSample(sampleId); - Preconditions.checkNotNull(sample); - String sortedSampleId = sampleIdToSortedSampleId.get(sampleId); Preconditions.checkNotNull(sortedSampleId); Sample sortedSample = persistenceService.getSample(sortedSampleId); Preconditions.checkNotNull(sortedSample); - VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(operation, sample, sortedSample); + VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(sample, sortedSample); if (sample.isSampleDataNotEmpty()) { diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java index 6a96174..62b029f 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java @@ -88,15 +88,6 @@ public class VoyageCommonsTransectImportAction extends VoyageCommonsImportDataAc addProcessedRow(result, row); addId(result, EchoBaseUserEntityEnum.Transect, createdTransect, rowNumber); -// if (collectIds) { -// -// // collect id of the import -// result.addId(EchoBaseUserEntityEnum.Transect, createdTransect); -// } else { -// -// result.incrementsNumberCreated(EchoBaseUserEntityEnum.Transect); -// } - } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java index f8ccd1a..29367df 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java @@ -70,17 +70,6 @@ public class VoyageCommonsTransitImportAction extends VoyageCommonsImportDataAct addId(result, EchoBaseUserEntityEnum.Transit, createdTransit, rowNumber); addProcessedRow(result, row); -// if (collectIds) { -// -// // collect id of the import -// result.addId(EchoBaseUserEntityEnum.Transit, createdTransit); -// -// -// } else { -// -// result.incrementsNumberCreated(EchoBaseUserEntityEnum.Transit); -// } - } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java index 27e89fd..c8f7447 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java @@ -1,11 +1,8 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.GearMetadataValue; import fr.ifremer.echobase.entities.data.Operation; -import fr.ifremer.echobase.entities.references.Vessel; import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; import fr.ifremer.echobase.services.service.importdata.contexts.VoyageOperationsImportDataContext; @@ -15,9 +12,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.Map; -import java.util.TreeMap; - /** * Created on 25/03/16. * @@ -28,8 +22,6 @@ public class VoyageOperationsGearMetadataImportAction extends VoyageOperationsIm /** Logger. */ private static final Log log = LogFactory.getLog(VoyageOperationsGearMetadataImportAction.class); - private final Map<String, String> gearMetadataValueIdToOperationId = new TreeMap<>(); - private final Map<String, String> operationIdToVesselId = new TreeMap<>(); public VoyageOperationsGearMetadataImportAction(VoyageOperationsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getGearMetadataFile()); @@ -69,11 +61,6 @@ public class VoyageOperationsGearMetadataImportAction extends VoyageOperationsIm addProcessedRow(result, row); - gearMetadataValueIdToOperationId.put(gearMetadataValue.getTopiaId(), operation.getTopiaId()); - - Vessel vessel = row.getVessel(); - operationIdToVesselId.put(operation.getTopiaId(), vessel.getTopiaId()); - } } @@ -83,30 +70,13 @@ public class VoyageOperationsGearMetadataImportAction extends VoyageOperationsIm @Override protected void computeImportedExport(VoyageOperationsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : getImportFileIds(result)) { - - String gearMetadataValueId = importFileId.getEntityId(); + for (GearMetadataValue gearMetadataValue : getImportedEntities(GearMetadataValue.class, result)) { if (log.isInfoEnabled()) { - log.info("Adding gear metadata value : " + gearMetadataValueId + " to imported export."); + log.info("Adding gear metadata value : " + gearMetadataValue.getTopiaId() + " to imported export."); } - String operationId = gearMetadataValueIdToOperationId.get(gearMetadataValueId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - String vesselId = operationIdToVesselId.get(operationId); - Preconditions.checkNotNull(vesselId); - - Vessel vessel = persistenceService.getVessel(vesselId); - Preconditions.checkNotNull(vessel); - - GearMetadataValue gearMetadataValue = operation.getGearMetadataValueByTopiaId(gearMetadataValueId); - Preconditions.checkNotNull(gearMetadataValue); - - VoyageOperationsGearMetadataValueImportRow importedRow = VoyageOperationsGearMetadataValueImportRow.of(operation, vessel, gearMetadataValue); + VoyageOperationsGearMetadataValueImportRow importedRow = VoyageOperationsGearMetadataValueImportRow.of(gearMetadataValue); addImportedRow(result, importedRow); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java index eb78547..0bb72a5 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java @@ -1,8 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Transect; import fr.ifremer.echobase.entities.data.Transit; @@ -18,8 +16,6 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; import java.util.Date; -import java.util.Map; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -31,7 +27,6 @@ public class VoyageOperationsOperationImportAction extends VoyageOperationsImpor /** Logger. */ private static final Log log = LogFactory.getLog(VoyageOperationsOperationImportAction.class); - private final Map<String, String> opperationIdToTransectId = new TreeMap<>(); public VoyageOperationsOperationImportAction(VoyageOperationsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getOperationFile()); @@ -84,10 +79,6 @@ public class VoyageOperationsOperationImportAction extends VoyageOperationsImpor addProcessedRow(result, row); - // add to operation ids cache (need to compute imported export later) - opperationIdToTransectId.put(operation.getTopiaId(), transect.getTopiaId()); - - } } @@ -97,21 +88,13 @@ public class VoyageOperationsOperationImportAction extends VoyageOperationsImpor @Override protected void computeImportedExport(VoyageOperationsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : getImportFileIds(result)) { - String operationId = importFileId.getEntityId(); + for (Operation operation : getImportedEntities(Operation.class, result)) { + if (log.isInfoEnabled()) { - log.info("Adding operation: " + operationId + " to imported export."); + log.info("Adding operation: " + operation.getTopiaId() + " to imported export."); } - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - String transectId = opperationIdToTransectId.get(operationId); - Preconditions.checkNotNull(transectId); - - Transect transect = persistenceService.getTransect(transectId); - Preconditions.checkNotNull(transect); - VoyageOperationsOperationImportRow importedRow = VoyageOperationsOperationImportRow.of(transect, operation); + VoyageOperationsOperationImportRow importedRow = VoyageOperationsOperationImportRow.of(operation); addImportedRow(result, importedRow); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java index 46423fc..d65576c 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java @@ -1,11 +1,8 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.OperationMetadataValue; -import fr.ifremer.echobase.entities.references.Vessel; import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; import fr.ifremer.echobase.services.service.importdata.contexts.VoyageOperationsImportDataContext; @@ -15,9 +12,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.Map; -import java.util.TreeMap; - /** * Created on 25/03/16. * @@ -28,8 +22,6 @@ public class VoyageOperationsOperationMetadataImportAction extends VoyageOperati /** Logger. */ private static final Log log = LogFactory.getLog(VoyageOperationsOperationMetadataImportAction.class); - private final Map<String, String> operationMetadataValueIdToOperationId = new TreeMap<>(); - private final Map<String, String> operationIdToVesselId = new TreeMap<>(); public VoyageOperationsOperationMetadataImportAction(VoyageOperationsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getOperationMetadataFile()); @@ -69,11 +61,6 @@ public class VoyageOperationsOperationMetadataImportAction extends VoyageOperati addProcessedRow(result, row); addId(result, EchoBaseUserEntityEnum.OperationMetadataValue, operationMetadataValue, rowNumber); - operationMetadataValueIdToOperationId.put(operationMetadataValue.getTopiaId(), operation.getTopiaId()); - - Vessel vessel = row.getVessel(); - operationIdToVesselId.put(operation.getTopiaId(), vessel.getTopiaId()); - } } @@ -83,28 +70,12 @@ public class VoyageOperationsOperationMetadataImportAction extends VoyageOperati @Override protected void computeImportedExport(VoyageOperationsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : getImportFileIds(result)) { - String operationMetadataValueId = importFileId.getEntityId(); + for (OperationMetadataValue operationMetadataValue : getImportedEntities(OperationMetadataValue.class, result)) { + if (log.isInfoEnabled()) { - log.info("Adding operation metadata value : " + operationMetadataValueId + " to imported export."); + log.info("Adding operation metadata value : " + operationMetadataValue.getTopiaId() + " to imported export."); } - - String operationId = operationMetadataValueIdToOperationId.get(operationMetadataValueId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - String vesselId = operationIdToVesselId.get(operationId); - Preconditions.checkNotNull(vesselId); - - Vessel vessel = persistenceService.getVessel(vesselId); - Preconditions.checkNotNull(vessel); - - OperationMetadataValue operationMetadataValue = operation.getOperationMetadataValueByTopiaId(operationMetadataValueId); - Preconditions.checkNotNull(operationMetadataValue); - - VoyageOperationsOperationMetadataValueImportRow importedRow = VoyageOperationsOperationMetadataValueImportRow.of(operation, vessel, operationMetadataValue); + VoyageOperationsOperationMetadataValueImportRow importedRow = VoyageOperationsOperationMetadataValueImportRow.of(operationMetadataValue); addImportedRow(result, importedRow); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java index 7112c9d..df50de1 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java @@ -1,6 +1,5 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Category; import fr.ifremer.echobase.entities.data.Cell; @@ -19,8 +18,6 @@ import org.nuiton.csv.Import; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -32,8 +29,8 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA /** Logger. */ private static final Log log = LogFactory.getLog(VoyageResultsCellImportDataActionSupport.class); + protected final List<DataMetadata> metas; - private final Map<String, String> resultIdToCellId = new TreeMap<>(); protected VoyageResultsCellImportDataActionSupport(VoyageResultsImportDataContext importDataContext, InputFile inputFile, String... columnNamesToExclude) { super(importDataContext, inputFile); @@ -66,11 +63,7 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA Category category = getResultCategory(result, resultCategoryCache, row); Cell cell = row.getCell(); - String cellTopiaId = cell.getTopiaId(); - List<Result> cellResults = addResults(row, cell, category, resultLabel, result, true, true, rowNumber); - Result lastResult = cellResults.get(cellResults.size() - 1); - - resultIdToCellId.put(lastResult.getTopiaId(), cellTopiaId); + addResults(row, cell, category, resultLabel, result, true, true, rowNumber); addProcessedRow(result, row); @@ -90,53 +83,64 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA int count = 0; int max = result.sizeImportFileIds(); - for (ImportFileId importFileId : getImportFileIds(result)) { + int currentLineNumber = -1; + + for (ImportFileId importFileId : filterIds(Result.class, result)) { + String resultId = importFileId.getEntityId(); if (log.isDebugEnabled()) { log.debug("Treat imported result: " + resultId + " (" + (++count) + "/" + max + ")"); } - cellResultIds.add(resultId); + if (currentLineNumber == -1) { - String cellId = resultIdToCellId.get(resultId); + currentLineNumber = importFileId.getLineNumber(); - if (cellId != null) { + } else if (currentLineNumber != importFileId.getLineNumber()) { + // flush cell + flushCell(voyage, result, cellResultIds); - // on the last result of a row, flush current cell + cellResultIds.clear(); - Preconditions.checkState(!cellResultIds.isEmpty(), "Can't flush a cell (" + cellId + " ) with no result..."); + currentLineNumber = importFileId.getLineNumber(); - Cell cell = persistenceService.getCell(cellId); - Preconditions.checkNotNull(cell); + } - if (log.isInfoEnabled()) { - log.info("Flush Cell " + cell + " with " + cellResultIds.size() + " results"); - } + cellResultIds.add(resultId); - List<Result> cellResults = new LinkedList<>(); + } - Category category = null; - for (Result cellResult : cell.getResult()) { + flushCell(voyage, result, cellResultIds); - if (cellResultIds.contains(cellResult.getTopiaId())) { - if (category == null) { - category = cellResult.getCategory(); - } - cellResults.add(cellResult); - } + } - } + protected void flushCell(Voyage voyage, ImportDataFileResult result, List<String> cellResultIds) { - E row = newImportedRow(voyage, cell, category, cellResults); - addImportedRow(result, row); + String firstResultId = cellResultIds.get(0); + Result firstCellResult = persistenceService.getResult(firstResultId); + Cell cell = firstCellResult.getCell(); - cellResultIds.clear(); + if (log.isInfoEnabled()) { + log.info("Flush " + cell + " with " + cellResultIds.size() + " result(s)."); + } + List<Result> cellResults = new LinkedList<>(); + + Category category = null; + for (Result cellResult : cell.getResult()) { + if (cellResultIds.contains(cellResult.getTopiaId())) { + if (category == null) { + category = cellResult.getCategory(); + } + cellResults.add(cellResult); } } + E row = newImportedRow(voyage, cell, category, cellResults); + addImportedRow(result, row); + } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java index b9f75de..2faaf91 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java @@ -13,8 +13,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.LinkedHashSet; +import java.util.Set; /** * Created on 25/03/16. @@ -26,7 +26,7 @@ public class VoyageResultsRegionAssociationImportAction extends VoyageResultsImp /** Logger. */ private static final Log log = LogFactory.getLog(VoyageResultsRegionAssociationImportAction.class); - private final Map<String, String> esduCellDataIdToRegionCellId = new LinkedHashMap<>(); + private final Set<String> regionCellIds = new LinkedHashSet<>(); public VoyageResultsRegionAssociationImportAction(VoyageResultsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getRegionAssociationFile()); @@ -61,12 +61,10 @@ public class VoyageResultsRegionAssociationImportAction extends VoyageResultsImp Cell esduCell = row.getEsduCell(); regionCell.addChilds(esduCell); - if (!esduCellDataIdToRegionCellId.containsValue(regionCell.getTopiaId())) { - result.incrementsNumberUpdated(EchoBaseUserEntityEnum.Cell); - } + result.incrementsNumberUpdated(EchoBaseUserEntityEnum.Cell); addProcessedRow(result, row); - esduCellDataIdToRegionCellId.put(esduCell.getTopiaId(), regionCell.getTopiaId()); + regionCellIds.add(regionCell.getTopiaId()); } @@ -80,23 +78,20 @@ public class VoyageResultsRegionAssociationImportAction extends VoyageResultsImp String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (Map.Entry<String, String> entry : esduCellDataIdToRegionCellId.entrySet()) { - - String esduCellId = entry.getKey(); - String regionCellId = entry.getValue(); + for (String regionCellId : regionCellIds) { if (log.isInfoEnabled()) { - log.info("Treat imported esduCell association: " + esduCellId + " to regionCell: " + regionCellId); + log.info("Treat imported region regionCell: " + regionCellId); } - - Cell esduCell = persistenceService.getCell(esduCellId); - Preconditions.checkNotNull(esduCell); - Cell regionCell = persistenceService.getCell(regionCellId); Preconditions.checkNotNull(regionCell); - VoyageResultsRegionCellAssociationImportRow row = VoyageResultsRegionCellAssociationImportRow.of(voyage, regionCell, esduCell); - addImportedRow(result, row); + for (Cell esduCell : regionCell.getChilds()) { + + VoyageResultsRegionCellAssociationImportRow row = VoyageResultsRegionCellAssociationImportRow.of(voyage, regionCell, esduCell); + addImportedRow(result, row); + + } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java index cedfc12..57ea597 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java @@ -5,7 +5,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Data; import fr.ifremer.echobase.entities.data.Datas; @@ -21,9 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.Map; -import java.util.TreeMap; - /** * Created on 25/03/16. * @@ -34,7 +30,6 @@ public class VoyageResultsRegionImportAction extends VoyageResultsImportDataActi /** Logger. */ private static final Log log = LogFactory.getLog(VoyageResultsRegionImportAction.class); - private final Map<String, String> cellDataCoordinateIdToCellId = new TreeMap<>(); public VoyageResultsRegionImportAction(VoyageResultsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getRegionsFile()); @@ -96,15 +91,12 @@ public class VoyageResultsRegionImportAction extends VoyageResultsImportDataActi } // add coordinate data - Data cellData = createCellData(cell, - dataCoordinateMeta, - row.getDataCoordinate(), - dataQuality, - result, - true, rowNumber); - - cellDataCoordinateIdToCellId.put(cellData.getTopiaId(), cell.getTopiaId()); - + createCellData(cell, + dataCoordinateMeta, + row.getDataCoordinate(), + dataQuality, + result, + true, rowNumber); addProcessedRow(result, row); @@ -125,38 +117,27 @@ public class VoyageResultsRegionImportAction extends VoyageResultsImportDataActi String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : filterIds(Data.class, result)) { + for (Data coordinateCellData : getImportedEntities(Data.class, result)) { - String cellDataId = importFileId.getEntityId(); - String cellId = cellDataCoordinateIdToCellId.get(cellDataId); - if (cellId == null) { + if (!dataCoordinateMeta.equals(coordinateCellData.getDataMetadata())) { // Treat only coordindate data, since there is exactly one per row // The surface data is the same for all rows of a same region cell continue; } + Cell cell = coordinateCellData.getCell(); if (log.isInfoEnabled()) { - log.info("Treat imported coordinate data: " + cellDataId + " for cell: " + cellId); + log.info("Treat imported coordinate data: " + coordinateCellData.getTopiaId() + " for cell: " + cell.getTopiaId()); } - Preconditions.checkNotNull(cellId); - - Cell cell = persistenceService.getCell(cellId); - Preconditions.checkNotNull(cell); - Preconditions.checkState(cell.isDataNotEmpty()); - - Data coordinateData = cell.getDataByTopiaId(cellDataId); - Preconditions.checkNotNull(coordinateData); - Preconditions.checkState(dataCoordinateMeta.equals(coordinateData.getDataMetadata())); Optional<Data> optionalSurfaceData = Iterables.tryFind(cell.getData(), dataPredicate); Preconditions.checkState(optionalSurfaceData.isPresent()); String surfaceDataValue = optionalSurfaceData.get().getDataValue(); - Preconditions.checkNotNull(coordinateData); - VoyageResultsRegionCellImportRow row = VoyageResultsRegionCellImportRow.of(voyage, cell, coordinateData, Float.valueOf(surfaceDataValue)); + VoyageResultsRegionCellImportRow row = VoyageResultsRegionCellImportRow.of(voyage, coordinateCellData, Float.valueOf(surfaceDataValue)); addImportedRow(result, row); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java index f900e76..5afb54d 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java @@ -1,8 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.LengthAgeKey; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.io.InputFile; @@ -80,11 +78,7 @@ public class VoyageResultsVoyageLengthAgeKeyImportAction extends VoyageResultsIm String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : getImportFileIds(result)) { - - String lengthAgeKeyId = importFileId.getEntityId(); - LengthAgeKey lengthAgeKey = persistenceService.getLengthAgeKey(lengthAgeKeyId); - Preconditions.checkNotNull(lengthAgeKey); + for (LengthAgeKey lengthAgeKey : getImportedEntities(LengthAgeKey.class, result)) { VoyageResultsVoyageLengthAgeKeyImportRow row = VoyageResultsVoyageLengthAgeKeyImportRow.of(voyage, lengthAgeKey); addImportedRow(result, row); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java index 0442050..560fe2d 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java @@ -1,8 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.LengthWeightKey; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.SpeciesCategory; @@ -89,11 +87,7 @@ public class VoyageResultsVoyageLengthWeightKeyImportAction extends VoyageResult String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : getImportFileIds(result)) { - - String lengthWeightKeyId = importFileId.getEntityId(); - LengthWeightKey lengthWeightKey = persistenceService.getLengthWeightKey(lengthWeightKeyId); - Preconditions.checkNotNull(lengthWeightKey); + for (LengthWeightKey lengthWeightKey : getImportedEntities(LengthWeightKey.class, result)) { VoyageResultsVoyageLengthWeightKeyImportRow row = VoyageResultsVoyageLengthWeightKeyImportRow.of(voyage, lengthWeightKey); addImportedRow(result, row); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java index b68496a..bb0969c 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java @@ -49,10 +49,11 @@ public class VoyageCatchesBiometrySampleImportRow { protected int numFish; - public static VoyageCatchesBiometrySampleImportRow of(Operation operation, Sample sample, SampleData sampleData, int numFish) { + public static VoyageCatchesBiometrySampleImportRow of(SampleData sampleData, int numFish) { + Sample sample = sampleData.getSample(); VoyageCatchesBiometrySampleImportRow row = new VoyageCatchesBiometrySampleImportRow(sampleData); - row.setOperation(operation); + row.setOperation(sample.getOperation()); row.setSpecies(sample.getSpeciesCategory().getSpecies()); row.setNumFish(numFish); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java index 2f11800..d6363e7 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java @@ -140,9 +140,9 @@ public class VoyageCatchesTotalSampleImportRow { this.sortedWeight = sortedWeight; } - public static VoyageCatchesTotalSampleImportRow of(Operation operation, Sample sample, Sample sortedSample) { + public static VoyageCatchesTotalSampleImportRow of(Sample sample, Sample sortedSample) { VoyageCatchesTotalSampleImportRow row = new VoyageCatchesTotalSampleImportRow(sample); - row.setOperation(operation); + row.setOperation(sample.getOperation()); row.setSpecies(sample.getSpeciesCategory().getSpecies()); row.setSizeCategory(sample.getSpeciesCategory().getSizeCategory()); row.setSortedWeight(sortedSample.getSampleWeight()); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java index d8066d0..7dc0fc4 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java @@ -45,10 +45,11 @@ public class VoyageOperationsGearMetadataValueImportRow { protected Vessel vessel; protected Operation operation; - public static VoyageOperationsGearMetadataValueImportRow of(Operation operation, Vessel vessel, GearMetadataValue gearMetadataValue) { + public static VoyageOperationsGearMetadataValueImportRow of(GearMetadataValue gearMetadataValue) { VoyageOperationsGearMetadataValueImportRow row = new VoyageOperationsGearMetadataValueImportRow(gearMetadataValue); + Operation operation = gearMetadataValue.getOperation(); row.setOperation(operation); - row.setVessel(vessel); + row.setVessel(operation.getTransect().getVessel()); return row; } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java index 0b7f5b8..78b9b29 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java @@ -42,8 +42,9 @@ public class VoyageOperationsOperationImportRow { protected final Operation operation; protected Vessel vessel; - public static VoyageOperationsOperationImportRow of(Transect transect, Operation operation) { + public static VoyageOperationsOperationImportRow of(Operation operation) { VoyageOperationsOperationImportRow row = new VoyageOperationsOperationImportRow(operation); + Transect transect = operation.getTransect(); row.setVessel(transect.getVessel()); return row; } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java index d49a5d9..3ce0b7f 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java @@ -44,10 +44,11 @@ public class VoyageOperationsOperationMetadataValueImportRow { protected Operation operation; protected Vessel vessel; - public static VoyageOperationsOperationMetadataValueImportRow of(Operation operation, Vessel vessel, OperationMetadataValue operationMetadataValue) { + public static VoyageOperationsOperationMetadataValueImportRow of(OperationMetadataValue operationMetadataValue) { VoyageOperationsOperationMetadataValueImportRow row = new VoyageOperationsOperationMetadataValueImportRow(operationMetadataValue); + Operation operation = operationMetadataValue.getOperation(); row.setOperation(operation); - row.setVessel(vessel); + row.setVessel(operation.getTransect().getVessel()); return row; } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java index 8b16040..05fc438 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java @@ -109,9 +109,10 @@ public class VoyageResultsRegionCellImportRow { this.dataQuality = dataQuality; } - public static VoyageResultsRegionCellImportRow of(Voyage voyage, Cell cell, Data coordinateData, float surfaceDataValue) { + public static VoyageResultsRegionCellImportRow of(Voyage voyage, Data coordinateData, float surfaceDataValue) { VoyageResultsRegionCellImportRow row = new VoyageResultsRegionCellImportRow(); row.setVoyage(voyage); + Cell cell = coordinateData.getCell(); row.setName(cell.getName()); row.setCellType(cell.getCellType()); row.setDataSurface(surfaceDataValue); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java index edb522b..13a8a83 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java @@ -111,10 +111,18 @@ public class EchoBaseServiceFixtures { return NB_SAMPLE_TOTAL() + NB_SAMPLE_UNSORTED() + NB_SAMPLE_BIOMETRY(); } + public int NB_SAMPLE_ALL() { + return NB_SAMPLE_TOTAL() + NB_SAMPLE_UNSORTED_ALL() + NB_SAMPLE_BIOMETRY(); + } + public int NB_SAMPLE_DATA() { return NB_SAMPLE_DATA_TOTAL() + NB_SAMPLE_DATA_UNSORTED() + NB_SAMPLE_DATA_BIOMETRY(); } + public int NB_SAMPLE_DATA_ALL() { + return NB_SAMPLE_DATA_TOTAL() + NB_SAMPLE_DATA_UNSORTED_ALL() + NB_SAMPLE_DATA_BIOMETRY(); + } + public int NB_DATA_ACQUISITION() { return 1; } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java index 9be5649..06fe741 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java @@ -48,7 +48,7 @@ public class VoyageCatchesAllImportServiceIT extends VoyageCatchesImportServiceI protected VoyageCatchesImportConfiguration createConfiguration() throws IOException { VoyageCatchesImportConfiguration configuration = super.createConfiguration(); prepareInputFile(configuration.getTotalSampleFile(), getImportPath("totalsample.csv.gz")); - prepareInputFile(configuration.getSubSampleFile(), getImportPath("subsample.csv.gz")); + prepareInputFile(configuration.getSubSampleFile(), getImportPath("subsample_all.csv.gz")); prepareInputFile(configuration.getBiometrySampleFile(), getImportPath("biometrysample.csv.gz")); return configuration; } @@ -63,37 +63,37 @@ public class VoyageCatchesAllImportServiceIT extends VoyageCatchesImportServiceI @Override protected void assertAfertImport(ImportDataResult<VoyageCatchesImportConfiguration> result) throws IOException { - int nbSample = fixtures.NB_SAMPLE(); - int nbSampleData = fixtures.NB_SAMPLE_DATA(); + int nbSample = fixtures.NB_SAMPLE_ALL(); + int nbSampleData = fixtures.NB_SAMPLE_DATA_ALL(); int nbSampleTotal = fixtures.NB_SAMPLE_TOTAL(); int nbSampleDataTotal = fixtures.NB_SAMPLE_DATA_TOTAL(); - int nbSampleUnsorted = fixtures.NB_SAMPLE_UNSORTED(); - int nbSampleDataUnsorted = fixtures.NB_SAMPLE_DATA_UNSORTED(); + int nbSampleUnsorted = fixtures.NB_SAMPLE_UNSORTED_ALL(); + int nbSampleDataUnsorted = fixtures.NB_SAMPLE_DATA_UNSORTED_ALL(); int nbSampleBiometry = fixtures.NB_SAMPLE_BIOMETRY(); int nbSampleDataBiometry = fixtures.NB_SAMPLE_DATA_BIOMETRY(); { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 0); + assertCsvImportResult0(importDataFileResult, nbSampleTotal); assertCsvImportResultPerEntity(importDataFileResult, Sample.class, nbSampleTotal, 0, nbSample); assertCsvImportResultPerEntity(importDataFileResult, SampleData.class, nbSampleDataTotal, 0, nbSampleData); - assertCsvImportResult0(importDataFileResult, nbSampleTotal); } { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 1); + assertCsvImportResult0(importDataFileResult, nbSampleUnsorted + nbSampleDataUnsorted); assertCsvImportResultPerEntity(importDataFileResult, Sample.class, nbSampleUnsorted, 0, nbSample); assertCsvImportResultPerEntity(importDataFileResult, SampleData.class, nbSampleDataUnsorted, 0, nbSampleData); - assertCsvImportResult0(importDataFileResult, nbSampleUnsorted + nbSampleDataUnsorted); } { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 2); + assertCsvImportResult0(importDataFileResult, nbSampleBiometry + nbSampleDataBiometry); assertCsvImportResultPerEntity(importDataFileResult, Sample.class, nbSampleBiometry, 0, nbSample); assertCsvImportResultPerEntity(importDataFileResult, SampleData.class, nbSampleDataBiometry, 0, nbSampleData); - assertCsvImportResult0(importDataFileResult, nbSampleBiometry + nbSampleDataBiometry); } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java index bc5bf4a..109f463 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java @@ -66,6 +66,7 @@ public class VoyageResultsRegionCellImportServiceIT extends VoyageResultsImportS return configuration; } + // public static final int NB_ESDU_CELLS = 2073; public static final int NB_ESDU_CELLS = 22; @Override @@ -73,6 +74,10 @@ public class VoyageResultsRegionCellImportServiceIT extends VoyageResultsImportS int nbRegionCell = 10; int nbRegionCellData = 238; + +// //FIXME Comprendre pourquoi toutes les cellules esdu sont dans 2 régions différentes ? +// int nbUpdatedEsduCell = 2 * NB_ESDU_CELLS; + int nbUpdatedEsduCell = NB_ESDU_CELLS; int nbRegionCellResult = 2128/*1873*/; int nbCell = NB_ESDU_CELLS + nbRegionCell; @@ -86,8 +91,8 @@ public class VoyageResultsRegionCellImportServiceIT extends VoyageResultsImportS { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 1); - assertCsvImportResultPerEntity(importDataFileResult, Cell.class, 0, nbRegionCell, nbCell); assertCsvImportResult0(importDataFileResult, 0); + assertCsvImportResultPerEntity(importDataFileResult, Cell.class, 0, nbUpdatedEsduCell, nbCell); } { -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 952bddb60fbf609052fea6bca7ba244923c3d5d2 Author: Tony CHEMIT <chemit@codelutin.com> Date: Tue May 3 10:09:14 2016 +0200 Ajout bidirection sur le modèle (A reverter quand import branche Atlantos) --- echobase-domain/src/main/xmi/echobase.zargo | Bin 90457 -> 90458 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/echobase-domain/src/main/xmi/echobase.zargo b/echobase-domain/src/main/xmi/echobase.zargo index 43367ab..9b4f1f6 100644 Binary files a/echobase-domain/src/main/xmi/echobase.zargo and b/echobase-domain/src/main/xmi/echobase.zargo differ -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit bc5e510b7dac443a765e966334eca1feb8d4aac4 Author: Tony CHEMIT <chemit@codelutin.com> Date: Tue May 3 10:16:20 2016 +0200 Fin de l'optimisation des export (See #8170) --- .../echobase/entities/ImportFileIdTopiaDao.java | 57 +++++-- .../fr/ifremer/echobase/entities/ImportedCell.java | 41 +++++ .../echobase/entities/ImportedCellResult.java | 41 +++++ .../entities/ImportedSampleDataResult.java | 48 ++++++ .../services/service/UserDbPersistenceService.java | 23 ++- .../actions/ImportDataActionSupport.java | 6 - .../actions/VoyageAcousticsImportAction.java | 83 +++------ .../VoyageCatchesBiometrySampleImportAction.java | 7 +- .../VoyageCatchesSubSampleImportAction.java | 188 ++------------------- .../VoyageCatchesTotalSampleImportAction.java | 79 ++++----- .../actions/VoyageCommonsTransectImportAction.java | 6 +- .../actions/VoyageCommonsTransitImportAction.java | 6 +- .../actions/VoyageCommonsVoyageImportAction.java | 8 +- .../VoyageResultsCellImportDataActionSupport.java | 59 +++---- .../csv/VoyageCatchesBiometrySampleImportRow.java | 8 +- .../csv/VoyageCatchesTotalSampleImportRow.java | 11 +- ...VoyageResultsRegionCell_AllImportServiceIT.java | 108 ++++++++++++ 17 files changed, 415 insertions(+), 364 deletions(-) diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java index 5555d79..12e998e 100644 --- a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java @@ -1,11 +1,11 @@ package fr.ifremer.echobase.entities; import com.google.common.collect.ImmutableMap; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Result; +import fr.ifremer.echobase.entities.data.SampleData; import org.nuiton.topia.persistence.HqlAndParametersBuilder; import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.util.pagination.PaginationResult; - -import java.util.Map; public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFileId> { @@ -27,24 +27,51 @@ public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFil } + public long getImportFileIdsCountForImportFile(ImportFile importFile) { + return forImportFileEquals(importFile).count(); + } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { - HqlAndParametersBuilder<ImportFileId> builder = newHqlAndParametersBuilder(); - builder.setSelectClause(" SELECT entityId"); - builder.addEquals(ImportFileId.PROPERTY_IMPORT_FILE, importFile); - builder.addWhereClause(ImportFileId.PROPERTY_ENTITY_ID + " LIKE :" + ImportFileId.PROPERTY_ENTITY_ID, ImmutableMap.<String, Object>builder().put(ImportFileId.PROPERTY_ENTITY_ID, entityType.getName() + "#%").build()); - builder.setOrderByArguments(ImportFileId.PROPERTY_IMPORT_ORDER); + return findAllLazy(importFile, entityType, "e"); + + } - String idHql = builder.getHql(); - Map<String, Object> hqlParameters = builder.getHqlParameters(); - PaginationResult<ImportFileId> pager = initPagination(idHql, hqlParameters, batchSize); - String hql = "From " + entityType.getName() + " Where topiaId IN (" + idHql + ")"; - return findAllLazy(hql, hqlParameters, pager); + public Iterable<ImportedCellResult> getImportedCellResults(ImportFile importFile) { + + return findAllLazy(importFile, Result.class, "new fr.ifremer.echobase.entities.ImportedCellResult(i.lineNumber, e.cell, e.category, e)"); } - public long getImportFileIdsCountForImportFile(ImportFile importFile) { - return forImportFileEquals(importFile).count(); + + public Iterable<ImportedSampleDataResult> getImportedSampleDataResults(ImportFile importFile) { + + return findAllLazy(importFile, SampleData.class, "new fr.ifremer.echobase.entities.ImportedSampleDataResult(i.lineNumber, e.sample.operation, e.sample, e.sample.speciesCategory, e)"); + + } + + public Iterable<ImportedCell> getImportedAcousticCells(ImportFile importFile) { + + return findAllLazy(importFile, Cell.class, "new fr.ifremer.echobase.entities.ImportedCell(i.lineNumber, e)"); + + } + + protected <T, E extends TopiaEntity> Iterable<T> findAllLazy(ImportFile importFile, Class<E> entityType, String constructor) { + + String hql + = " Select " + constructor + " From " + entityType.getName() + " As e, " + ImportFileId.class.getName() + " As i" + + " Where e.id = i." + ImportFileId.PROPERTY_ENTITY_ID + + " And i." + ImportFileId.PROPERTY_IMPORT_FILE + " = :" + ImportFileId.PROPERTY_IMPORT_FILE + + " And i." + ImportFileId.PROPERTY_ENTITY_ID + " Like :" + ImportFileId.PROPERTY_ENTITY_ID + + " Order By i." + ImportFileId.PROPERTY_IMPORT_ORDER; + + ImmutableMap<String, Object> hqlParameters = ImmutableMap.<String, Object>builder() + .put(ImportFileId.PROPERTY_IMPORT_FILE, importFile) + .put(ImportFileId.PROPERTY_ENTITY_ID, entityType.getName() + "#%") + .build(); + + return findAllLazy(hql, hqlParameters, 10000); + } } diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCell.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCell.java new file mode 100644 index 0000000..1d084f3 --- /dev/null +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCell.java @@ -0,0 +1,41 @@ +package fr.ifremer.echobase.entities; + +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.DataAcquisition; +import fr.ifremer.echobase.entities.data.DataProcessing; + +/** + * Created on 02/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportedCell { + + private final int lineNumber; + private final DataProcessing dataProcessing; + private final DataAcquisition dataAcquisition; + private final Cell cell; + + public ImportedCell(int lineNumber, Cell cell) { + this.lineNumber = lineNumber; + this.dataProcessing = cell.getDataProcessing(); + this.dataAcquisition = null == dataProcessing ? null : dataProcessing.getDataAcquisition(); + this.cell = cell; + } + + public int getLineNumber() { + return lineNumber; + } + + public DataProcessing getDataProcessing() { + return dataProcessing; + } + + public DataAcquisition getDataAcquisition() { + return dataAcquisition; + } + + public Cell getCell() { + return cell; + } +} diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCellResult.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCellResult.java new file mode 100644 index 0000000..fc8618c --- /dev/null +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCellResult.java @@ -0,0 +1,41 @@ +package fr.ifremer.echobase.entities; + +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Result; + +/** + * Created on 02/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportedCellResult { + + private final int lineNumber; + private final Cell cell; + private final Category category; + private final Result result; + + public ImportedCellResult(int lineNumber, Cell cell, Category category, Result result) { + this.lineNumber = lineNumber; + this.cell = cell; + this.category = category; + this.result = result; + } + + public int getLineNumber() { + return lineNumber; + } + + public Cell getCell() { + return cell; + } + + public Category getCategory() { + return category; + } + + public Result getResult() { + return result; + } +} diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedSampleDataResult.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedSampleDataResult.java new file mode 100644 index 0000000..0465e59 --- /dev/null +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedSampleDataResult.java @@ -0,0 +1,48 @@ +package fr.ifremer.echobase.entities; + +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.entities.references.SpeciesCategory; + +/** + * Created on 02/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportedSampleDataResult { + + private final int lineNumber; + private final Operation operation; + private final Sample sample; + private final SampleData sampleData; + private final SpeciesCategory speciesCategory; + + public ImportedSampleDataResult(int lineNumber, Operation operation, Sample sample,SpeciesCategory speciesCategory, SampleData sampleData) { + this.lineNumber = lineNumber; + this.operation = operation; + this.sample = sample; + this.speciesCategory = speciesCategory; + this.sampleData = sampleData; + } + + public int getLineNumber() { + return lineNumber; + } + + public Operation getOperation() { + return operation; + } + + public Sample getSample() { + return sample; + } + + public SpeciesCategory getSpeciesCategory() { + return speciesCategory; + } + + public SampleData getSampleData() { + return sampleData; + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java index 7f49194..90524e7 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java @@ -37,6 +37,9 @@ import fr.ifremer.echobase.entities.ImportFileIdTopiaDao; import fr.ifremer.echobase.entities.ImportLog; import fr.ifremer.echobase.entities.ImportLogTopiaDao; import fr.ifremer.echobase.entities.ImportType; +import fr.ifremer.echobase.entities.ImportedCell; +import fr.ifremer.echobase.entities.ImportedCellResult; +import fr.ifremer.echobase.entities.ImportedSampleDataResult; import fr.ifremer.echobase.entities.data.Category; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.CellTopiaDao; @@ -213,7 +216,8 @@ public class UserDbPersistenceService extends EchoBaseServiceSupport { public Cell getCell(String id) { return persistenceContext.getCellDao().forTopiaIdEquals(id).findUnique(); } - public Optional<Cell >getOptionalCell(String id) { + + public Optional<Cell> getOptionalCell(String id) { return persistenceContext.getCellDao().forTopiaIdEquals(id).tryFindUnique(); } @@ -1029,13 +1033,28 @@ public class UserDbPersistenceService extends EchoBaseServiceSupport { } - public <E extends TopiaEntity> Iterable<ImportFileId> getImportFileIdsForImportfileAndType(ImportFile importFile, Class<E> entityType) { + public <E extends TopiaEntity> Iterable<ImportFileId> getImportFileIdsForImportFileAndType(ImportFile importFile, Class<E> entityType) { ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); return importFileIdDao.getImportFileIdsForImportFileAndType(importFile, entityType); } + public Iterable<ImportedCell> getImportedAcousticCells(ImportFile importFile) { + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedAcousticCells(importFile); + } + + public Iterable<ImportedCellResult> getImportedCellResults(ImportFile importFile) { + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedCellResults(importFile); + } + + public Iterable<ImportedSampleDataResult> getImportedSampleDataResults(ImportFile importFile) { + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedSampleDataResults(importFile); + } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java index 9c225db..bab2269 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java @@ -401,12 +401,6 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS return persistenceService.getImportFileIdsForImportFile(result.getImportFile()); } - protected <EE extends TopiaEntity> Iterable<ImportFileId> filterIds(Class<EE> type, ImportDataFileResult result) { - - return persistenceService.getImportFileIdsForImportfileAndType(result.getImportFile(), type); - - } - private void flushProcessedExport(ImportDataFileResult result) { try { diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java index 5911ec1..5c3a5f4 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java @@ -6,7 +6,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; +import fr.ifremer.echobase.entities.ImportedCell; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Data; import fr.ifremer.echobase.entities.data.DataAcquisition; @@ -48,7 +48,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -63,8 +62,6 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction private final DateFormat cellDateFormat; private final DataMetadataProvider dataMetadataProvider; - private final Map<String, String> esduCellIdToDataProcessingId = new TreeMap<>(); - private final Map<String, String> dataProcessingIdToDataAcquisitionId = new TreeMap<>(); private String lastEsduId; @@ -284,10 +281,6 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction // add it to data processing dataProcessing.addCell(esduCell); - esduCellIdToDataProcessingId.put(esduCell.getTopiaId(), dataProcessing.getTopiaId()); - - dataProcessingIdToDataAcquisitionId.put(dataProcessing.getTopiaId(), dataAcquisition.getTopiaId()); - // create esdu cell data createEsduCellData(esduCell, row, dataQuality, result, rowNumber); @@ -337,56 +330,31 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction List<Cell> elementaryCells = new LinkedList<>(); - for (ImportFileId importedFileId : persistenceService.getImportFileIdsForImportFile(result.getImportFile())) { + for (ImportedCell importedCell : persistenceService.getImportedAcousticCells(result.getImportFile())) { - String importedId = importedFileId.getEntityId(); - - if (log.isTraceEnabled()) { - log.trace("Imported id: " + importedId); + if (log.isInfoEnabled()) { + log.info("(Line :" + importedCell.getLineNumber() + " - Imported id: " + importedCell.getCell()); } - if (importedId.startsWith(DataAcquisition.class.getName() + "#")) { + Cell cell = importedCell.getCell(); + + if (esduCellType.equals(cell.getCellType())) { if (log.isDebugEnabled()) { - log.debug("DataAcquisition created: " + importedId); + log.debug("Esdu Cell created: " + cell); } - continue; - } + // flush previous esdu cell + flushImportedEsduCell(result, processingTemplate, importedCell.getDataAcquisition(), importedCell.getDataProcessing(), cell, elementaryCells); + elementaryCells.clear(); - if (importedId.startsWith(DataProcessing.class.getName() + "#")) { + } else { if (log.isDebugEnabled()) { - log.debug("DataProcessing created: " + importedId); + log.debug("Elementary Cell created: " + cell); } - continue; - } - - if (importedId.startsWith(Cell.class.getName() + "#")) { - - - Cell cell = persistenceService.getCell(importedId); - - if (esduCellType.equals(cell.getCellType())) { - - if (log.isDebugEnabled()) { - log.debug("Esdu Cell created: " + importedId); - } - - // flush previous esdu cell - flushImportedEsduCell(result, processingTemplate, cell, elementaryCells); - elementaryCells.clear(); - - } else { - - if (log.isDebugEnabled()) { - log.debug("Elementary Cell created: " + importedId); - } - - elementaryCells.add(cell); - - } + elementaryCells.add(cell); } @@ -405,21 +373,12 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction } } - private void flushImportedEsduCell(ImportDataFileResult result, String processingTemplate, Cell esduCell, List<Cell> elementaryCells) { + private void flushImportedEsduCell(ImportDataFileResult result, String processingTemplate, DataAcquisition dataAcquisition, DataProcessing dataProcessing, Cell esduCell, List<Cell> elementaryCells) { - Preconditions.checkArgument(!elementaryCells.isEmpty(), "cant have esdu with ni elementary cells"); + Preconditions.checkArgument(!elementaryCells.isEmpty(), "cant have esdu with no elementary cells"); - String dataProcessingId = esduCellIdToDataProcessingId.get(esduCell.getTopiaId()); - Preconditions.checkNotNull(dataProcessingId); - - DataProcessing dataProcessing = persistenceService.getDataProcessing(dataProcessingId); - Preconditions.checkNotNull(dataProcessing); - - String dataAcquisitionId = dataProcessingIdToDataAcquisitionId.get(dataProcessingId); - Preconditions.checkNotNull(dataAcquisitionId); - - DataAcquisition dataAcquisition = persistenceService.getDataAcquisition(dataAcquisitionId); Preconditions.checkNotNull(dataAcquisition); + Preconditions.checkNotNull(dataProcessing); VoyageAcousticImportRow esduRow = VoyageAcousticImportRow.ofEsduCell(processingTemplate, dataAcquisition, dataProcessing, esduCell); @@ -504,7 +463,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction } { Data data = elementaryCellDataByMeta.get(dataMetadataProvider.getDepthRefSurfaceStartDataMetadata()); - Preconditions.checkState(data==null,"Elementary cell can't use depthRefSurface dataMeta"); + Preconditions.checkState(data == null, "Elementary cell can't use depthRefSurface dataMeta"); } { Data data = elementaryCellDataByMeta.get(dataMetadataProvider.getDepthRefEndDataMetadata(true)); @@ -832,7 +791,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction private final DataMetadata numberOfSamplesEchoIntegratedDataMetadata; private final DataMetadata depthRefSurfaceStartDataMetadata; private final DataMetadata depthRefSurfaceEndDataMetadata; - private final DataMetadata depthRefBottomStartDataMetadata; +// private final DataMetadata depthRefBottomStartDataMetadata; private final DataMetadata depthRefBottomEndDataMetadata; public DataMetadataProvider(VoyageAcousticsImportDataContext importDataContext) { @@ -850,7 +809,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction numberOfSamplesEchoIntegratedDataMetadata = dataMetadatasByName.get("NumberOfSamplesEchoIntegrated"); depthRefSurfaceStartDataMetadata = dataMetadatasByName.get("DepthRefSurface" + suffix); depthRefSurfaceEndDataMetadata = dataMetadatasByName.get("DepthRefSurfaceEnd"); - depthRefBottomStartDataMetadata = dataMetadatasByName.get("DepthRefBottomStart"); +// depthRefBottomStartDataMetadata = dataMetadatasByName.get("DepthRefBottomStart"); depthRefBottomEndDataMetadata = dataMetadatasByName.get("DepthRefBottomEnd"); } @@ -895,7 +854,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction } public DataMetadata getDepthRefSurfaceStartDataMetadata() { - return depthRefSurfaceStartDataMetadata ; + return depthRefSurfaceStartDataMetadata; } public DataMetadata getDepthRefEndDataMetadata(boolean surface) { diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java index bcd88eb..dd6e0b3 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java @@ -2,6 +2,7 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.ImportedSampleDataResult; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -120,8 +121,9 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport SampleData sampleDataFishIndex = null; - for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { + for (ImportedSampleDataResult sampleDataRow : persistenceService.getImportedSampleDataResults(result.getImportFile())) { + SampleData sampleData = sampleDataRow.getSampleData(); if (sampleDataTypeFishIndex.equals(sampleData.getSampleDataType())) { sampleDataFishIndex = sampleData; continue; @@ -136,11 +138,12 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport Float numFishSampleDataValue = sampleDataFishIndex.getDataValue(); Preconditions.checkNotNull(numFishSampleDataValue); - VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(sampleData, numFishSampleDataValue.intValue()); + VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(sampleDataRow.getOperation(), sampleDataRow.getSpeciesCategory().getSpecies(), sampleData, numFishSampleDataValue.intValue()); addImportedRow(result, importedRow); } + } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java index 6852f5f..be33c91 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java @@ -1,6 +1,7 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; +import fr.ifremer.echobase.entities.ImportedSampleDataResult; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -67,8 +68,6 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA SampleDataType sampleDataTypeNumberAtLength1cm = importDataContext.getSampleDataTypeNumberAtLength1cm(); SampleDataType sampleDataTypeWeightAtLength = importDataContext.getSampleDataTypeWeightAtLength(); -// Set<String> createdSampleIds = new TreeSet<>(); - try (Import<VoyageCatchesSubSampleImportRow> importer = open()) { incrementsProgress(); @@ -98,17 +97,11 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA sample.setSampleType(sampleTypeSubsample); sample = addSample(operation, sample, result, rowNumber); -// createdSampleIds.add(sample.getTopiaId()); } -// // add to sample ids cache (need to compute imported export later) -// sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); - addProcessedRow(result, row); -// boolean sampleWasCreated = createdSampleIds.contains(sample.getTopiaId()); - Integer round = row.getRound(); SampleDataType dataType; @@ -125,17 +118,13 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA { //create numberAtLength data -// SampleData sampleData = addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result, true, rowNumber); -// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } if (row.getWeightAtLength() != null) { //create weightAtLength data -// SampleData sampleData = addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result, true, rowNumber); -// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } @@ -148,46 +137,48 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA @Override protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { -// String currentSampleId = null; - - List<SampleData> sampleDataIds = new LinkedList<>(); + List<SampleData> sampleDatas = new LinkedList<>(); Sample sample = null; - for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { + for (ImportedSampleDataResult sampleDataRow : persistenceService.getImportedSampleDataResults(result.getImportFile())) { + + SampleData sampleData = sampleDataRow.getSampleData(); if (log.isInfoEnabled()) { log.info("Test imported id: " + sampleData.getTopiaId()); } + Sample currentSample = sampleDataRow.getSample(); + if (sample == null) { // first row - sample = sampleData.getSample(); + sample = currentSample; } else { - if (!sample.equals(sampleData.getSample())) { + if (!sample.equals(currentSample)) { // flush sample - flushSample(importDataContext, sample, result, sampleDataIds); - sampleDataIds.clear(); + flushSample(importDataContext, sample, result, sampleDatas); - sample = sampleData.getSample(); + sample = currentSample; + sampleDatas.clear(); } } - sampleDataIds.add(sampleData); + sampleDatas.add(sampleData); } - if (!sampleDataIds.isEmpty()) { + if (!sampleDatas.isEmpty()) { // flush last sample - flushSample(importDataContext, sample, result, sampleDataIds); + flushSample(importDataContext, sample, result, sampleDatas); } @@ -256,155 +247,6 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA } -// @Override -// protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { -// -// String currentSampleId = null; -// -// Set<String> sampleDataIds = new LinkedHashSet<>(); -// -// for (ImportFileId importedFileId : getImportFileIds(result)) { -// -// String importedId = importedFileId.getEntityId(); -// if (log.isInfoEnabled()) { -// log.info("Test imported id: " + importedId); -// } -// -// if (importedId.startsWith(Sample.class.getName() + "#")) { -// -// // new sample to import -// -// if (currentSampleId == null) { -// -// // first sample to collect -// currentSampleId = importedId; -// -// } else { -// -// // flush previous sample -// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); -// -// // starts to collect new sample -// currentSampleId = importedId; -// sampleDataIds.clear(); -// -// } -// -// } else if (importedId.startsWith(SampleData.class.getName() + "#")) { -// -// // coming from an existing sample -// String sampleId = sampleDataIdToSampleId.get(importedId); -// Preconditions.checkNotNull(sampleId); -// -// if (currentSampleId == null) { -// -// // first sample to treat -// currentSampleId = sampleId; -// -// } else { -// -// if (!currentSampleId.equals(sampleId)) { -// -// // flush previous sample -// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); -// -// // starts to collect new sample -// currentSampleId = sampleId; -// sampleDataIds.clear(); -// -// } -// -// } -// -// // add this sample data to current sample to flush -// sampleDataIds.add(importedId); -// -// } else { -// throw new IllegalStateException("Cant deal with imported id: " + importedId); -// } -// -// -// } -// -// if (!sampleDataIds.isEmpty()) { -// -// // flush last sample -// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); -// -// } -// -// } -// -// protected void addImportedSample(VoyageCatchesImportDataContext importDataContext, String sampleId, ImportDataFileResult result, Set<String> sampleDataIds) { -// -// String operationId = sampleIdToOperationId.get(sampleId); -// Preconditions.checkNotNull(operationId); -// -// Operation operation = persistenceService.getOperation(operationId); -// Preconditions.checkNotNull(operation); -// -// Sample sample = persistenceService.getSample(sampleId); -// Preconditions.checkNotNull(sample); -// -// if (log.isInfoEnabled()) { -// log.info("Adding sample: " + sample + " to imported export."); -// } -// -// Set<SampleDataType> lengthSampleDataTypes = new LinkedHashSet<>(); -// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength()); -// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength05cm()); -// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength1cm()); -// -// Preconditions.checkState(sample.isSampleDataNotEmpty()); -// -// VoyageCatchesSubSampleImportRow importedRow = null; -// -// String lastLengthClass = null; -// boolean lengthDataTypeFound = false; -// for (String sampleDataId : sampleDataIds) { -// -// SampleData aSampleData = sample.getSampleDataByTopiaId(sampleDataId); -// -// String lengthClass = aSampleData.getDataLabel(); -// -// boolean isLengthDataType = lengthSampleDataTypes.contains(aSampleData.getSampleDataType()); -// -// if (lastLengthClass == null) { -// -// // first row -// lastLengthClass = lengthClass; -// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); -// lengthDataTypeFound = false; -// -// } else { -// -// if ((isLengthDataType && lengthDataTypeFound) || !lengthClass.equals(lastLengthClass)) { -// -// // changing length class or length data type already found -// addImportedRow(result, importedRow); -// -// lastLengthClass = lengthClass; -// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); -// lengthDataTypeFound = false; -// } -// } -// -// if (log.isInfoEnabled()) { -// log.info("Treated sample data: " + aSampleData); -// } -// -// addImportedSampleData(importDataContext, importedRow, aSampleData); -// -// if (isLengthDataType) { -// lengthDataTypeFound = true; -// } -// -// } -// -// addImportedRow(result, importedRow); -// -// } - protected void addImportedSampleData(VoyageCatchesImportDataContext importDataContext, VoyageCatchesSubSampleImportRow importedRow, SampleData sampleData) { SampleDataType sampleDataTypeNumberAtLength = importDataContext.getSampleDataTypeNumberAtLength(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java index 49d8098..b57c603 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java @@ -25,8 +25,6 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; import java.util.Collection; -import java.util.Map; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -38,7 +36,7 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesTotalSampleImportAction.class); - private final Map<String, String> sampleIdToSortedSampleId = new TreeMap<>(); + public VoyageCatchesTotalSampleImportAction(VoyageCatchesImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getTotalSampleFile()); @@ -61,8 +59,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat log.info("Starts import of totalSample from file " + inputFile.getFileName()); } - sampleIdToSortedSampleId.clear(); - SpeciesCategoryCache speciesCategoryCache = importDataContext.getSpeciesCategoryCache(); SampleType sampleTypeTotal = importDataContext.getSampleTypeTotal(); @@ -141,10 +137,7 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat createdSortedSample.setSampleWeight(row.getSortedWeight()); // create sorted sample - Sample sortedSample = addSample(operation, createdSortedSample, result, rowNumber); - - // add to sample ids cache (need to compte imported export later) - sampleIdToSortedSampleId.put(sample.getTopiaId(), sortedSample.getTopiaId()); + addSample(operation, createdSortedSample, result, rowNumber); } @@ -155,62 +148,62 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat @Override protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { + SampleType sampleTypeSorted = importDataContext.getSampleTypeSorted(); + SampleDataType sampleDataTypeMeanLength = importDataContext.getSampleDataTypeMeanLength(); SampleDataType sampleDataTypeMeanWeight = importDataContext.getSampleDataTypeMeanWeight(); SampleDataType sampleDataTypeNoPerKg = importDataContext.getSampleDataTypeNoPerKg(); - Collection<String> sortedSampleIds = sampleIdToSortedSampleId.values(); - for (Sample sample : getImportedEntities(Sample.class, result)) { + Sample sample = null; - String sampleId = sample.getTopiaId(); - if (sortedSampleIds.contains(sampleId)) { + for (Sample currentSample : getImportedEntities(Sample.class, result)) { - // skip sorted sample - continue; - } + boolean currentSampleIsSorted = sampleTypeSorted.equals(currentSample.getSampleType()); - if (log.isInfoEnabled()) { - log.info("Adding sample: " + sampleId + " to imported export."); - } + if (currentSampleIsSorted) { - String sortedSampleId = sampleIdToSortedSampleId.get(sampleId); - Preconditions.checkNotNull(sortedSampleId); + // flush samples - Sample sortedSample = persistenceService.getSample(sortedSampleId); - Preconditions.checkNotNull(sortedSample); + Preconditions.checkState(sample != null); + Preconditions.checkState(currentSample.isSampleDataEmpty()); - VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(sample, sortedSample); + VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(sample.getOperation(), sample.getSpeciesCategory(), sample, currentSample.getSampleWeight()); - if (sample.isSampleDataNotEmpty()) { + if (sample.isSampleDataNotEmpty()) { - Collection<SampleData> sampleData = sample.getSampleData(); - ImmutableMap<SampleDataType, SampleData> sampleDataByType = Maps.uniqueIndex(sampleData, SampleDatas.TO_SAMPLE_DATA_TYPE); + Collection<SampleData> sampleData = sample.getSampleData(); + ImmutableMap<SampleDataType, SampleData> sampleDataByType = Maps.uniqueIndex(sampleData, SampleDatas.TO_SAMPLE_DATA_TYPE); - { - SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanLength); - if (sampleData1 != null) { - importedRow.setMeanLength(sampleData1.getDataValue()); + { + SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanLength); + if (sampleData1 != null) { + importedRow.setMeanLength(sampleData1.getDataValue()); + } } - } - { - SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanWeight); - if (sampleData1 != null) { - importedRow.setMeanWeight(sampleData1.getDataValue()); + { + SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanWeight); + if (sampleData1 != null) { + importedRow.setMeanWeight(sampleData1.getDataValue()); + } } - } - { - SampleData sampleData1 = sampleDataByType.get(sampleDataTypeNoPerKg); - if (sampleData1 != null) { - importedRow.setNoPerKg(sampleData1.getDataValue()); + { + SampleData sampleData1 = sampleDataByType.get(sampleDataTypeNoPerKg); + if (sampleData1 != null) { + importedRow.setNoPerKg(sampleData1.getDataValue()); + } } } + + addImportedRow(result, importedRow); + + continue; + } - addImportedRow(result, importedRow); + sample = currentSample; } - } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java index 62b029f..fddf006 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Transect; import fr.ifremer.echobase.entities.data.Transit; import fr.ifremer.echobase.entities.data.Voyage; @@ -100,13 +99,12 @@ public class VoyageCommonsTransectImportAction extends VoyageCommonsImportDataAc String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : filterIds(Transect.class, result)) { + for (Transect transect : getImportedEntities(Transect.class, result)) { - String transectId = importFileId.getEntityId(); + String transectId = transect.getTopiaId(); if (log.isInfoEnabled()) { log.info("Adding transect: " + transectId + " to imported export."); } - Transect transect = persistenceService.getTransect(transectId); VoyageCommonsTransectImportRow importedRow = VoyageCommonsTransectImportRow.of(voyage, transect); addImportedRow(result, importedRow); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java index 29367df..2065371 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Transit; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.io.InputFile; @@ -82,12 +81,11 @@ public class VoyageCommonsTransitImportAction extends VoyageCommonsImportDataAct String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : filterIds(Transit.class, result)) { - String transitId = importFileId.getEntityId(); + for (Transit transit : getImportedEntities(Transit.class, result)) { + String transitId = transit.getTopiaId(); if (log.isInfoEnabled()) { log.info("Adding transit: " + transitId + " to imported export."); } - Transit transit = persistenceService.getTransit(transitId); VoyageCommonsTransitImportRow importedRow = VoyageCommonsTransitImportRow.of(voyage, transit); addImportedRow(result, importedRow); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java index d9f126f..619aa02 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.AreaOfOperation; import fr.ifremer.echobase.entities.references.Mission; @@ -105,15 +104,14 @@ public class VoyageCommonsVoyageImportAction extends VoyageCommonsImportDataActi @Override protected void computeImportedExport(VoyageCommonsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : filterIds(Voyage.class, result)) { + for (Voyage voyage : getImportedEntities(Voyage.class, result)) { - String voyageId = importFileId.getEntityId(); + String voyageId = voyage.getTopiaId(); if (log.isInfoEnabled()) { log.info("Adding voyage: " + voyageId + " to imported export."); } - Voyage importedRow = persistenceService.getVoyage(voyageId); - addImportedRow(result, importedRow); + addImportedRow(result, voyage); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java index df50de1..f1c2255 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java @@ -1,6 +1,7 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import fr.ifremer.echobase.entities.ImportFileId; +import com.google.common.base.Preconditions; +import fr.ifremer.echobase.entities.ImportedCellResult; import fr.ifremer.echobase.entities.data.Category; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Result; @@ -79,63 +80,45 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - List<String> cellResultIds = new LinkedList<>(); - int count = 0; - int max = result.sizeImportFileIds(); - + List<Result> cellResults = new LinkedList<>(); + Cell cell = null; + Category category = null; int currentLineNumber = -1; + for (ImportedCellResult resultRow : persistenceService.getImportedCellResults(result.getImportFile())) { - for (ImportFileId importFileId : filterIds(Result.class, result)) { - - String resultId = importFileId.getEntityId(); - if (log.isDebugEnabled()) { - log.debug("Treat imported result: " + resultId + " (" + (++count) + "/" + max + ")"); - } + int lineNumber = resultRow.getLineNumber(); if (currentLineNumber == -1) { - currentLineNumber = importFileId.getLineNumber(); - - } else if (currentLineNumber != importFileId.getLineNumber()) { + currentLineNumber = lineNumber; + cell = resultRow.getCell(); + category = resultRow.getCategory(); - // flush cell - flushCell(voyage, result, cellResultIds); + } else if (currentLineNumber != lineNumber) { - cellResultIds.clear(); + flushCell(currentLineNumber, result, voyage, cell, category, cellResults); - currentLineNumber = importFileId.getLineNumber(); + currentLineNumber = lineNumber; + cell = resultRow.getCell(); + category = resultRow.getCategory(); + cellResults.clear(); } - cellResultIds.add(resultId); + cellResults.add(resultRow.getResult()); } - flushCell(voyage, result, cellResultIds); + flushCell(currentLineNumber, result, voyage, cell, category, cellResults); } - protected void flushCell(Voyage voyage, ImportDataFileResult result, List<String> cellResultIds) { + protected void flushCell(int lineNumber, ImportDataFileResult result, Voyage voyage, Cell cell, Category category, List<Result> cellResults) { - String firstResultId = cellResultIds.get(0); - Result firstCellResult = persistenceService.getResult(firstResultId); - Cell cell = firstCellResult.getCell(); + Preconditions.checkState(!cellResults.isEmpty()); if (log.isInfoEnabled()) { - log.info("Flush " + cell + " with " + cellResultIds.size() + " result(s)."); - } - List<Result> cellResults = new LinkedList<>(); - - Category category = null; - for (Result cellResult : cell.getResult()) { - - if (cellResultIds.contains(cellResult.getTopiaId())) { - if (category == null) { - category = cellResult.getCategory(); - } - cellResults.add(cellResult); - } - + log.info("Flush " + cell + " (" + lineNumber + ") with " + cellResults.size() + " result(s) with category: " + category + "."); } E row = newImportedRow(voyage, cell, category, cellResults); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java index bb0969c..ee89f29 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java @@ -21,7 +21,6 @@ package fr.ifremer.echobase.services.service.importdata.csv; import fr.ifremer.echobase.entities.data.Operation; -import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; import fr.ifremer.echobase.entities.data.SampleDataImpl; import fr.ifremer.echobase.entities.references.SampleDataType; @@ -49,12 +48,11 @@ public class VoyageCatchesBiometrySampleImportRow { protected int numFish; - public static VoyageCatchesBiometrySampleImportRow of(SampleData sampleData, int numFish) { + public static VoyageCatchesBiometrySampleImportRow of(Operation operation, Species species, SampleData sampleData, int numFish) { - Sample sample = sampleData.getSample(); VoyageCatchesBiometrySampleImportRow row = new VoyageCatchesBiometrySampleImportRow(sampleData); - row.setOperation(sample.getOperation()); - row.setSpecies(sample.getSpeciesCategory().getSpecies()); + row.setOperation(operation); + row.setSpecies(species); row.setNumFish(numFish); return row; diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java index d6363e7..c515049 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java @@ -25,6 +25,7 @@ import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleImpl; import fr.ifremer.echobase.entities.references.SizeCategory; import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.SpeciesCategory; /** * Bean used as a row for import of {@link VoyageCatchesTotalSampleImportExportModel}. @@ -140,12 +141,12 @@ public class VoyageCatchesTotalSampleImportRow { this.sortedWeight = sortedWeight; } - public static VoyageCatchesTotalSampleImportRow of(Sample sample, Sample sortedSample) { + public static VoyageCatchesTotalSampleImportRow of(Operation operation, SpeciesCategory speciesCategory, Sample sample, Float sampleSortedWeight) { VoyageCatchesTotalSampleImportRow row = new VoyageCatchesTotalSampleImportRow(sample); - row.setOperation(sample.getOperation()); - row.setSpecies(sample.getSpeciesCategory().getSpecies()); - row.setSizeCategory(sample.getSpeciesCategory().getSizeCategory()); - row.setSortedWeight(sortedSample.getSampleWeight()); + row.setOperation(operation); + row.setSpecies(speciesCategory.getSpecies()); + row.setSizeCategory(speciesCategory.getSizeCategory()); + row.setSortedWeight(sampleSortedWeight); return row; } } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCell_AllImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCell_AllImportServiceIT.java new file mode 100644 index 0000000..f181c1c --- /dev/null +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCell_AllImportServiceIT.java @@ -0,0 +1,108 @@ +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services.service.importdata; + +import com.google.common.collect.Iterables; +import fr.ifremer.echobase.entities.ImportType; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.Result; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.services.ImportDataFixtures; +import fr.ifremer.echobase.services.service.importdata.configurations.VoyageResultsImportConfiguration; + +import java.io.IOException; + +/** + * @author Tony Chemit - chemit@codelutin.com + * @since 0.3 + */ +public class VoyageResultsRegionCell_AllImportServiceIT extends VoyageResultsImportServiceITSupport { + + public VoyageResultsRegionCell_AllImportServiceIT() { + super(3); + } + + @Override + public ImportDataFixtures getImportDataFixture() { + return ImportDataFixtures.IMPORT_DATA_ECHOBASE_CATCHES_AND_VOYAGE_RESULT; + } + + protected String[] getImportPath(String filename) { + return new String[]{"/import-data", "result", "region", filename}; + } + + @Override + protected VoyageResultsImportConfiguration createConfiguration() throws IOException { + VoyageResultsImportConfiguration configuration = super.createConfiguration(); + configuration.setImportType(ImportType.RESULT_REGION); + prepareInputFile(configuration.getRegionsFile(), getImportPath("regions.csv.gz")); + prepareInputFile(configuration.getRegionAssociationFile(), getImportPath("regionAssociations.csv.gz")); + prepareInputFile(configuration.getRegionResultFile(), getImportPath("regionResults.csv.gz")); + + addMissingEsduCells("esduName", configuration.getVoyageId(), configuration.getRegionAssociationFile()); + + fakeServiceContext.assertNbEntities(Cell.class, NB_ESDU_CELLS); + + return configuration; + } + + public static final int NB_ESDU_CELLS = 2073; +// public static final int NB_ESDU_CELLS = 22; + + @Override + protected void assertAfertImport(ImportDataResult<VoyageResultsImportConfiguration> result) throws IOException { + + int nbRegionCell = 10; + int nbRegionCellData = 238; + +// //FIXME Comprendre pourquoi toutes les cellules esdu sont dans 2 régions différentes ? + int nbUpdatedEsduCell = 2 * NB_ESDU_CELLS; +// int nbUpdatedEsduCell = NB_ESDU_CELLS; + int nbRegionCellResult = 2128/*1873*/; + + int nbCell = NB_ESDU_CELLS + nbRegionCell; + + { + ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 0); + assertCsvImportResultPerEntity(importDataFileResult, Cell.class, nbRegionCell, 0, nbCell); + assertCsvImportResultPerEntity(importDataFileResult, Data.class, nbRegionCellData, 0, nbRegionCellData); + assertCsvImportResult0(importDataFileResult, nbRegionCell + nbRegionCellData); + } + + { + ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 1); + assertCsvImportResult0(importDataFileResult, 0); + assertCsvImportResultPerEntity(importDataFileResult, Cell.class, 0, nbUpdatedEsduCell, nbCell); + } + + { + ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 2); + assertCsvImportResult0(importDataFileResult, nbRegionCellResult); + assertCsvImportResultPerEntity(importDataFileResult, Result.class, nbRegionCellResult, 0, nbRegionCellResult); + assertCsvImportResultPerEntity(importDataFileResult, Category.class, 27, 0, 27); + assertCsvImportResultPerEntity(importDataFileResult, SpeciesCategory.class, 1, 0, 457); + } + + } + +} -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 4500852eae783c33d02c3a18472d7ab3455bcacc Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed May 4 11:31:27 2016 +0200 Utilisation ToPIA 3.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dbc5be7..2e0de86 100644 --- a/pom.xml +++ b/pom.xml @@ -128,7 +128,7 @@ <!-- libraries version --> - <topiaVersion>3.1-SNAPSHOT</topiaVersion> + <topiaVersion>3.1</topiaVersion> <nuitonUtilsVersion>3.0-rc-12</nuitonUtilsVersion> <nuitonI18nVersion>3.4.1</nuitonI18nVersion> <nuitonWebVersion>1.17</nuitonWebVersion> -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 3d921412e28327e8c60d497b939c8c1fff2b4723 Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed May 4 11:34:02 2016 +0200 On doit rajouter un order by à la main + correction de la fenetre à calculer --- .../echobase/services/service/DbEditorService.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/DbEditorService.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/DbEditorService.java index 7274449..6eff2e3 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/DbEditorService.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/DbEditorService.java @@ -25,12 +25,12 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import fr.ifremer.echobase.EchoBaseTechnicalException; -import fr.ifremer.echobase.services.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.entities.AbstractEchoBaseDao; import fr.ifremer.echobase.entities.EchoBaseUser; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUserPersistenceContext; import fr.ifremer.echobase.services.EchoBaseServiceSupport; +import fr.ifremer.echobase.services.csv.EchoBaseCsvUtil; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; @@ -135,6 +135,9 @@ public class DbEditorService extends EchoBaseServiceSupport { public <E extends TopiaEntity> List<E> getEntities(TableMeta<EchoBaseUserEntityEnum> tableMeta, TopiaPagerBean pager) { + Preconditions.checkNotNull(tableMeta); + Preconditions.checkNotNull(pager); + EchoBaseUserEntityEnum entityEnum = tableMeta.getSource(); try { AbstractEchoBaseDao<E> dao = (AbstractEchoBaseDao<E>) persistenceContext.getDao(entityEnum.getContract()); @@ -143,7 +146,7 @@ public class DbEditorService extends EchoBaseServiceSupport { String hql = dao.newFromClause("e"); - if (pager != null && pager.canFilter()) { + if (pager.canFilter()) { String filterHql = pagerToHql(tableMeta, pager, paramsList); hql += " WHERE " + filterHql; } @@ -158,14 +161,14 @@ public class DbEditorService extends EchoBaseServiceSupport { Map<String, Object> hqlParameters = TopiaUtil.convertPropertiesArrayToMap(params); Long count = dao.findAny("SELECT COUNT(*) " + hql, hqlParameters); - if (pager != null) { + pager.setRecords(count); + PagerBeanUtil.computeRecordIndexesAndPagesNumber(pager); - pager.setRecords(count); - PagerBeanUtil.computeRecordIndexesAndPagesNumber(pager); + if (!hql.toLowerCase().contains("order by")) { + hql+=" Order By e.id"; } - //FIXME echatellier 20160309 use proper paginationparameter everywhere - PaginationParameter param = PaginationParameter.of(pager.getPageIndex(), pager.getPageSize()); + PaginationParameter param = PaginationParameter.of(pager.getPageIndex() - 1, pager.getPageSize()); return dao.find(hql, hqlParameters, param.getStartIndex(), param.getEndIndex()); } catch (TopiaException eee) { throw new EchoBaseTechnicalException("Could not obtain data", eee); -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 5fc7551f8266a512b1b823e99adc98a5d4e7cd1e Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed May 4 12:26:41 2016 +0200 Afficher les fichiers résultats d'import dans le tableau de bord --- .../ui/actions/workingDb/DownloadExportFile.java | 48 +++++++++++++ ...oadImportFile.java => DownloadFileSupport.java} | 9 ++- .../ui/actions/workingDb/DownloadImportFile.java | 80 ++-------------------- .../src/main/resources/config/struts-workingDb.xml | 11 +++ .../resources/i18n/echobase-ui_en_GB.properties | 1 + .../resources/i18n/echobase-ui_fr_FR.properties | 1 + .../webapp/WEB-INF/jsp/workingDb/dashboard.jsp | 70 +++++++++++++------ 7 files changed, 122 insertions(+), 98 deletions(-) diff --git a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadExportFile.java b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadExportFile.java new file mode 100644 index 0000000..7f5ccb8 --- /dev/null +++ b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadExportFile.java @@ -0,0 +1,48 @@ +package fr.ifremer.echobase.ui.actions.workingDb; + +/* + * #%L + * EchoBase :: UI + * %% + * Copyright (C) 2011 - 2015 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import fr.ifremer.echobase.entities.ImportFile; + +import java.sql.Blob; + +/** + * Created on 12/30/14. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 2.7.2 + */ +public class DownloadExportFile extends DownloadFileSupport { + + private static final long serialVersionUID = 1L; + + @Override + protected String getFilename(ImportFile importFile) { + return "check-"+importFile.getName(); + } + + @Override + protected Blob getFile(ImportFile importFile) { + return importFile.getCheckFile(); + } + +} \ No newline at end of file diff --git a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadImportFile.java b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java similarity index 91% copy from echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadImportFile.java copy to echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java index 849285c..dad8dfb 100644 --- a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadImportFile.java +++ b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java @@ -38,7 +38,7 @@ import java.util.zip.GZIPInputStream; * @author Tony Chemit - chemit@codelutin.com * @since 2.7.2 */ -public class DownloadImportFile extends EchoBaseActionSupport { +public abstract class DownloadFileSupport extends EchoBaseActionSupport { private static final long serialVersionUID = 1L; @@ -80,6 +80,9 @@ public class DownloadImportFile extends EchoBaseActionSupport { return contentType; } + protected abstract Blob getFile(ImportFile importFile); + protected abstract String getFilename(ImportFile importFile); + @Override public String execute() throws Exception { @@ -87,11 +90,11 @@ public class DownloadImportFile extends EchoBaseActionSupport { String id = URLDecoder.decode(importFileId, "UTF-8"); ImportFile importFile = userDbPersistenceService.getImportFile(id); - filename = importFile.getName(); + filename = getFilename(importFile); contentType = "application/text"; - Blob file = importFile.getFile(); + Blob file = getFile(importFile); { byte[] bytes; try (InputStream binaryStream = file.getBinaryStream()) { diff --git a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadImportFile.java b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadImportFile.java index 849285c..78b594c 100644 --- a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadImportFile.java +++ b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadImportFile.java @@ -22,15 +22,8 @@ package fr.ifremer.echobase.ui.actions.workingDb; */ import fr.ifremer.echobase.entities.ImportFile; -import fr.ifremer.echobase.services.service.UserDbPersistenceService; -import fr.ifremer.echobase.ui.actions.EchoBaseActionSupport; -import org.apache.commons.io.IOUtils; -import javax.inject.Inject; -import java.io.InputStream; -import java.net.URLDecoder; import java.sql.Blob; -import java.util.zip.GZIPInputStream; /** * Created on 12/30/14. @@ -38,79 +31,18 @@ import java.util.zip.GZIPInputStream; * @author Tony Chemit - chemit@codelutin.com * @since 2.7.2 */ -public class DownloadImportFile extends EchoBaseActionSupport { +public class DownloadImportFile extends DownloadFileSupport { private static final long serialVersionUID = 1L; - /** id of import file to download. */ - protected String importFileId; - - protected String filename; - - public void setImportFileId(String importFileId) { - this.importFileId = importFileId; - } - - public String getImportFileId() { - return importFileId; - } - - public String getFilename() { - return filename; - } - - /** Input stream of the file to download. */ - protected transient InputStream inputStream; - - public InputStream getInputStream() { - return inputStream; - } - - /** Length of the file to download. */ - protected long contentLength; - - public long getContentLength() { - return contentLength; - } - - /** Content type of the file to download. */ - protected String contentType; - - public String getContentType() { - return contentType; + @Override + protected String getFilename(ImportFile importFile) { + return importFile.getName(); } @Override - public String execute() throws Exception { - - //FIXME Find out why we need to decode id? - String id = URLDecoder.decode(importFileId, "UTF-8"); - ImportFile importFile = userDbPersistenceService.getImportFile(id); - - filename = importFile.getName(); - - contentType = "application/text"; - - Blob file = importFile.getFile(); - { - byte[] bytes; - try (InputStream binaryStream = file.getBinaryStream()) { - bytes = IOUtils.toString(new GZIPInputStream(binaryStream)).getBytes(); - } - contentLength = bytes.length; - } - - inputStream = new GZIPInputStream(file.getBinaryStream(), 65535); - - return SUCCESS; - + protected Blob getFile(ImportFile importFile) { + return importFile.getFile(); } - //------------------------------------------------------------------------// - //-- Injected objects // - //------------------------------------------------------------------------// - - @Inject - protected transient UserDbPersistenceService userDbPersistenceService; - } \ No newline at end of file diff --git a/echobase-ui/src/main/resources/config/struts-workingDb.xml b/echobase-ui/src/main/resources/config/struts-workingDb.xml index 4cb4d00..201af08 100644 --- a/echobase-ui/src/main/resources/config/struts-workingDb.xml +++ b/echobase-ui/src/main/resources/config/struts-workingDb.xml @@ -194,6 +194,17 @@ </result> </action> + <!-- Download export file --> + <action name="downloadExportFile" method="execute" + class="fr.ifremer.echobase.ui.actions.workingDb.DownloadExportFile"> + <interceptor-ref name="prepareParamsStackLogguedWithDb"/> + <result type="stream"> + <param name="contentType">${contentType}</param> + <param name="contentLength">${contentLength}</param> + <param name="contentDisposition">attachment; filename="${filename}"</param> + </result> + </action> + </package> </struts> diff --git a/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties b/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties index 1baf9b3..b05accf 100644 --- a/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties +++ b/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties @@ -411,6 +411,7 @@ echobase.title.exportCoserResult=Coser export results echobase.title.exportDb=Export a complete database echobase.title.exportDbProgress=Export database in progress echobase.title.exportDbResult=Database export results +echobase.title.exportLog.files= echobase.title.exportTable=Export a table echobase.title.importData.selectImportType=Select type of import echobase.title.importDb=Database import diff --git a/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties b/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties index f8fb5ee..7a38c9b 100644 --- a/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties +++ b/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties @@ -414,6 +414,7 @@ echobase.title.exportCoserResult=Résultats de l'export Coser echobase.title.exportDb=Export complet de la base echobase.title.exportDbProgress=Export de la base de données en cours echobase.title.exportDbResult=Résultats de l'export de la base de données +echobase.title.exportLog.files=Fichiers d'export echobase.title.exportTable=Exporter une table echobase.title.importData.selectImportType=Sélection du type d'import echobase.title.importDb=Import d'une base diff --git a/echobase-ui/src/main/webapp/WEB-INF/jsp/workingDb/dashboard.jsp b/echobase-ui/src/main/webapp/WEB-INF/jsp/workingDb/dashboard.jsp index e9a75b0..cf025c1 100644 --- a/echobase-ui/src/main/webapp/WEB-INF/jsp/workingDb/dashboard.jsp +++ b/echobase-ui/src/main/webapp/WEB-INF/jsp/workingDb/dashboard.jsp @@ -56,34 +56,45 @@ $('#importLogDate').html(json['importDate']); $('#importLogImportType').html(json['importType']); - loadImportFiles(json['importFile']); - } - }); - return false; - } + var importFiles = json['importFile']; + var importUrlPrefix = '<s:url action="downloadImportFile" namespace="/workingDb"/>'; - function loadImportFiles(importFiles) { - var importUrlPrefix ='<s:url action="downloadImportFile" namespace="/workingDb"/>'; + var importResult=""; + $(importFiles).each(function () { + var importFile = this; + var importFileId = importFile['topiaId']; + var filename = importFile['name']; + var importUrl = encodeURI($.prepareUrl(importUrlPrefix , {importFileId:importFileId})); - var result=""; - $(importFiles).each(function () { - var importFile = this; - var importFileId = importFile['topiaId']; - var filename = importFile['name']; - var importUrl = encodeURI($.prepareUrl(importUrlPrefix , {importFileId:importFileId})); + importResult += "<li><a href='"+importUrl+"' class='fontsize11'>" + filename + "</a></li>"; + }); - result += "<li><a href='"+importUrl+"' class='fontsize11'>" + filename + "</a></li>"; - }); + if (importResult == "") { + importResult = "<s:text name='echobase.message.noImportFilesFound'/>"; + } + $('#importLogFiles').html(importResult); - if (result == "") { - result = "<s:text name='echobase.message.noImportFilesFound'/>"; - } - $('#importLogFiles').html(result); - } + var exportResult = ""; + var exportUrlPrefix = '<s:url action="downloadExportFile" namespace="/workingDb"/>'; + $(importFiles).each(function () { + var exportFile = this; + var importFileId = exportFile['topiaId']; + var filename = 'check-' + exportFile['name']; + var importUrl = encodeURI($.prepareUrl(exportUrlPrefix, {importFileId: importFileId})); - function downloadImportFile(importFileId) { + exportResult += "<li><a href='" + importUrl + "' class='fontsize11'>" + filename + "</a></li>"; + }); + if (exportResult == "") { + exportResult = "<s:text name='echobase.message.noImportFilesFound'/>"; + } + $('#exportLogFiles').html(exportResult); + + } + }); + return false; } + function loadVoyageDetail(voyageId) { $.ajax( { @@ -265,3 +276,20 @@ </fieldset> + +<fieldset id="exportLogFilesFieldset"> + <legend><s:text name="echobase.title.exportLog.files"/></legend> + + <div class="noImportLogDetail"> + <pre> + <s:text name="echobase.message.noImportLogSelected"/> + </pre> + </div> + <div class="importLogDetail"> + + <ul id='exportLogFiles' style="font-weight: bold;"></ul> + + </div> + + +</fieldset> -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 3495e8a1d06126c5c68c3966252a35cd97d96e50 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 9 12:27:59 2016 +0200 Bien vérifier que les fichiers importés sont correctes (See #8170) --- .../fr/ifremer/echobase/io/EchoBaseIOUtil.java | 82 ++++++++++++++++++++++ .../service/importdata/CheckFileException.java | 32 +++++++++ .../service/importdata/ImportException.java | 26 +++++++ .../actions/ImportDataActionSupport.java | 36 +++++++++- .../ui/actions/importData/DownloadInputFile.java | 73 +++++++++++++++++++ .../ui/actions/workingDb/DownloadFileSupport.java | 17 ++--- .../main/resources/config/struts-importData.xml | 11 +++ .../resources/i18n/echobase-ui_en_GB.properties | 4 ++ .../resources/i18n/echobase-ui_fr_FR.properties | 4 ++ .../jsp/importData/importDataActionResult.jsp | 62 ++++++++++++++++ .../WEB-INF/jsp/importData/resultCommonImport.jsp | 2 +- 11 files changed, 334 insertions(+), 15 deletions(-) diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java b/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java index 9f1c4ea..3c7c006 100644 --- a/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java @@ -29,8 +29,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.util.GZUtil; import org.nuiton.util.ZipUtil; +import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -41,9 +43,14 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; +import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Blob; +import java.sql.SQLException; import java.util.Collection; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -281,6 +288,81 @@ public class EchoBaseIOUtil { } } + public static File copyAndGzipFileToDirectory(File sourceFile, File targetDirectory) throws IOException { + + try (InputStream inputStream = java.nio.file.Files.newInputStream(sourceFile.toPath())) { + + FileUtils.forceMkdir(targetDirectory); + File targetFile = new File(targetDirectory, sourceFile.getName()); + + try (OutputStream outputStream = new GZIPOutputStream(java.nio.file.Files.newOutputStream(targetFile.toPath()))) { + IOUtils.copy(inputStream, outputStream); + } + + return targetFile; + + } + + } + + public static long fileLength(File sourceFile) throws IOException { + + boolean gzip; + + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(sourceFile))) { + gzip = GZUtil.isGzipStream(inputStream); + } + + long fileLength; + + if (gzip) { + + try (GZIPInputStream inputStream = new GZIPInputStream(new FileInputStream(sourceFile))) { + + byte[] bytes = IOUtils.toString(inputStream, StandardCharsets.UTF_8).getBytes(); + fileLength = new String(bytes).length(); + + } + + } else { + + fileLength = sourceFile.length(); + + } + + return fileLength; + + } + + public static long blobLength(Blob sourceFile) throws IOException, SQLException { + + boolean gzip; + + try (InputStream inputStream = new BufferedInputStream(sourceFile.getBinaryStream())) { + gzip = GZUtil.isGzipStream(inputStream); + } + + long fileLength; + + if (gzip) { + + try (GZIPInputStream inputStream = new GZIPInputStream(sourceFile.getBinaryStream())) { + + byte[] bytes = IOUtils.toString(inputStream, StandardCharsets.UTF_8).getBytes(); + fileLength = new String(bytes).length(); + + } + + } else { + + fileLength = sourceFile.length(); + + } + + return fileLength; + + } + protected EchoBaseIOUtil() { // no instance of helper class } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/CheckFileException.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/CheckFileException.java new file mode 100644 index 0000000..310281d --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/CheckFileException.java @@ -0,0 +1,32 @@ +package fr.ifremer.echobase.services.service.importdata; + +/** + * Created on 07/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class CheckFileException extends RuntimeException { + + private final String importFilename; + private final String expectedFileSuffix; + private final String actualFileSuffix; + + public CheckFileException(String message, String importFilename, String expectedFileSuffix, String actualFileSuffix) { + super(message); + this.importFilename = importFilename; + this.expectedFileSuffix = expectedFileSuffix; + this.actualFileSuffix = actualFileSuffix; + } + + public String getImportFilename() { + return importFilename; + } + + public String getExpectedFileSuffix() { + return expectedFileSuffix; + } + + public String getActualFileSuffix() { + return actualFileSuffix; + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java index 2b53f47..e9e1aa3 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java @@ -47,4 +47,30 @@ public class ImportException extends Exception { public ImportException(Locale locale, InputFile inputfile, Exception importError) { this(l(locale, "echobase.importError.fromFile", inputfile.getFile().getName(), importError.getMessage()), importError); } + + public boolean isCheckImportError() { + Throwable cause = getCause(); + return cause != null && cause instanceof CheckFileException; + } + + public String getImportFilename() { + return isCheckImportError() ? ((CheckFileException) getCause()).getImportFilename() : null; + } + + public String getExpectedFileName() { + return isCheckImportError() ? "expected-" + ((CheckFileException) getCause()).getImportFilename() : null; + } + + public String getActualFileName() { + return isCheckImportError() ? "actual-" + ((CheckFileException) getCause()).getImportFilename() : null; + } + + public String getExpectedFileSuffix() { + return isCheckImportError() ? ((CheckFileException) getCause()).getExpectedFileSuffix() : null; + } + + public String getActualFileSuffix() { + return isCheckImportError() ? ((CheckFileException) getCause()).getActualFileSuffix() : null; + } + } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java index bab2269..8811fdd 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java @@ -9,13 +9,16 @@ import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Data; import fr.ifremer.echobase.entities.references.DataMetadata; import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.io.EchoBaseIOUtil; import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.csv.BatchCsvExport; import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CheckFileException; import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; import fr.ifremer.echobase.services.service.importdata.ImportException; import fr.ifremer.echobase.services.service.importdata.configurations.ImportDataConfigurationSupport; import fr.ifremer.echobase.services.service.importdata.contexts.ImportDataContextSupport; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -27,6 +30,7 @@ import org.nuiton.topia.persistence.TopiaEntity; import java.io.BufferedWriter; import java.io.Closeable; +import java.io.File; import java.io.IOException; import java.io.Reader; import java.nio.charset.StandardCharsets; @@ -186,7 +190,11 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS flushImportedExport(result); } - checkImport(result.getProcessedImportFile(), result.getImportedExportFile()); + try { + checkImport(result, result.getProcessedImportFile(), result.getImportedExportFile()); + } catch (Exception e) { + throw new ImportException(importDataContext.getLocale(), inputFile, e); + } flushResult(result); @@ -305,11 +313,35 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS return csvImport = Import.newImport(importModel, reader); } - protected void checkImport(InputFile processedExportFile, InputFile importedExportFile) { + protected void checkImport(ImportDataFileResult result, InputFile processedExportFile, InputFile importedExportFile) { //TODO checkt that processed and imported file are the same if (log.isInfoEnabled()) { log.info("Check processed export file: " + processedExportFile + " vs imported export file: " + importedExportFile); } + try { + File dataDirectory = persistenceService.getConfiguration().getDataDirectory(); + String expectedContent = new String(Files.readAllBytes(processedExportFile.getFile().toPath()), StandardCharsets.UTF_8); + String actualContent = new String(Files.readAllBytes(importedExportFile.getFile().toPath()), StandardCharsets.UTF_8); + if (!expectedContent.equals(actualContent)) { + + // On recopie les fichiers pour que l'utilisateur puisse les télécharger... + File workingDirectory = new File(dataDirectory, "import-check-" + System.nanoTime()); + FileUtils.forceMkdir(workingDirectory); + + File targetProcessedExportFile = EchoBaseIOUtil.copyAndGzipFileToDirectory(processedExportFile.getFile(), workingDirectory); + File targetImportedExportFile = EchoBaseIOUtil.copyAndGzipFileToDirectory(importedExportFile.getFile(), workingDirectory); + + int directoryPrefix = dataDirectory.getAbsolutePath().length() + 1; + String targetProcessedExportFileSuffix = targetProcessedExportFile.getAbsolutePath().substring(directoryPrefix); + String targetImportedExportFileSuffix = targetImportedExportFile.getAbsolutePath().substring(directoryPrefix); + + throw new CheckFileException("Check file for file: " + importedExportFile.getFileName() + " failed, contents are not the same.", result.getImportFile().getName(), targetProcessedExportFileSuffix, targetImportedExportFileSuffix); + + } + } catch (IOException e) { + throw new EchoBaseTechnicalException("Could not read files", e); + } + } protected final Locale getLocale() { diff --git a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importData/DownloadInputFile.java b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importData/DownloadInputFile.java new file mode 100644 index 0000000..dc64eb3 --- /dev/null +++ b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importData/DownloadInputFile.java @@ -0,0 +1,73 @@ +package fr.ifremer.echobase.ui.actions.importData; + +import fr.ifremer.echobase.io.EchoBaseIOUtil; +import fr.ifremer.echobase.ui.actions.EchoBaseActionSupport; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; + +/** + * Created on 08/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class DownloadInputFile extends EchoBaseActionSupport { + + private static final long serialVersionUID = 1L; + + /** Prefix of input file to download. */ + protected String fileSuffix; + + /** Input stream of the file to download. */ + protected transient InputStream inputStream; + + /** Length of the file to download. */ + protected long contentLength; + + /** Content type of the file to download. */ + protected String contentType; + + private String filename; + + public void setFileSuffix(String fileSuffix) { + this.fileSuffix = fileSuffix; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public String getFilename() { + return filename; + } + + public InputStream getInputStream() { + return inputStream; + } + + public long getContentLength() { + return contentLength; + } + + public String getContentType() { + return contentType; + } + + @Override + public String execute() throws Exception { + + File dataDirectory = getEchoBaseApplicationContext().getConfiguration().getDataDirectory(); + File file = new File(dataDirectory, fileSuffix); + + contentType = "text/csv"; + contentLength = EchoBaseIOUtil.fileLength(file); + + inputStream = new GZIPInputStream(new FileInputStream(file), 65535); + + return SUCCESS; + + } + +} diff --git a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java index dad8dfb..0151552 100644 --- a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java +++ b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java @@ -22,9 +22,9 @@ package fr.ifremer.echobase.ui.actions.workingDb; */ import fr.ifremer.echobase.entities.ImportFile; +import fr.ifremer.echobase.io.EchoBaseIOUtil; import fr.ifremer.echobase.services.service.UserDbPersistenceService; import fr.ifremer.echobase.ui.actions.EchoBaseActionSupport; -import org.apache.commons.io.IOUtils; import javax.inject.Inject; import java.io.InputStream; @@ -92,18 +92,11 @@ public abstract class DownloadFileSupport extends EchoBaseActionSupport { filename = getFilename(importFile); - contentType = "application/text"; + contentType = "text/csv"; + Blob blob = getFile(importFile); + contentLength = EchoBaseIOUtil.blobLength(blob); - Blob file = getFile(importFile); - { - byte[] bytes; - try (InputStream binaryStream = file.getBinaryStream()) { - bytes = IOUtils.toString(new GZIPInputStream(binaryStream)).getBytes(); - } - contentLength = bytes.length; - } - - inputStream = new GZIPInputStream(file.getBinaryStream(), 65535); + inputStream = new GZIPInputStream(blob.getBinaryStream(), 65535); return SUCCESS; diff --git a/echobase-ui/src/main/resources/config/struts-importData.xml b/echobase-ui/src/main/resources/config/struts-importData.xml index a383702..67c1dc5 100644 --- a/echobase-ui/src/main/resources/config/struts-importData.xml +++ b/echobase-ui/src/main/resources/config/struts-importData.xml @@ -68,6 +68,17 @@ <result>/WEB-INF/jsp/importData/result{1}Import.jsp</result> </action> + <!-- Download checked file file --> + <action name="downloadCheckedFile" method="execute" + class="fr.ifremer.echobase.ui.actions.importData.DownloadInputFile"> + <interceptor-ref name="prepareParamsStackLogguedWithDb"/> + <result type="stream"> + <param name="contentType">${contentType}</param> + <param name="contentLength">${contentLength}</param> + <param name="contentDisposition">attachment; filename="${filename}"</param> + </result> + </action> + <!-- Get all vessels used by any transects of a voyage --> <action name="getVesselsForVoyage" class="fr.ifremer.echobase.ui.actions.importData.GetVesselsForVoyage"> diff --git a/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties b/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties index b05accf..728b201 100644 --- a/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties +++ b/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties @@ -254,6 +254,7 @@ echobase.error.workingDbConfiguration.createOnlyOnPostgresql=Database creation i echobase.error.workingDbConfiguration.description.required=Mandatory description echobase.error.workingDbConfiguration.url.already.exists=A configuration with same url already exists echobase.error.workingDbConfiguration.url.required=Mandatory jdbc url +echobase.exported.file=Imported file echobase.header.request.result=SQL query result echobase.importDb.freeResult=Free db import was succesful in %s. echobase.importDb.referentialResult=Referential db import was succesul in %s. @@ -391,6 +392,9 @@ echobase.message.warnEmbeddedApplicationInProgress=Please do not close the windo echobase.message.warnExportInProgress=Please do not close the window to access the export file echobase.message.warnImportInProgress=Please do not close the window to access the imported file echobase.message.warnRemoveDataInProgress=Please do not close the window to access remove data result +echobase.mismatch.import.file=A file (%s) was just imported, but the check file produced is not equals to input file. +echobase.mismatch.import.file.error= +echobase.processed.file=File to import echobase.spatialView.loading=Loading data... echobase.title.confirm.deleteImportLogs=Delete data echobase.title.confirm.deleteQuery=Delete a query diff --git a/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties b/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties index 7a38c9b..01edb49 100644 --- a/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties +++ b/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties @@ -255,6 +255,7 @@ echobase.error.workingDbConfiguration.createOnlyOnPostgresql=La création d'une echobase.error.workingDbConfiguration.description.required=Description obligatoire echobase.error.workingDbConfiguration.url.already.exists=Une configuration existe déjà avec cette url echobase.error.workingDbConfiguration.url.required=Url jdbc obligatoire +echobase.exported.file=Fichier importé echobase.header.request.result=Résultat de la requête SQL echobase.importDb.freeResult=L'import de type libre s'est déroulé avec succès en %s. echobase.importDb.referentialResult=L'import de type référentiel s'est déroulé avec succès en %s. @@ -394,6 +395,9 @@ echobase.message.warnEmbeddedApplicationInProgress=Merci de ne pas fermer la fen echobase.message.warnExportInProgress=Merci de ne pas fermer la fenêtre pour pouvoir accéder aux résultats de l'export echobase.message.warnImportInProgress=Merci de ne pas fermer la fenêtre pour pouvoir accéder aux résultats de l'import echobase.message.warnRemoveDataInProgress=Merci de ne pas fermer la fenètre pour pouvoir accéder aux résultats de la suppression de l'import +echobase.mismatch.import.file=Le fichier (%s) vient d'être importé mais le fichier de validation correspondant produit n'est pas identique à ce fichier. +echobase.mismatch.import.file.error=Problème de validation de données importées. +echobase.processed.file=Fichier à importer echobase.spatialView.loading=Chargement des données... echobase.title.confirm.deleteImportLogs=Supprimer des données echobase.title.confirm.deleteQuery=Supprimer une requête diff --git a/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/importDataActionResult.jsp b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/importDataActionResult.jsp new file mode 100644 index 0000000..ae085ec --- /dev/null +++ b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/importDataActionResult.jsp @@ -0,0 +1,62 @@ +<%-- + #%L + EchoBase :: UI + %% + Copyright (C) 2011 Ifremer, Codelutin + %% + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + #L% + --%> +<%@page contentType="text/html" pageEncoding="UTF-8" %> +<%@ taglib prefix="s" uri="/struts-tags" %> + +<s:if test="error.checkImportError"> + + <fieldset> + <legend> + <s:text name="echobase.mismatch.import.file.error"/> + </legend> + + <p> + <s:text name="echobase.mismatch.import.file"> + <s:param value="%{error.importFilename}"/> + </s:text> + </p> + + <br/> + + <div> + <ul> + <li> + <s:a action="downloadCheckedFile" namespace="/importData"> + <s:param name="fileSuffix" value="%{error.expectedFileSuffix}"/> + <s:param name="filename" value="%{error.expectedFileName}"/> + <s:text name="echobase.processed.file"/> + </s:a> + </li> + <li> + <s:a action="downloadCheckedFile" namespace="/importData"> + <s:param name="fileSuffix" value="%{error.actualFileSuffix}"/> + <s:param name="filename" value="%{error.actualFileName}"/> + <s:text name="echobase.exported.file"/> + </s:a> + </li> + </ul> + </div> + + </fieldset> + +</s:if> + +<%@ include file="/WEB-INF/includes/actionResult.jsp" %> diff --git a/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp index b1b20cf..1dfe589 100644 --- a/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp +++ b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp @@ -25,4 +25,4 @@ <s:text name="echobase.common.importType.common"/> </title> -<%@ include file="/WEB-INF/includes/actionResult.jsp" %> +<%@ include file="importDataActionResult.jsp" %> -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit e9095b622ed3bad89d9b5dc5a9854d242b237f4c Merge: 4c7c2a1 3495e8a Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 9 12:30:16 2016 +0200 Livrable #8170 Merge branch 'feature/8170' into develop .../echobase/entities/ImportFileIdTopiaDao.java | 46 +++++++++ .../fr/ifremer/echobase/entities/ImportedCell.java | 41 ++++++++ .../echobase/entities/ImportedCellResult.java | 41 ++++++++ .../entities/ImportedSampleDataResult.java | 48 +++++++++ .../fr/ifremer/echobase/io/EchoBaseIOUtil.java | 82 ++++++++++++++++ echobase-domain/src/main/xmi/echobase.zargo | Bin 90472 -> 90458 bytes .../echobase/services/service/DbEditorService.java | 17 ++-- .../services/service/UserDbPersistenceService.java | 30 +++++- .../service/importdata/CheckFileException.java | 32 ++++++ .../service/importdata/ImportException.java | 26 +++++ .../actions/ImportDataActionSupport.java | 46 +++++++-- .../actions/VoyageAcousticsImportAction.java | 83 ++++------------ .../VoyageCatchesBiometrySampleImportAction.java | 34 ++----- .../VoyageCatchesSubSampleImportAction.java | 107 ++++++--------------- .../VoyageCatchesTotalSampleImportAction.java | 92 +++++++----------- .../actions/VoyageCommonsTransectImportAction.java | 15 +-- .../actions/VoyageCommonsTransitImportAction.java | 17 +--- .../actions/VoyageCommonsVoyageImportAction.java | 8 +- .../VoyageOperationsGearMetadataImportAction.java | 36 +------ .../VoyageOperationsOperationImportAction.java | 25 +---- ...ageOperationsOperationMetadataImportAction.java | 37 +------ .../VoyageResultsCellImportDataActionSupport.java | 75 ++++++--------- ...VoyageResultsRegionAssociationImportAction.java | 31 +++--- .../actions/VoyageResultsRegionImportAction.java | 41 +++----- ...oyageResultsVoyageLengthAgeKeyImportAction.java | 8 +- ...geResultsVoyageLengthWeightKeyImportAction.java | 8 +- .../csv/VoyageCatchesBiometrySampleImportRow.java | 5 +- .../csv/VoyageCatchesTotalSampleImportRow.java | 9 +- ...VoyageOperationsGearMetadataValueImportRow.java | 5 +- .../csv/VoyageOperationsOperationImportRow.java | 3 +- ...eOperationsOperationMetadataValueImportRow.java | 5 +- .../csv/VoyageResultsRegionCellImportRow.java | 3 +- .../echobase/services/EchoBaseServiceFixtures.java | 8 ++ .../VoyageCatchesAllImportServiceIT.java | 16 +-- .../VoyageResultsRegionCellImportServiceIT.java | 7 +- ...oyageResultsRegionCell_AllImportServiceIT.java} | 15 ++- .../ui/actions/importData/DownloadInputFile.java | 73 ++++++++++++++ .../ui/actions/workingDb/DownloadExportFile.java | 34 +++---- ...oadImportFile.java => DownloadFileSupport.java} | 24 ++--- .../ui/actions/workingDb/DownloadImportFile.java | 80 ++------------- .../main/resources/config/struts-importData.xml | 11 +++ .../src/main/resources/config/struts-workingDb.xml | 11 +++ .../resources/i18n/echobase-ui_en_GB.properties | 5 + .../resources/i18n/echobase-ui_fr_FR.properties | 5 + .../jsp/importData/importDataActionResult.jsp | 62 ++++++++++++ .../WEB-INF/jsp/importData/resultCommonImport.jsp | 2 +- .../webapp/WEB-INF/jsp/workingDb/dashboard.jsp | 70 ++++++++++---- pom.xml | 2 +- 48 files changed, 859 insertions(+), 622 deletions(-) -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
participants (1)
-
codelutin.com scm