This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit bc5e510b7dac443a765e966334eca1feb8d4aac4 Author: Tony CHEMIT <chemit@codelutin.com> Date: Tue May 3 10:16:20 2016 +0200 Fin de l'optimisation des export (See #8170) --- .../echobase/entities/ImportFileIdTopiaDao.java | 57 +++++-- .../fr/ifremer/echobase/entities/ImportedCell.java | 41 +++++ .../echobase/entities/ImportedCellResult.java | 41 +++++ .../entities/ImportedSampleDataResult.java | 48 ++++++ .../services/service/UserDbPersistenceService.java | 23 ++- .../actions/ImportDataActionSupport.java | 6 - .../actions/VoyageAcousticsImportAction.java | 83 +++------ .../VoyageCatchesBiometrySampleImportAction.java | 7 +- .../VoyageCatchesSubSampleImportAction.java | 188 ++------------------- .../VoyageCatchesTotalSampleImportAction.java | 79 ++++----- .../actions/VoyageCommonsTransectImportAction.java | 6 +- .../actions/VoyageCommonsTransitImportAction.java | 6 +- .../actions/VoyageCommonsVoyageImportAction.java | 8 +- .../VoyageResultsCellImportDataActionSupport.java | 59 +++---- .../csv/VoyageCatchesBiometrySampleImportRow.java | 8 +- .../csv/VoyageCatchesTotalSampleImportRow.java | 11 +- ...VoyageResultsRegionCell_AllImportServiceIT.java | 108 ++++++++++++ 17 files changed, 415 insertions(+), 364 deletions(-) diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java index 5555d79..12e998e 100644 --- a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java @@ -1,11 +1,11 @@ package fr.ifremer.echobase.entities; import com.google.common.collect.ImmutableMap; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Result; +import fr.ifremer.echobase.entities.data.SampleData; import org.nuiton.topia.persistence.HqlAndParametersBuilder; import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.util.pagination.PaginationResult; - -import java.util.Map; public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFileId> { @@ -27,24 +27,51 @@ public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFil } + public long getImportFileIdsCountForImportFile(ImportFile importFile) { + return forImportFileEquals(importFile).count(); + } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { - HqlAndParametersBuilder<ImportFileId> builder = newHqlAndParametersBuilder(); - builder.setSelectClause(" SELECT entityId"); - builder.addEquals(ImportFileId.PROPERTY_IMPORT_FILE, importFile); - builder.addWhereClause(ImportFileId.PROPERTY_ENTITY_ID + " LIKE :" + ImportFileId.PROPERTY_ENTITY_ID, ImmutableMap.<String, Object>builder().put(ImportFileId.PROPERTY_ENTITY_ID, entityType.getName() + "#%").build()); - builder.setOrderByArguments(ImportFileId.PROPERTY_IMPORT_ORDER); + return findAllLazy(importFile, entityType, "e"); + + } - String idHql = builder.getHql(); - Map<String, Object> hqlParameters = builder.getHqlParameters(); - PaginationResult<ImportFileId> pager = initPagination(idHql, hqlParameters, batchSize); - String hql = "From " + entityType.getName() + " Where topiaId IN (" + idHql + ")"; - return findAllLazy(hql, hqlParameters, pager); + public Iterable<ImportedCellResult> getImportedCellResults(ImportFile importFile) { + + return findAllLazy(importFile, Result.class, "new fr.ifremer.echobase.entities.ImportedCellResult(i.lineNumber, e.cell, e.category, e)"); } - public long getImportFileIdsCountForImportFile(ImportFile importFile) { - return forImportFileEquals(importFile).count(); + + public Iterable<ImportedSampleDataResult> getImportedSampleDataResults(ImportFile importFile) { + + return findAllLazy(importFile, SampleData.class, "new fr.ifremer.echobase.entities.ImportedSampleDataResult(i.lineNumber, e.sample.operation, e.sample, e.sample.speciesCategory, e)"); + + } + + public Iterable<ImportedCell> getImportedAcousticCells(ImportFile importFile) { + + return findAllLazy(importFile, Cell.class, "new fr.ifremer.echobase.entities.ImportedCell(i.lineNumber, e)"); + + } + + protected <T, E extends TopiaEntity> Iterable<T> findAllLazy(ImportFile importFile, Class<E> entityType, String constructor) { + + String hql + = " Select " + constructor + " From " + entityType.getName() + " As e, " + ImportFileId.class.getName() + " As i" + + " Where e.id = i." + ImportFileId.PROPERTY_ENTITY_ID + + " And i." + ImportFileId.PROPERTY_IMPORT_FILE + " = :" + ImportFileId.PROPERTY_IMPORT_FILE + + " And i." + ImportFileId.PROPERTY_ENTITY_ID + " Like :" + ImportFileId.PROPERTY_ENTITY_ID + + " Order By i." + ImportFileId.PROPERTY_IMPORT_ORDER; + + ImmutableMap<String, Object> hqlParameters = ImmutableMap.<String, Object>builder() + .put(ImportFileId.PROPERTY_IMPORT_FILE, importFile) + .put(ImportFileId.PROPERTY_ENTITY_ID, entityType.getName() + "#%") + .build(); + + return findAllLazy(hql, hqlParameters, 10000); + } } diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCell.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCell.java new file mode 100644 index 0000000..1d084f3 --- /dev/null +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCell.java @@ -0,0 +1,41 @@ +package fr.ifremer.echobase.entities; + +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.DataAcquisition; +import fr.ifremer.echobase.entities.data.DataProcessing; + +/** + * Created on 02/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportedCell { + + private final int lineNumber; + private final DataProcessing dataProcessing; + private final DataAcquisition dataAcquisition; + private final Cell cell; + + public ImportedCell(int lineNumber, Cell cell) { + this.lineNumber = lineNumber; + this.dataProcessing = cell.getDataProcessing(); + this.dataAcquisition = null == dataProcessing ? null : dataProcessing.getDataAcquisition(); + this.cell = cell; + } + + public int getLineNumber() { + return lineNumber; + } + + public DataProcessing getDataProcessing() { + return dataProcessing; + } + + public DataAcquisition getDataAcquisition() { + return dataAcquisition; + } + + public Cell getCell() { + return cell; + } +} diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCellResult.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCellResult.java new file mode 100644 index 0000000..fc8618c --- /dev/null +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedCellResult.java @@ -0,0 +1,41 @@ +package fr.ifremer.echobase.entities; + +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Result; + +/** + * Created on 02/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportedCellResult { + + private final int lineNumber; + private final Cell cell; + private final Category category; + private final Result result; + + public ImportedCellResult(int lineNumber, Cell cell, Category category, Result result) { + this.lineNumber = lineNumber; + this.cell = cell; + this.category = category; + this.result = result; + } + + public int getLineNumber() { + return lineNumber; + } + + public Cell getCell() { + return cell; + } + + public Category getCategory() { + return category; + } + + public Result getResult() { + return result; + } +} diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedSampleDataResult.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedSampleDataResult.java new file mode 100644 index 0000000..0465e59 --- /dev/null +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportedSampleDataResult.java @@ -0,0 +1,48 @@ +package fr.ifremer.echobase.entities; + +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.entities.references.SpeciesCategory; + +/** + * Created on 02/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportedSampleDataResult { + + private final int lineNumber; + private final Operation operation; + private final Sample sample; + private final SampleData sampleData; + private final SpeciesCategory speciesCategory; + + public ImportedSampleDataResult(int lineNumber, Operation operation, Sample sample,SpeciesCategory speciesCategory, SampleData sampleData) { + this.lineNumber = lineNumber; + this.operation = operation; + this.sample = sample; + this.speciesCategory = speciesCategory; + this.sampleData = sampleData; + } + + public int getLineNumber() { + return lineNumber; + } + + public Operation getOperation() { + return operation; + } + + public Sample getSample() { + return sample; + } + + public SpeciesCategory getSpeciesCategory() { + return speciesCategory; + } + + public SampleData getSampleData() { + return sampleData; + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java index 7f49194..90524e7 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java @@ -37,6 +37,9 @@ import fr.ifremer.echobase.entities.ImportFileIdTopiaDao; import fr.ifremer.echobase.entities.ImportLog; import fr.ifremer.echobase.entities.ImportLogTopiaDao; import fr.ifremer.echobase.entities.ImportType; +import fr.ifremer.echobase.entities.ImportedCell; +import fr.ifremer.echobase.entities.ImportedCellResult; +import fr.ifremer.echobase.entities.ImportedSampleDataResult; import fr.ifremer.echobase.entities.data.Category; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.CellTopiaDao; @@ -213,7 +216,8 @@ public class UserDbPersistenceService extends EchoBaseServiceSupport { public Cell getCell(String id) { return persistenceContext.getCellDao().forTopiaIdEquals(id).findUnique(); } - public Optional<Cell >getOptionalCell(String id) { + + public Optional<Cell> getOptionalCell(String id) { return persistenceContext.getCellDao().forTopiaIdEquals(id).tryFindUnique(); } @@ -1029,13 +1033,28 @@ public class UserDbPersistenceService extends EchoBaseServiceSupport { } - public <E extends TopiaEntity> Iterable<ImportFileId> getImportFileIdsForImportfileAndType(ImportFile importFile, Class<E> entityType) { + public <E extends TopiaEntity> Iterable<ImportFileId> getImportFileIdsForImportFileAndType(ImportFile importFile, Class<E> entityType) { ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); return importFileIdDao.getImportFileIdsForImportFileAndType(importFile, entityType); } + public Iterable<ImportedCell> getImportedAcousticCells(ImportFile importFile) { + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedAcousticCells(importFile); + } + + public Iterable<ImportedCellResult> getImportedCellResults(ImportFile importFile) { + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedCellResults(importFile); + } + + public Iterable<ImportedSampleDataResult> getImportedSampleDataResults(ImportFile importFile) { + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedSampleDataResults(importFile); + } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java index 9c225db..bab2269 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java @@ -401,12 +401,6 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS return persistenceService.getImportFileIdsForImportFile(result.getImportFile()); } - protected <EE extends TopiaEntity> Iterable<ImportFileId> filterIds(Class<EE> type, ImportDataFileResult result) { - - return persistenceService.getImportFileIdsForImportfileAndType(result.getImportFile(), type); - - } - private void flushProcessedExport(ImportDataFileResult result) { try { diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java index 5911ec1..5c3a5f4 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageAcousticsImportAction.java @@ -6,7 +6,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; +import fr.ifremer.echobase.entities.ImportedCell; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Data; import fr.ifremer.echobase.entities.data.DataAcquisition; @@ -48,7 +48,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -63,8 +62,6 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction private final DateFormat cellDateFormat; private final DataMetadataProvider dataMetadataProvider; - private final Map<String, String> esduCellIdToDataProcessingId = new TreeMap<>(); - private final Map<String, String> dataProcessingIdToDataAcquisitionId = new TreeMap<>(); private String lastEsduId; @@ -284,10 +281,6 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction // add it to data processing dataProcessing.addCell(esduCell); - esduCellIdToDataProcessingId.put(esduCell.getTopiaId(), dataProcessing.getTopiaId()); - - dataProcessingIdToDataAcquisitionId.put(dataProcessing.getTopiaId(), dataAcquisition.getTopiaId()); - // create esdu cell data createEsduCellData(esduCell, row, dataQuality, result, rowNumber); @@ -337,56 +330,31 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction List<Cell> elementaryCells = new LinkedList<>(); - for (ImportFileId importedFileId : persistenceService.getImportFileIdsForImportFile(result.getImportFile())) { + for (ImportedCell importedCell : persistenceService.getImportedAcousticCells(result.getImportFile())) { - String importedId = importedFileId.getEntityId(); - - if (log.isTraceEnabled()) { - log.trace("Imported id: " + importedId); + if (log.isInfoEnabled()) { + log.info("(Line :" + importedCell.getLineNumber() + " - Imported id: " + importedCell.getCell()); } - if (importedId.startsWith(DataAcquisition.class.getName() + "#")) { + Cell cell = importedCell.getCell(); + + if (esduCellType.equals(cell.getCellType())) { if (log.isDebugEnabled()) { - log.debug("DataAcquisition created: " + importedId); + log.debug("Esdu Cell created: " + cell); } - continue; - } + // flush previous esdu cell + flushImportedEsduCell(result, processingTemplate, importedCell.getDataAcquisition(), importedCell.getDataProcessing(), cell, elementaryCells); + elementaryCells.clear(); - if (importedId.startsWith(DataProcessing.class.getName() + "#")) { + } else { if (log.isDebugEnabled()) { - log.debug("DataProcessing created: " + importedId); + log.debug("Elementary Cell created: " + cell); } - continue; - } - - if (importedId.startsWith(Cell.class.getName() + "#")) { - - - Cell cell = persistenceService.getCell(importedId); - - if (esduCellType.equals(cell.getCellType())) { - - if (log.isDebugEnabled()) { - log.debug("Esdu Cell created: " + importedId); - } - - // flush previous esdu cell - flushImportedEsduCell(result, processingTemplate, cell, elementaryCells); - elementaryCells.clear(); - - } else { - - if (log.isDebugEnabled()) { - log.debug("Elementary Cell created: " + importedId); - } - - elementaryCells.add(cell); - - } + elementaryCells.add(cell); } @@ -405,21 +373,12 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction } } - private void flushImportedEsduCell(ImportDataFileResult result, String processingTemplate, Cell esduCell, List<Cell> elementaryCells) { + private void flushImportedEsduCell(ImportDataFileResult result, String processingTemplate, DataAcquisition dataAcquisition, DataProcessing dataProcessing, Cell esduCell, List<Cell> elementaryCells) { - Preconditions.checkArgument(!elementaryCells.isEmpty(), "cant have esdu with ni elementary cells"); + Preconditions.checkArgument(!elementaryCells.isEmpty(), "cant have esdu with no elementary cells"); - String dataProcessingId = esduCellIdToDataProcessingId.get(esduCell.getTopiaId()); - Preconditions.checkNotNull(dataProcessingId); - - DataProcessing dataProcessing = persistenceService.getDataProcessing(dataProcessingId); - Preconditions.checkNotNull(dataProcessing); - - String dataAcquisitionId = dataProcessingIdToDataAcquisitionId.get(dataProcessingId); - Preconditions.checkNotNull(dataAcquisitionId); - - DataAcquisition dataAcquisition = persistenceService.getDataAcquisition(dataAcquisitionId); Preconditions.checkNotNull(dataAcquisition); + Preconditions.checkNotNull(dataProcessing); VoyageAcousticImportRow esduRow = VoyageAcousticImportRow.ofEsduCell(processingTemplate, dataAcquisition, dataProcessing, esduCell); @@ -504,7 +463,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction } { Data data = elementaryCellDataByMeta.get(dataMetadataProvider.getDepthRefSurfaceStartDataMetadata()); - Preconditions.checkState(data==null,"Elementary cell can't use depthRefSurface dataMeta"); + Preconditions.checkState(data == null, "Elementary cell can't use depthRefSurface dataMeta"); } { Data data = elementaryCellDataByMeta.get(dataMetadataProvider.getDepthRefEndDataMetadata(true)); @@ -832,7 +791,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction private final DataMetadata numberOfSamplesEchoIntegratedDataMetadata; private final DataMetadata depthRefSurfaceStartDataMetadata; private final DataMetadata depthRefSurfaceEndDataMetadata; - private final DataMetadata depthRefBottomStartDataMetadata; +// private final DataMetadata depthRefBottomStartDataMetadata; private final DataMetadata depthRefBottomEndDataMetadata; public DataMetadataProvider(VoyageAcousticsImportDataContext importDataContext) { @@ -850,7 +809,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction numberOfSamplesEchoIntegratedDataMetadata = dataMetadatasByName.get("NumberOfSamplesEchoIntegrated"); depthRefSurfaceStartDataMetadata = dataMetadatasByName.get("DepthRefSurface" + suffix); depthRefSurfaceEndDataMetadata = dataMetadatasByName.get("DepthRefSurfaceEnd"); - depthRefBottomStartDataMetadata = dataMetadatasByName.get("DepthRefBottomStart"); +// depthRefBottomStartDataMetadata = dataMetadatasByName.get("DepthRefBottomStart"); depthRefBottomEndDataMetadata = dataMetadatasByName.get("DepthRefBottomEnd"); } @@ -895,7 +854,7 @@ public class VoyageAcousticsImportAction extends VoyageAcousticsImportDataAction } public DataMetadata getDepthRefSurfaceStartDataMetadata() { - return depthRefSurfaceStartDataMetadata ; + return depthRefSurfaceStartDataMetadata; } public DataMetadata getDepthRefEndDataMetadata(boolean surface) { diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java index bcd88eb..dd6e0b3 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java @@ -2,6 +2,7 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; +import fr.ifremer.echobase.entities.ImportedSampleDataResult; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -120,8 +121,9 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport SampleData sampleDataFishIndex = null; - for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { + for (ImportedSampleDataResult sampleDataRow : persistenceService.getImportedSampleDataResults(result.getImportFile())) { + SampleData sampleData = sampleDataRow.getSampleData(); if (sampleDataTypeFishIndex.equals(sampleData.getSampleDataType())) { sampleDataFishIndex = sampleData; continue; @@ -136,11 +138,12 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport Float numFishSampleDataValue = sampleDataFishIndex.getDataValue(); Preconditions.checkNotNull(numFishSampleDataValue); - VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(sampleData, numFishSampleDataValue.intValue()); + VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(sampleDataRow.getOperation(), sampleDataRow.getSpeciesCategory().getSpecies(), sampleData, numFishSampleDataValue.intValue()); addImportedRow(result, importedRow); } + } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java index 6852f5f..be33c91 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java @@ -1,6 +1,7 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; +import fr.ifremer.echobase.entities.ImportedSampleDataResult; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -67,8 +68,6 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA SampleDataType sampleDataTypeNumberAtLength1cm = importDataContext.getSampleDataTypeNumberAtLength1cm(); SampleDataType sampleDataTypeWeightAtLength = importDataContext.getSampleDataTypeWeightAtLength(); -// Set<String> createdSampleIds = new TreeSet<>(); - try (Import<VoyageCatchesSubSampleImportRow> importer = open()) { incrementsProgress(); @@ -98,17 +97,11 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA sample.setSampleType(sampleTypeSubsample); sample = addSample(operation, sample, result, rowNumber); -// createdSampleIds.add(sample.getTopiaId()); } -// // add to sample ids cache (need to compute imported export later) -// sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); - addProcessedRow(result, row); -// boolean sampleWasCreated = createdSampleIds.contains(sample.getTopiaId()); - Integer round = row.getRound(); SampleDataType dataType; @@ -125,17 +118,13 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA { //create numberAtLength data -// SampleData sampleData = addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result, true, rowNumber); -// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } if (row.getWeightAtLength() != null) { //create weightAtLength data -// SampleData sampleData = addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result, true, rowNumber); -// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } @@ -148,46 +137,48 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA @Override protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { -// String currentSampleId = null; - - List<SampleData> sampleDataIds = new LinkedList<>(); + List<SampleData> sampleDatas = new LinkedList<>(); Sample sample = null; - for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { + for (ImportedSampleDataResult sampleDataRow : persistenceService.getImportedSampleDataResults(result.getImportFile())) { + + SampleData sampleData = sampleDataRow.getSampleData(); if (log.isInfoEnabled()) { log.info("Test imported id: " + sampleData.getTopiaId()); } + Sample currentSample = sampleDataRow.getSample(); + if (sample == null) { // first row - sample = sampleData.getSample(); + sample = currentSample; } else { - if (!sample.equals(sampleData.getSample())) { + if (!sample.equals(currentSample)) { // flush sample - flushSample(importDataContext, sample, result, sampleDataIds); - sampleDataIds.clear(); + flushSample(importDataContext, sample, result, sampleDatas); - sample = sampleData.getSample(); + sample = currentSample; + sampleDatas.clear(); } } - sampleDataIds.add(sampleData); + sampleDatas.add(sampleData); } - if (!sampleDataIds.isEmpty()) { + if (!sampleDatas.isEmpty()) { // flush last sample - flushSample(importDataContext, sample, result, sampleDataIds); + flushSample(importDataContext, sample, result, sampleDatas); } @@ -256,155 +247,6 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA } -// @Override -// protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { -// -// String currentSampleId = null; -// -// Set<String> sampleDataIds = new LinkedHashSet<>(); -// -// for (ImportFileId importedFileId : getImportFileIds(result)) { -// -// String importedId = importedFileId.getEntityId(); -// if (log.isInfoEnabled()) { -// log.info("Test imported id: " + importedId); -// } -// -// if (importedId.startsWith(Sample.class.getName() + "#")) { -// -// // new sample to import -// -// if (currentSampleId == null) { -// -// // first sample to collect -// currentSampleId = importedId; -// -// } else { -// -// // flush previous sample -// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); -// -// // starts to collect new sample -// currentSampleId = importedId; -// sampleDataIds.clear(); -// -// } -// -// } else if (importedId.startsWith(SampleData.class.getName() + "#")) { -// -// // coming from an existing sample -// String sampleId = sampleDataIdToSampleId.get(importedId); -// Preconditions.checkNotNull(sampleId); -// -// if (currentSampleId == null) { -// -// // first sample to treat -// currentSampleId = sampleId; -// -// } else { -// -// if (!currentSampleId.equals(sampleId)) { -// -// // flush previous sample -// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); -// -// // starts to collect new sample -// currentSampleId = sampleId; -// sampleDataIds.clear(); -// -// } -// -// } -// -// // add this sample data to current sample to flush -// sampleDataIds.add(importedId); -// -// } else { -// throw new IllegalStateException("Cant deal with imported id: " + importedId); -// } -// -// -// } -// -// if (!sampleDataIds.isEmpty()) { -// -// // flush last sample -// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); -// -// } -// -// } -// -// protected void addImportedSample(VoyageCatchesImportDataContext importDataContext, String sampleId, ImportDataFileResult result, Set<String> sampleDataIds) { -// -// String operationId = sampleIdToOperationId.get(sampleId); -// Preconditions.checkNotNull(operationId); -// -// Operation operation = persistenceService.getOperation(operationId); -// Preconditions.checkNotNull(operation); -// -// Sample sample = persistenceService.getSample(sampleId); -// Preconditions.checkNotNull(sample); -// -// if (log.isInfoEnabled()) { -// log.info("Adding sample: " + sample + " to imported export."); -// } -// -// Set<SampleDataType> lengthSampleDataTypes = new LinkedHashSet<>(); -// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength()); -// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength05cm()); -// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength1cm()); -// -// Preconditions.checkState(sample.isSampleDataNotEmpty()); -// -// VoyageCatchesSubSampleImportRow importedRow = null; -// -// String lastLengthClass = null; -// boolean lengthDataTypeFound = false; -// for (String sampleDataId : sampleDataIds) { -// -// SampleData aSampleData = sample.getSampleDataByTopiaId(sampleDataId); -// -// String lengthClass = aSampleData.getDataLabel(); -// -// boolean isLengthDataType = lengthSampleDataTypes.contains(aSampleData.getSampleDataType()); -// -// if (lastLengthClass == null) { -// -// // first row -// lastLengthClass = lengthClass; -// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); -// lengthDataTypeFound = false; -// -// } else { -// -// if ((isLengthDataType && lengthDataTypeFound) || !lengthClass.equals(lastLengthClass)) { -// -// // changing length class or length data type already found -// addImportedRow(result, importedRow); -// -// lastLengthClass = lengthClass; -// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); -// lengthDataTypeFound = false; -// } -// } -// -// if (log.isInfoEnabled()) { -// log.info("Treated sample data: " + aSampleData); -// } -// -// addImportedSampleData(importDataContext, importedRow, aSampleData); -// -// if (isLengthDataType) { -// lengthDataTypeFound = true; -// } -// -// } -// -// addImportedRow(result, importedRow); -// -// } - protected void addImportedSampleData(VoyageCatchesImportDataContext importDataContext, VoyageCatchesSubSampleImportRow importedRow, SampleData sampleData) { SampleDataType sampleDataTypeNumberAtLength = importDataContext.getSampleDataTypeNumberAtLength(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java index 49d8098..b57c603 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java @@ -25,8 +25,6 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; import java.util.Collection; -import java.util.Map; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -38,7 +36,7 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesTotalSampleImportAction.class); - private final Map<String, String> sampleIdToSortedSampleId = new TreeMap<>(); + public VoyageCatchesTotalSampleImportAction(VoyageCatchesImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getTotalSampleFile()); @@ -61,8 +59,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat log.info("Starts import of totalSample from file " + inputFile.getFileName()); } - sampleIdToSortedSampleId.clear(); - SpeciesCategoryCache speciesCategoryCache = importDataContext.getSpeciesCategoryCache(); SampleType sampleTypeTotal = importDataContext.getSampleTypeTotal(); @@ -141,10 +137,7 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat createdSortedSample.setSampleWeight(row.getSortedWeight()); // create sorted sample - Sample sortedSample = addSample(operation, createdSortedSample, result, rowNumber); - - // add to sample ids cache (need to compte imported export later) - sampleIdToSortedSampleId.put(sample.getTopiaId(), sortedSample.getTopiaId()); + addSample(operation, createdSortedSample, result, rowNumber); } @@ -155,62 +148,62 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat @Override protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { + SampleType sampleTypeSorted = importDataContext.getSampleTypeSorted(); + SampleDataType sampleDataTypeMeanLength = importDataContext.getSampleDataTypeMeanLength(); SampleDataType sampleDataTypeMeanWeight = importDataContext.getSampleDataTypeMeanWeight(); SampleDataType sampleDataTypeNoPerKg = importDataContext.getSampleDataTypeNoPerKg(); - Collection<String> sortedSampleIds = sampleIdToSortedSampleId.values(); - for (Sample sample : getImportedEntities(Sample.class, result)) { + Sample sample = null; - String sampleId = sample.getTopiaId(); - if (sortedSampleIds.contains(sampleId)) { + for (Sample currentSample : getImportedEntities(Sample.class, result)) { - // skip sorted sample - continue; - } + boolean currentSampleIsSorted = sampleTypeSorted.equals(currentSample.getSampleType()); - if (log.isInfoEnabled()) { - log.info("Adding sample: " + sampleId + " to imported export."); - } + if (currentSampleIsSorted) { - String sortedSampleId = sampleIdToSortedSampleId.get(sampleId); - Preconditions.checkNotNull(sortedSampleId); + // flush samples - Sample sortedSample = persistenceService.getSample(sortedSampleId); - Preconditions.checkNotNull(sortedSample); + Preconditions.checkState(sample != null); + Preconditions.checkState(currentSample.isSampleDataEmpty()); - VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(sample, sortedSample); + VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(sample.getOperation(), sample.getSpeciesCategory(), sample, currentSample.getSampleWeight()); - if (sample.isSampleDataNotEmpty()) { + if (sample.isSampleDataNotEmpty()) { - Collection<SampleData> sampleData = sample.getSampleData(); - ImmutableMap<SampleDataType, SampleData> sampleDataByType = Maps.uniqueIndex(sampleData, SampleDatas.TO_SAMPLE_DATA_TYPE); + Collection<SampleData> sampleData = sample.getSampleData(); + ImmutableMap<SampleDataType, SampleData> sampleDataByType = Maps.uniqueIndex(sampleData, SampleDatas.TO_SAMPLE_DATA_TYPE); - { - SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanLength); - if (sampleData1 != null) { - importedRow.setMeanLength(sampleData1.getDataValue()); + { + SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanLength); + if (sampleData1 != null) { + importedRow.setMeanLength(sampleData1.getDataValue()); + } } - } - { - SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanWeight); - if (sampleData1 != null) { - importedRow.setMeanWeight(sampleData1.getDataValue()); + { + SampleData sampleData1 = sampleDataByType.get(sampleDataTypeMeanWeight); + if (sampleData1 != null) { + importedRow.setMeanWeight(sampleData1.getDataValue()); + } } - } - { - SampleData sampleData1 = sampleDataByType.get(sampleDataTypeNoPerKg); - if (sampleData1 != null) { - importedRow.setNoPerKg(sampleData1.getDataValue()); + { + SampleData sampleData1 = sampleDataByType.get(sampleDataTypeNoPerKg); + if (sampleData1 != null) { + importedRow.setNoPerKg(sampleData1.getDataValue()); + } } } + + addImportedRow(result, importedRow); + + continue; + } - addImportedRow(result, importedRow); + sample = currentSample; } - } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java index 62b029f..fddf006 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Transect; import fr.ifremer.echobase.entities.data.Transit; import fr.ifremer.echobase.entities.data.Voyage; @@ -100,13 +99,12 @@ public class VoyageCommonsTransectImportAction extends VoyageCommonsImportDataAc String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : filterIds(Transect.class, result)) { + for (Transect transect : getImportedEntities(Transect.class, result)) { - String transectId = importFileId.getEntityId(); + String transectId = transect.getTopiaId(); if (log.isInfoEnabled()) { log.info("Adding transect: " + transectId + " to imported export."); } - Transect transect = persistenceService.getTransect(transectId); VoyageCommonsTransectImportRow importedRow = VoyageCommonsTransectImportRow.of(voyage, transect); addImportedRow(result, importedRow); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java index 29367df..2065371 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Transit; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.io.InputFile; @@ -82,12 +81,11 @@ public class VoyageCommonsTransitImportAction extends VoyageCommonsImportDataAct String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : filterIds(Transit.class, result)) { - String transitId = importFileId.getEntityId(); + for (Transit transit : getImportedEntities(Transit.class, result)) { + String transitId = transit.getTopiaId(); if (log.isInfoEnabled()) { log.info("Adding transit: " + transitId + " to imported export."); } - Transit transit = persistenceService.getTransit(transitId); VoyageCommonsTransitImportRow importedRow = VoyageCommonsTransitImportRow.of(voyage, transit); addImportedRow(result, importedRow); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java index d9f126f..619aa02 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsVoyageImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.AreaOfOperation; import fr.ifremer.echobase.entities.references.Mission; @@ -105,15 +104,14 @@ public class VoyageCommonsVoyageImportAction extends VoyageCommonsImportDataActi @Override protected void computeImportedExport(VoyageCommonsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : filterIds(Voyage.class, result)) { + for (Voyage voyage : getImportedEntities(Voyage.class, result)) { - String voyageId = importFileId.getEntityId(); + String voyageId = voyage.getTopiaId(); if (log.isInfoEnabled()) { log.info("Adding voyage: " + voyageId + " to imported export."); } - Voyage importedRow = persistenceService.getVoyage(voyageId); - addImportedRow(result, importedRow); + addImportedRow(result, voyage); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java index df50de1..f1c2255 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java @@ -1,6 +1,7 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import fr.ifremer.echobase.entities.ImportFileId; +import com.google.common.base.Preconditions; +import fr.ifremer.echobase.entities.ImportedCellResult; import fr.ifremer.echobase.entities.data.Category; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Result; @@ -79,63 +80,45 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - List<String> cellResultIds = new LinkedList<>(); - int count = 0; - int max = result.sizeImportFileIds(); - + List<Result> cellResults = new LinkedList<>(); + Cell cell = null; + Category category = null; int currentLineNumber = -1; + for (ImportedCellResult resultRow : persistenceService.getImportedCellResults(result.getImportFile())) { - for (ImportFileId importFileId : filterIds(Result.class, result)) { - - String resultId = importFileId.getEntityId(); - if (log.isDebugEnabled()) { - log.debug("Treat imported result: " + resultId + " (" + (++count) + "/" + max + ")"); - } + int lineNumber = resultRow.getLineNumber(); if (currentLineNumber == -1) { - currentLineNumber = importFileId.getLineNumber(); - - } else if (currentLineNumber != importFileId.getLineNumber()) { + currentLineNumber = lineNumber; + cell = resultRow.getCell(); + category = resultRow.getCategory(); - // flush cell - flushCell(voyage, result, cellResultIds); + } else if (currentLineNumber != lineNumber) { - cellResultIds.clear(); + flushCell(currentLineNumber, result, voyage, cell, category, cellResults); - currentLineNumber = importFileId.getLineNumber(); + currentLineNumber = lineNumber; + cell = resultRow.getCell(); + category = resultRow.getCategory(); + cellResults.clear(); } - cellResultIds.add(resultId); + cellResults.add(resultRow.getResult()); } - flushCell(voyage, result, cellResultIds); + flushCell(currentLineNumber, result, voyage, cell, category, cellResults); } - protected void flushCell(Voyage voyage, ImportDataFileResult result, List<String> cellResultIds) { + protected void flushCell(int lineNumber, ImportDataFileResult result, Voyage voyage, Cell cell, Category category, List<Result> cellResults) { - String firstResultId = cellResultIds.get(0); - Result firstCellResult = persistenceService.getResult(firstResultId); - Cell cell = firstCellResult.getCell(); + Preconditions.checkState(!cellResults.isEmpty()); if (log.isInfoEnabled()) { - log.info("Flush " + cell + " with " + cellResultIds.size() + " result(s)."); - } - List<Result> cellResults = new LinkedList<>(); - - Category category = null; - for (Result cellResult : cell.getResult()) { - - if (cellResultIds.contains(cellResult.getTopiaId())) { - if (category == null) { - category = cellResult.getCategory(); - } - cellResults.add(cellResult); - } - + log.info("Flush " + cell + " (" + lineNumber + ") with " + cellResults.size() + " result(s) with category: " + category + "."); } E row = newImportedRow(voyage, cell, category, cellResults); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java index bb0969c..ee89f29 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java @@ -21,7 +21,6 @@ package fr.ifremer.echobase.services.service.importdata.csv; import fr.ifremer.echobase.entities.data.Operation; -import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; import fr.ifremer.echobase.entities.data.SampleDataImpl; import fr.ifremer.echobase.entities.references.SampleDataType; @@ -49,12 +48,11 @@ public class VoyageCatchesBiometrySampleImportRow { protected int numFish; - public static VoyageCatchesBiometrySampleImportRow of(SampleData sampleData, int numFish) { + public static VoyageCatchesBiometrySampleImportRow of(Operation operation, Species species, SampleData sampleData, int numFish) { - Sample sample = sampleData.getSample(); VoyageCatchesBiometrySampleImportRow row = new VoyageCatchesBiometrySampleImportRow(sampleData); - row.setOperation(sample.getOperation()); - row.setSpecies(sample.getSpeciesCategory().getSpecies()); + row.setOperation(operation); + row.setSpecies(species); row.setNumFish(numFish); return row; diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java index d6363e7..c515049 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java @@ -25,6 +25,7 @@ import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleImpl; import fr.ifremer.echobase.entities.references.SizeCategory; import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.entities.references.SpeciesCategory; /** * Bean used as a row for import of {@link VoyageCatchesTotalSampleImportExportModel}. @@ -140,12 +141,12 @@ public class VoyageCatchesTotalSampleImportRow { this.sortedWeight = sortedWeight; } - public static VoyageCatchesTotalSampleImportRow of(Sample sample, Sample sortedSample) { + public static VoyageCatchesTotalSampleImportRow of(Operation operation, SpeciesCategory speciesCategory, Sample sample, Float sampleSortedWeight) { VoyageCatchesTotalSampleImportRow row = new VoyageCatchesTotalSampleImportRow(sample); - row.setOperation(sample.getOperation()); - row.setSpecies(sample.getSpeciesCategory().getSpecies()); - row.setSizeCategory(sample.getSpeciesCategory().getSizeCategory()); - row.setSortedWeight(sortedSample.getSampleWeight()); + row.setOperation(operation); + row.setSpecies(speciesCategory.getSpecies()); + row.setSizeCategory(speciesCategory.getSizeCategory()); + row.setSortedWeight(sampleSortedWeight); return row; } } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCell_AllImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCell_AllImportServiceIT.java new file mode 100644 index 0000000..f181c1c --- /dev/null +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCell_AllImportServiceIT.java @@ -0,0 +1,108 @@ +/* + * #%L + * EchoBase :: Services + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services.service.importdata; + +import com.google.common.collect.Iterables; +import fr.ifremer.echobase.entities.ImportType; +import fr.ifremer.echobase.entities.data.Category; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.Result; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.services.ImportDataFixtures; +import fr.ifremer.echobase.services.service.importdata.configurations.VoyageResultsImportConfiguration; + +import java.io.IOException; + +/** + * @author Tony Chemit - chemit@codelutin.com + * @since 0.3 + */ +public class VoyageResultsRegionCell_AllImportServiceIT extends VoyageResultsImportServiceITSupport { + + public VoyageResultsRegionCell_AllImportServiceIT() { + super(3); + } + + @Override + public ImportDataFixtures getImportDataFixture() { + return ImportDataFixtures.IMPORT_DATA_ECHOBASE_CATCHES_AND_VOYAGE_RESULT; + } + + protected String[] getImportPath(String filename) { + return new String[]{"/import-data", "result", "region", filename}; + } + + @Override + protected VoyageResultsImportConfiguration createConfiguration() throws IOException { + VoyageResultsImportConfiguration configuration = super.createConfiguration(); + configuration.setImportType(ImportType.RESULT_REGION); + prepareInputFile(configuration.getRegionsFile(), getImportPath("regions.csv.gz")); + prepareInputFile(configuration.getRegionAssociationFile(), getImportPath("regionAssociations.csv.gz")); + prepareInputFile(configuration.getRegionResultFile(), getImportPath("regionResults.csv.gz")); + + addMissingEsduCells("esduName", configuration.getVoyageId(), configuration.getRegionAssociationFile()); + + fakeServiceContext.assertNbEntities(Cell.class, NB_ESDU_CELLS); + + return configuration; + } + + public static final int NB_ESDU_CELLS = 2073; +// public static final int NB_ESDU_CELLS = 22; + + @Override + protected void assertAfertImport(ImportDataResult<VoyageResultsImportConfiguration> result) throws IOException { + + int nbRegionCell = 10; + int nbRegionCellData = 238; + +// //FIXME Comprendre pourquoi toutes les cellules esdu sont dans 2 régions différentes ? + int nbUpdatedEsduCell = 2 * NB_ESDU_CELLS; +// int nbUpdatedEsduCell = NB_ESDU_CELLS; + int nbRegionCellResult = 2128/*1873*/; + + int nbCell = NB_ESDU_CELLS + nbRegionCell; + + { + ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 0); + assertCsvImportResultPerEntity(importDataFileResult, Cell.class, nbRegionCell, 0, nbCell); + assertCsvImportResultPerEntity(importDataFileResult, Data.class, nbRegionCellData, 0, nbRegionCellData); + assertCsvImportResult0(importDataFileResult, nbRegionCell + nbRegionCellData); + } + + { + ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 1); + assertCsvImportResult0(importDataFileResult, 0); + assertCsvImportResultPerEntity(importDataFileResult, Cell.class, 0, nbUpdatedEsduCell, nbCell); + } + + { + ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 2); + assertCsvImportResult0(importDataFileResult, nbRegionCellResult); + assertCsvImportResultPerEntity(importDataFileResult, Result.class, nbRegionCellResult, 0, nbRegionCellResult); + assertCsvImportResultPerEntity(importDataFileResult, Category.class, 27, 0, 27); + assertCsvImportResultPerEntity(importDataFileResult, SpeciesCategory.class, 1, 0, 457); + } + + } + +} -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.