This is an automated email from the git hooks/post-receive script. New commit to branch feature/8170 in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 5edc41de112f0c1fb6401dcc5e71bd2303cceddf Author: Tony CHEMIT <chemit@codelutin.com> Date: Sat Apr 2 12:26:10 2016 +0200 Sub sample import ok \o/ (See #8170) --- .../VoyageCatchesSubSampleImportAction.java | 221 ++++++++++++++++++++- .../importdata/csv/SubSampleImportExportModel.java | 13 +- .../service/importdata/csv/SubSampleImportRow.java | 34 +++- .../echobase/services/EchoBaseServiceFixtures.java | 10 + .../VoyageCatchesImportServiceOnlySubSampleIT.java | 25 ++- 5 files changed, 275 insertions(+), 28 deletions(-) diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java index 2850772..8db9d65 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java @@ -1,7 +1,9 @@ package fr.ifremer.echobase.services.service.importdata.actions; +import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.SampleData; import fr.ifremer.echobase.entities.references.SampleDataType; import fr.ifremer.echobase.entities.references.SampleType; import fr.ifremer.echobase.entities.references.SexCategory; @@ -19,6 +21,11 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; /** * Created on 25/03/16. @@ -45,6 +52,9 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA return SubSampleImportExportModel.forExport(importDataContext); } + private final Map<String, String> sampleIdToOperationId = new TreeMap<>(); + private final Map<String, String> sampleDataIdToSampleId = new TreeMap<>(); + @Override protected void performImport(VoyageCatchesImportDataContext importDataContext, InputFile inputFile, ImportDataFileResult result) { @@ -61,6 +71,8 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA SampleDataType sampleDataTypeNumberAtLength1cm = importDataContext.getSampleDataTypeNumberAtLength1cm(); SampleDataType sampleDataTypeWeightAtLength = importDataContext.getSampleDataTypeWeightAtLength(); + Set<String> createdSampleIds = new TreeSet<>(); + try (Import<SubSampleImportRow> importer = open()) { incrementsProgress(); @@ -81,7 +93,7 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA // find the sample with this category Sample sample = operation.getSample(category, sampleTypeSubsample); - if (sample == null) { + if (sample == null || (sample.getSampleWeight() != null && !sample.getSampleWeight().equals(row.getSampleWeight()))) { // must create it sample = row.getSample(); @@ -90,10 +102,19 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA sample.setSampleType(sampleTypeSubsample); sample = addSample(operation, sample, result); + createdSampleIds.add(sample.getTopiaId()); + } + // add to sample ids cache (need to compute imported export later) + sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); + + addProcessedRow(result, row); + + boolean sampleWasCreated = createdSampleIds.contains(sample.getTopiaId()); + Integer round = row.getRound(); - SampleDataType dataType = sampleDataTypeNumberAtLength; + SampleDataType dataType; if (round == null) { dataType = sampleDataTypeNumberAtLength; @@ -101,15 +122,23 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA dataType = sampleDataTypeNumberAtLength05cm; } else if (round == 1) { dataType = sampleDataTypeNumberAtLength1cm; + } else { + throw new IllegalStateException("Can't deal with round value: " + round); } - //create numberAtLength data - addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result); + { + //create numberAtLength data + SampleData sampleData = addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result, true); + sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); + + } if (row.getWeightAtLength() != null) { //create weightAtLength data - addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result); + SampleData sampleData = addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result, true); + sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); + } } @@ -121,7 +150,187 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA @Override protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { - // TODO + String currentSampleId = null; + + Set<String> sampleDataIds = new LinkedHashSet<>(); + + for (String importedId : result.getIds()) { + + if (log.isInfoEnabled()) { + log.info("Test imported id: " + importedId); + } + + if (importedId.startsWith(Sample.class.getName() + "#")) { + + // new sample to import + + if (currentSampleId == null) { + + // first sample to collect + currentSampleId = importedId; + + } else { + + // flush previous sample + addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); + + // starts to collect new sample + currentSampleId = importedId; + sampleDataIds.clear(); + + } + + } else if (importedId.startsWith(SampleData.class.getName() + "#")) { + + // coming from an existing sample + String sampleId = sampleDataIdToSampleId.get(importedId); + Preconditions.checkNotNull(sampleId); + + if (currentSampleId == null) { + + // first sample to treat + currentSampleId = sampleId; + + } else { + + if (!currentSampleId.equals(sampleId)) { + + // flush previous sample + addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); + + // starts to collect new sample + currentSampleId = sampleId; + sampleDataIds.clear(); + + } + + } + + // add this sample data to current sample to flush + sampleDataIds.add(importedId); + + } else { + throw new IllegalStateException("Cant deal with imported id: " + importedId); + } + + + } + + if (!sampleDataIds.isEmpty()) { + + // flush last sample + addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); + + } + + } + + protected void addImportedSample(VoyageCatchesImportDataContext importDataContext, String sampleId, ImportDataFileResult result, Set<String> sampleDataIds) { + + String operationId = sampleIdToOperationId.get(sampleId); + Preconditions.checkNotNull(operationId); + + Operation operation = persistenceService.getOperation(operationId); + Preconditions.checkNotNull(operation); + + Sample sample = persistenceService.getSample(sampleId); + Preconditions.checkNotNull(sample); + + if (log.isInfoEnabled()) { + log.info("Adding sample: " + sample + " to imported export."); + } + + Set<SampleDataType> lengthSampleDataTypes = new LinkedHashSet<>(); + lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength()); + lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength05cm()); + lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength1cm()); + + Preconditions.checkState(sample.isSampleDataNotEmpty()); + + SubSampleImportRow importedRow = null; + + String lastLengthClass = null; + boolean lengthDataTypeFound = false; + for (String sampleDataId : sampleDataIds) { + + SampleData aSampleData = sample.getSampleDataByTopiaId(sampleDataId); + + String lengthClass = aSampleData.getDataLabel(); + + boolean isLengthDataType = lengthSampleDataTypes.contains(aSampleData.getSampleDataType()); + + if (lastLengthClass == null) { + + // first row + lastLengthClass = lengthClass; + importedRow = SubSampleImportRow.of(operation, sample); + lengthDataTypeFound = false; + + } else { + + if ((isLengthDataType && lengthDataTypeFound) || !lengthClass.equals(lastLengthClass)) { + + // changing length class or length data type alredy found + addImportedRow(result, importedRow); + + lastLengthClass = lengthClass; + importedRow = SubSampleImportRow.of(operation, sample); + lengthDataTypeFound = false; + } + } + + if (log.isInfoEnabled()) { + log.info("Treated sample data: " + aSampleData); + } + + addImportedSampleData(importDataContext, importedRow, aSampleData); + + if (isLengthDataType) { + lengthDataTypeFound = true; + } + + } + + addImportedRow(result, importedRow); + + } + + protected void addImportedSampleData(VoyageCatchesImportDataContext importDataContext, SubSampleImportRow importedRow, SampleData sampleData) { + + SampleDataType sampleDataTypeNumberAtLength = importDataContext.getSampleDataTypeNumberAtLength(); + SampleDataType sampleDataTypeNumberAtLength05cm = importDataContext.getSampleDataTypeNumberAtLength05cm(); + SampleDataType sampleDataTypeNumberAtLength1cm = importDataContext.getSampleDataTypeNumberAtLength1cm(); + SampleDataType sampleDataTypeWeightAtLength = importDataContext.getSampleDataTypeWeightAtLength(); + + SampleDataType sampleDataType = sampleData.getSampleDataType(); + + if (sampleDataTypeNumberAtLength.equals(sampleDataType)) { + + importedRow.setNumberAtLength(sampleData.getDataValue()); +// importedRow.setUnits(sampleDataType.getUnits()); + + } else if (sampleDataTypeNumberAtLength05cm.equals(sampleDataType)) { + + importedRow.setNumberAtLength(sampleData.getDataValue()); + importedRow.setRound(5); +// importedRow.setUnits(sampleDataType.getUnits()); + + } else if (sampleDataTypeNumberAtLength1cm.equals(sampleDataType)) { + + importedRow.setNumberAtLength(sampleData.getDataValue()); + importedRow.setRound(1); +// importedRow.setUnits(sampleDataType.getUnits()); + + } else if (sampleDataTypeWeightAtLength.equals(sampleDataType)) { + + importedRow.setWeightAtLength(sampleData.getDataValue()); + importedRow.setUnits(sampleDataType.getUnits()); + + } else { + throw new IllegalStateException("Can't deal with this sampleData: " + sampleData); + } + + importedRow.setLengthClass(sampleData.getDataLabel()); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportExportModel.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportExportModel.java index dda2685..840a107 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportExportModel.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportExportModel.java @@ -49,12 +49,13 @@ public class SubSampleImportExportModel extends EchoBaseImportExportModelSupport model.newForeignKeyColumn(Species.PROPERTY_BARACOUDA_CODE, SubSampleImportRow.PROPERTY_SPECIES, Species.class, Species.PROPERTY_BARACOUDA_CODE, importDataContext.getSpeciesByBaracoudaCode()); model.newForeignKeyColumn(SubSampleImportRow.PROPERTY_SIZE_CATEGORY, SizeCategory.class, SizeCategory.PROPERTY_NAME, importDataContext.getSizeCategoriesByName()); model.newForeignKeyColumn(SubSampleImportRow.PROPERTY_SEX_CATEGORY, SexCategory.class, SexCategory.PROPERTY_NAME, importDataContext.getSexCategoriesByName()); + model.newMandatoryColumn(SubSampleImportRow.PROPERTY_LENGTH_CLASS); model.newMandatoryColumn(Sample.PROPERTY_SAMPLE_WEIGHT, EchoBaseCsvUtil.PRIMITIVE_FLOAT); model.newMandatoryColumn(Sample.PROPERTY_NUMBER_SAMPLED, EchoBaseCsvUtil.PRIMITIVE_INTEGER); model.newMandatoryColumn(SubSampleImportRow.PROPERTY_NUMBER_AT_LENGTH, EchoBaseCsvUtil.PRIMITIVE_FLOAT); model.newMandatoryColumn(SubSampleImportRow.PROPERTY_WEIGHT_AT_LENGTH, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); - model.newMandatoryColumn(SubSampleImportRow.PROPERTY_LENGTH_CLASS); - model.newMandatoryColumn(SubSampleImportRow.PROPERTY_UNITS); +// model.newMandatoryColumn(SubSampleImportRow.PROPERTY_UNITS); + model.newIgnoredColumn(SubSampleImportRow.PROPERTY_UNITS); model.newMandatoryColumn(SubSampleImportRow.PROPERTY_ROUND, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER); return model; @@ -64,16 +65,16 @@ public class SubSampleImportExportModel extends EchoBaseImportExportModelSupport SubSampleImportExportModel model = new SubSampleImportExportModel(importDataContext.getCsvSeparator()); // model.newIgnoredColumn("subHaul"); - model.newColumnForExport(EchoBaseCsvUtil.OPERATION_ID, EchoBaseCsvUtil.OPERATION_FORMATTER); - model.newColumnForExport(Species.PROPERTY_BARACOUDA_CODE, EchoBaseCsvUtil.SPECIES_FORMATTER); + model.newColumnForExport(EchoBaseCsvUtil.OPERATION_ID, TotalSampleImportRow.PROPERTY_OPERATION, EchoBaseCsvUtil.OPERATION_FORMATTER); + model.newColumnForExport(Species.PROPERTY_BARACOUDA_CODE, TotalSampleImportRow.PROPERTY_SPECIES, EchoBaseCsvUtil.SPECIES_FORMATTER); model.newColumnForExport(SubSampleImportRow.PROPERTY_SIZE_CATEGORY, EchoBaseCsvUtil.SIZE_CATEGORY_FORMATTER); model.newColumnForExport(SubSampleImportRow.PROPERTY_SEX_CATEGORY, EchoBaseCsvUtil.SEX_CATEGORY_FORMATTER); + model.newColumnForExport(SubSampleImportRow.PROPERTY_LENGTH_CLASS); model.newColumnForExport(Sample.PROPERTY_SAMPLE_WEIGHT, EchoBaseCsvUtil.PRIMITIVE_FLOAT); model.newColumnForExport(Sample.PROPERTY_NUMBER_SAMPLED, EchoBaseCsvUtil.PRIMITIVE_INTEGER); model.newColumnForExport(SubSampleImportRow.PROPERTY_NUMBER_AT_LENGTH, EchoBaseCsvUtil.PRIMITIVE_FLOAT); model.newColumnForExport(SubSampleImportRow.PROPERTY_WEIGHT_AT_LENGTH, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER_FORMATTER); - model.newColumnForExport(SubSampleImportRow.PROPERTY_LENGTH_CLASS); - model.newColumnForExport(SubSampleImportRow.PROPERTY_UNITS); +// model.newColumnForExport(SubSampleImportRow.PROPERTY_UNITS); model.newColumnForExport(SubSampleImportRow.PROPERTY_ROUND, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER_FORMATTER); return model; diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportRow.java index 1774c8f..c2f75e0 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/SubSampleImportRow.java @@ -43,8 +43,6 @@ public class SubSampleImportRow { public static final String PROPERTY_SEX_CATEGORY = "sexCategory"; - public static final String PROPERTY_SAMPLE = "sample"; - public static final String PROPERTY_NUMBER_AT_LENGTH = "numberAtLength"; public static final String PROPERTY_WEIGHT_AT_LENGTH = "weightAtLength"; @@ -55,22 +53,32 @@ public class SubSampleImportRow { public static final String PROPERTY_ROUND = "round"; protected final Sample sample; + protected Operation operation; protected Species species; protected SizeCategory sizeCategory; protected SexCategory sexCategory; protected float numberAtLength; - protected Float weightAtLength; - protected String lengthClass; - protected Integer round; - protected String units; + public static SubSampleImportRow of(Operation operation, Sample sample) { + SubSampleImportRow row = new SubSampleImportRow(sample); + row.setOperation(operation); + row.setSpecies(sample.getSpeciesCategory().getSpecies()); + row.setSizeCategory(sample.getSpeciesCategory().getSizeCategory()); + row.setSexCategory(sample.getSpeciesCategory().getSexCategory()); + return row; + } + + public SubSampleImportRow(Sample sample) { + this.sample = sample; + } + public SubSampleImportRow() { - sample = new SampleImpl(); + this(new SampleImpl()); } public Sample getSample() { @@ -117,14 +125,22 @@ public class SubSampleImportRow { this.units = units; } - public void setNumberSampled(Integer numberSampled) { - sample.setNumberSampled(numberSampled); + public Float getSampleWeight() { + return sample.getSampleWeight(); } public void setSampleWeight(Float sampleWeight) { sample.setSampleWeight(sampleWeight); } + public Integer getNumberSampled() { + return sample.getNumberSampled(); + } + + public void setNumberSampled(Integer numberSampled) { + sample.setNumberSampled(numberSampled); + } + public SexCategory getSexCategory() { return sexCategory; } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java index 86daaeb..54cf630 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java @@ -45,6 +45,10 @@ public class EchoBaseServiceFixtures { return "/import-data/echobase-operation.h2.db.gz"; } + public String IMPORT_DATA_ECHOBASE_OPERATION_TOTAL_SAMPLES() { + return "/import-data/echobase-operation-total-samples.h2.db.gz"; + } + public String IMPORT_DATA_ECHOBASE_CATCHES() { return "/import-data/echobase-catches.h2.db.gz"; } @@ -117,6 +121,12 @@ public class EchoBaseServiceFixtures { return 200; } + public int NB_SAMPLE_UNSORTED_ALL() { + return 706; + } + + public int NB_SAMPLE_DATA_UNSORTED_ALL() { return 8057; } + public int NB_SAMPLE_BIOMETRY() { return 15; } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesImportServiceOnlySubSampleIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesImportServiceOnlySubSampleIT.java index 8ea75db..852f915 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesImportServiceOnlySubSampleIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesImportServiceOnlySubSampleIT.java @@ -20,12 +20,13 @@ */ package fr.ifremer.echobase.services.service.importdata; +import com.google.common.collect.Iterables; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.services.FakeEchoBaseServiceContext; import fr.ifremer.echobase.services.service.importdata.configurations.VoyageCatchesImportConfiguration; import java.io.IOException; -import java.util.Set; /** * @author Tony Chemit - chemit@codelutin.com @@ -34,9 +35,14 @@ import java.util.Set; public class VoyageCatchesImportServiceOnlySubSampleIT extends VoyageCatchesImportServiceITSupport { @Override + protected FakeEchoBaseServiceContext initContext() { + return new FakeEchoBaseServiceContext(fixtures.IMPORT_DATA_ECHOBASE_OPERATION_TOTAL_SAMPLES()); + } + + @Override protected VoyageCatchesImportConfiguration createConfiguration() throws IOException { VoyageCatchesImportConfiguration configuration = super.createConfiguration(); - prepareInputFile(configuration.getSubSampleFile(), getImportPath("subsample.csv.gz")); + prepareInputFile(configuration.getSubSampleFile(), getImportPath("subsample_all.csv.gz")); return configuration; } @@ -49,16 +55,21 @@ public class VoyageCatchesImportServiceOnlySubSampleIT extends VoyageCatchesImpo protected void assertBeforeImport() { assertImportCommonData(); assertImportOperations(); - assertNoEntities(Sample.class, SampleData.class); + assertNbEntities(Sample.class, fixtures.NB_SAMPLE_TOTAL()); + assertNbEntities(SampleData.class, fixtures.NB_SAMPLE_DATA_TOTAL()); } @Override protected void assertAfertImport(ImportDataResult<VoyageCatchesImportConfiguration> result) throws IOException { - Set<ImportDataFileResult> importDataFileResults = result.getImportResults(); - assertNbIds(importDataFileResults, 0, fixtures.NB_SAMPLE_UNSORTED()); - assertCsvImportResult(importDataFileResults, 0, Sample.class, fixtures.NB_SAMPLE_UNSORTED()); - assertCsvImportResult(importDataFileResults, 0, SampleData.class, fixtures.NB_SAMPLE_DATA_UNSORTED()); + ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 0); + + int nbSamples = fixtures.NB_SAMPLE_UNSORTED_ALL(); + int nbSampleData = fixtures.NB_SAMPLE_DATA_UNSORTED_ALL(); + + assertCsvImportResultPerEntity(importDataFileResult, Sample.class, nbSamples, 0, nbSamples + fixtures.NB_SAMPLE_TOTAL()); + assertCsvImportResultPerEntity(importDataFileResult, SampleData.class, nbSampleData, 0, nbSampleData + fixtures.NB_SAMPLE_DATA_TOTAL()); + assertCsvImportResult0(importDataFileResult, nbSamples + nbSampleData); } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.