Author: tchemit Date: 2014-05-15 18:05:21 +0200 (Thu, 15 May 2014) New Revision: 1779 Url: http://forge.codelutin.com/projects/tutti/repository/revisions/1779 Log: refs #evo_5007 [IMPORT PSION] 3?\195?\168me r?\195?\168gle ?\195?\160 compl?\195?\169ter (report from version 3.4.3) Added: branches/tutti-3.4.x/tutti-service/src/test/resources/psion/evo-5077.IWA Modified: branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportBatchModel.java branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportModel.java branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportService.java branches/tutti-3.4.x/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties branches/tutti-3.4.x/tutti-service/src/test/java/fr/ifremer/tutti/service/psionimport/PsionImportServiceTest.java Modified: branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportBatchModel.java =================================================================== --- branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportBatchModel.java 2014-05-15 14:57:38 UTC (rev 1778) +++ branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportBatchModel.java 2014-05-15 16:05:21 UTC (rev 1779) @@ -77,6 +77,8 @@ protected String categoryCode; + private boolean applyBothWeightOnCategorizedBatch; + public PsionImportBatchModel(Species species, Integer lengthStepCaracteristicId) { this.species = species; this.lengthStepCaracteristicId = lengthStepCaracteristicId; @@ -150,6 +152,14 @@ return frequencies.size(); } + public void setApplyBothWeightOnCategorizedBatch(boolean applyBothWeightOnCategorizedBatch) { + this.applyBothWeightOnCategorizedBatch = applyBothWeightOnCategorizedBatch; + } + + public boolean isApplyBothWeightOnCategorizedBatch() { + return applyBothWeightOnCategorizedBatch; + } + void merge(PsionImportBatchModel batchModel) { setWeight(getWeight() + batchModel.getWeight()); setSampleWeight(getSampleWeight() + batchModel.getSampleWeight()); Modified: branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportModel.java =================================================================== --- branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportModel.java 2014-05-15 14:57:38 UTC (rev 1778) +++ branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportModel.java 2014-05-15 16:05:21 UTC (rev 1779) @@ -37,12 +37,11 @@ import org.apache.commons.logging.LogFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; import static org.nuiton.i18n.I18n.t; @@ -152,7 +151,7 @@ Species species = batchModel.getSpecies(); speciesSet.add(species); - + String categoryCode = batchModel.getCategoryCode(); Float weight = batchModel.getWeight(); Float sampleWeight = batchModel.getSampleWeight(); @@ -233,6 +232,7 @@ } } + } void cleanUnsortedBatches() { @@ -250,102 +250,88 @@ void checkSortedBatches() throws IOException { - Set<Species> speciesSet = sortedBatchsBySpecies.keySet(); +// for (Species species : sortedBatchsBySpecies.keySet()) { +// +// Collection<PsionImportBatchModel> sortedBatches = sortedBatchsBySpecies.get(species); +// +// Map<String, Float> weightByCategory = new HashMap<>(); +// +// for (PsionImportBatchModel sortedBatch : sortedBatches) { +// Float weight = sortedBatch.getWeight(); +// if (TuttiEntities.isGreaterWeight(weight, 0)) { +// +// Float rootweight = weightByCategory.get(sortedBatch.getCategoryCode()); +// +// if (rootweight == null) { +// +// // first time +// weightByCategory.put(sortedBatch.getCategoryCode(), weight); +// } else if (!TuttiEntities.isEqualWeight(rootweight, weight)) { +// +// // can't have 2 batches with different vrac batch weight +// throw new IOException( +// t("tutti.service.psionimport.error.inconsistentVracWeight.message", species.getSurveyCode())); +// } +// +// } +// } +// } - for (Species species : speciesSet) { + for (Species species : sortedBatchsBySpecies.keySet()) { - Collection<PsionImportBatchModel> sortedBatches = sortedBatchsBySpecies.get(species); + Collection<PsionImportBatchModel> speciesBatchesWithDoubleWeight = new ArrayList<>(); + Collection<PsionImportBatchModel> speciesBatchesWithDoubleWeightAndCat = new ArrayList<>(); - Map<String, Float> weightByCategory = new HashMap<>(); + for (PsionImportBatchModel batchModel : sortedBatchsBySpecies.get(species)) { - for (PsionImportBatchModel sortedBatch : sortedBatches) { - Float weight = sortedBatch.getWeight(); - if (TuttiEntities.isGreaterWeight(weight, 0)) { + Float weight = batchModel.getWeight(); + Float sampleWeight = batchModel.getSampleWeight(); - Float rootweight = weightByCategory.get(sortedBatch.getCategoryCode()); + if (TuttiEntities.isEqualWeight(weight, 0) && TuttiEntities.isGreaterWeight(sampleWeight, 0)) { - if (rootweight == null) { + // POID = 0 et TAIL != POID : un seul poids à positionner + continue; + } - // first time - weightByCategory.put(sortedBatch.getCategoryCode(), weight); - } else if (!TuttiEntities.isEqualWeight(rootweight, weight)) { + if (TuttiEntities.isGreaterWeight(weight, 0) && TuttiEntities.isGreaterWeight(weight, sampleWeight)) { - // can't have 2 batches with different vrac batch weight - throw new IOException( - t("tutti.service.psionimport.error.inconsistentVracWeight.message", species.getSurveyCode())); + // POID > 0 et POID > TAIL : deux poids à positionner + speciesBatchesWithDoubleWeight.add(batchModel); + + if (batchModel.withCategories()) { + + speciesBatchesWithDoubleWeightAndCat.add(batchModel); } } + } + + if (!speciesBatchesWithDoubleWeight.isEmpty()) { + + if (speciesBatchesWithDoubleWeightAndCat.size() == speciesBatchesWithDoubleWeight.size()) { + + // tous les lots sont categories + // on marque les lots pour que weight soit sur la categorie et sampleWeight comme poids de sous echantillon + + for (PsionImportBatchModel batchModel : speciesBatchesWithDoubleWeight) { + batchModel.setApplyBothWeightOnCategorizedBatch(true); + } + } else if (speciesBatchesWithDoubleWeight.size() == 1 && speciesBatchesWithDoubleWeightAndCat.isEmpty()) { + + // un seul lot non categorise + } else { + + // on bloque l'import + throw new IOException( + t("tutti.service.psionimport.error.inconsistentVracCategory.message", species.getSurveyCode())); + } + + + } + } } -// public boolean withBatchs() { -// return !batchsByCategory.isEmpty(); -// } -// -// public Set<Species> getSpecies() { -// Set<Species> result = Sets.newLinkedHashSet(); -// for (PsionImportBatchModel batch : batchsByCategory.values()) { -// result.add(batch.getSpecies()); -// } -// return result; -// } -// -// public List<PsionImportBatchModel> getBatchs(Species species) { -// List<PsionImportBatchModel> result = Lists.newArrayList(); -// for (PsionImportBatchModel batch : batchsByCategory.values()) { -// if (species.equals(batch.getSpecies())) { -// result.add(batch); -// } -// } -// return result; -// } -// -// public Set<Integer> getSampleCategoryIdUsed() { -// Set<Integer> result = Sets.newHashSet(); -// for (PsionImportBatchModel batch : batchsByCategory.values()) { -// Iterator<PsionImportBatchModel.SampleCategory> categoryIterator = batch.getCategoryIterator(); -// while (categoryIterator.hasNext()) { -// PsionImportBatchModel.SampleCategory next = categoryIterator.next(); -// -// result.add(next.getCategoryId()); -// } -// } -// return result; -// } -// -// void addBatch(PsionImportBatchModel batchModel) { -// -// String cacheCode = batchModel.getSpecies().getSurveyCode() + "_" + batchModel.getCategoryCode(); -// -// PsionImportBatchModel mergeBatch = batchsByCategory.get(cacheCode); -// -// if (mergeBatch == null) { -// -// // new batch -// batchsByCategory.put(cacheCode, batchModel); -// -// if (log.isInfoEnabled()) { -// log.info("Added " + batchModel); -// } -// } else { -// -// // merge data with this batch -// -// mergeBatch.setWeight(mergeBatch.getWeight() + batchModel.getWeight()); -// mergeBatch.setSampleWeight(mergeBatch.getSampleWeight() + batchModel.getSampleWeight()); -// -// for (Map.Entry<Float, MutableInt> entry : batchModel.getFrequencies().entrySet()) { -// Float stepClass = entry.getKey(); -// int number = entry.getValue().intValue(); -// mergeBatch.addFrequency(stepClass, number); -// } -// if (log.isInfoEnabled()) { -// log.info("Merged " + batchModel + " to " + mergeBatch); -// } -// } -// } - } Modified: branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportService.java =================================================================== --- branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportService.java 2014-05-15 14:57:38 UTC (rev 1778) +++ branches/tutti-3.4.x/tutti-service/src/main/java/fr/ifremer/tutti/service/psionimport/PsionImportService.java 2014-05-15 16:05:21 UTC (rev 1779) @@ -508,8 +508,6 @@ // insert all imported species batches -// TuttiEnumerationFile enumerationFile = persistenceService.getEnumerationFile(); - Set<Species> species = importModel.getSpecies(); for (Species specy : species) { @@ -530,103 +528,6 @@ result.incrementNbUnsortedImported(); } -// List<PsionImportBatchModel> batchs = importModel.getSortedBatches(specy); -// -// if (batchs.size() == 1 && !batchs.get(0).withCategories()) { -// -// PsionImportBatchModel batchModel = batchs.get(0); -// -// // simple batch with no category -// SpeciesBatch batch = createSpeciesBatch(operation, -// batchModel.getSpecies(), -// batchModel.getWeight(), -// batchModel.getSampleWeight(), -// enumerationFile.PMFM_ID_SORTED_UNSORTED, -// sortedCaracteristic); -// -// batch = persistenceService.createSpeciesBatch(batch, null); -// -// persistFrequencies(batch, batchModel); -// -// } else { -// -// // batch with categories -// -// SpeciesBatch rootBatch = createSpeciesBatch(operation, -// specy, -// null, -// null, -// enumerationFile.PMFM_ID_SORTED_UNSORTED, -// sortedCaracteristic); -// -// rootBatch = persistenceService.createSpeciesBatch(rootBatch, null); -// -// -// for (PsionImportBatchModel batchModel : batchs) { -// -// SpeciesBatch parentBatch = rootBatch; -// -// SpeciesBatch childBatch = null; -// -// Iterator<PsionImportBatchModel.SampleCategory> categoryIterator = batchModel.getCategoryIterator(); -// -// while (categoryIterator.hasNext()) { -// PsionImportBatchModel.SampleCategory sampleCategory = categoryIterator.next(); -// -// boolean lastCategory = !categoryIterator.hasNext(); -// -// Integer categoryId = sampleCategory.getCategoryId(); -// Serializable categoryValue = sampleCategory.getCategoryValue(); -// -// if (lastCategory) { -// -// // always create the leaf -// childBatch = createSpeciesBatch(operation, -// specy, -// batchModel.getWeight(), -// batchModel.getSampleWeight(), -// categoryId, -// categoryValue); -// } else { -// -// // try to find child in parent children -// -// childBatch = null; -// for (SpeciesBatch speciesBatch : parentBatch.getChildBatchs()) { -// -// if (speciesBatch.getSampleCategoryId().equals(categoryId) && -// speciesBatch.getSampleCategoryValue().equals(categoryValue)) { -// childBatch = speciesBatch; -// break; -// } -// } -// -// if (childBatch == null) { -// -// // must create it -// childBatch = createSpeciesBatch(operation, -// specy, -// null, -// null, -// categoryId, -// categoryValue); -// } -// } -// -// if (TuttiEntities.isNew(childBatch)) { -// -// // persist it -// childBatch = persistenceService.createSpeciesBatch(childBatch, parentBatch.getId()); -// parentBatch.addChildBatchs(childBatch); -// } -// -// parentBatch = childBatch; -// } -// -// persistFrequencies(childBatch, batchModel); -// } -// } -// result.incrementNbImported(); } persistenceService.saveCatchBatch(catchBatch); @@ -636,8 +537,6 @@ Species specy, List<PsionImportBatchModel> batchs) { - CaracteristicQualitativeValue caracteristicQualitativeValue = sortedCaracteristic; - if (batchs.size() == 1 && !batchs.get(0).withCategories()) { PsionImportBatchModel batchModel = batchs.get(0); @@ -648,7 +547,7 @@ batchModel.getWeight(), batchModel.getSampleWeight(), enumerationFile.PMFM_ID_SORTED_UNSORTED, - caracteristicQualitativeValue); + sortedCaracteristic); batch = persistenceService.createSpeciesBatch(batch, null); @@ -658,17 +557,26 @@ // batch with categories - // Is there two weights ? If so then the weight has to be placed in the sorted batch + // Is there two weights ? + // If so and applyOnLeaf is off then the weight has to be placed in the sorted batch + // Otherwise apply both weight on categorized batch Float sortedBatchWeight = null; Float weight = batchs.get(0).getWeight(); Float sampleWeight = batchs.get(0).getSampleWeight(); + boolean applyOnLeaf = batchs.get(0).isApplyBothWeightOnCategorizedBatch(); if (sampleWeight != null) { // use the weight as sorted batch weight - sortedBatchWeight = weight; + + if (!applyOnLeaf) { + + // use the weight as sorted batch weight + sortedBatchWeight = weight; + } + } SpeciesBatch rootBatch = createSpeciesBatch(operation, @@ -676,7 +584,7 @@ sortedBatchWeight, null, enumerationFile.PMFM_ID_SORTED_UNSORTED, - caracteristicQualitativeValue); + sortedCaracteristic); rootBatch = persistenceService.createSpeciesBatch(rootBatch, null); @@ -690,8 +598,6 @@ Species specy, List<PsionImportBatchModel> batchs) { - CaracteristicQualitativeValue caracteristicQualitativeValue = unsortedCaracteristic; - if (batchs.size() == 1 && !batchs.get(0).withCategories()) { PsionImportBatchModel batchModel = batchs.get(0); @@ -702,7 +608,7 @@ batchModel.getWeight(), batchModel.getSampleWeight(), enumerationFile.PMFM_ID_SORTED_UNSORTED, - caracteristicQualitativeValue); + unsortedCaracteristic); batch = persistenceService.createSpeciesBatch(batch, null); @@ -717,7 +623,7 @@ null, null, enumerationFile.PMFM_ID_SORTED_UNSORTED, - caracteristicQualitativeValue); + unsortedCaracteristic); rootBatch = persistenceService.createSpeciesBatch(rootBatch, null); Modified: branches/tutti-3.4.x/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties =================================================================== --- branches/tutti-3.4.x/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties 2014-05-15 14:57:38 UTC (rev 1778) +++ branches/tutti-3.4.x/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties 2014-05-15 16:05:21 UTC (rev 1779) @@ -160,6 +160,7 @@ tutti.service.protocol.import.taxonUsed.error=Le taxon référent d'id %s est déjà utilisé tutti.service.psion.import.attachment.comment=Import Psion du %s tutti.service.psionimport.error.inconsistentVracWeight.message=Pour l'espèce '%s', il existe deux enregistrements de lot vrac avec le champs 'POID' différent, ce qui est interdit +tutti.service.psionimport.error.inconsistentVracCategory.message=Pour l'espèce '%s', la cétégorisation des lots vrac n'est pas cohérente (POID != 0 et TAIL > 0) (on autorise uniquement que tous les lots soient catégorisés ou un seul lot non catégorisé) tutti.service.psionimport.error.invalid.category.syntax=Ligne %s, catégorisation '%s' inconnue, l'espèce %s sera ignorée tutti.service.psionimport.error.invalid.command.syntax=Ligne %s, la commande '%s' n'est pas reconnue tutti.service.psionimport.error.invalid.date.format=Format de la date du trait incorrecte (mm-dd-aaaa) Modified: branches/tutti-3.4.x/tutti-service/src/test/java/fr/ifremer/tutti/service/psionimport/PsionImportServiceTest.java =================================================================== --- branches/tutti-3.4.x/tutti-service/src/test/java/fr/ifremer/tutti/service/psionimport/PsionImportServiceTest.java 2014-05-15 14:57:38 UTC (rev 1778) +++ branches/tutti-3.4.x/tutti-service/src/test/java/fr/ifremer/tutti/service/psionimport/PsionImportServiceTest.java 2014-05-15 16:05:21 UTC (rev 1779) @@ -262,4 +262,65 @@ BatchContainer<SpeciesBatch> rootSpeciesBatchAfter = persistenceService.getRootSpeciesBatch(operation.getId(), null); Assert.assertEquals(oldNbBatchs, rootSpeciesBatchAfter.sizeChildren()); } + + @Test + public void importEvo5077() throws IOException { + + File importFile = dbResource.copyClassPathResource("psion/evo-5077.IWA", "evo-5077.IWA"); + + FishingOperation operation = dataContext.operations.get(1); + CatchBatch catchBatch = persistenceService.getCatchBatchFromFishingOperation(operation.getId()); + catchBatch.setFishingOperation(operation); + + BatchContainer<SpeciesBatch> rootSpeciesBatch = persistenceService.getRootSpeciesBatch(operation.getId(), null); + int oldSortedBatchs = 0; + int oldUnsortedBatchs = 0; + for (SpeciesBatch speciesBatch : rootSpeciesBatch.getChildren()) { + + boolean sorted = vracPredicate.apply(speciesBatch); + + if (sorted) { + oldSortedBatchs++; + } else { + oldUnsortedBatchs++; + } + } + + PsionImportResult importResult = service.importFile(importFile, operation, catchBatch); + + int nbSortedAdded = importResult.getNbSortedImported(); + int nbUnsortedAdded = importResult.getNbUnsortedImported(); + List<String> errors = importResult.getErrors(); + + if (log.isInfoEnabled()) { + log.info("Sorted Imported: " + nbSortedAdded); + log.info("Unsorted Imported: " + nbUnsortedAdded); + log.info("Errors: " + errors.size()); + } + + int nbNewSortedBatchs = 3; + int nbNewUnsortedBatchs = 1; + Assert.assertEquals(nbNewSortedBatchs, nbSortedAdded); + Assert.assertEquals(nbNewUnsortedBatchs, nbUnsortedAdded); + Assert.assertEquals(0, errors.size()); + + // no batch imported + BatchContainer<SpeciesBatch> rootSpeciesBatchAfter = persistenceService.getRootSpeciesBatch(operation.getId(), null); + + int totalSortedBatchs = 0; + int totalUnsortedBatchs = 0; + for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { + + boolean sorted = vracPredicate.apply(speciesBatch); + + if (sorted) { + totalSortedBatchs++; + } else { + totalUnsortedBatchs++; + } + } + + Assert.assertEquals(oldSortedBatchs + nbNewSortedBatchs, totalSortedBatchs); + Assert.assertEquals(oldUnsortedBatchs + nbNewUnsortedBatchs, totalUnsortedBatchs); + } } Added: branches/tutti-3.4.x/tutti-service/src/test/resources/psion/evo-5077.IWA =================================================================== --- branches/tutti-3.4.x/tutti-service/src/test/resources/psion/evo-5077.IWA (rev 0) +++ branches/tutti-3.4.x/tutti-service/src/test/resources/psion/evo-5077.IWA 2014-05-15 16:05:21 UTC (rev 1779) @@ -0,0 +1,31 @@ +cc +1 +07-01-2013 +07:19:11 + +ESPE : MERLMER +POID : 50 +TAIL : 50 +CATE : I +LONG : 12.5 +LONG : 15 +ESPE : SPICMAE +POID : 0 +TAIL : 20 +CATE : N +LONG : 12 +ESPE : TRIPLAS +POID : 40 +TAIL : 20 +CATE : N +LONG : 15 +ESPE : PAGEERY +POID : 100 +TAIL : 50 +CATE : M +LONG : 16.5 +ESPE : PAGEERY +POID : 150 +TAIL : 80 +CATE : F +LONG : 16.5