This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository tutti. See http://git.codelutin.com/tutti.git commit 3c37b63abcce24f64a581dce47a4655fbdc35efc Author: Kevin Morin <morin@codelutin.com> Date: Wed Sep 3 16:55:06 2014 +0200 refs #5411 [CAPTURE] Import BIGFIN --- .../tutti/service/bigfin/BigfinDataRowModel.java | 10 +- .../tutti/service/bigfin/BigfinImportService.java | 130 ++++++++++++++------- .../resources/i18n/tutti-service_en_GB.properties | 4 + .../resources/i18n/tutti-service_fr_FR.properties | 6 +- .../resources/i18n/tutti-ui-swing_fr_FR.properties | 6 +- 5 files changed, 104 insertions(+), 52 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java index d16de6d..dd592e6 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java @@ -39,6 +39,7 @@ public class BigfinDataRowModel extends TuttiCsvUtil.AbstractTuttiImportModel<Bi final Map<String, Species> foundSpecies = new HashMap<>(); + newMandatoryColumn("Record_id", BigfinDataRow.PROPERTY_RECORD_ID); newMandatoryColumn("length(mm)", BigfinDataRow.PROPERTY_LENGTH, Common.PRIMITIVE_FLOAT); newMandatoryColumn("weight(g)", BigfinDataRow.PROPERTY_WEIGHT, Common.FLOAT); @@ -72,6 +73,11 @@ public class BigfinDataRowModel extends TuttiCsvUtil.AbstractTuttiImportModel<Bi // record the code in the found codes foundSpecies.put(value, result); } + // if the species is not found, create a new one with the value in external code + if (result == null) { + result = Speciess.newSpecies(); + result.setExternalCode(value); + } return result; } }); @@ -88,9 +94,6 @@ public class BigfinDataRowModel extends TuttiCsvUtil.AbstractTuttiImportModel<Bi @Override public Signs parse(String value) throws ParseException { Signs result = Signs.getSign(value.toUpperCase()); - if (result == null) { - throw new ParseException("Could not parse Sign value: " + value, 0); - } return result; } }); @@ -108,7 +111,6 @@ public class BigfinDataRowModel extends TuttiCsvUtil.AbstractTuttiImportModel<Bi } }); - newIgnoredColumn("Record_id"); newIgnoredColumn("cruise"); newIgnoredColumn("pan"); newIgnoredColumn("text"); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java index 161e14b..7f18157 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java @@ -27,6 +27,7 @@ import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchBean; import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequency; import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequencyBean; import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequencys; +import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchs; import fr.ifremer.tutti.persistence.entities.protocol.SpeciesProtocol; import fr.ifremer.tutti.persistence.entities.protocol.TuttiProtocol; import fr.ifremer.tutti.persistence.entities.referential.Caracteristic; @@ -34,6 +35,7 @@ import fr.ifremer.tutti.persistence.entities.referential.CaracteristicQualitativ import fr.ifremer.tutti.persistence.entities.referential.CaracteristicQualitativeValues; import fr.ifremer.tutti.persistence.entities.referential.Species; import fr.ifremer.tutti.service.AbstractTuttiService; +import fr.ifremer.tutti.service.DecoratorService; import fr.ifremer.tutti.service.PersistenceService; import fr.ifremer.tutti.service.TuttiServiceContext; import fr.ifremer.tutti.service.psionimport.PsionImportResult; @@ -48,6 +50,7 @@ import org.nuiton.jaxx.application.ApplicationBusinessException; import java.io.File; import java.io.Reader; +import java.io.Serializable; import java.text.DateFormat; import java.util.ArrayList; import java.util.Collection; @@ -148,10 +151,14 @@ public class BigfinImportService extends AbstractTuttiService { alreadyUsedSpecies.add(speciesBatch.getSpecies()); } - Set<Species> speciesNotrecognized = new HashSet<>(); + // set of not found species already added in the errors + Set<Species> speciesNotRecognized = new HashSet<>(); + // set of species without lengthstep alreay added in the errors Set<Species> speciesInProtocolButWithoutLengthStepPmfmId = new HashSet<>(); List<BigfinDataRow> rows = new ArrayList<>(); + BigfinImportResult result = new BigfinImportResult(bigfinFile); + // load model BigfinDataRowModel importModel = new BigfinDataRowModel(';', speciesBySurveyCode); Reader reader = null; @@ -171,15 +178,37 @@ public class BigfinImportService extends AbstractTuttiService { if (station != null && station.toString().equals(operation.getStationNumber()) && dt != null && dt.after(operation.getGearShootingStartDate()) && dt.before(operation.getGearShootingEndDate())) { - if (species == null) { + if (bean.getSzClass() == null) { + String error = t("tutti.service.bigfinImport.error.szClass.unknwon", bean.getRecordId()); + if (log.isWarnEnabled()) { + log.warn(error); + } + result.addError(error); + } + if (species == null || species.getId() == null) { // bloquer tout si un "species" ne match pas le référentiel de Tutti : lister dans ce cas les codes non reconnus - speciesNotrecognized.add(species); + if (speciesNotRecognized.add(species)) { + String error = t("tutti.service.bigfinImport.error.species.not.found", species.getExternalCode()); + if (log.isWarnEnabled()) { + log.warn(error); + } + result.addError(error); + } } else { // bloquer toute espèce reconnue du protocole mais qui n'a pas de méthode de mesure - SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(species.getSurveyCode()); - if (speciesProtocol != null && speciesProtocol.getLengthStepPmfmId() == null) { - speciesInProtocolButWithoutLengthStepPmfmId.add(species); + String code = species.getSurveyCode(); + if (code == null) { + code = species.getReferenceTaxonId().toString(); + } + SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(code); + if (speciesProtocol != null && speciesProtocol.getLengthStepPmfmId() == null + && speciesInProtocolButWithoutLengthStepPmfmId.add(species)) { + String error = t("tutti.service.bigfinImport.error.species.without.lengthstep", code); + if (log.isWarnEnabled()) { + log.warn(error); + } + result.addError(error); } else if (alreadyUsedSpecies.contains(species)) { // TODO on fait quoi ? @@ -202,13 +231,7 @@ public class BigfinImportService extends AbstractTuttiService { IOUtils.closeQuietly(reader); } - BigfinImportResult result = new BigfinImportResult(bigfinFile); - - if (!speciesInProtocolButWithoutLengthStepPmfmId.isEmpty() || !speciesNotrecognized.isEmpty()) { - //TODO kmorin 20140901 return errors - - - } else { + if (result.isDone()) { Multimap<Species, BigfinDataRow> rowsBySpecies = Multimaps.index(rows, new Function<BigfinDataRow, Species>() { @Override public Species apply(BigfinDataRow bigfinDataRow) { @@ -217,19 +240,22 @@ public class BigfinImportService extends AbstractTuttiService { }); for (Species species : rowsBySpecies.keySet()) { - - SpeciesBatch batch = new SpeciesBatchBean(); - batch.setSpecies(species); - batch.setFishingOperation(operation); - //TODO check in text if hv - batch.setSampleCategoryId(PmfmId.SORTED_UNSORTED.getValue()); - batch.setSampleCategoryValue(sortedCaracteristic); - - batch = persistenceService.createSpeciesBatch(batch, null); + SpeciesBatch batch = createSpeciesBatch(species, + operation, + PmfmId.SORTED_UNSORTED.getValue(), + sortedCaracteristic, + null); // new sorted batch imported result.incrementNbSortedImported(); + String code = batch.getSpecies().getSurveyCode(); + if (code == null) { + code = species.getReferenceTaxonId().toString(); + } + SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(code); + Caracteristic lengthStepPmfm = persistenceService.getCaracteristic(Integer.parseInt(speciesProtocol.getLengthStepPmfmId())); + Collection<BigfinDataRow> speciesRows = rowsBySpecies.get(species); Multimap<CaracteristicQualitativeValue, BigfinDataRow> rowsBySize = Multimaps.index(speciesRows, new Function<BigfinDataRow, CaracteristicQualitativeValue>() { @@ -241,12 +267,12 @@ public class BigfinImportService extends AbstractTuttiService { }); for (CaracteristicQualitativeValue size : rowsBySize.keySet()) { - SpeciesBatch sizeBatch = new SpeciesBatchBean(); - sizeBatch.setSpecies(species); - sizeBatch.setFishingOperation(operation); - sizeBatch.setSampleCategoryId(PmfmId.SIZE_CATEGORY.getValue()); - sizeBatch.setSampleCategoryValue(size); - sizeBatch = persistenceService.createSpeciesBatch(sizeBatch, batch.getId()); + + SpeciesBatch sizeBatch = createSpeciesBatch(species, + operation, + PmfmId.SIZE_CATEGORY.getValue(), + size, + batch.getId()); Collection<BigfinDataRow> sizeRows = rowsBySize.get(size); Multimap<CaracteristicQualitativeValue, BigfinDataRow> rowsByGender = @@ -257,33 +283,47 @@ public class BigfinImportService extends AbstractTuttiService { return result; } }); + for (CaracteristicQualitativeValue gender : rowsByGender.keySet()) { Collection<BigfinDataRow> bigfinDataRows = rowsByGender.get(gender); - SpeciesBatch genderBatch = new SpeciesBatchBean(); - genderBatch.setSpecies(species); - genderBatch.setFishingOperation(operation); - genderBatch.setSampleCategoryId(PmfmId.SEX.getValue()); - genderBatch.setSampleCategoryValue(gender); - genderBatch = persistenceService.createSpeciesBatch(genderBatch, sizeBatch.getId()); + SpeciesBatch genderBatch = createSpeciesBatch(species, + operation, + PmfmId.SEX.getValue(), + gender, + sizeBatch.getId()); - List<SpeciesBatchFrequency> frequencies = createFrequencies(genderBatch, bigfinDataRows); + List<SpeciesBatchFrequency> frequencies = createFrequencies(genderBatch, bigfinDataRows, lengthStepPmfm); persistenceService.saveSpeciesBatchFrequency(genderBatch.getId(), frequencies); } } } - } - addFileAsAttachment(bigfinFile, catchBatch); + addFileAsAttachment(bigfinFile, catchBatch); + } return result; } - protected List<SpeciesBatchFrequency> createFrequencies(SpeciesBatch batch, Collection<BigfinDataRow> rows) { - SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(batch.getSpecies().getSurveyCode()); - Caracteristic lengthStepPmfm = persistenceService.getCaracteristic(Integer.parseInt(speciesProtocol.getLengthStepPmfmId())); - //TODO on fait quoi si c'est nul? + protected SpeciesBatch createSpeciesBatch(Species species, + FishingOperation operation, + Integer categoryId, + Serializable categoryValue, + String parentBatchId) { + SpeciesBatch batch = SpeciesBatchs.newSpeciesBatch(); + batch.setSpecies(species); + batch.setFishingOperation(operation); + //TODO check in text if hv + batch.setSampleCategoryId(categoryId); + batch.setSampleCategoryValue(categoryValue); + + batch = persistenceService.createSpeciesBatch(batch, parentBatchId); + return batch; + } + + protected List<SpeciesBatchFrequency> createFrequencies(SpeciesBatch batch, Collection<BigfinDataRow> rows, Caracteristic lengthStepPmfm) { + //TODO on fait quoi si lengthStepPmfm est nul? String unit = lengthStepPmfm.getUnit(); Float precision = lengthStepPmfm.getPrecision(); if (precision == null) { @@ -328,9 +368,11 @@ public class BigfinImportService extends AbstractTuttiService { totalWeight += weight; } } - // convert grams to kilograms - totalWeight = Weights.roundKiloGram(totalWeight / 1000); - frequency.setWeight(totalWeight); + if (totalWeight > 0f) { + // convert grams to kilograms + totalWeight = Weights.roundKiloGram(totalWeight / 1000); + frequency.setWeight(totalWeight); + } } return frequencies; } diff --git a/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties b/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties index 7b41d9c..5232704 100644 --- a/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties +++ b/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties @@ -46,6 +46,10 @@ tutti.report.step.export.fishingOperation= tutti.report.step.generateReport= tutti.report.step.load.fishingOperation= tutti.service.arp.import.attachment.comment= +tutti.service.bigfin.import.attachment.comment= +tutti.service.bigfinImport.error.species.not.found= +tutti.service.bigfinImport.error.species.without.lengthstep= +tutti.service.bigfinImport.error.szClass.unknwon= tutti.service.bigfinimport.error.no.protocol= tutti.service.compressZipFile.error= tutti.service.context.serviceInstanciation.error= diff --git a/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties b/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties index beaca46..6cd20b4 100644 --- a/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties +++ b/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties @@ -43,7 +43,11 @@ tutti.propety.vessel.nation.registrationCode=%s (nat.) tutti.report.step.export.fishingOperation=Exporter le trait sélectionné tutti.report.step.generateReport=Générer le rapport tutti.report.step.load.fishingOperation=Charger le trait sélectionné -tutti.service.bigfinimport.error.no.protocol= +tutti.service.bigfin.import.attachment.comment=Import Bigfin du %s +tutti.service.bigfinImport.error.species.not.found=L'espèce '<strong>%s</strong>' est inconnue +tutti.service.bigfinImport.error.species.without.lengthstep=L'espèce '<strong>%s</strong>' n'a pas de classe de taille associée dans le protocole. +tutti.service.bigfinImport.error.szClass.unknwon=Ligne <i>%s</i>, code inconnu (doit être 0, 1 ou 2) +tutti.service.bigfinimport.error.no.protocol=Impossible de faire un import Bigfin sans protocol. tutti.service.compressZipFile.error=Erreur lors de la compression du dossier %1s dans le fichier %2s tutti.service.context.serviceInstanciation.error=Erreur lors de l'instanciation du service %s tutti.service.csv.parse.entityNotFound=L'entité de type %1s avec la propriété %2s de valeur %3s n'a pas été trouvée diff --git a/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties b/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties index 34edbdc..c2a6941 100644 --- a/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties +++ b/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties @@ -979,9 +979,9 @@ tutti.editSpeciesBatch.action.exportMultiPost.success=Les lots d'espèces ont é tutti.editSpeciesBatch.action.exportMultiPost.tip=Exporter les lots d'espèces pour les importer sur le poste maître tutti.editSpeciesBatch.action.importBigfin=Import Bigfin tutti.editSpeciesBatch.action.importBigfin.mnemonic=B -tutti.editSpeciesBatch.action.importBigfin.no.matching.data= -tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation= -tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation.title= +tutti.editSpeciesBatch.action.importBigfin.no.matching.data=Import Bigfin non réalisé (des erreurs ont été détectées lors de la lecture du fichier) +tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation=L'import Bigfin n'a pas été réalisé, des erreurs ont été détectées \:<ul>%s</ul><br/>Aucun lot n'a donc été importé. +tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation.title=Import Bigfin tutti.editSpeciesBatch.action.importBigfin.success=Import Bigfin réussi \: %1s espèces importées (Vrac), %2s espèces importées (Hors-Vrac) tutti.editSpeciesBatch.action.importBigfin.tip=Import Bigfin tutti.editSpeciesBatch.action.importMultiPost=Importer des lots d'espèces -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.