Author: chatellier Date: 2010-10-28 15:07:48 +0000 (Thu, 28 Oct 2010) New Revision: 116 Log: Validation plus completes (sur plusieurs lignes) Modified: trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java Modified: trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java =================================================================== --- trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java 2010-10-28 13:42:45 UTC (rev 115) +++ trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java 2010-10-28 15:07:48 UTC (rev 116) @@ -82,6 +82,11 @@ return result; } + public double getObervationNobsmin() { + double result = getOptionAsDouble(CoserBusinessOption.OBSERVATION_NOBSMIN.key); + return result; + } + public static enum CoserBusinessOption implements OptionDef { DATABASE_DIRECTORY("coser.database.directory", _("coser.config.database.directory.description"), "${user.home}" + File.separator + "coser", String.class, false, false), @@ -89,8 +94,10 @@ VALIDATOR_DIRECTORY("coser.validator.directory", _("coser.config.validator.directory.description"), "${" + DATABASE_DIRECTORY.key + "}" + File.separator + "validators", String.class, false, false), REFERENCE_SPECIES("coser.reference.species", _("coser.config.reference.species.description"), "", String.class, false, false), - REFERENCE_ZONES("coser.reference.zones", _("coser.config.reference.zones.description"), "", String.class, false, false); + REFERENCE_ZONES("coser.reference.zones", _("coser.config.reference.zones.description"), "", String.class, false, false), + OBSERVATION_NOBSMIN("coser.observation.nobsmin", _("coser.config.observation.nobsmin.description"), "", Double.class, false, false); + protected String key; protected String description; protected String defaultValue; Modified: trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java =================================================================== --- trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java 2010-10-28 13:42:45 UTC (rev 115) +++ trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java 2010-10-28 15:07:48 UTC (rev 116) @@ -28,6 +28,7 @@ import static org.nuiton.i18n.I18n._; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -175,7 +176,24 @@ * @return les erreurs de validation */ public List<ValidationError> validateCategory(Control control, Category category, ProgressMonitor progress) { + List<ValidationError> errors = validateCategoryXWork(control, category, progress); + List<ValidationError> specificErrors = validateCategorySpecific(control, category, progress); + if (specificErrors != null) { + errors.addAll(specificErrors); + } + return errors; + } + /** + * Valide une category entière d'un project. + * + * @param control control a valider + * @param category category a valider + * @param progress progress monitor + * @return les erreurs de validation (not null) + */ + public List<ValidationError> validateCategoryXWork(Control control, Category category, ProgressMonitor progress) { + // instance des bean utilisé lors de la validation Catch beanCatch = new Catch(); Haul beanHaul = new Haul(); @@ -248,7 +266,6 @@ // check for duplicated lines String uniqueDataKey = getSignificantData(line); - System.out.println(uniqueDataKey); if (uniqueDataKeys.contains(uniqueDataKey)) { ValidationError error = new ValidationError(); error.setLevel(ValidationLevel.ERROR); @@ -266,8 +283,8 @@ /** * Retourne sous forme de clé la partie significative des données. - * * Donc sans le numéro de ligne. + * Utilisé pour la detection des doublons. * * @param data data * @return string key @@ -281,4 +298,253 @@ } return sb.toString(); } + + /** + * Effectue un calcul global, mais specific a chaque categorie. + * + * @param control control + * @param category category + * @param progress progress monitor + * @return + */ + protected List<ValidationError> validateCategorySpecific(Control control, + Category category, ProgressMonitor progress) { + + List<ValidationError> validationErrors = null; + switch (category) { + case CATCH: + validationErrors = validateCategorySpecificCatch(control, progress); + break; + case HAUL: + validationErrors = validateCategorySpecificHaul(control, progress); + break; + case LENGTH: + validationErrors = validateCategorySpecificLength(control, progress); + break; + case STRATA: + validationErrors = validateCategorySpecificStrata(control, progress); + break; + } + return validationErrors; + } + + /** + * Alerte si Somme(CAPTURES$Nombre par CAPTURES$Annee|Strate|Espece) < nobsmin. + * Erreur si CAPT$Nombre n'est pas 0 si le poids dans CAPT$Poids>0 + * + * @param control + * @param progress + * @return + */ + protected List<ValidationError> validateCategorySpecificCatch( + Control control, ProgressMonitor progress) { + + List<ValidationError> validationErrors = new ArrayList<ValidationError>(); + + int total = control.getCatch().size() - 1; + if (progress != null) { + progress.setText(_("coser.business.control.step.observation", 0)); + progress.setTotal(total); + } + + // "Campagne","Annee","Trait","Espece","Nombre","Poids" + // regroupement par campagne/annee/trait + Map<String, Double> nombreForKey = new HashMap<String, Double>(); + Map<String, String> firstLineForKey = new HashMap<String, String>(); + + // parcours des elements + Iterator<String[]> itTuple = control.getCatch().iterator(); + itTuple.next(); // skip header + int lineIndex = 1; // skip header + while (itTuple.hasNext()) { + String[] tuple = itTuple.next(); + + // update progress + if (progress != null) { + int progressPercent = (int)((double)lineIndex / (double)total * 100.0); + progress.setText(_("coser.business.control.step.observation", progressPercent)); + progress.setCurrent(lineIndex); + ++lineIndex; + } + + // compute key + StringBuffer sb = new StringBuffer(); + for (int tupleIndex = 0 ; tupleIndex < tuple.length; ++tupleIndex) { + if (tupleIndex == Catch.INDEX_YEAR || tupleIndex == Catch.INDEX_HAUL || + tupleIndex == Catch.INDEX_SPECIES) { + sb.append(tuple[tupleIndex]).append(';'); + } + } + + // store number sum + String nombreValue = tuple[Catch.INDEX_NUMBER]; + String lineNumber = tuple[AbstractDataEntity.INDEX_LINE]; + try { + Double nombre = Double.valueOf(nombreValue); + String key = sb.toString(); + if (nombreForKey.containsKey(key)) { + Double oldValue = nombreForKey.get(key); + Double newValue = oldValue + nombre; + nombreForKey.put(key, newValue); + + // sauvegarde la premiere ligne qui correspond a un regroupement de clé + firstLineForKey.put(key, lineNumber); + } + else { + nombreForKey.put(key, nombre); + } + + + // autre validation (rien a voir avec le reste attention) + // Vérifier que le nombre d'individus dans CAPT$Nombre n'est pas 0 si le poids dans CAPT$Poids>0. + String poidsValue = tuple[Catch.INDEX_WEIGHT]; + Double poids = Double.valueOf(poidsValue); + if (poids > 0 && nombre <= 0) { + ValidationError error = new ValidationError(); + error.setLevel(ValidationLevel.ERROR); + error.setLineNumber(lineNumber); + error.setMessage(_("coser.business.control.error.noCatchNumberWithWeight")); + validationErrors.add(error); + } + } + catch (NumberFormatException ex) { + // par trop grave, normalement les données deviennet + // valide au fil de la validation + if (log.isWarnEnabled()) { + log.warn("Can't parse " + nombreValue + " as double"); + } + } + } + + // now look for invalid data + for (Map.Entry<String, Double> sumObservation : nombreForKey.entrySet()) { + String key = sumObservation.getKey(); + Double value = sumObservation.getValue(); + if (value < config.getObervationNobsmin()) { + + String lineNumber = firstLineForKey.get(key); + + ValidationError error = new ValidationError(); + error.setLevel(ValidationLevel.ERROR); + error.setLineNumber(lineNumber); + error.setMessage(_("coser.business.control.error.minObservationCount", key)); + validationErrors.add(error); + } + } + + return validationErrors; + } + + /** + * Alerte si Somme(TAILLES$Nombre par TAILLES$Annee|Strate|Espece) < nobsmin + * + * @param control + * @param progress + * @return + */ + protected List<ValidationError> validateCategorySpecificLength( + Control control, ProgressMonitor progress) { + List<ValidationError> validationErrors = new ArrayList<ValidationError>(); + + int total = control.getLength().size() - 1; + if (progress != null) { + progress.setText(_("coser.business.control.step.observation", 0)); + progress.setTotal(total); + } + + // "Campagne","Annee","Trait","Espece","Nombre","Poids" + // regroupement par campagne/annee/trait + Map<String, Double> nombreForKey = new HashMap<String, Double>(); + Map<String, String> firstLineForKey = new HashMap<String, String>(); + + // parcours des elements + Iterator<String[]> itTuple = control.getLength().iterator(); + itTuple.next(); // skip header + int lineIndex = 1; // skip header + while (itTuple.hasNext()) { + String[] tuple = itTuple.next(); + + // update progress + if (progress != null) { + int progressPercent = (int)((double)lineIndex / (double)total * 100.0); + progress.setText(_("coser.business.control.step.observation", progressPercent)); + progress.setCurrent(lineIndex); + ++lineIndex; + } + + // compute key + StringBuffer sb = new StringBuffer(); + for (int tupleIndex = 0 ; tupleIndex < tuple.length; ++tupleIndex) { + if (tupleIndex == Length.INDEX_YEAR || tupleIndex == Length.INDEX_HAUL || + tupleIndex == Length.INDEX_SPECIES) { + sb.append(tuple[tupleIndex]).append(';'); + } + } + + // store number sum + String nombreValue = tuple[Length.INDEX_NUMBER]; + String lineNumber = tuple[AbstractDataEntity.INDEX_LINE]; + try { + Double nombre = Double.valueOf(nombreValue); + String key = sb.toString(); + if (nombreForKey.containsKey(key)) { + Double oldValue = nombreForKey.get(key); + Double newValue = oldValue + nombre; + nombreForKey.put(key, newValue); + + // sauvegarde la premiere ligne qui correspond a un regroupement de clé + firstLineForKey.put(key, lineNumber); + } + else { + nombreForKey.put(key, nombre); + } + } + catch (NumberFormatException ex) { + // par trop grave, normalement les données deviennet + // valide au fil de la validation + if (log.isWarnEnabled()) { + log.warn("Can't parse " + nombreValue + " as double"); + } + } + } + + // now look for invalid data + for (Map.Entry<String, Double> sumObservation : nombreForKey.entrySet()) { + String key = sumObservation.getKey(); + Double value = sumObservation.getValue(); + if (value < config.getObervationNobsmin()) { + + String lineNumber = firstLineForKey.get(key); + + ValidationError error = new ValidationError(); + error.setLevel(ValidationLevel.WARNING); + error.setLineNumber(lineNumber); + error.setMessage(_("coser.business.control.error.minObservationCount", key)); + validationErrors.add(error); + } + } + + return validationErrors; + } + + /** + * @param control + * @param progress + * @return + */ + protected List<ValidationError> validateCategorySpecificHaul(Control control, + ProgressMonitor progress) { + // TODO Auto-generated method stub + return null; + } + + /** + * @param control + * @param progress + * @return + */ + protected List<ValidationError> validateCategorySpecificStrata( + Control control, ProgressMonitor progress) { + return null; + } } Modified: trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties =================================================================== --- trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties 2010-10-28 13:42:45 UTC (rev 115) +++ trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties 2010-10-28 15:07:48 UTC (rev 116) @@ -1,3 +1,4 @@ +Age\ attribute\ is\ required= Can't\ create\ project= Can't\ find\ line\ %s\ for\ deletion= Can't\ find\ line\ %s\ for\ undeletion= @@ -2,10 +3,28 @@ Can't\ read\ file\ '%s'.\ Check\ CSV\ file\ separator\ (%c)= +Depth\ attribute\ is\ not\ a\ valid\ double= +Maturity\ attribute\ is\ required= Missing\ file\ %s= +Number\ attribute\ is\ not\ a\ valid\ double= Original\ line\ already\ exists\!= Project\ %s\ already\ exist= Project\ %s\ doesn't\ exists\ \!= +Project\ name\ is\ required= Selection\ %s\ already\ exists= +Selection\ name\ is\ required= +Sex\ is\ mandatory= Specy\ %s\ doesn't\ exist\ in\ referential= +Weight\ attribute\ is\ not\ a\ valid\ double= +Weight\ n'est\ pas\ compris\ entre\ ${min}\ en\ ${max}= Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s= Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s\ or\ %s= +You\ must\ enter\ a\ campagne\ name.= +You\ must\ enter\ a\ espece\ name.= +You\ must\ enter\ a\ haul\ name.= +You\ must\ enter\ a\ month\ name.= +You\ must\ enter\ a\ strata\ name.= +You\ must\ enter\ a\ stratum\ name.= +You\ must\ enter\ a\ survey\ name.= +You\ must\ enter\ a\ sweptSurface\ name.= +You\ must\ enter\ a\ trait\ name.= +You\ must\ enter\ a\ year\ name.= coser.business.category.catch=Catch @@ -15,9 +34,23 @@ coser.business.category.length=Length coser.business.category.reftax.species= coser.business.category.strata=Stata +coser.business.control.error.duplicatedLine= +coser.business.control.error.minObservationCount= +coser.business.control.error.noCatchNumberWithWeight= +coser.business.control.step.observation= +coser.business.control.step.observation.catch= +coser.business.control.step.xworks= coser.business.line=Line coser.config.database.directory.description= +coser.config.observation.nobsmin.description= coser.config.projects.directory.description= coser.config.reference.species.description= coser.config.reference.zones.description= coser.config.validator.directory.description= +lat\ attribute\ is\ not\ a\ valid\ double= +lat\ must\ contains\ at\ least\ 5\ decimal= +length\ attribute\ is\ not\ a\ valid\ double= +long\ attribute\ is\ not\ a\ valid\ double= +long\ must\ contains\ at\ least\ 5\ decimal= +number\ attribute\ is\ not\ a\ valid\ double= +sweptSurface\ attribute\ is\ not\ a\ valid\ double= Modified: trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties =================================================================== --- trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties 2010-10-28 13:42:45 UTC (rev 115) +++ trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties 2010-10-28 15:07:48 UTC (rev 116) @@ -1,3 +1,4 @@ +Age\ attribute\ is\ required= Can't\ create\ project=Impossible de cr\u00E9er le projet Can't\ find\ line\ %s\ for\ deletion= Can't\ find\ line\ %s\ for\ undeletion= @@ -2,10 +3,28 @@ Can't\ read\ file\ '%s'.\ Check\ CSV\ file\ separator\ (%c)=Impossible de lire le fichier '%s'.\nMerci de v\u00E9rifier le s\u00E9parateur utilis\u00E9 (%c). +Depth\ attribute\ is\ not\ a\ valid\ double= +Maturity\ attribute\ is\ required= Missing\ file\ %s=Fichier manquant \: %s +Number\ attribute\ is\ not\ a\ valid\ double= Original\ line\ already\ exists\!= Project\ %s\ already\ exist=Le projet %s existe d\u00E9j\u00E0 \! Project\ %s\ doesn't\ exists\ \!=Le projet %s n'existe pas \! +Project\ name\ is\ required= Selection\ %s\ already\ exists=La s\u00E9lection %s existe d\u00E9j\u00E0 \! +Selection\ name\ is\ required= +Sex\ is\ mandatory= Specy\ %s\ doesn't\ exist\ in\ referential=L'esp\u00E8ce %s n'existe pas dans le r\u00E9f\u00E9rentiel +Weight\ attribute\ is\ not\ a\ valid\ double= +Weight\ n'est\ pas\ compris\ entre\ ${min}\ en\ ${max}= Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s=Mauvais ent\u00EAte de fichier d\u00E9tect\u00E9.\n\nTrouv\u00E9\u2009\:\n\t%s\nAttendu\u2009\:\n\t%s. Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s\ or\ %s=Mauvais ent\u00EAte de fichier d\u00E9tect\u00E9.\n\nTrouv\u00E9\u2009\:\n\t%s\nAttendu\u2009\:\n\t%s\nou\u2009\:\n\t%s. +You\ must\ enter\ a\ campagne\ name.= +You\ must\ enter\ a\ espece\ name.= +You\ must\ enter\ a\ haul\ name.= +You\ must\ enter\ a\ month\ name.= +You\ must\ enter\ a\ strata\ name.= +You\ must\ enter\ a\ stratum\ name.= +You\ must\ enter\ a\ survey\ name.= +You\ must\ enter\ a\ sweptSurface\ name.= +You\ must\ enter\ a\ trait\ name.= +You\ must\ enter\ a\ year\ name.= coser.business.category.catch=Captures @@ -15,9 +34,22 @@ coser.business.category.length=Tailles coser.business.category.reftax.species=Reftax (esp\u00E8ce) coser.business.category.strata=Strates +coser.business.control.error.duplicatedLine=Ligne en doublon +coser.business.control.error.minObservationCount=Nombre minimal d'observation non atteint (%s) \: %.2f +coser.business.control.error.noCatchNumberWithWeight=Poids renseign\u00E9s, mais capture non pr\u00E9sente +coser.business.control.step.observation=Verfication du nombre d'obervation (%d%%) +coser.business.control.step.xworks=Validation par lignes (%d%%) coser.business.line=Ligne coser.config.database.directory.description=Emplacement de la base de campagnes de coser +coser.config.observation.nobsmin.description=Nombre minimal d'observation coser.config.projects.directory.description=Emplacement des projets Coser coser.config.reference.species.description=Emplacement du fichier de r\u00E9f\u00E9rence des esp\u00E8ces coser.config.reference.zones.description=Emplacement du fichier de r\u00E9f\u00E9rence des zones coser.config.validator.directory.description=Emplacement des fichiers de validations +lat\ attribute\ is\ not\ a\ valid\ double= +lat\ must\ contains\ at\ least\ 5\ decimal= +length\ attribute\ is\ not\ a\ valid\ double= +long\ attribute\ is\ not\ a\ valid\ double= +long\ must\ contains\ at\ least\ 5\ decimal= +number\ attribute\ is\ not\ a\ valid\ double= +sweptSurface\ attribute\ is\ not\ a\ valid\ double= Modified: trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java =================================================================== --- trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java 2010-10-28 13:42:45 UTC (rev 115) +++ trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java 2010-10-28 15:07:48 UTC (rev 116) @@ -32,9 +32,13 @@ import org.junit.Assert; import org.junit.Test; +import fr.ifremer.coser.CoserConstants.Category; +import fr.ifremer.coser.bean.Control; import fr.ifremer.coser.control.ValidationError; import fr.ifremer.coser.data.Catch; import fr.ifremer.coser.data.Haul; +import fr.ifremer.coser.storage.DataStorage; +import fr.ifremer.coser.storage.MemoryListStorage; /** * Test abour validation. @@ -61,17 +65,11 @@ myCatch.setData(new String[]{"1", "","","","","",""}); List<ValidationError> errors = validationService.validate(myCatch); - if (log.isDebugEnabled()) { - log.debug("Validation errors = " + errors); - } Assert.assertNotNull(errors); Assert.assertEquals(4, errors.size()); myCatch.setData(new String[]{"1", "Toto","","","","999",""}); errors = validationService.validate(myCatch); - if (log.isDebugEnabled()) { - log.debug("Validation errors = " + errors); - } Assert.assertNotNull(errors); Assert.assertEquals(3, errors.size()); } @@ -84,10 +82,7 @@ Catch myCatch = new Catch(); myCatch.setData(new String[]{"1", "Test survey","1999","Test trait","Test sp","NA","12"}); List<ValidationError> errors = validationService.validate(myCatch); - if (log.isDebugEnabled()) { - log.debug("Validation errors = " + errors); - } - Assert.assertNull(errors); + Assert.assertNull(errors); } /** @@ -102,7 +97,23 @@ haulBean.setData(new String[]{"1","COSER_TEST","2010","TRAIT3","10","STR3","0.06000","43.89000","-1.73234","115.00"}); errors = validationService.validate(haulBean); - System.out.println(errors); Assert.assertNull(errors); } + + /** + * Test, si un poids est reseigné, la capture doit l'etre. + */ + @Test + public void testWeigthWithNonCapture() { + Control control = new Control(); + DataStorage dataCatch = new MemoryListStorage(); + dataCatch.add(new String[]{"Line", "Campagne","Annee","Trait","Espece","Nombre","Poids"}); + dataCatch.add(new String[]{"1", "Test survey","1999","Test trait","Test sp","0","12"}); + control.setCatch(dataCatch); + List<ValidationError> errors = validationService.validateCategory(control, Category.CATCH, null); + if (log.isDebugEnabled()) { + log.debug("Validation errors = " + errors); + } + Assert.assertEquals(1, errors.size()); + } }