Author: fdesbois Date: 2010-01-19 18:02:46 +0000 (Tue, 19 Jan 2010) New Revision: 242 Modified: trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java trunk/suiviobsmer-business/src/main/xmi/suiviobsmer.zargo Log: Evol #2025 : Some fields in Csv file are "NA" but not reading correctly in ActivityCalendar import Modified: trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java =================================================================== --- trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java 2010-01-19 16:20:59 UTC (rev 241) +++ trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java 2010-01-19 18:02:46 UTC (rev 242) @@ -66,7 +66,7 @@ /** * CSV headers for Boat */ - public static enum BOAT implements ImportHeader { + public enum BOAT implements ImportHeader { /** Boat immatriculation **/ NAVS_COD(20), /** Boat name **/ @@ -106,7 +106,7 @@ /** * CSV headers for SamplingPlan */ - public static enum SAMPLING implements ImportHeader { + public enum SAMPLING implements ImportHeader { /** SampleRow code **/ PLAN_CODE(6), /** Company name **/ @@ -160,7 +160,7 @@ /** * CSV headers for FishingZone */ - public static enum FISHING_ZONE implements ImportHeader { + public enum FISHING_ZONE implements ImportHeader { /** FishingZone facade **/ PECHE_FACADE(10), /** FishingZone sector **/ @@ -188,7 +188,7 @@ /** * CSV headers for Contact */ - public static enum CONTACT implements ImportHeader { + public enum CONTACT implements ImportHeader { /** Contact code (create date time for existing contact) **/ CONT_CODE(0), /** Contact create date **/ @@ -239,6 +239,55 @@ } } + public enum ACTIVITY_CALENDAR implements ImportHeader { + /** ActivityCalendar year **/ + SYNA_AN, + /** ActivityCalendar fiability **/ + INDQ_COD, + /** ActivityMonth month **/ + SYNA_MOI, + /** ActivityMonth harbourCode **/ + SYNA_POR_COD, + /** ActivityMonth nbSeaDays **/ + SYNA_NOMJDM, + /** ActivityMonth nbFishingDays **/ + SYNA_NOMJDP, + /** ActivityMonth nbBoardingPersons **/ + SYNA_NOMHE, + /** ActivityMonth harbourId **/ + SYNA_TPOR_COD, + /** ActivityMonth harbourLibelle **/ + SYNA_POR_LIB, + /** ActivityProfession professionOrder **/ + META_ORDRE, + /** ActivityProfession code **/ + MET_COD, + /** ActivityProfession id **/ + MET_ID, + /** ActivityProfession libelle **/ + MET_LIB, + /** ActivityZone code **/ + SECT_COD, + /** ActivityZone gradiantCode **/ + GRA_COD, + /** ActivityZone gradiantLibelle **/ + GRA_LIB, + /** ActivityZone zoneId **/ + TSECT_COD, + /** ActivityZone zoneLibelle **/ + SECT_LIB; + + @Override + public int forContactCsv() { + return -1; + } + + @Override + public String datePattern() { + return ""; + } + } + public static int CONTACT_NB_HEADERS = 33; public static String getHeaderForContactCsv(int index) { @@ -276,15 +325,6 @@ } return valid; } -// -// public static Boolean parseContactValidation(String validation) { -// if (validation.equals("A")) { -// return Boolean.TRUE; -// } else if (validation.equals("R")) { -// return Boolean.FALSE; -// } -// return null; -// } public static String formatContactMammals(boolean mammals) { return mammals ? "X" : ""; @@ -346,28 +386,96 @@ return System.currentTimeMillis(); } + /** + * Read a string value in CsvReader from {@code header} column. + * In ActivityCalendarImport the result value can be null if the read value is "NA". + * + * @param reader CsvReader used to read the value + * @param header Column header in the CsvReader + * @return the String value read or null if set to "NA" in ActivityCalendar import. + * @throws IOException + */ public static String read(CsvReader reader, ImportHeader header) throws IOException { - return reader.get(header.name()).trim(); + String result = reader.get(header.name()).trim(); + if (header instanceof ACTIVITY_CALENDAR && result.equals("NA")) { + return null; + } + return result; } + /** + * Read an int value in CsvReader from {@code header} column. + * + * @param reader CsvReader used to read the value + * @param header Column header in the CsvReader + * @return the int value or -1 if the value read is null + * @throws IOException + * @see ImportHelper#read(CsvReader, ImportHeader) + */ public static int readInt(CsvReader reader, ImportHeader header) throws IOException { String str = read(reader, header); + if (str == null) { + return -1; + } return Integer.parseInt(str); } + /** + * Read an Integer value in CsvReader from {@code header} column. + * + * @param reader CsvReader used to read the value + * @param header Column header in the CsvReader + * @return the Integer value or null if the value read is null + * @throws IOException + * @see ImportHelper#readInt(CsvReader, ImportHeader) + */ + public static Integer readInteger(CsvReader reader, ImportHeader header) throws IOException { + int result = readInt(reader, header); + return result != -1 ? result : null; + } + + /** + * Read two values in CsvReader corresponding to a period from {@code headerBegin} and {@code headEnd} column. + * + * @param reader CsvReader used to read the value + * @param headerBegin Column header in the CsvReader for periodBegin + * @param headerEnd Column header in the CsvReader for periodEnd + * @return the PeriodDates created from periodBegin and periodEnd read from Csv file + * @throws IOException + * @throws ParseException + * @throws IllegalArgumentException + * @see ImportHelper#readDate(CsvReader, ImportHeader) + */ public static PeriodDates readPeriod(CsvReader reader, ImportHeader headerBegin, ImportHeader headerEnd) throws IOException, ParseException, IllegalArgumentException { Date end = readDate(reader, headerEnd); Date begin = readDate(reader, headerBegin); - PeriodDates period = new PeriodDates(begin, end); - period.initDayOfMonthExtremities(); - return period; + if (begin != null && end != null) { + PeriodDates period = new PeriodDates(begin, end); + period.initDayOfMonthExtremities(); + return period; + } + return null; } + /** + * Read a Date value in CsvReader from {@code header} column. + * + * @param reader CsvReader used to read the value + * @param header Column header in the CsvReader + * @return the Date value or null if the value read is null + * @throws IOException + * @throws ParseException + * @see ImportHelper#read(CsvReader, ImportHeader) + */ public static Date readDate(CsvReader reader, ImportHeader header) throws IOException, ParseException { DateFormat dateFormat = new SimpleDateFormat(header.datePattern()); - return dateFormat.parse(read(reader, header)); + String str = read(reader, header); + if (str == null) { + return null; + } + return dateFormat.parse(str); } } Modified: trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java =================================================================== --- trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java 2010-01-19 16:20:59 UTC (rev 241) +++ trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java 2010-01-19 18:02:46 UTC (rev 242) @@ -23,6 +23,7 @@ import fr.ifremer.suiviobsmer.*; import com.csvreader.CsvReader; +import fr.ifremer.suiviobsmer.ImportHelper.ACTIVITY_CALENDAR; import fr.ifremer.suiviobsmer.ImportHelper.BOAT; import fr.ifremer.suiviobsmer.entity.ActivityCalendar; import fr.ifremer.suiviobsmer.entity.ActivityCalendarDAO; @@ -161,39 +162,26 @@ while(reader.readRecord()) { currRow++; - //long firstTic = System.currentTimeMillis(); int boatImmatriculation = Integer.parseInt(reader.get(BOAT.NAVS_COD.name()).trim()); -// long tic1 = System.currentTimeMillis(); - - //Boat boat = boatDAO.findByImmatriculation(boatImmatriculation); -// TopiaContextImplementor topia = (TopiaContextImplementor)transaction; -// SQLQuery query = topia.getHibernate().createSQLQuery( -// "SELECT * FROM boat WHERE immatriculation = " + boatImmatriculation); -// -// Boat boat = (Boat)query.addEntity(BoatImpl.class).uniqueResult(); - Boat boat = boatDAO.findByImmatriculation(boatImmatriculation); //.add(Boat.IMMATRICULATION, boatImmatriculation).executeToEntity(transaction); + Boat boat = boatDAO.findByImmatriculation(boatImmatriculation); -// long tic2 = System.currentTimeMillis(); -// log.info("findByImmatriculation : " + (tic2 - tic1)); if (boat != null) { - int year = Integer.parseInt(reader.get("SYNA_AN").trim()); +// int year = Integer.parseInt(reader.get("SYNA_AN").trim()); + int year = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.SYNA_AN); + if (year == -1) { + error("Ligne non sauvegardé car l'année (SYNA_AN) n'est pas renseigné", currRow); + continue; + } ActivityCalendarKey key = new ActivityCalendarKey(boat, year); ActivityCalendar calendar = availableCalendars.get(key); -// tic1 = System.currentTimeMillis(); - // If not available select it from Database if (calendar == null) { -// TopiaContextImplementor topia = (TopiaContextImplementor)transaction; -// SQLQuery query = topia.getHibernate().createSQLQuery( -// "SELECT * FROM activityCalendar WHERE boat = '" + key.boatId() + "' AND year = " + key.year()); -// -// calendar = (ActivityCalendar)query.addEntity(ActivityCalendarImpl.class).uniqueResult(); calendar = calendarDAO.findByProperties( ActivityCalendar.BOAT, boat, @@ -201,7 +189,9 @@ // Create it if not exist in Database if (calendar == null) { - int fiability = Integer.parseInt(reader.get("INDQ_COD").trim()); + //int fiability = Integer.parseInt(reader.get("INDQ_COD").trim()); + // fiability can be -1 if NA is set in CSV file + int fiability = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.INDQ_COD); calendar = calendarDAO.create( ActivityCalendar.BOAT, key.boat(), @@ -218,21 +208,17 @@ availableCalendars.put(key, calendar); } -// ActivityCalendar calendar = calendarDAO.findByProperties( -// ActivityCalendar.BOAT, boat, -// ActivityCalendar.YEAR, year); + //int monthNum = Integer.parseInt(reader.get("SYNA_MOI").trim()); + int monthNum = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.SYNA_MOI); + if (monthNum == -1) { + error("Ligne non sauvegardé car le mois (SYNA_MOI) n'est pas renseigné", currRow); + continue; + } -// tic2 = System.currentTimeMillis(); -// log.info("calendar.findByProperties : " + (tic2 - tic1)); + //String harbourCode = reader.get("SYNA_POR_COD").trim(); + String harbourCode = ImportHelper.read(reader, ACTIVITY_CALENDAR.SYNA_POR_COD); + boolean active = harbourCode != null && !harbourCode.equals("INA"); - - String calendarCode = "[ calendrier " + year + ", navire " + boatImmatriculation + " ]"; - - int monthNum = Integer.parseInt(reader.get("SYNA_MOI").trim()); - - String harbourCode = reader.get("SYNA_POR_COD").trim(); - boolean active = !harbourCode.equals("INA"); - ActivityMonth month = calendar.getActivityMonth(monthNum); if (month == null) { @@ -242,34 +228,37 @@ month.setActivityProfession(new ArrayList<ActivityProfession>()); -// if (log.isDebugEnabled()) { -// info("Création du mois " + monthNum + " actif(" + active + ") " + calendarCode, currRow); -// } - calendar.addActivityMonth(month); } month.setActive(active); if (active) { - String nbSeaDaysString = reader.get("SYNA_NOMJDM").trim(); - Integer nbSeaDays = !nbSeaDaysString.equals("NA") ? - Integer.valueOf(nbSeaDaysString) : null; +// String nbSeaDaysString = reader.get("SYNA_NOMJDM").trim(); +// Integer nbSeaDays = !nbSeaDaysString.equals("NA") ? +// Integer.valueOf(nbSeaDaysString) : null; +// +// String nbFishingDaysString = reader.get("SYNA_NOMJDP").trim(); +// Integer nbFishingDays = !nbFishingDaysString.equals("NA") ? +// Integer.valueOf(nbFishingDaysString) : null; +// +// String nbBoardingPersonsString = reader.get("SYNA_NOMHE").trim(); +// Integer nbBoardingPersons = !nbBoardingPersonsString.equals("NA") ? +// Integer.valueOf(nbBoardingPersonsString) : null; +// +// String harbourIdString = reader.get("SYNA_TPOR_COD").trim(); +// int harbourId = !harbourIdString.equals("NA") ? Integer.parseInt(harbourIdString) : -1; +// String harbourLibelle = reader.get("SYNA_POR_LIB").trim(); +// harbourLibelle = !harbourLibelle.equals("NA") ? harbourLibelle : null; +// harbourCode = !harbourCode.equals("NA") ? harbourCode : null; - String nbFishingDaysString = reader.get("SYNA_NOMJDP").trim(); - Integer nbFishingDays = !nbFishingDaysString.equals("NA") ? - Integer.valueOf(nbFishingDaysString) : null; + Integer nbSeaDays = ImportHelper.readInteger(reader, ACTIVITY_CALENDAR.SYNA_NOMJDM); + Integer nbFishingDays = ImportHelper.readInteger(reader, ACTIVITY_CALENDAR.SYNA_NOMJDP); + Integer nbBoardingPersons = ImportHelper.readInteger(reader, ACTIVITY_CALENDAR.SYNA_NOMHE); + + int harbourId = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.SYNA_TPOR_COD); + String harbourLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.SYNA_POR_LIB); - String nbBoardingPersonsString = reader.get("SYNA_NOMHE").trim(); - Integer nbBoardingPersons = !nbBoardingPersonsString.equals("NA") ? - Integer.valueOf(nbBoardingPersonsString) : null; - - String harbourIdString = reader.get("SYNA_TPOR_COD").trim(); - int harbourId = !harbourIdString.equals("NA") ? Integer.parseInt(harbourIdString) : -1; - String harbourLibelle = reader.get("SYNA_POR_LIB").trim(); - harbourLibelle = !harbourLibelle.equals("NA") ? harbourLibelle : null; - harbourCode = !harbourCode.equals("NA") ? harbourCode : null; - month.setHarbourId(harbourId); month.setHarbourCode(harbourCode); month.setHarbourLibelle(harbourLibelle); @@ -277,10 +266,17 @@ month.setNbFishingDays(nbFishingDays); month.setNbSeaDays(nbSeaDays); - int professionOrder = Integer.parseInt(reader.get("META_ORDRE").trim()); - String professionCode = reader.get("MET_COD").trim(); - int professionId = Integer.parseInt(reader.get("MET_ID").trim()); - String professionLibelle = reader.get("MET_LIB").trim(); +// int professionOrder = Integer.parseInt(reader.get("META_ORDRE").trim()); // NA +// String professionCode = reader.get("MET_COD").trim(); +// int professionId = Integer.parseInt(reader.get("MET_ID").trim()); +// String professionLibelle = reader.get("MET_LIB").trim(); + + // Can be equals to -1 if set to NA in Csv File (not a problem for ordering Profession) + int professionOrder = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.META_ORDRE); + String professionCode = ImportHelper.read(reader, ACTIVITY_CALENDAR.MET_COD); + int professionId = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.MET_ID); + String professionLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.MET_LIB); + ActivityProfession profession = month.getActivityProfession(professionOrder); if (profession == null) { @@ -291,12 +287,6 @@ profession.setActivityZone(new ArrayList<ActivityZone>()); -// if (log.isDebugEnabled()) { -// info("Création du métier " + professionOrder + " code(" + professionCode + ") " + -// "[ mois " + monthNum + " ] " + -// calendarCode, currRow); -// } - month.addActivityProfession(profession); } @@ -304,12 +294,18 @@ profession.setCode(professionCode); profession.setLibelle(professionLibelle); - String zoneCode = reader.get("SECT_COD").trim(); - int gradiantCode = Integer.parseInt(reader.get("GRA_COD").trim()); - String gradiantLibelle = reader.get("GRA_LIB").trim(); - int zoneId = Integer.parseInt(reader.get("TSECT_COD").trim()); - String zoneLibelle = reader.get("SECT_LIB").trim(); +// String zoneCode = reader.get("SECT_COD").trim(); +// int gradiantCode = Integer.parseInt(reader.get("GRA_COD").trim()); +// String gradiantLibelle = reader.get("GRA_LIB").trim(); +// int zoneId = Integer.parseInt(reader.get("TSECT_COD").trim()); +// String zoneLibelle = reader.get("SECT_LIB").trim(); + String zoneCode = ImportHelper.read(reader, ACTIVITY_CALENDAR.SECT_COD); + int gradiantCode = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.GRA_COD); + String gradiantLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.GRA_LIB); + int zoneId = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.TSECT_COD); + String zoneLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.SECT_LIB); + ActivityZone zone = profession.getActivityZone(zoneCode); if (zone == null) { @@ -317,19 +313,7 @@ ActivityZone.ZONE_CODE, zoneCode, ActivityZone.ACTIVITY_PROFESSION, profession); -// if (log.isDebugEnabled()) { -// info("Création de la zone " + zoneCode + " [ mois " + monthNum + " ] " + -// calendarCode, currRow); -// } - profession.addActivityZone(zone); - - } else { -// if (log.isDebugEnabled()) { -// info("Ecrasement de la zone " + zoneCode + " [ mois " + monthNum + " ] " + -// calendarCode, currRow); -// } - //warn("Zone déjà existante [" + zoneCode + "] ! :: " + Arrays.toString(reader.getValues()), currRow); } zone.setZoneId(zoneId); @@ -344,11 +328,6 @@ warn("Navire inexistant dans le référentiel : " + boatImmatriculation, currRow); } -// if (log.isDebugEnabled()) { -// long tic = System.currentTimeMillis(); -// log.debug("Time : " + DurationFormatUtils.formatDurationHMS(tic - firstTic)); -// } - if (result % 1000 == 0) { tic1 = System.currentTimeMillis(); @@ -360,16 +339,6 @@ log.debug("commit : " + (tic2 - tic1)); } firstTic = ImportHelper.logTimeAndMemory(log, firstTic, "calendrier d'activité ligne " + currRow); -// if (log.isInfoEnabled()) { -// log.info("RUNNING... Import calendrier d'activité ligne " + currRow); -// Runtime runtime = Runtime.getRuntime(); -// long mem = runtime.totalMemory() - runtime.freeMemory(); -// long memMega = mem / 1024 / 1024; -// log.info("Memory : " + memMega + " Mo"); -// long tic = System.currentTimeMillis(); -// log.info("Time : " + DurationFormatUtils.formatDurationHMS(tic - firstTic)); -// firstTic = System.currentTimeMillis(); -// } } } @@ -378,16 +347,9 @@ long stopTime = System.currentTimeMillis(); - Runtime runtime = Runtime.getRuntime(); + // Force garbage collector + System.gc(); - long mem = (runtime.totalMemory() - runtime.freeMemory()) / 1048576; - log.info("Memory before gc : " + mem + " Mo"); - - System.gc(); - - mem = (runtime.totalMemory() - runtime.freeMemory()) / 1048576; - log.info("Memory after gc : " + mem + " Mo"); - String execTime = DurationFormatUtils.formatDurationHMS(stopTime - startTime); info("Nombre de ligne ajouté : " + result, -1); Modified: trunk/suiviobsmer-business/src/main/xmi/suiviobsmer.zargo =================================================================== (Binary files differ)