Author: fdesbois Date: 2010-03-29 16:17:48 +0000 (Mon, 29 Mar 2010) New Revision: 396 Added: trunk/wao-business/src/test/java/fr/ifremer/wao/io/ trunk/wao-business/src/test/java/fr/ifremer/wao/io/ImportHelperTest.java trunk/wao-business/src/test/resources/import/dates.csv Modified: trunk/README.txt trunk/changelog.txt trunk/pom.xml trunk/wao-business/src/main/java/fr/ifremer/wao/WaoContext.java trunk/wao-business/src/main/java/fr/ifremer/wao/io/ImportHelper.java trunk/wao-business/src/main/java/fr/ifremer/wao/service/ActivityCalendarImport.java trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceBoatImpl.java trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceContactImpl.java trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceReferentialImpl.java trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceSamplingImpl.java trunk/wao-business/src/test/resources/log4j.properties Log: Evo #2233 : Improve parsing dates from Csv import Modified: trunk/README.txt =================================================================== --- trunk/README.txt 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/README.txt 2010-03-29 16:17:48 UTC (rev 396) @@ -19,5 +19,5 @@ Configuration au lancement -------------------------- --Dconfig.file=/path/to/Wao.properties +-Dconfig.path=/path/to/properties/file (file named Wao.properties) -Dwao.log.home=/path/to/log/folder Modified: trunk/changelog.txt =================================================================== --- trunk/changelog.txt 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/changelog.txt 2010-03-29 16:17:48 UTC (rev 396) @@ -9,6 +9,8 @@ Evolutions ++++++++++ +- [fdesbois] Evo #2233 : Gérer les formats de dates par défaut d'OpenOffice pour + l'import CSV (JJ/MM/AA) - [fdesbois] Evo #2229 : Gestion du chemin vers les logs de l'application - [fdesbois] Evo #2226 : Amélioration gestion interne des enumérations (ContactState) @@ -19,9 +21,9 @@ * commons-lang 2.4 -> 2.5 * commons-beanutils 1.8.0 -> 1.8.2 * junit 4.7 -> 4.8.1 - * topia 2.3.0-beta-8 -> 2.3 + * topia 2.3.0-beta-8 -> 2.3.1 * eugene 2.0.0-beta-4 -> 2.0 - * nuiton-utils 1.1.5 -> 1.2 + * nuiton-utils 1.1.5 -> 1.2.1 * h2 1.2.128 -> 1.2.132 * postgresql 8.3-603.jdbc4 -> 8.4-701.jdbc4 Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/pom.xml 2010-03-29 16:17:48 UTC (rev 396) @@ -215,7 +215,7 @@ <labs.id>154</labs.id> <!-- libraries version --> - <nuitonutils.version>1.2</nuitonutils.version> + <nuitonutils.version>1.2.1-SNAPSHOT</nuitonutils.version> <topia.version>2.3.1-SNAPSHOT</topia.version> <eugene.version>2.0</eugene.version> <tapestry.version>5.1.0.5</tapestry.version> Modified: trunk/wao-business/src/main/java/fr/ifremer/wao/WaoContext.java =================================================================== --- trunk/wao-business/src/main/java/fr/ifremer/wao/WaoContext.java 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/main/java/fr/ifremer/wao/WaoContext.java 2010-03-29 16:17:48 UTC (rev 396) @@ -25,6 +25,7 @@ import fr.ifremer.wao.entity.ActivityCalendar; import java.io.IOException; +import java.nio.charset.Charset; import java.util.Date; import java.util.Properties; import org.apache.commons.lang.RandomStringUtils; @@ -207,6 +208,10 @@ return ""; } + public static Charset getCsvCharset() { + return Charset.forName("UTF-8"); + } + public static String prepareSampleRowCode(String code) { // Replace single number after year by a double one : 2010_111 -> 2010_0111 code = code.replaceFirst("_(\\d{3})$", "_0$1"); Modified: trunk/wao-business/src/main/java/fr/ifremer/wao/io/ImportHelper.java =================================================================== --- trunk/wao-business/src/main/java/fr/ifremer/wao/io/ImportHelper.java 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/main/java/fr/ifremer/wao/io/ImportHelper.java 2010-03-29 16:17:48 UTC (rev 396) @@ -28,7 +28,6 @@ import fr.ifremer.wao.io.WaoCsvHeader.DatedHeader; import fr.ifremer.wao.io.WaoCsvHeader.IOHeader; import fr.ifremer.wao.io.WaoCsvHeader.SAMPLING; -import fr.ifremer.wao.io.WaoCsvHeader.SamplingHeader; import java.io.IOException; import java.text.DateFormat; import java.text.ParseException; @@ -37,11 +36,11 @@ import java.util.Date; import java.util.GregorianCalendar; import java.util.Locale; -import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DurationFormatUtils; import org.nuiton.util.PeriodDates; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * ImportHelper @@ -56,8 +55,17 @@ */ public class ImportHelper { + private static final Logger logger = + LoggerFactory.getLogger(ImportHelper.class); + protected CsvReader reader; + // 2000 -> 2099 + public static final int CURRENT_MILLENIUM = 2000; + + public static final String DEFAULT_PATTERN = "dd/MM/yyyy"; + public static final String MONTH_PATTERN = "MM/yyyy"; + public ImportHelper(CsvReader reader) { this.reader = reader; } @@ -196,17 +204,70 @@ */ public static Date readDate(CsvReader reader, DatedHeader header) throws IOException, ParseException { - DateFormat dateFormat = new SimpleDateFormat(header.datePattern()); + String str = read(reader, header); - if (StringUtils.isEmpty(str)) { + if (logger.isDebugEnabled()) { + logger.debug("Read date " + header.name() + " : " + str); + } + return parseDate(str, header.datePattern()); + } + + /** + * Parse a date from a Csv file. Depends on pattern wanted. The default + * pattern from OpenOffice or Excel is considered for parsing (JJ/MM/AA). + * + * @param value of the reading date + * @param pattern for parsing + * @return the parsed date + * @throws ParseException + */ + public static Date parseDate(String value, String pattern) + throws ParseException { + + if (StringUtils.isEmpty(value)) { return null; } - // Parsing problem with Pattern dd/MM/yyyy which is valid for dd/MM/yy date - // The length is tested to avoid accepting wrong dates - if (str.length() != header.datePattern().length()) { - throw new ParseException("Date error : " + str, str.length()); + + DateFormat dateFormat = null; + Date result = null; + Calendar calendar = Calendar.getInstance(); + + // Case of MONTH_PATTERN : MM/yyyy + if (pattern.equals(MONTH_PATTERN)) { + dateFormat = new SimpleDateFormat(DEFAULT_PATTERN); + try { + // Try to parse with default_pattern (ok for default OpenOffice + // or Excel date format) + result = dateFormat.parse(value); + calendar.setTime(result); + // In this case, we must suppress the day from the date + // Put it to 1 + calendar.set(Calendar.DAY_OF_MONTH, 1); + } catch (ParseException eee) { + // If the date can't be parse, it must be ok with header pattern + logger.debug("Date parse error : " + eee.getMessage()); + dateFormat = new SimpleDateFormat(pattern); + result = dateFormat.parse(value); + calendar.setTime(result); + } + } else { + dateFormat = new SimpleDateFormat(pattern); + result = dateFormat.parse(value); + calendar.setTime(result); } - return dateFormat.parse(str); + + // Check if the year is correct + int year = calendar.get(Calendar.YEAR); + if (logger.isDebugEnabled()) { + logger.debug("Read date year : " + year); + } + // If year < to the current millenium, the year is incorrect, must + // add the millenium to the year + // It's because of century problem : /10 must be /2010 + if (year < CURRENT_MILLENIUM) { + calendar.add(Calendar.YEAR, CURRENT_MILLENIUM); + } + return calendar.getTime(); } public static long logTimeAndMemory(Logger log, long tic1, String msg) { Modified: trunk/wao-business/src/main/java/fr/ifremer/wao/service/ActivityCalendarImport.java =================================================================== --- trunk/wao-business/src/main/java/fr/ifremer/wao/service/ActivityCalendarImport.java 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/main/java/fr/ifremer/wao/service/ActivityCalendarImport.java 2010-03-29 16:17:48 UTC (rev 396) @@ -142,7 +142,7 @@ try { transaction = WaoContext.getTopiaRootContext().beginTransaction(); - CsvReader reader = new CsvReader(input, Charset.forName("UTF-8")); + CsvReader reader = new CsvReader(input, WaoContext.getCsvCharset()); long startTime = System.currentTimeMillis(); Modified: trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceBoatImpl.java =================================================================== --- trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceBoatImpl.java 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceBoatImpl.java 2010-03-29 16:17:48 UTC (rev 396) @@ -383,7 +383,7 @@ try { transaction = rootContext.beginTransaction(); - CsvReader reader = new CsvReader(input, Charset.forName("UTF-8")); + CsvReader reader = new CsvReader(input, WaoContext.getCsvCharset()); reader.readHeaders(); long startTime = System.currentTimeMillis(); @@ -499,7 +499,7 @@ file.deleteOnExit(); FileOutputStream output = new FileOutputStream(file); - writer = new CsvWriter(output, ',', Charset.forName("UTF-8")); + writer = new CsvWriter(output, ',', WaoContext.getCsvCharset()); ExportHelper<BOAT> export = new ExportHelper<BOAT>(writer, BOAT.getTotalHeaders()) { @Override Modified: trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceContactImpl.java =================================================================== --- trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceContactImpl.java 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceContactImpl.java 2010-03-29 16:17:48 UTC (rev 396) @@ -209,7 +209,7 @@ } FileOutputStream output = new FileOutputStream(file); - writer = new CsvWriter(output, ',', Charset.forName("UTF-8")); + writer = new CsvWriter(output, ',', WaoContext.getCsvCharset()); ExportHelper<ContactHeader> export = new ExportHelper<ContactHeader>(writer, CONTACT.getTotalHeaders()) { @@ -337,7 +337,7 @@ try { transaction = rootContext.beginTransaction(); - reader = new CsvReader(input, Charset.forName("UTF-8")); + reader = new CsvReader(input, WaoContext.getCsvCharset()); reader.readHeaders(); ContactDAO dao = WaoModelDAOHelper.getContactDAO(transaction); Modified: trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceReferentialImpl.java =================================================================== --- trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceReferentialImpl.java 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceReferentialImpl.java 2010-03-29 16:17:48 UTC (rev 396) @@ -182,7 +182,7 @@ try { transaction = rootContext.beginTransaction(); - CsvReader reader = new CsvReader(input, Charset.forName("UTF-8")); + CsvReader reader = new CsvReader(input, WaoContext.getCsvCharset()); reader.readHeaders(); FishingZoneDAO dao = WaoModelDAOHelper.getFishingZoneDAO(transaction); Modified: trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceSamplingImpl.java =================================================================== --- trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceSamplingImpl.java 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/main/java/fr/ifremer/wao/service/ServiceSamplingImpl.java 2010-03-29 16:17:48 UTC (rev 396) @@ -595,7 +595,7 @@ file.deleteOnExit(); FileOutputStream output = new FileOutputStream(file); - writer = new CsvWriter(output, ',', Charset.forName("UTF-8")); + writer = new CsvWriter(output, ',', WaoContext.getCsvCharset()); // Get min and max dates to have the biggest period // Date min = (Date)query.executeToObject("MIN(S." + SampleRow.PERIOD_BEGIN + ")"); @@ -727,7 +727,7 @@ try { transaction = rootContext.beginTransaction(); - CsvReader reader = new CsvReader(input, Charset.forName("UTF-8")); + CsvReader reader = new CsvReader(input, WaoContext.getCsvCharset()); reader.readHeaders(); SampleRowDAO dao = WaoModelDAOHelper.getSampleRowDAO(transaction); @@ -749,7 +749,8 @@ continue; } - String districts = reader.get(FISHING_ZONE.PECHE_DIVISION.name()); + String districts = ImportHelper.read(reader, + FISHING_ZONE.PECHE_DIVISION); row = dao.findByCode(code); // Refuse existing SampleRow @@ -965,11 +966,11 @@ " _ lastMonthId = " + lastMonthColumnId); } - DateFormat dateFormat = SAMPLING.getDateFormat(); - for (int i = firstMonthColumnId; i < lastMonthColumnId; i++) { - Date monthDate = dateFormat.parse(reader.getHeader(i)); + Date monthDate = ImportHelper.parseDate(reader.getHeader(i), + SAMPLING.defaultDatePattern()); + Integer monthValue = StringUtils.isEmpty(reader.get(i)) ? null : Integer.parseInt(reader.get(i)); Added: trunk/wao-business/src/test/java/fr/ifremer/wao/io/ImportHelperTest.java =================================================================== --- trunk/wao-business/src/test/java/fr/ifremer/wao/io/ImportHelperTest.java (rev 0) +++ trunk/wao-business/src/test/java/fr/ifremer/wao/io/ImportHelperTest.java 2010-03-29 16:17:48 UTC (rev 396) @@ -0,0 +1,201 @@ + +package fr.ifremer.wao.io; + +import com.csvreader.CsvReader; +import fr.ifremer.wao.io.WaoCsvHeader.DatedHeader; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.Date; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.nuiton.util.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * @author fdesbois + */ +public class ImportHelperTest { + + private static final Logger logger = + LoggerFactory.getLogger(ImportHelperTest.class); + + public ImportHelperTest() { + } + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of parseContactValidation method, of class ImportHelper. + */ + //@Test + public void testParseContactValidation() throws Exception { + System.out.println("parseContactValidation"); + } + + /** + * Test of parseContactMammals method, of class ImportHelper. + */ + //@Test + public void testParseContactMammals() throws Exception { + System.out.println("parseContactMammals"); + } + + /** + * Test of parseContactCreateDate method, of class ImportHelper. + */ + //@Test + public void testParseContactCreateDate() throws Exception { + System.out.println("parseContactCreateDate"); + } + + /** + * Test of readSampleRowCode method, of class ImportHelper. + */ + //@Test + public void testReadSampleRowCode() throws Exception { + System.out.println("readSampleRowCode"); + } + + /** + * Test of read method, of class ImportHelper. + */ + //@Test + public void testRead() throws Exception { + System.out.println("read"); + } + + /** + * Test of readInt method, of class ImportHelper. + */ + //@Test + public void testReadInt() throws Exception { + System.out.println("readInt"); + } + + /** + * Test of readInteger method, of class ImportHelper. + */ + //@Test + public void testReadInteger() throws Exception { + System.out.println("readInteger"); + } + + /** + * Test of readPeriod method, of class ImportHelper. + */ + //@Test + public void testReadPeriod() throws Exception { + System.out.println("readPeriod"); + } + + /** + * Test of readDate method, of class ImportHelper. + */ + @Test + public void testReadDate() throws Exception { + logger.info("readDate"); + + InputStream stream = + getClass().getResourceAsStream("/import/dates.csv"); + CsvReader reader = new CsvReader(stream, Charset.forName("UTF-8")); + reader.readHeaders(); + + reader.readRecord(); + + // TEST1 : Date like JJ/MM/AAAA + // normal pattern : dd/MM/yyyy + DatedHeader header1 = new DatedHeader() { + + @Override + public String datePattern() { + return "dd/MM/yyyy"; + } + + @Override + public String name() { + return "DATE_1"; + } + }; + + Date res = ImportHelper.readDate(reader, header1); + Date ref = DateUtils.createDate(10, 2, 2010); + Assert.assertEquals(ref, res); + + // TEST2 : Date like JJ/MM/AA + // same pattern as header1 : dd/MM/yyyy + DatedHeader header2 = new DatedHeader() { + + @Override + public String datePattern() { + return "dd/MM/yyyy"; + } + + @Override + public String name() { + return "DATE_2"; + } + }; + + res = ImportHelper.readDate(reader, header2); + Assert.assertEquals(ref, res); + + // TEST3: date like MM/AAAA + // pattern without day : MM/yyyy + DatedHeader header3 = new DatedHeader() { + + @Override + public String datePattern() { + return "MM/yyyy"; + } + + @Override + public String name() { + return "DATE_3"; + } + }; + + res = ImportHelper.readDate(reader, header3); + Date ref2 = DateUtils.createDate(1, 2, 2010); + Assert.assertEquals(ref2, res); + + // TEST4: date like JJ/MM/AA + // same pattern as header3 : MM/yyyy + DatedHeader header4 = new DatedHeader() { + + @Override + public String datePattern() { + return "MM/yyyy"; + } + + @Override + public String name() { + return "DATE_4"; + } + }; + + res = ImportHelper.readDate(reader, header4); + Assert.assertEquals(ref2, res); + + } + +} \ No newline at end of file Property changes on: trunk/wao-business/src/test/java/fr/ifremer/wao/io/ImportHelperTest.java ___________________________________________________________________ Added: svn:keywords + "Author Date Id Revision HeadURL" Added: trunk/wao-business/src/test/resources/import/dates.csv =================================================================== --- trunk/wao-business/src/test/resources/import/dates.csv (rev 0) +++ trunk/wao-business/src/test/resources/import/dates.csv 2010-03-29 16:17:48 UTC (rev 396) @@ -0,0 +1,2 @@ +DATE_1,DATE_2,DATE_3,DATE_4 +10/02/2010,10/02/10,02/2010,10/02/10 \ No newline at end of file Modified: trunk/wao-business/src/test/resources/log4j.properties =================================================================== --- trunk/wao-business/src/test/resources/log4j.properties 2010-03-29 12:59:43 UTC (rev 395) +++ trunk/wao-business/src/test/resources/log4j.properties 2010-03-29 16:17:48 UTC (rev 396) @@ -8,6 +8,6 @@ log4j.appender.A1.layout=org.apache.log4j.PatternLayout log4j.appender.A1.layout.ConversionPattern=%d [%p] %c{2} %m%n -log4j.logger.fr.ifremer.wao=INFO +log4j.logger.fr.ifremer.wao=DEBUG log4j.logger.org.nuiton.util=INFO log4j.logger.org.nuiton.util.beans.BinderProvider=ERROR