- Nuiton-csv-commits - mailman.cloud.codelutin.com

Jenkins build became unstable: nuiton-csv-ci » Nuiton CSV #6
by admin+ci-nuiton.org＠codelutin.com 23 May '14

23 May '14

See <http://ci.nuiton.org/jenkins/job/nuiton-csv-ci/org.nuiton$nuiton-csv/6/chan…>

1 1

r2617 - trunk/src/main/java/org/nuiton/csv
by bleny＠users.nuiton.org 23 May '14

23 May '14

Author: bleny Date: 2014-05-23 17:13:18 +0200 (Fri, 23 May 2014) New Revision: 2617 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2617 Log: fixes #3144 remove BOM from header Modified: trunk/src/main/java/org/nuiton/csv/Import.java Modified: trunk/src/main/java/org/nuiton/csv/Import.java =================================================================== --- trunk/src/main/java/org/nuiton/csv/Import.java 2014-05-23 14:54:53 UTC (rev 2616) +++ trunk/src/main/java/org/nuiton/csv/Import.java 2014-05-23 15:13:18 UTC (rev 2617) @@ -64,6 +64,13 @@ /** Logger. */ private static final Log log = LogFactory.getLog(Import.class); + /** + * BOM is a character you can find in the beginning of a UTF-8 encoded file that should not be interpreted. + * + * See http://en.wikipedia.org/wiki/Byte_order_mark + */ + protected static final char BOM_CHARACTER = '\uFEFF'; + /** Csv import model. */ protected ImportModel<E> model; @@ -331,6 +338,9 @@ try { String[] result = reader.getHeaders(); + if (result.length > 0) { + result[0] = removeBomCharacter(result[0]); + } return result; } catch (IOException e) { throw new ImportRuntimeException( @@ -338,6 +348,10 @@ } } + protected String removeBomCharacter(String s) { + return s.replaceFirst(BOM_CHARACTER + "?", ""); + } + protected List<ImportableColumn<E, Object>> getNonIgnoredHeaders() { List<ImportableColumn<E, Object>> nonIgnoredHeaders = new ArrayList<ImportableColumn<E, Object>>();

1 0

r2616 - in trunk/src/test: java/org/nuiton/csv resources
by bleny＠users.nuiton.org 23 May '14

23 May '14

Author: bleny Date: 2014-05-23 16:54:53 +0200 (Fri, 23 May 2014) New Revision: 2616 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2616 Log: refs #3144 add test to prove issue Added: trunk/src/test/java/org/nuiton/csv/Utf8WithBomTest.java trunk/src/test/resources/csv-file-encoded-in-UTF-8-with-BOM.csv Added: trunk/src/test/java/org/nuiton/csv/Utf8WithBomTest.java =================================================================== --- trunk/src/test/java/org/nuiton/csv/Utf8WithBomTest.java (rev 0) +++ trunk/src/test/java/org/nuiton/csv/Utf8WithBomTest.java 2014-05-23 14:54:53 UTC (rev 2616) @@ -0,0 +1,63 @@ +package org.nuiton.csv; + +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.InputStream; +import java.util.List; + +public class Utf8WithBomTest { + + private static final Log log = LogFactory.getLog(Utf8WithBomTest.class); + + protected ImportModel<ObjectUtils.Null> importModel = new ImportModel<ObjectUtils.Null>() { + + @Override + public char getSeparator() { + return ';'; + } + + @Override + public void pushCsvHeaderNames(List<String> headerNames) { + // do nothing + } + + @Override + public ObjectUtils.Null newEmptyInstance() { + throw new UnsupportedOperationException(); + } + + @Override + public Iterable<ImportableColumn<ObjectUtils.Null, Object>> getColumnsForImport() { + ModelBuilder<ObjectUtils.Null> builder = new ModelBuilder<ObjectUtils.Null>(); + + builder.newMandatoryColumn("N° 1", "whatever"); + builder.newMandatoryColumn("N° 2", "whatever"); + builder.newMandatoryColumn("Nom", "whatever"); + + return (Iterable) builder.getColumnsForImport(); + } + + }; + + @Test + public void testImport() { + + InputStream csv = getClass().getResourceAsStream("/csv-file-encoded-in-UTF-8-with-BOM.csv"); + + Import<ObjectUtils.Null> csvImport = Import.newImport(importModel, csv); + + try { + csvImport.prepareAndValidate(); + } catch (ImportRuntimeException e) { + if (log.isDebugEnabled()) { + log.debug("exception caught", e); + } + Assert.fail("no exception should have been raised"); + } + + } +} Added: trunk/src/test/resources/csv-file-encoded-in-UTF-8-with-BOM.csv =================================================================== --- trunk/src/test/resources/csv-file-encoded-in-UTF-8-with-BOM.csv (rev 0) +++ trunk/src/test/resources/csv-file-encoded-in-UTF-8-with-BOM.csv 2014-05-23 14:54:53 UTC (rev 2616) @@ -0,0 +1,3 @@ +N° 1;N° 2;Nom +1;2;Taiste +

1 0

r2615 - trunk/src/main/java/org/nuiton/csv
by bleny＠users.nuiton.org 23 May '14

23 May '14

Author: bleny Date: 2014-05-23 16:42:34 +0200 (Fri, 23 May 2014) New Revision: 2615 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2615 Log: fixes #3203 Modified: trunk/src/main/java/org/nuiton/csv/Common.java Modified: trunk/src/main/java/org/nuiton/csv/Common.java =================================================================== --- trunk/src/main/java/org/nuiton/csv/Common.java 2014-03-26 07:44:09 UTC (rev 2614) +++ trunk/src/main/java/org/nuiton/csv/Common.java 2014-05-23 14:42:34 UTC (rev 2615) @@ -182,9 +182,12 @@ public static class DateValue implements ValueParserFormatter<Date> { + protected String dateFormatPattern; + protected DateFormat dateFormat; public DateValue(String dateFormatPattern) { + this.dateFormatPattern = dateFormatPattern; dateFormat = new SimpleDateFormat(dateFormatPattern); } @@ -192,7 +195,11 @@ public Date parse(String value) throws ParseException { Date date = null; if (StringUtils.isNotBlank(value)) { - date = dateFormat.parse(value); + try { + date = dateFormat.parse(value); + } catch (ParseException e) { + throw new ParseException("unable to parse date, expected format is " + dateFormatPattern, 0); + } } return date; }

1 0

[ANN] - Nuiton CSV 3.0-rc-1 released
by nuiton.org Release Notification 26 Mar '14

26 Mar '14

The Nuiton CSV team is pleased to announce the nuiton-csv-3.0-rc-1 release! Simple CSV API Documentation of the project can be found here: https://doc.nuiton.org/nuiton-csv Changes ------- Changes in this version include: New features: o Add a *YEAR* ValueParserformatter Issue: 3133. Thanks to Tony Chemit. Resolved by tchemit. o Be able to parse more integer and long number Issue: 3131. Thanks to Tony Chemit. Resolved by tchemit. Changes: o Updates mavenpom to 5.0.4 Issue: 3132. Thanks to Tony Chemit. Resolved by tchemit. Downloads --------- For a manual installation, you can download files here: https://forge.nuiton.org/projects/nuiton-csv/files * nuiton-csv-3.0-rc-1-deps.zip - https://forge.nuiton.org/attachments/download/1016 * nuiton-csv-3.0-rc-1-full.zip - https://forge.nuiton.org/attachments/download/1017 Maven artifacts --------------- Artifacts are deployed in Maven Central Repository http://repo1.maven.org/maven2/ Find us at * http://search.maven.org/#artifactdetails|org.nuiton|nuiton-csv|3.0-rc-1|jar Have fun! -Nuiton CSV team

1 0

r2614 - trunk
by maven-release＠users.nuiton.org 26 Mar '14

26 Mar '14

Author: maven-release Date: 2014-03-26 08:44:09 +0100 (Wed, 26 Mar 2014) New Revision: 2614 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2614 Log: [maven-release-plugin] prepare for next development iteration Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2014-03-26 07:44:06 UTC (rev 2613) +++ trunk/pom.xml 2014-03-26 07:44:09 UTC (rev 2614) @@ -9,7 +9,7 @@ </parent> <artifactId>nuiton-csv</artifactId> - <version>3.0-rc-1</version> + <version>3.0-SNAPSHOT</version> <name>Nuiton CSV</name> <description>Simple CSV API</description> @@ -46,12 +46,12 @@ <scm> <connection> - scm:svn:https://svn.nuiton.org/nuiton-csv/tags/nuiton-csv-3.0-rc-1 + scm:svn:https://svn.nuiton.org/nuiton-csv/trunk </connection> <developerConnection> - scm:svn:https://svn.nuiton.org/nuiton-csv/tags/nuiton-csv-3.0-rc-1 + scm:svn:https://svn.nuiton.org/nuiton-csv/trunk </developerConnection> - <url>https://forge.nuiton.org/projects/nuiton-csv/repository/show/tags/nuiton-cs…</url> + <url>https://forge.nuiton.org/projects/nuiton-csv/repository/show/trunk</url> </scm> <distributionManagement> <site>

1 0

r2613 - in tags: . nuiton-csv-3.0-rc-1
by maven-release＠users.nuiton.org 26 Mar '14

26 Mar '14

Author: maven-release Date: 2014-03-26 08:44:06 +0100 (Wed, 26 Mar 2014) New Revision: 2613 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2613 Log: [maven-release-plugin] copy for tag nuiton-csv-3.0-rc-1 Added: tags/nuiton-csv-3.0-rc-1/ Property changes on: tags/nuiton-csv-3.0-rc-1 ___________________________________________________________________ Added: svn:ignore + target velocity.log maven.log .eclipse .classpath *.iml *.ipr *.iws .settings .project testdir .idea Added: svn:mergeinfo +

1 0

r2612 - trunk
by maven-release＠users.nuiton.org 26 Mar '14

26 Mar '14

Author: maven-release Date: 2014-03-26 08:44:03 +0100 (Wed, 26 Mar 2014) New Revision: 2612 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2612 Log: [maven-release-plugin] prepare release nuiton-csv-3.0-rc-1 Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2014-03-26 07:42:52 UTC (rev 2611) +++ trunk/pom.xml 2014-03-26 07:44:03 UTC (rev 2612) @@ -9,7 +9,7 @@ </parent> <artifactId>nuiton-csv</artifactId> - <version>3.0-SNAPSHOT</version> + <version>3.0-rc-1</version> <name>Nuiton CSV</name> <description>Simple CSV API</description> @@ -46,12 +46,12 @@ <scm> <connection> - scm:svn:https://svn.nuiton.org/nuiton-csv/trunk + scm:svn:https://svn.nuiton.org/nuiton-csv/tags/nuiton-csv-3.0-rc-1 </connection> <developerConnection> - scm:svn:https://svn.nuiton.org/nuiton-csv/trunk + scm:svn:https://svn.nuiton.org/nuiton-csv/tags/nuiton-csv-3.0-rc-1 </developerConnection> - <url>https://forge.nuiton.org/projects/nuiton-csv/repository/show/trunk</url> + <url>https://forge.nuiton.org/projects/nuiton-csv/repository/show/tags/nuiton-cs…</url> </scm> <distributionManagement> <site>

1 0

r2611 - trunk/src/main/java/org/nuiton/csv
by tchemit＠users.nuiton.org 26 Mar '14

26 Mar '14

Author: tchemit Date: 2014-03-26 08:42:52 +0100 (Wed, 26 Mar 2014) New Revision: 2611 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2611 Log: fixes #3131: Be able to parse more integer and long number fixes #3133: Add a *YEAR* ValueParserformatter Modified: trunk/src/main/java/org/nuiton/csv/Common.java Modified: trunk/src/main/java/org/nuiton/csv/Common.java =================================================================== --- trunk/src/main/java/org/nuiton/csv/Common.java 2014-03-26 07:42:29 UTC (rev 2610) +++ trunk/src/main/java/org/nuiton/csv/Common.java 2014-03-26 07:42:52 UTC (rev 2611) @@ -100,6 +100,9 @@ public static final ValueParserFormatter<Date> WEEK = new DateValue("w/yyyy"); + public static final ValueParserFormatter<Date> YEAR = + new DateValue("yyyy"); + public static <E extends Map<String, Object>, T> MapProperty<E, T> newMapProperty(String propertyName) { return new MapProperty<E, T>(propertyName); } @@ -251,7 +254,7 @@ if (StringUtils.isBlank(value)) { result = null; } else { - Integer ordinal = null; + Integer ordinal; try { ordinal = Integer.valueOf(value); } catch (NumberFormatException e) { @@ -263,7 +266,8 @@ "Ordinal value [" + ordinal + "] not inbound (possible value from [0.." + maxValue + "] for enum " + Arrays.toString(universe), - 0); + 0 + ); } result = universe[ordinal]; } @@ -464,7 +468,13 @@ @Override protected Integer parseNoneEmptyValue(String value) { - return Integer.valueOf(value); + // use a float to be able to parse for example 6e+06 (see https://forge.nuiton.org/issues/3131) + Float aFloat = Float.valueOf(value); + if (aFloat > Integer.MAX_VALUE) { + // too big + throw new IllegalArgumentException(value + " is too big to be an int, should be a long."); + } + return aFloat.intValue(); } } @@ -485,7 +495,13 @@ @Override protected Long parseNoneEmptyValue(String value) { - return Long.valueOf(value); + // use a double to be able to parse for example 6e+06 (see https://forge.nuiton.org/issues/3131) + Double aDouble = Double.valueOf(value); + if (aDouble > Long.MAX_VALUE) { + // too big + throw new IllegalArgumentException(value + " is too big to be an long."); + } + return aDouble.longValue(); } }

1 0

r2610 - trunk/src/main/java/org/nuiton/csv
by tchemit＠users.nuiton.org 26 Mar '14

26 Mar '14

Author: tchemit Date: 2014-03-26 08:42:29 +0100 (Wed, 26 Mar 2014) New Revision: 2610 Url: http://forge.nuiton.org/projects/nuiton-csv/repository/revisions/2610 Log: add comment about execption Modified: trunk/src/main/java/org/nuiton/csv/ExportableColumn.java Modified: trunk/src/main/java/org/nuiton/csv/ExportableColumn.java =================================================================== --- trunk/src/main/java/org/nuiton/csv/ExportableColumn.java 2014-03-26 07:34:52 UTC (rev 2609) +++ trunk/src/main/java/org/nuiton/csv/ExportableColumn.java 2014-03-26 07:42:29 UTC (rev 2610) @@ -34,6 +34,7 @@ String getHeaderName(); + //FIXME tchemit-2014-03-26 Should use a specific expection. T getValue(E object) throws Exception; String formatValue(T value);

1 0