This is an automated email from the git hooks/post-receive script. New commit to branch feature/3798_new_export_API in repository nuiton-csv. See http://git.nuiton.org/nuiton-csv.git commit c939fbeb360041286e8ceb2438f8371f7c449f54 Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Dec 10 11:22:16 2015 +0100 Fix encoding pb, always use array of bytes is the safest way to fix that --- src/main/java/org/nuiton/csv/Exporter.java | 155 +++++++--------------- src/main/java/org/nuiton/csv/ExporterBuilder.java | 3 +- src/test/java/org/nuiton/csv/ExporterTest.java | 91 +++++++------ 3 files changed, 98 insertions(+), 151 deletions(-) diff --git a/src/main/java/org/nuiton/csv/Exporter.java b/src/main/java/org/nuiton/csv/Exporter.java index 60193c2..c869be3 100644 --- a/src/main/java/org/nuiton/csv/Exporter.java +++ b/src/main/java/org/nuiton/csv/Exporter.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; import java.io.StringWriter; @@ -337,9 +338,14 @@ public class Exporter<O> { protected boolean firstRowConsumed; /** - * The current row to export. + * Current row to export as an array of bytes. */ - protected String currentRow; + protected byte[] currentRow; + + /** + * Position to read in the current row array. + */ + protected int currentRowPosition; protected static <E> ExporterAction<E> of(ExporterConfiguration<E> configuration, Iterable<E> data, boolean writeHeader) { return new ExporterAction<>(configuration, writeHeader, data); @@ -354,16 +360,16 @@ public class Exporter<O> { public void export(OutputStream outputStream) throws IOException { - for (String row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { - outputStream.write(row.getBytes()); + for (byte[] row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { + outputStream.write(row); } } public void export(Writer writer) throws IOException { - for (String row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { - writer.write(row); + for (byte[] row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { + writer.write(new String(row)); } } @@ -373,9 +379,9 @@ public class Exporter<O> { return new InputStream() { @Override - public int read() throws IOException { + public int read() { - String row = readFirstRowOrGetCurrentRow(); + byte[] row = readFirstRowOrGetCurrentRow(); int result; if (row == null) { @@ -385,11 +391,13 @@ public class Exporter<O> { } else { - int length = row.length(); + int length = row.length; - result = getCharAt(0); + result = currentRow[currentRowPosition++]; - updateCurrentRow(1, length == 1); + if (currentRowPosition >= length) { + currentRow = readNextRow(); + } } @@ -401,139 +409,66 @@ public class Exporter<O> { public void close() throws IOException { // nothing to do } + }; } public Reader toReader() { - return new Reader() { - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - String row = readFirstRowOrGetCurrentRow(); - - int result; - if (row == null) { - - // end of stream - result = -1; - - } else { - - int length = row.length(); - if (length >= len) { - - // enough in this row to fill buffer - fillBuffer(cbuf, off, len); - updateCurrentRow(len, false); - - result = len; - - } else { - - // fill with that we got - fillBuffer(cbuf, off, length); - row = updateCurrentRow(length, true); - if (row == null) { - - // no more stuff - result = length; - - } else { - - // ask for the rest - result = length + read(cbuf, off + length, len - length); - - } - - } - - } - - return result; - - } - - @Override - public void close() throws IOException { - // nothing to do - } - }; + return new InputStreamReader(toInputStream(), configuration.getCharset()); } - protected String readFirstRowOrGetCurrentRow() { + protected byte[] readFirstRowOrGetCurrentRow() { if (!firstRowConsumed) { firstRowConsumed = true; if (writeHeader) { - currentRow = readHeader(); + readHeader(); } else { - currentRow = readNextRow(); + readNextRow(); } } return currentRow; } - protected String readNextRow() { + protected byte[] readNextRow() { - String nextRow = null; + currentRowPosition = 0; if (dataIterator.hasNext()) { E row = dataIterator.next(); try { - nextRow = readRow(row); + currentRow = readRow(row); } catch (Exception e) { throw new ImportRuntimeException("Could not obtain row", e); } - } - - return nextRow; - - } - - protected void fillBuffer(char[] cbuf, int off, int len) { - for (int i = 0; i < len; i++) { - char c = getCharAt(i); - cbuf[i + off] = c; - } - } - - protected char getCharAt(int index) { - char charAt = currentRow.charAt(index); - //FIXME-tc-2015-12-10 See if we can do better - char encodedChar = (char) configuration.getCharset().encode(String.valueOf(charAt)).get(); - return encodedChar; - } - - protected String updateCurrentRow(int len, boolean computeNext) { - - if (computeNext) { - currentRow = readNextRow(); } else { - currentRow = currentRow.substring(len); - if (currentRow.isEmpty()) { - currentRow = null; - } + currentRow = null; } + return currentRow; } - protected String readHeader() { + protected void readHeader() { + + currentRowPosition = 0; + String cellSeparator = configuration.getCellSeparator(); StringBuilder rowBuilder = new StringBuilder(); int index = 0; for (ExportableColumn<E, ?> column : configuration.getColumns()) { String formattedCell = column.getHeaderName(); - addCellToRowBuilder(rowBuilder, ++index, formattedCell); + addCellToRowBuilder(rowBuilder, ++index, formattedCell, cellSeparator); } - return toFormattedRow(rowBuilder); + currentRow = toCurrentRow(rowBuilder); } - protected String readRow(E row) throws Exception { + protected byte[] readRow(E row) throws Exception { + String cellSeparator = configuration.getCellSeparator(); StringBuilder rowBuilder = new StringBuilder(); int index = 0; for (ExportableColumn<E, Object> column : configuration.getColumns()) { @@ -541,27 +476,27 @@ public class Exporter<O> { Object cell = column.getValue(row); String formattedCell = column.formatValue(cell); Preconditions.checkNotNull(formattedCell, - String.format("column for header %s returned a null value.%s", + String.format("column for header %s (%s) returned a null value.", column.getHeaderName(), column.toString())); - addCellToRowBuilder(rowBuilder, ++index, formattedCell); + addCellToRowBuilder(rowBuilder, ++index, formattedCell, cellSeparator); } - return toFormattedRow(rowBuilder); + return toCurrentRow(rowBuilder); } - protected void addCellToRowBuilder(StringBuilder rowBuilder, int index, String formattedCell) { - String csvCell = StringUtil.escapeCsvValue(formattedCell, configuration.getCellSeparator()); + protected void addCellToRowBuilder(StringBuilder rowBuilder, int index, String formattedCell, String cellSeparator) { + String csvCell = StringUtil.escapeCsvValue(formattedCell, cellSeparator); rowBuilder.append(csvCell); if (index < nbCellsPerRow) { - rowBuilder.append(configuration.getCellSeparator()); + rowBuilder.append(cellSeparator); } } - protected String toFormattedRow(StringBuilder rowBuilder) { + protected byte[] toCurrentRow(StringBuilder rowBuilder) { rowBuilder.append(configuration.getEndOfLineSeparator()); String formattedRow = rowBuilder.toString(); - return formattedRow; + return formattedRow.getBytes(configuration.getCharset()); } } diff --git a/src/main/java/org/nuiton/csv/ExporterBuilder.java b/src/main/java/org/nuiton/csv/ExporterBuilder.java index b55fe6c..5444ef0 100644 --- a/src/main/java/org/nuiton/csv/ExporterBuilder.java +++ b/src/main/java/org/nuiton/csv/ExporterBuilder.java @@ -4,6 +4,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; /** * To build an {@link Exporter}. @@ -19,7 +20,7 @@ public class ExporterBuilder<O> { public static final String DEFAULT_END_OF_LINE_SEPARATOR = "\n"; - public static final Charset DEFAULT_CHARSET = Charset.defaultCharset(); + public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; protected final ExporterConfiguration<O> configuration; diff --git a/src/test/java/org/nuiton/csv/ExporterTest.java b/src/test/java/org/nuiton/csv/ExporterTest.java index a3768e8..6c0a6cd 100644 --- a/src/test/java/org/nuiton/csv/ExporterTest.java +++ b/src/test/java/org/nuiton/csv/ExporterTest.java @@ -7,7 +7,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.nuiton.util.DateUtil; @@ -32,28 +31,40 @@ public class ExporterTest { public static final Charset CHARSET = Charset.forName("UTF-8"); public static final Iterable<String> RESULTS = ImmutableList.of( - "DATE~TITLE~NUMBER--\n", - "DATE~TITLE~NUMBER--\n" + - "01/12/2011~Batman~1--\n", - "DATE~TITLE~NUMBER--\n" + - "01/12/2011~Batman~1--\n" + + "\"D~ATE\"~TITLEé~NUMBER--\n", + "\"D~ATE\"~TITLEé~NUMBER--\n" + + "01/12/2011~Batéman~1--\n", + "\"D~ATE\"~TITLEé~NUMBER--\n" + + "01/12/2011~Batéman~1--\n" + "02/12/2011~~7--\n", - "DATE~TITLE~NUMBER--\n" + - "01/12/2011~Batman~1--\n" + + "\"D~ATE\"~TITLEé~NUMBER--\n" + + "01/12/2011~Batéman~1--\n" + "02/12/2011~~7--\n" + "07/12/2011~~9--\n" + "18/12/2011~~18--\n" + "23/12/2011~~4--\n" ); - protected Iterable<Set<RowBean>> sets; + protected static final ExportModel<RowBean> EXPORT_MODEL = new ExportModel<RowBean>() { - protected ExportModel<RowBean> model = new RowBeanExportModel(); + @Override + public char getSeparator() { + return 0; + } + + @Override + public Iterable<ExportableColumn<RowBean, Object>> getColumnsForExport() { + ModelBuilder<RowBean> modelBuilder = new ModelBuilder<>(); + modelBuilder.newColumnForExport("D~ATE", "date", Common.DAY); + modelBuilder.newColumnForExport("TITLEé", "title"); + modelBuilder.newColumnForExport("NUMBER", "number", Common.INTEGER); + return (Iterable) modelBuilder.getColumnsForExport(); + } + }; - protected Exporter<RowBean> exporter; + protected static final Iterable<Set<RowBean>> SETS; - @Before - public void setUp() { + static { ImmutableSet.Builder<RowBean> builder = ImmutableSet.builder(); @@ -61,7 +72,7 @@ public class ExporterTest { setBuilder.add(Collections.<RowBean>emptySet()); - builder.add(new RowBean(DateUtil.createDate(1, 12, 2011), "Batman", 1, RowBeanEnum.ONE)); + builder.add(new RowBean(DateUtil.createDate(1, 12, 2011), "Batéman", 1, RowBeanEnum.ONE)); setBuilder.add(builder.build()); builder.add(new RowBean(DateUtil.createDate(2, 12, 2011), "", 7, RowBeanEnum.TWO)); @@ -72,26 +83,26 @@ public class ExporterTest { builder.add((new RowBean(DateUtil.createDate(23, 12, 2011), "", 4, RowBeanEnum.TWO))); setBuilder.add(builder.build()); - sets = setBuilder.build(); - - exporter = Exporter - .builder(model) - .setCellSeparator("~") - .setEndOfLineSeparator("--\n") - .setCharset(CHARSET) - .build(); + SETS = setBuilder.build(); } + protected static final Exporter<RowBean> EXPORTER = Exporter + .builder(EXPORT_MODEL) + .setCellSeparator("~") + .setEndOfLineSeparator("--\n") + .setCharset(CHARSET) + .build(); + @Test public void testExportToString() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { - String csv = exporter.exportToString(set); + String csv = EXPORTER.exportToString(set); - String csvWithoutHeader = exporter.exportToStringWithoutHeader(set); + String csvWithoutHeader = EXPORTER.exportToStringWithoutHeader(set); assertExportResultEquals(resultIterator, csv, csvWithoutHeader); @@ -103,17 +114,17 @@ public class ExporterTest { public void testExportToWriter() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { String csv; try (StringWriter writer = new StringWriter()) { - exporter.exportToWriter(set, writer); + EXPORTER.exportToWriter(set, writer); csv = writer.toString(); } String csvWithoutHeader; try (StringWriter writer = new StringWriter()) { - exporter.exportToWriterWithoutHeader(set, writer); + EXPORTER.exportToWriterWithoutHeader(set, writer); csvWithoutHeader = writer.toString(); } @@ -131,16 +142,16 @@ public class ExporterTest { int index = 0; Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { File exportFile = new File(parentFile, (index++) + ".csv"); try (OutputStream outputStream = new FileOutputStream(exportFile)) { - exporter.exportToOutputStream(set, outputStream); + EXPORTER.exportToOutputStream(set, outputStream); } File exportFileWithoutHeader = new File(parentFile, (index) + "-withoutHeader.csv"); try (OutputStream outputStream = new FileOutputStream(exportFileWithoutHeader)) { - exporter.exportToOutputStreamWithoutHeader(set, outputStream); + EXPORTER.exportToOutputStreamWithoutHeader(set, outputStream); } String csv = Files.toString(exportFile, CHARSET); @@ -160,15 +171,15 @@ public class ExporterTest { Iterator<String> resultIterator = RESULTS.iterator(); int index = 0; - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { File exportFile = new File(parentFile, (index++) + ".csv"); - exporter.exportToFile(set, exportFile); + EXPORTER.exportToFile(set, exportFile); String csv = Files.toString(exportFile, CHARSET); File exportFileWithoutHeader = new File(parentFile, (index) + "-withoutHeader.csv"); - exporter.exportToFileWithoutHeader(set, exportFileWithoutHeader); + EXPORTER.exportToFileWithoutHeader(set, exportFileWithoutHeader); String csvWithoutHeader = Files.toString(exportFileWithoutHeader, CHARSET); @@ -182,15 +193,15 @@ public class ExporterTest { public void testExportToReader() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { String csv; - try (Reader reader = exporter.exportToReader(set)) { + try (Reader reader = EXPORTER.exportToReader(set)) { csv = IOUtils.toString(reader); } String csvWithoutHeader; - try (Reader reader = exporter.exportToReaderWithoutHeader(set)) { + try (Reader reader = EXPORTER.exportToReaderWithoutHeader(set)) { csvWithoutHeader = IOUtils.toString(reader); } @@ -204,14 +215,14 @@ public class ExporterTest { public void testExportToInputStream() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { String csv; - try (InputStream inputStream = exporter.exportToInputStream(set)) { + try (InputStream inputStream = EXPORTER.exportToInputStream(set)) { csv = IOUtils.toString(inputStream); } String csvWithoutHeader; - try (InputStream inputStream = exporter.exportToInputStreamWithoutHeader(set)) { + try (InputStream inputStream = EXPORTER.exportToInputStreamWithoutHeader(set)) { csvWithoutHeader = IOUtils.toString(inputStream); } assertExportResultEquals(resultIterator, csv, csvWithoutHeader); -- To stop receiving notification emails like this one, please contact nuiton.org SCM administrator <admin+scm@nuiton.org>.