branch feature/3798_new_export_API updated (ada693e -> c939fbe)
This is an automated email from the git hooks/post-receive script. New change to branch feature/3798_new_export_API in repository nuiton-csv. See http://git.nuiton.org/nuiton-csv.git from ada693e Prefer use a StringWriter to export to string, rather than using a ByteArrayOutputStream new c939fbe Fix encoding pb, always use array of bytes is the safest way to fix that The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit c939fbeb360041286e8ceb2438f8371f7c449f54 Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Dec 10 11:22:16 2015 +0100 Fix encoding pb, always use array of bytes is the safest way to fix that Summary of changes: src/main/java/org/nuiton/csv/Exporter.java | 155 +++++++--------------- src/main/java/org/nuiton/csv/ExporterBuilder.java | 3 +- src/test/java/org/nuiton/csv/ExporterTest.java | 91 +++++++------ 3 files changed, 98 insertions(+), 151 deletions(-) -- To stop receiving notification emails like this one, please contact nuiton.org SCM administrator <admin+scm@nuiton.org>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/3798_new_export_API in repository nuiton-csv. See http://git.nuiton.org/nuiton-csv.git commit c939fbeb360041286e8ceb2438f8371f7c449f54 Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Dec 10 11:22:16 2015 +0100 Fix encoding pb, always use array of bytes is the safest way to fix that --- src/main/java/org/nuiton/csv/Exporter.java | 155 +++++++--------------- src/main/java/org/nuiton/csv/ExporterBuilder.java | 3 +- src/test/java/org/nuiton/csv/ExporterTest.java | 91 +++++++------ 3 files changed, 98 insertions(+), 151 deletions(-) diff --git a/src/main/java/org/nuiton/csv/Exporter.java b/src/main/java/org/nuiton/csv/Exporter.java index 60193c2..c869be3 100644 --- a/src/main/java/org/nuiton/csv/Exporter.java +++ b/src/main/java/org/nuiton/csv/Exporter.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; import java.io.StringWriter; @@ -337,9 +338,14 @@ public class Exporter<O> { protected boolean firstRowConsumed; /** - * The current row to export. + * Current row to export as an array of bytes. */ - protected String currentRow; + protected byte[] currentRow; + + /** + * Position to read in the current row array. + */ + protected int currentRowPosition; protected static <E> ExporterAction<E> of(ExporterConfiguration<E> configuration, Iterable<E> data, boolean writeHeader) { return new ExporterAction<>(configuration, writeHeader, data); @@ -354,16 +360,16 @@ public class Exporter<O> { public void export(OutputStream outputStream) throws IOException { - for (String row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { - outputStream.write(row.getBytes()); + for (byte[] row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { + outputStream.write(row); } } public void export(Writer writer) throws IOException { - for (String row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { - writer.write(row); + for (byte[] row = readFirstRowOrGetCurrentRow(); row != null; row = readNextRow()) { + writer.write(new String(row)); } } @@ -373,9 +379,9 @@ public class Exporter<O> { return new InputStream() { @Override - public int read() throws IOException { + public int read() { - String row = readFirstRowOrGetCurrentRow(); + byte[] row = readFirstRowOrGetCurrentRow(); int result; if (row == null) { @@ -385,11 +391,13 @@ public class Exporter<O> { } else { - int length = row.length(); + int length = row.length; - result = getCharAt(0); + result = currentRow[currentRowPosition++]; - updateCurrentRow(1, length == 1); + if (currentRowPosition >= length) { + currentRow = readNextRow(); + } } @@ -401,139 +409,66 @@ public class Exporter<O> { public void close() throws IOException { // nothing to do } + }; } public Reader toReader() { - return new Reader() { - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - String row = readFirstRowOrGetCurrentRow(); - - int result; - if (row == null) { - - // end of stream - result = -1; - - } else { - - int length = row.length(); - if (length >= len) { - - // enough in this row to fill buffer - fillBuffer(cbuf, off, len); - updateCurrentRow(len, false); - - result = len; - - } else { - - // fill with that we got - fillBuffer(cbuf, off, length); - row = updateCurrentRow(length, true); - if (row == null) { - - // no more stuff - result = length; - - } else { - - // ask for the rest - result = length + read(cbuf, off + length, len - length); - - } - - } - - } - - return result; - - } - - @Override - public void close() throws IOException { - // nothing to do - } - }; + return new InputStreamReader(toInputStream(), configuration.getCharset()); } - protected String readFirstRowOrGetCurrentRow() { + protected byte[] readFirstRowOrGetCurrentRow() { if (!firstRowConsumed) { firstRowConsumed = true; if (writeHeader) { - currentRow = readHeader(); + readHeader(); } else { - currentRow = readNextRow(); + readNextRow(); } } return currentRow; } - protected String readNextRow() { + protected byte[] readNextRow() { - String nextRow = null; + currentRowPosition = 0; if (dataIterator.hasNext()) { E row = dataIterator.next(); try { - nextRow = readRow(row); + currentRow = readRow(row); } catch (Exception e) { throw new ImportRuntimeException("Could not obtain row", e); } - } - - return nextRow; - - } - - protected void fillBuffer(char[] cbuf, int off, int len) { - for (int i = 0; i < len; i++) { - char c = getCharAt(i); - cbuf[i + off] = c; - } - } - - protected char getCharAt(int index) { - char charAt = currentRow.charAt(index); - //FIXME-tc-2015-12-10 See if we can do better - char encodedChar = (char) configuration.getCharset().encode(String.valueOf(charAt)).get(); - return encodedChar; - } - - protected String updateCurrentRow(int len, boolean computeNext) { - - if (computeNext) { - currentRow = readNextRow(); } else { - currentRow = currentRow.substring(len); - if (currentRow.isEmpty()) { - currentRow = null; - } + currentRow = null; } + return currentRow; } - protected String readHeader() { + protected void readHeader() { + + currentRowPosition = 0; + String cellSeparator = configuration.getCellSeparator(); StringBuilder rowBuilder = new StringBuilder(); int index = 0; for (ExportableColumn<E, ?> column : configuration.getColumns()) { String formattedCell = column.getHeaderName(); - addCellToRowBuilder(rowBuilder, ++index, formattedCell); + addCellToRowBuilder(rowBuilder, ++index, formattedCell, cellSeparator); } - return toFormattedRow(rowBuilder); + currentRow = toCurrentRow(rowBuilder); } - protected String readRow(E row) throws Exception { + protected byte[] readRow(E row) throws Exception { + String cellSeparator = configuration.getCellSeparator(); StringBuilder rowBuilder = new StringBuilder(); int index = 0; for (ExportableColumn<E, Object> column : configuration.getColumns()) { @@ -541,27 +476,27 @@ public class Exporter<O> { Object cell = column.getValue(row); String formattedCell = column.formatValue(cell); Preconditions.checkNotNull(formattedCell, - String.format("column for header %s returned a null value.%s", + String.format("column for header %s (%s) returned a null value.", column.getHeaderName(), column.toString())); - addCellToRowBuilder(rowBuilder, ++index, formattedCell); + addCellToRowBuilder(rowBuilder, ++index, formattedCell, cellSeparator); } - return toFormattedRow(rowBuilder); + return toCurrentRow(rowBuilder); } - protected void addCellToRowBuilder(StringBuilder rowBuilder, int index, String formattedCell) { - String csvCell = StringUtil.escapeCsvValue(formattedCell, configuration.getCellSeparator()); + protected void addCellToRowBuilder(StringBuilder rowBuilder, int index, String formattedCell, String cellSeparator) { + String csvCell = StringUtil.escapeCsvValue(formattedCell, cellSeparator); rowBuilder.append(csvCell); if (index < nbCellsPerRow) { - rowBuilder.append(configuration.getCellSeparator()); + rowBuilder.append(cellSeparator); } } - protected String toFormattedRow(StringBuilder rowBuilder) { + protected byte[] toCurrentRow(StringBuilder rowBuilder) { rowBuilder.append(configuration.getEndOfLineSeparator()); String formattedRow = rowBuilder.toString(); - return formattedRow; + return formattedRow.getBytes(configuration.getCharset()); } } diff --git a/src/main/java/org/nuiton/csv/ExporterBuilder.java b/src/main/java/org/nuiton/csv/ExporterBuilder.java index b55fe6c..5444ef0 100644 --- a/src/main/java/org/nuiton/csv/ExporterBuilder.java +++ b/src/main/java/org/nuiton/csv/ExporterBuilder.java @@ -4,6 +4,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; /** * To build an {@link Exporter}. @@ -19,7 +20,7 @@ public class ExporterBuilder<O> { public static final String DEFAULT_END_OF_LINE_SEPARATOR = "\n"; - public static final Charset DEFAULT_CHARSET = Charset.defaultCharset(); + public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; protected final ExporterConfiguration<O> configuration; diff --git a/src/test/java/org/nuiton/csv/ExporterTest.java b/src/test/java/org/nuiton/csv/ExporterTest.java index a3768e8..6c0a6cd 100644 --- a/src/test/java/org/nuiton/csv/ExporterTest.java +++ b/src/test/java/org/nuiton/csv/ExporterTest.java @@ -7,7 +7,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.nuiton.util.DateUtil; @@ -32,28 +31,40 @@ public class ExporterTest { public static final Charset CHARSET = Charset.forName("UTF-8"); public static final Iterable<String> RESULTS = ImmutableList.of( - "DATE~TITLE~NUMBER--\n", - "DATE~TITLE~NUMBER--\n" + - "01/12/2011~Batman~1--\n", - "DATE~TITLE~NUMBER--\n" + - "01/12/2011~Batman~1--\n" + + "\"D~ATE\"~TITLEé~NUMBER--\n", + "\"D~ATE\"~TITLEé~NUMBER--\n" + + "01/12/2011~Batéman~1--\n", + "\"D~ATE\"~TITLEé~NUMBER--\n" + + "01/12/2011~Batéman~1--\n" + "02/12/2011~~7--\n", - "DATE~TITLE~NUMBER--\n" + - "01/12/2011~Batman~1--\n" + + "\"D~ATE\"~TITLEé~NUMBER--\n" + + "01/12/2011~Batéman~1--\n" + "02/12/2011~~7--\n" + "07/12/2011~~9--\n" + "18/12/2011~~18--\n" + "23/12/2011~~4--\n" ); - protected Iterable<Set<RowBean>> sets; + protected static final ExportModel<RowBean> EXPORT_MODEL = new ExportModel<RowBean>() { - protected ExportModel<RowBean> model = new RowBeanExportModel(); + @Override + public char getSeparator() { + return 0; + } + + @Override + public Iterable<ExportableColumn<RowBean, Object>> getColumnsForExport() { + ModelBuilder<RowBean> modelBuilder = new ModelBuilder<>(); + modelBuilder.newColumnForExport("D~ATE", "date", Common.DAY); + modelBuilder.newColumnForExport("TITLEé", "title"); + modelBuilder.newColumnForExport("NUMBER", "number", Common.INTEGER); + return (Iterable) modelBuilder.getColumnsForExport(); + } + }; - protected Exporter<RowBean> exporter; + protected static final Iterable<Set<RowBean>> SETS; - @Before - public void setUp() { + static { ImmutableSet.Builder<RowBean> builder = ImmutableSet.builder(); @@ -61,7 +72,7 @@ public class ExporterTest { setBuilder.add(Collections.<RowBean>emptySet()); - builder.add(new RowBean(DateUtil.createDate(1, 12, 2011), "Batman", 1, RowBeanEnum.ONE)); + builder.add(new RowBean(DateUtil.createDate(1, 12, 2011), "Batéman", 1, RowBeanEnum.ONE)); setBuilder.add(builder.build()); builder.add(new RowBean(DateUtil.createDate(2, 12, 2011), "", 7, RowBeanEnum.TWO)); @@ -72,26 +83,26 @@ public class ExporterTest { builder.add((new RowBean(DateUtil.createDate(23, 12, 2011), "", 4, RowBeanEnum.TWO))); setBuilder.add(builder.build()); - sets = setBuilder.build(); - - exporter = Exporter - .builder(model) - .setCellSeparator("~") - .setEndOfLineSeparator("--\n") - .setCharset(CHARSET) - .build(); + SETS = setBuilder.build(); } + protected static final Exporter<RowBean> EXPORTER = Exporter + .builder(EXPORT_MODEL) + .setCellSeparator("~") + .setEndOfLineSeparator("--\n") + .setCharset(CHARSET) + .build(); + @Test public void testExportToString() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { - String csv = exporter.exportToString(set); + String csv = EXPORTER.exportToString(set); - String csvWithoutHeader = exporter.exportToStringWithoutHeader(set); + String csvWithoutHeader = EXPORTER.exportToStringWithoutHeader(set); assertExportResultEquals(resultIterator, csv, csvWithoutHeader); @@ -103,17 +114,17 @@ public class ExporterTest { public void testExportToWriter() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { String csv; try (StringWriter writer = new StringWriter()) { - exporter.exportToWriter(set, writer); + EXPORTER.exportToWriter(set, writer); csv = writer.toString(); } String csvWithoutHeader; try (StringWriter writer = new StringWriter()) { - exporter.exportToWriterWithoutHeader(set, writer); + EXPORTER.exportToWriterWithoutHeader(set, writer); csvWithoutHeader = writer.toString(); } @@ -131,16 +142,16 @@ public class ExporterTest { int index = 0; Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { File exportFile = new File(parentFile, (index++) + ".csv"); try (OutputStream outputStream = new FileOutputStream(exportFile)) { - exporter.exportToOutputStream(set, outputStream); + EXPORTER.exportToOutputStream(set, outputStream); } File exportFileWithoutHeader = new File(parentFile, (index) + "-withoutHeader.csv"); try (OutputStream outputStream = new FileOutputStream(exportFileWithoutHeader)) { - exporter.exportToOutputStreamWithoutHeader(set, outputStream); + EXPORTER.exportToOutputStreamWithoutHeader(set, outputStream); } String csv = Files.toString(exportFile, CHARSET); @@ -160,15 +171,15 @@ public class ExporterTest { Iterator<String> resultIterator = RESULTS.iterator(); int index = 0; - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { File exportFile = new File(parentFile, (index++) + ".csv"); - exporter.exportToFile(set, exportFile); + EXPORTER.exportToFile(set, exportFile); String csv = Files.toString(exportFile, CHARSET); File exportFileWithoutHeader = new File(parentFile, (index) + "-withoutHeader.csv"); - exporter.exportToFileWithoutHeader(set, exportFileWithoutHeader); + EXPORTER.exportToFileWithoutHeader(set, exportFileWithoutHeader); String csvWithoutHeader = Files.toString(exportFileWithoutHeader, CHARSET); @@ -182,15 +193,15 @@ public class ExporterTest { public void testExportToReader() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { String csv; - try (Reader reader = exporter.exportToReader(set)) { + try (Reader reader = EXPORTER.exportToReader(set)) { csv = IOUtils.toString(reader); } String csvWithoutHeader; - try (Reader reader = exporter.exportToReaderWithoutHeader(set)) { + try (Reader reader = EXPORTER.exportToReaderWithoutHeader(set)) { csvWithoutHeader = IOUtils.toString(reader); } @@ -204,14 +215,14 @@ public class ExporterTest { public void testExportToInputStream() throws Exception { Iterator<String> resultIterator = RESULTS.iterator(); - for (Set<RowBean> set : sets) { + for (Set<RowBean> set : SETS) { String csv; - try (InputStream inputStream = exporter.exportToInputStream(set)) { + try (InputStream inputStream = EXPORTER.exportToInputStream(set)) { csv = IOUtils.toString(inputStream); } String csvWithoutHeader; - try (InputStream inputStream = exporter.exportToInputStreamWithoutHeader(set)) { + try (InputStream inputStream = EXPORTER.exportToInputStreamWithoutHeader(set)) { csvWithoutHeader = IOUtils.toString(inputStream); } assertExportResultEquals(resultIterator, csv, csvWithoutHeader); -- To stop receiving notification emails like this one, please contact nuiton.org SCM administrator <admin+scm@nuiton.org>.
participants (1)
-
nuiton.org scm