This is an automated email from the git hooks/post-receive script. New commit to branch feature/3820_rethink_import_api in repository nuiton-csv. See https://gitlab.nuiton.org/nuiton/nuiton-csv.git commit 2bc7dbc31a0fe52fc37736ecffcfbbfb81a4582c Author: Tony CHEMIT <chemit@codelutin.com> Date: Sat Dec 19 06:04:17 2015 +0100 First flush of a new Importer API (See #3820) --- src/main/java/org/nuiton/csv/Importer.java | 406 +++++++++++++++++++++ src/main/java/org/nuiton/csv/ImporterBuilder.java | 208 +++++++++++ .../org/nuiton/csv/ImporterColumnsAdapter.java | 47 +++ .../java/org/nuiton/csv/ImporterConfiguration.java | 139 +++++++ .../java/org/nuiton/csv/ImporterRowFactory.java | 12 + .../org/nuiton/csv/ImporterExporterFixtures.java | 79 ++++ src/test/java/org/nuiton/csv/ImporterTest.java | 149 ++++++++ 7 files changed, 1040 insertions(+) diff --git a/src/main/java/org/nuiton/csv/Importer.java b/src/main/java/org/nuiton/csv/Importer.java new file mode 100644 index 0000000..bdf6460 --- /dev/null +++ b/src/main/java/org/nuiton/csv/Importer.java @@ -0,0 +1,406 @@ +package org.nuiton.csv; + +import com.csvreader.CsvReader; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Files; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.ext.CsvReaders; +import org.nuiton.util.StringUtil; + +import java.io.Closeable; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; + +import static org.nuiton.i18n.I18n.t; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class Importer<O> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(Importer.class); + + /** + * Creates an importer from the given model, and using default configuration values. + * + * @param rowType type of data to import + * @param model model of import + * @param <O> type of data to import + * @return the importer ready to use + */ + public static <O> Importer<O> of(Class<O> rowType, ImportModel<O> model) { + return builder(rowType, model).build(); + } + + /** + * Creates an importer from the given configuration. + * + * @param configuration import configuration + * @param <O> type of data to import + * @return the importer ready to use + */ + public static <O> Importer<O> of(ImporterConfiguration<O> configuration) { + return builder(configuration).build(); + } + + /** + * Creates a new importer builder from the given model. + * + * @param model model of import + * @param rowType type of data to import + * @param <O> type of data to import + * @return the importer builder + */ + public static <O> ImporterBuilder<O> builder(Class<O> rowType, ImportModel<O> model) { + return Importer + .builder(rowType) + .addColumns(model.getColumnsForImport()) + .setCellSeparator(String.valueOf(model.getSeparator())); + } + + /** + * Creates a new importer builder from the given configuration. + * + * @param configuration configuration of import + * @param <O> type of data to import + * @return the importer builder + */ + public static <O> ImporterBuilder<O> builder(ImporterConfiguration<O> configuration) { + return new ImporterBuilder<>(configuration); + } + + /** + * Creates a new empty importer builder. + * + * @param rowType type of data to import + * @param <O> type of data to import + * @return the importer builder + */ + public static <O> ImporterBuilder<O> builder(Class<O> rowType) { + return new ImporterBuilder<>(rowType); + } + + public ImporterAction<O> from(InputStream inputStream) { + return new ImporterAction<>(configuration, inputStream); + } + + public ImporterAction<O> from(Reader reader) { + return new ImporterAction<>(configuration, reader); + } + + public ImporterAction<O> from(File file) throws FileNotFoundException { + return new ImporterAction<>(configuration, file); + } + + public ImporterAction<O> from(String string) { + return new ImporterAction<>(configuration, string); + } + + public static class ImporterAction<O> implements Iterator<O>, Closeable { + + /** + * Importer configuration. + */ + protected final ImporterConfiguration<O> configuration; + + /** Csv reader (this is the input). */ + protected final CsvReader reader; + + protected final ImporterRowFactory<O> rowFactory; + + // read first line since first line is header + protected boolean hasNext; + + // to stock the current line number + protected int lineNumber; + + protected ImmutableList<ImportableColumn<O, ?>> columns; + + protected ImporterAction(ImporterConfiguration<O> configuration, InputStream inputStream) { + this(configuration, new CsvReader(inputStream, configuration.getCellSeparator().charAt(0), configuration.getCharset())); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, Reader reader) { + this(configuration, new CsvReader(reader, configuration.getCellSeparator().charAt(0))); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, File file) throws FileNotFoundException { + this(configuration, new CsvReader(Files.newReader(file, configuration.getCharset()), configuration.getCellSeparator().charAt(0))); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, String string) { + this(configuration, new CsvReader(new StringReader(string), configuration.getCellSeparator().charAt(0))); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, CsvReader csvReader) { + this.configuration = configuration; + this.reader = csvReader; + this.reader.setTrimWhitespace(true); + this.reader.setSafetySwitch(configuration.isSafetySwitch()); + this.rowFactory = configuration.getRowFactory(); + + // obtains headers + ImmutableList<String> headers = getHeaders(); + + if (log.isTraceEnabled()) { + log.trace("headers of the CSV file are : " + headers); + } + + ImmutableSet<ImportableColumn<O, ?>> configurationColumns = configuration.getColumns(); + + ImporterColumnsAdapter<O> columnsAdapter = configuration.getColumnsAdapter(); + if (columnsAdapter != null) { + + // hook to do some stuff from the model + ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder = ImmutableSet.builder(); + columnsAdapter.adapt(columnBuilder, configurationColumns, headers); + configurationColumns = columnBuilder.build(); + + } + + // check model columns name are unique + checkUniqueModelColumnNames(configurationColumns); + + // check that given headers from csv file are all known + checkHeaderNamesAreAllKnown(headers, configurationColumns); + + // check all mandatories column are on csv header + checkAllMandatoryHeadersArePresent(headers, configurationColumns); + + this.columns = getNonIgnoredHeaders(configurationColumns); + this.hasNext = readRow(); + + } + + @Override + public void close() throws IOException { + reader.close(); + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public O next() throws NoSuchElementException { + + if (!hasNext) { + throw new NoSuchElementException(); + } + + lineNumber += 1; + + O element = newRowInstance(); + + for (ImportableColumn<O, ?> field : columns) { + + // read value from csv cell + String value = readValue(field, lineNumber); + + // contravariance ftw + Object parsedValue = parseValue(field, lineNumber, value); + + // set value to element + setValue((ImportableColumn) field, lineNumber, element, parsedValue); + } + + // check if there is a next row to read + hasNext = readRow(); + + return element; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + protected O newRowInstance() { + try { + return rowFactory.newInstance(); + } catch (IllegalAccessException | InstantiationException e) { + throw new ImportRuntimeException("Could not instanciate new row", e); + } + } + + protected <T> String readValue(ImportableColumn<O, T> field, int lineNumber) { + try { + String value = reader.get(field.getHeaderName()); + return value; + } catch (Exception e) { + reader.close(); + throw new ImportRuntimeException( + t("csv.import.error.unableToReadField", + field.getHeaderName(), lineNumber), e); + } + } + + protected <T> T parseValue(ImportableColumn<O, T> field, int lineNumber, String value) { + try { + T parsedValue = field.parseValue(value); + return parsedValue; + } catch (Exception e) { + String message = t("csv.import.error.unableToParseValue", + value, field.getHeaderName(), lineNumber) + + "\n" + e.getMessage(); + throw new ImportRuntimeException(message, e); + } + } + + protected <T> void setValue(ImportableColumn<O, T> field, int lineNumber, O element, T parsedValue) { + try { + field.setValue(element, parsedValue); + } catch (Exception e) { + String message = t("csv.import.error.unableToSetValue", + parsedValue, + element.toString(), + lineNumber, field.getHeaderName()); + if (log.isErrorEnabled()) { + log.error(message); + } + throw new ImportRuntimeException(message, e); + } + } + + /** + * Read the next row from the reader and return {@code true} if line + * was successfully read. + * + * @return {@code true} if line was successfully read, says in fact there is + * something after this line. + * @throws ImportRuntimeException if could not read line + */ + protected boolean readRow() throws ImportRuntimeException { + try { + boolean hasNext = reader.readRecord(); + return hasNext; + } catch (IOException e) { + reader.close(); + throw new ImportRuntimeException(t("csv.import.error.unableToReadLine", 1), e); + } + } + + protected ImmutableList<String> getHeaders() throws ImportRuntimeException { + try { + boolean canReadHeaders = reader.readHeaders(); + if (!canReadHeaders) { + throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders")); + } + } catch (IOException e) { + throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders"), e); + } + + try { + String[] headers = reader.getHeaders(); + if (headers.length > 0) { + headers[0] = CsvReaders.removeBomCharacter(headers[0]); + } + return ImmutableList.copyOf(headers); + } catch (IOException e) { + throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders"), e); + } + } + + protected ImmutableList<ImportableColumn<O, ?>> getNonIgnoredHeaders(ImmutableSet<ImportableColumn<O, ?>> columns) { + ImmutableList.Builder<ImportableColumn<O, ?>> nonIgnoredHeaders = ImmutableList.builder(); + for (ImportableColumn<O, ?> field : columns) { + if (!field.isIgnored()) { + nonIgnoredHeaders.add(field); + } + } + return nonIgnoredHeaders.build(); + } + + protected List<ImportableColumn<O, ?>> getAllMandatoryHeaders(ImmutableSet<ImportableColumn<O, ?>> columns) { + List<ImportableColumn<O, ?>> allMandatoryHeaders = + new ArrayList<>(); + for (ImportableColumn<O, ?> field : columns) { + if (field.isMandatory()) { + allMandatoryHeaders.add(field); + } + } + return allMandatoryHeaders; + } + + protected void checkHeaderNamesAreAllKnown(ImmutableList<String> headers, ImmutableSet<ImportableColumn<O, ?>> columns) { + List<String> csvHeaders = new ArrayList<>(headers); + + for (ImportableColumn<O, ?> field : columns) { + csvHeaders.remove(field.getHeaderName()); + } + if (!csvHeaders.isEmpty()) { + List<String> validHeaderNames = new LinkedList<>(); + for (ImportableColumn<O, ?> importableColumn : columns) { + validHeaderNames.add(importableColumn.getHeaderName()); + } + String validationMessage = + t("csv.import.error.unrecognizedHeaders", + StringUtil.join(csvHeaders, ", ", true), + StringUtil.join(validHeaderNames, ", ", true)); + throw new ImportRuntimeException(validationMessage); + } + } + + protected void checkUniqueModelColumnNames(ImmutableSet<ImportableColumn<O, ?>> columns) { + Set<String> headerNames = new HashSet<>(); + Set<String> doubleHeaderNames = new HashSet<>(); + for (ImportableColumn<O, ?> importableColumn : columns) { + String headerName = importableColumn.getHeaderName(); + boolean alreadyUsed = !headerNames.add(headerName); + if (alreadyUsed) { + doubleHeaderNames.add(headerName); + } + } + if (!doubleHeaderNames.isEmpty()) { + String message = t("csv.import.error.duplicatedHeaders", + StringUtil.join(doubleHeaderNames, ", ", true)); + + throw new ImportRuntimeException(message); + } + } + + protected void checkAllMandatoryHeadersArePresent(ImmutableList<String> headers, ImmutableSet<ImportableColumn<O, ?>> columns) { + + List<String> csvHeaders = new ArrayList<>(headers); + + List<String> mandatoryHeadersNames = new ArrayList<>(); + for (ImportableColumn<O, ?> field : getAllMandatoryHeaders(columns)) { + mandatoryHeadersNames.add(field.getHeaderName()); + } + mandatoryHeadersNames.removeAll(csvHeaders); + + if (!mandatoryHeadersNames.isEmpty()) { + String validationMessage = + t("csv.import.error.missingMandatoryHeaders", + StringUtil.join(mandatoryHeadersNames, ", ", true)); + throw new ImportRuntimeException(validationMessage); + } + } + } + + /** + * Immutable importer configuration. + */ + protected final ImporterConfiguration<O> configuration; + + protected Importer(ImporterConfiguration<O> configuration) { + this.configuration = configuration; + } +} diff --git a/src/main/java/org/nuiton/csv/ImporterBuilder.java b/src/main/java/org/nuiton/csv/ImporterBuilder.java new file mode 100644 index 0000000..87d6ba2 --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterBuilder.java @@ -0,0 +1,208 @@ +package org.nuiton.csv; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterBuilder<O> { + + public static final String DEFAULT_CELL_SEPARATOR = ";"; + + public static final String DEFAULT_END_OF_LINE_SEPARATOR = "\n"; + + public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; + + public static final boolean DEFAULT_SAFETY_SWITCH = false; + + public static final boolean DEFAULT_STRICT_MODE = true; + + public static final boolean DEFAULT_IGNORE_UNKNOWN_HEADER = false; + + protected final ImporterConfiguration<O> configuration; + + protected final ImmutableSet.Builder<ImportableColumn<O, ?>> columnsBuilder = ImmutableSet.builder(); + + /** + * Set the {@link ImporterConfiguration#cellSeparator} and return the builder. + * + * @param cellSeparator new value for {@link ImporterConfiguration#cellSeparator}. + * @return the builder + */ + public ImporterBuilder<O> setCellSeparator(String cellSeparator) { + configuration.setCellSeparator(cellSeparator); + return this; + } + + /** + * Set the {@link ImporterConfiguration#endOfLineSeparator} and return the builder. + * + * @param endOfLineSeparator new value for {@link ImporterConfiguration#endOfLineSeparator}. + * @return the builder + */ + public ImporterBuilder<O> setEndOfLineSeparator(String endOfLineSeparator) { + configuration.setEndOfLineSeparator(endOfLineSeparator); + return this; + } + + /** + * Set the {@link ImporterConfiguration#charset} and return the builder. + * + * @param charset new value for {@link ImporterConfiguration#charset}. + * @return the builder + */ + public ImporterBuilder<O> setCharset(Charset charset) { + configuration.setCharset(charset); + return this; + } + + /** + * Set the {@link ImporterConfiguration#ignoreUnknownHeader} and return the builder. + * + * @param ignoreUnknownHeader new value for {@link ImporterConfiguration#ignoreUnknownHeader}. + * @return the builder + */ + public ImporterBuilder<O> setIgnoreUnknownHeader(boolean ignoreUnknownHeader) { + configuration.setIgnoreUnknownHeader(ignoreUnknownHeader); + return this; + } + + /** + * Set the {@link ImporterConfiguration#safetySwitch} and return the builder. + * + * @param safetySwitch new value for {@link ImporterConfiguration#safetySwitch}. + * @return the builder + */ + public ImporterBuilder<O> setSafetySwitch(boolean safetySwitch) { + configuration.setSafetySwitch(safetySwitch); + return this; + } + + /** + * Set the {@link ImporterConfiguration#strictMode} and return the builder. + * + * @param strictMode new value for {@link ImporterConfiguration#strictMode}. + * @return the builder + */ + public ImporterBuilder<O> setStrictMode(boolean strictMode) { + configuration.setStrictMode(strictMode); + return this; + } + + /** + * Set the {@link ImporterConfiguration#rowFactory} and return the builder. + * + * @param importerRowFactory new value for {@link ImporterConfiguration#rowFactory}. + * @return the builder + */ + public ImporterBuilder<O> setRowFactory(ImporterRowFactory<O> importerRowFactory) { + configuration.setRowFactory(importerRowFactory); + return this; + } + + /** + * Set the {@link ImporterConfiguration#columnsAdapter} and return the builder. + * + * @param columnsAdapter new value for {@link ImporterConfiguration#columnsAdapter}. + * @return the builder + */ + public ImporterBuilder<O> setColumnsAdapter(ImporterColumnsAdapter<O> columnsAdapter) { + configuration.setColumnsAdapter(columnsAdapter); + return this; + } + + /** + * Add the given columns. + * + * @param columns columns to add to the builder + * @return the builder + */ + public ImporterBuilder<O> addColumns(Iterable<ImportableColumn<O, Object>> columns) { + columnsBuilder.addAll(columns); + return this; + } + + public <T> ImporterBuilder<O> addColumn(ImportableColumn<O, T> newColumn) { + columnsBuilder.add(newColumn); + return this; + } + + public ImporterBuilder<O> addIgnoredColumn(String headerName) { + ImportableColumn<O, Object> newColumn = Column.newImportableColumn(headerName, null, null, true, false); + return addColumn(newColumn); + } + + public ImporterBuilder<O> addMandatoryColumn(String headerName, String propertyName) { + return addMandatoryColumn(headerName, propertyName, Common.STRING); + } + + public <T> ImporterBuilder<O> addMandatoryColumn(String headerName, String propertyName, ValueParser<T> valueParser) { + return addMandatoryColumn(headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + public <T> ImporterBuilder<O> addMandatoryColumn(String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, true); + return addColumn(newColumn); + } + + public ImporterBuilder<O> addOptionamColumn(String headerName, String propertyName) { + return addOptionamColumn(headerName, propertyName, Common.STRING); + } + + public <T> ImporterBuilder<O> addOptionamColumn(String headerName, String propertyName, ValueParser<T> valueParser) { + return addOptionamColumn(headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + public <T> ImporterBuilder<O> addOptionamColumn(String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, false); + return addColumn(newColumn); + } + + public Importer<O> build() { + configuration.setColumns(columnsBuilder.build()); + Preconditions.checkNotNull(configuration.getColumns(), "No columns defined."); + Preconditions.checkNotNull(configuration.getCellSeparator(), "No charset defined."); + Preconditions.checkNotNull(configuration.getCellSeparator(), "No cellSeparator defined."); + Preconditions.checkNotNull(configuration.getEndOfLineSeparator(), "No endOfLineSeparator defined."); + Preconditions.checkNotNull(configuration.getRowFactory(), "No rowFactory defined."); + + return new Importer<>(configuration); + } + + public ImporterBuilder(final Class<O> rowType) { + this.configuration = new ImporterConfiguration<>(rowType); + setCharset(DEFAULT_CHARSET); + setCellSeparator(DEFAULT_CELL_SEPARATOR); + setEndOfLineSeparator(DEFAULT_END_OF_LINE_SEPARATOR); + setStrictMode(DEFAULT_STRICT_MODE); + setSafetySwitch(DEFAULT_SAFETY_SWITCH); + setIgnoreUnknownHeader(DEFAULT_IGNORE_UNKNOWN_HEADER); + setRowFactory(new ImporterRowFactory<O>() { + + @Override + public O newInstance() throws IllegalAccessException, InstantiationException { + return rowType.newInstance(); + } + }); + } + + public ImporterBuilder(ImporterConfiguration<O> configuration) { + this.configuration = new ImporterConfiguration<>(configuration.rowType); + addColumns((Iterable) configuration.getColumns()); + setCharset(configuration.getCharset()); + setCellSeparator(configuration.getCellSeparator()); + setEndOfLineSeparator(configuration.getEndOfLineSeparator()); + setRowFactory(configuration.getRowFactory()); + setIgnoreUnknownHeader(configuration.isIgnoreUnknownHeader()); + setStrictMode(configuration.isStrictMode()); + setSafetySwitch(configuration.isSafetySwitch()); + setColumnsAdapter(configuration.getColumnsAdapter()); + } + +} diff --git a/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java b/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java new file mode 100644 index 0000000..b4a6a13 --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java @@ -0,0 +1,47 @@ +package org.nuiton.csv; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public abstract class ImporterColumnsAdapter<O> { + + public abstract void adapt(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, ImmutableSet<ImportableColumn<O, ?>> columns, ImmutableList<String> headers); + + protected void addIgnoredColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName) { + ImportableColumn<O, String> newColumn = Column.newImportableColumn(headerName, null, null, true, false); + columnBuilder.add(newColumn); + } + + protected void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName) { + addMandatoryColumn(columnBuilder, headerName, propertyName, Common.STRING); + } + + protected <T> void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName, ValueParser<T> valueParser) { + addMandatoryColumn(columnBuilder, headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + protected <T> void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, true); + columnBuilder.add(newColumn); + } + + protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName) { + addOptionalColumn(columnBuilder, headerName, propertyName, Common.STRING); + } + + protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName, ValueParser<T> valueParser) { + addOptionalColumn(columnBuilder, headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, false); + columnBuilder.add(newColumn); + + } + +} diff --git a/src/main/java/org/nuiton/csv/ImporterConfiguration.java b/src/main/java/org/nuiton/csv/ImporterConfiguration.java new file mode 100644 index 0000000..f8a12df --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterConfiguration.java @@ -0,0 +1,139 @@ +package org.nuiton.csv; + +import com.csvreader.CsvReader; +import com.google.common.collect.ImmutableSet; + +import java.nio.charset.Charset; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterConfiguration<O> { + + protected final Class<O> rowType; + + /** + * Columns import model. + */ + protected ImmutableSet<ImportableColumn<O, ?>> columns; + + /** + * Cell separator. + */ + protected String cellSeparator; + + /** + * Eof of line separator. + */ + protected String endOfLineSeparator; + + /** + * Charset used to perform the export. + */ + protected Charset charset; + + /** + * Flag to turn or not the safetySwitch (see {@link CsvReader#getSafetySwitch()}). + * + * By default, not used. + */ + protected boolean safetySwitch; + + /** + * Flag to use a strict mode (says import will failed at the first error), + * if setted to {@link false}, then errors for each rows will be stored in + * the current row and import will continue to the end. + * + * By default, used (strict mode). + */ + protected boolean strictMode; + + /** + * Flag to ignore header found in a import file and not declared in the import model. + * + * By default, not used (strict mode). + */ + protected boolean ignoreUnknownHeader; + + protected ImporterRowFactory<O> rowFactory; + + private ImporterColumnsAdapter<O> columnsAdapter; + + public ImporterConfiguration(Class<O> rowType) { + this.rowType = rowType; + } + + public String getCellSeparator() { + return cellSeparator; + } + + public String getEndOfLineSeparator() { + return endOfLineSeparator; + } + + public Charset getCharset() { + return charset; + } + + public boolean isIgnoreUnknownHeader() { + return ignoreUnknownHeader; + } + + public boolean isSafetySwitch() { + return safetySwitch; + } + + public boolean isStrictMode() { + return strictMode; + } + + public ImporterRowFactory<O> getRowFactory() { + return rowFactory; + } + + public ImmutableSet<ImportableColumn<O, ?>> getColumns() { + return columns; + } + + public ImporterColumnsAdapter<O> getColumnsAdapter() { + return columnsAdapter; + } + + public void setCellSeparator(String cellSeparator) { + this.cellSeparator = cellSeparator; + } + + public void setCharset(Charset charset) { + this.charset = charset; + } + + public void setColumns(ImmutableSet<ImportableColumn<O, ?>> columns) { + this.columns = columns; + } + + public void setEndOfLineSeparator(String endOfLineSeparator) { + this.endOfLineSeparator = endOfLineSeparator; + } + + public void setIgnoreUnknownHeader(boolean ignoreUnknownHeader) { + this.ignoreUnknownHeader = ignoreUnknownHeader; + } + + public void setSafetySwitch(boolean safetySwitch) { + this.safetySwitch = safetySwitch; + } + + public void setStrictMode(boolean strictMode) { + this.strictMode = strictMode; + } + + public void setRowFactory(ImporterRowFactory<O> rowFactory) { + this.rowFactory = rowFactory; + } + + public void setColumnsAdapter(ImporterColumnsAdapter<O> columnsAdapter) { + this.columnsAdapter = columnsAdapter; + } +} \ No newline at end of file diff --git a/src/main/java/org/nuiton/csv/ImporterRowFactory.java b/src/main/java/org/nuiton/csv/ImporterRowFactory.java new file mode 100644 index 0000000..dff6b5e --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterRowFactory.java @@ -0,0 +1,12 @@ +package org.nuiton.csv; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public interface ImporterRowFactory<O> { + + O newInstance() throws IllegalAccessException, InstantiationException; + +} diff --git a/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java b/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java new file mode 100644 index 0000000..3792562 --- /dev/null +++ b/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java @@ -0,0 +1,79 @@ +package org.nuiton.csv; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.nuiton.util.DateUtil; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Set; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterExporterFixtures { + + public final Charset charset= StandardCharsets.UTF_8; + + protected final Iterable<Set<RowBean>> sets; + + protected final String cellSeparator = "~"; + + protected final Iterable<String> toImport; + + + public ImporterExporterFixtures() { + ImmutableSet.Builder<RowBean> builder = ImmutableSet.builder(); + + ImmutableList.Builder<Set<RowBean>> setBuilder = ImmutableList.builder(); + + setBuilder.add(Collections.<RowBean>emptySet()); + + builder.add(new RowBean(DateUtil.createDate(1, 12, 2011), "Batéman", 1, RowBeanEnum.ONE)); + setBuilder.add(builder.build()); + + builder.add(new RowBean(DateUtil.createDate(2, 12, 2011), "", 7, RowBeanEnum.TWO)); + setBuilder.add(builder.build()); + + builder.add((new RowBean(DateUtil.createDate(7, 12, 2011), "", 9, RowBeanEnum.ZERO))); + builder.add((new RowBean(DateUtil.createDate(18, 12, 2011), "", 18, RowBeanEnum.ONE))); + builder.add((new RowBean(DateUtil.createDate(23, 12, 2011), "", 4, RowBeanEnum.TWO))); + setBuilder.add(builder.build()); + + sets = setBuilder.build(); + + toImport = ImmutableList.of( + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n", + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" + + "01/12/2011~Batéman~1~ONE~jkfjk\n", + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" + + "01/12/2011~Batéman~1~ONE~jkfjk\n" + + "02/12/2011~~7~TWO~jkfjk\n", + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" + + "01/12/2011~Batéman~1~ONE~jkfjk\n" + + "02/12/2011~~7~TWO~jkfjk\n" + + "07/12/2011~~9~ZERO~jkfjk\n" + + "18/12/2011~~18~ONE~jkfjk\n" + + "23/12/2011~~4~TWO~jkfjk\n" + ); + } + + public Iterable<Set<RowBean>> getSets() { + return sets; + } + + public String getCellSeparator() { + return cellSeparator; + } + + public Charset getCharset() { + return charset; + } + + public Iterable<String> getToImport() { + return toImport; + } +} diff --git a/src/test/java/org/nuiton/csv/ImporterTest.java b/src/test/java/org/nuiton/csv/ImporterTest.java new file mode 100644 index 0000000..00ad999 --- /dev/null +++ b/src/test/java/org/nuiton/csv/ImporterTest.java @@ -0,0 +1,149 @@ +package org.nuiton.csv; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.StringReader; +import java.util.Iterator; +import java.util.Set; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterTest { + + protected static ImporterExporterFixtures FIXTURES; + + protected static Importer<RowBean> IMPORTER; + + @BeforeClass + public static void setUp() throws Exception { + + FIXTURES = new ImporterExporterFixtures(); + + IMPORTER = Importer + .builder(RowBean.class) + .setCharset(FIXTURES.getCharset()) + .setCellSeparator(FIXTURES.getCellSeparator()) + .addMandatoryColumn("D~ATE", "date", Common.DAY) + .addIgnoredColumn("title") + .addMandatoryColumn("TITLEé", "title") + .addMandatoryColumn("NUMBER", "number", Common.INTEGER) + .addMandatoryColumn("rowBeanEnum", "rowBeanEnum", Common.newEnumByNameParserFormatter(RowBeanEnum.class)) + .build(); + + + } + + @Test + public void testFromString() throws Exception { + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(resultIterator.next())) { + assertImportResultEquals(expected, actual); + } + } + + } + + @Test + public void testFromInputStream() throws Exception { + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(new ByteArrayInputStream(resultIterator.next().getBytes()))) { + assertImportResultEquals(expected, actual); + } + } + + } + + @Test + public void testFromReader() throws Exception { + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(new StringReader(resultIterator.next()))) { + assertImportResultEquals(expected, actual); + } + } + + } + + @Test + public void testFromFile() throws Exception { + + File parentFile = new File(FileUtils.getTempDirectory(), getClass().getName() + "-testFromFile"); + FileUtils.forceMkdir(parentFile); + + int index = 0; + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + + File importFile = new File(parentFile, (index++) + ".csv"); + Files.write(resultIterator.next().getBytes(), importFile); + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(importFile)) { + assertImportResultEquals(expected, actual); + } + + } + + } + + @Test + public void testFromStringWithColumnsAdapter() throws Exception { + + Importer<RowBean> importer = Importer + .builder(RowBean.class) + .setCharset(FIXTURES.getCharset()) + .setCellSeparator(FIXTURES.getCellSeparator()) + .addMandatoryColumn("D~ATE", "date", Common.DAY) + .setColumnsAdapter(new ImporterColumnsAdapter<RowBean>() { + + @Override + public void adapt(ImmutableSet.Builder<ImportableColumn<RowBean, ?>> columnBuilder, ImmutableSet<ImportableColumn<RowBean, ?>> importableColumns, ImmutableList<String> headers) { + columnBuilder.addAll(importableColumns); + addIgnoredColumn(columnBuilder, "title"); + addMandatoryColumn(columnBuilder, "TITLEé", "title"); + addMandatoryColumn(columnBuilder, "NUMBER", "number", Common.INTEGER); + addMandatoryColumn(columnBuilder, "rowBeanEnum", "rowBeanEnum", Common.newEnumByNameParserFormatter(RowBeanEnum.class)); + } + }) + .build(); + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = importer.from(resultIterator.next())) { + assertImportResultEquals(expected, actual); + } + } + + } + + protected void assertImportResultEquals(Iterable<RowBean> expected, Importer.ImporterAction<RowBean> actual) { + + Iterator<RowBean> expectedIterator = expected.iterator(); + + while (expectedIterator.hasNext()) { + + RowBean expectedRow = expectedIterator.next(); + RowBean actualRow = actual.next(); + Assert.assertEquals(expectedRow, actualRow); + + Assert.assertEquals(expectedIterator.hasNext(), actual.hasNext()); + + } + + } + +} \ No newline at end of file -- To stop receiving notification emails like this one, please contact nuiton.org SCM administrator <admin+scm@nuiton.org>.