branch feature/3820_rethink_import_api created (now e81761d)
This is an automated email from the git hooks/post-receive script. New change to branch feature/3820_rethink_import_api in repository nuiton-csv. See http://git.nuiton.org/nuiton-csv.git at e81761d First flush of a new Importer API (See #3820) This branch includes the following new commits: new e81761d First flush of a new Importer API (See #3820) The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit e81761d9a35c9fac7b34a8731a4c2131cf75e28a Author: Tony CHEMIT <chemit@codelutin.com> Date: Sat Dec 19 06:04:17 2015 +0100 First flush of a new Importer API (See #3820) -- To stop receiving notification emails like this one, please contact nuiton.org SCM administrator <admin+scm@nuiton.org>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/3820_rethink_import_api in repository nuiton-csv. See http://git.nuiton.org/nuiton-csv.git commit e81761d9a35c9fac7b34a8731a4c2131cf75e28a Author: Tony CHEMIT <chemit@codelutin.com> Date: Sat Dec 19 06:04:17 2015 +0100 First flush of a new Importer API (See #3820) --- src/main/java/org/nuiton/csv/Importer.java | 406 +++++++++++++++++++++ src/main/java/org/nuiton/csv/ImporterBuilder.java | 208 +++++++++++ .../org/nuiton/csv/ImporterColumnsAdapter.java | 47 +++ .../java/org/nuiton/csv/ImporterConfiguration.java | 139 +++++++ .../java/org/nuiton/csv/ImporterRowFactory.java | 12 + .../org/nuiton/csv/ImporterExporterFixtures.java | 79 ++++ src/test/java/org/nuiton/csv/ImporterTest.java | 149 ++++++++ 7 files changed, 1040 insertions(+) diff --git a/src/main/java/org/nuiton/csv/Importer.java b/src/main/java/org/nuiton/csv/Importer.java new file mode 100644 index 0000000..bdf6460 --- /dev/null +++ b/src/main/java/org/nuiton/csv/Importer.java @@ -0,0 +1,406 @@ +package org.nuiton.csv; + +import com.csvreader.CsvReader; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Files; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.csv.ext.CsvReaders; +import org.nuiton.util.StringUtil; + +import java.io.Closeable; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; + +import static org.nuiton.i18n.I18n.t; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class Importer<O> { + + /** Logger. */ + private static final Log log = LogFactory.getLog(Importer.class); + + /** + * Creates an importer from the given model, and using default configuration values. + * + * @param rowType type of data to import + * @param model model of import + * @param <O> type of data to import + * @return the importer ready to use + */ + public static <O> Importer<O> of(Class<O> rowType, ImportModel<O> model) { + return builder(rowType, model).build(); + } + + /** + * Creates an importer from the given configuration. + * + * @param configuration import configuration + * @param <O> type of data to import + * @return the importer ready to use + */ + public static <O> Importer<O> of(ImporterConfiguration<O> configuration) { + return builder(configuration).build(); + } + + /** + * Creates a new importer builder from the given model. + * + * @param model model of import + * @param rowType type of data to import + * @param <O> type of data to import + * @return the importer builder + */ + public static <O> ImporterBuilder<O> builder(Class<O> rowType, ImportModel<O> model) { + return Importer + .builder(rowType) + .addColumns(model.getColumnsForImport()) + .setCellSeparator(String.valueOf(model.getSeparator())); + } + + /** + * Creates a new importer builder from the given configuration. + * + * @param configuration configuration of import + * @param <O> type of data to import + * @return the importer builder + */ + public static <O> ImporterBuilder<O> builder(ImporterConfiguration<O> configuration) { + return new ImporterBuilder<>(configuration); + } + + /** + * Creates a new empty importer builder. + * + * @param rowType type of data to import + * @param <O> type of data to import + * @return the importer builder + */ + public static <O> ImporterBuilder<O> builder(Class<O> rowType) { + return new ImporterBuilder<>(rowType); + } + + public ImporterAction<O> from(InputStream inputStream) { + return new ImporterAction<>(configuration, inputStream); + } + + public ImporterAction<O> from(Reader reader) { + return new ImporterAction<>(configuration, reader); + } + + public ImporterAction<O> from(File file) throws FileNotFoundException { + return new ImporterAction<>(configuration, file); + } + + public ImporterAction<O> from(String string) { + return new ImporterAction<>(configuration, string); + } + + public static class ImporterAction<O> implements Iterator<O>, Closeable { + + /** + * Importer configuration. + */ + protected final ImporterConfiguration<O> configuration; + + /** Csv reader (this is the input). */ + protected final CsvReader reader; + + protected final ImporterRowFactory<O> rowFactory; + + // read first line since first line is header + protected boolean hasNext; + + // to stock the current line number + protected int lineNumber; + + protected ImmutableList<ImportableColumn<O, ?>> columns; + + protected ImporterAction(ImporterConfiguration<O> configuration, InputStream inputStream) { + this(configuration, new CsvReader(inputStream, configuration.getCellSeparator().charAt(0), configuration.getCharset())); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, Reader reader) { + this(configuration, new CsvReader(reader, configuration.getCellSeparator().charAt(0))); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, File file) throws FileNotFoundException { + this(configuration, new CsvReader(Files.newReader(file, configuration.getCharset()), configuration.getCellSeparator().charAt(0))); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, String string) { + this(configuration, new CsvReader(new StringReader(string), configuration.getCellSeparator().charAt(0))); + } + + protected ImporterAction(ImporterConfiguration<O> configuration, CsvReader csvReader) { + this.configuration = configuration; + this.reader = csvReader; + this.reader.setTrimWhitespace(true); + this.reader.setSafetySwitch(configuration.isSafetySwitch()); + this.rowFactory = configuration.getRowFactory(); + + // obtains headers + ImmutableList<String> headers = getHeaders(); + + if (log.isTraceEnabled()) { + log.trace("headers of the CSV file are : " + headers); + } + + ImmutableSet<ImportableColumn<O, ?>> configurationColumns = configuration.getColumns(); + + ImporterColumnsAdapter<O> columnsAdapter = configuration.getColumnsAdapter(); + if (columnsAdapter != null) { + + // hook to do some stuff from the model + ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder = ImmutableSet.builder(); + columnsAdapter.adapt(columnBuilder, configurationColumns, headers); + configurationColumns = columnBuilder.build(); + + } + + // check model columns name are unique + checkUniqueModelColumnNames(configurationColumns); + + // check that given headers from csv file are all known + checkHeaderNamesAreAllKnown(headers, configurationColumns); + + // check all mandatories column are on csv header + checkAllMandatoryHeadersArePresent(headers, configurationColumns); + + this.columns = getNonIgnoredHeaders(configurationColumns); + this.hasNext = readRow(); + + } + + @Override + public void close() throws IOException { + reader.close(); + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public O next() throws NoSuchElementException { + + if (!hasNext) { + throw new NoSuchElementException(); + } + + lineNumber += 1; + + O element = newRowInstance(); + + for (ImportableColumn<O, ?> field : columns) { + + // read value from csv cell + String value = readValue(field, lineNumber); + + // contravariance ftw + Object parsedValue = parseValue(field, lineNumber, value); + + // set value to element + setValue((ImportableColumn) field, lineNumber, element, parsedValue); + } + + // check if there is a next row to read + hasNext = readRow(); + + return element; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + protected O newRowInstance() { + try { + return rowFactory.newInstance(); + } catch (IllegalAccessException | InstantiationException e) { + throw new ImportRuntimeException("Could not instanciate new row", e); + } + } + + protected <T> String readValue(ImportableColumn<O, T> field, int lineNumber) { + try { + String value = reader.get(field.getHeaderName()); + return value; + } catch (Exception e) { + reader.close(); + throw new ImportRuntimeException( + t("csv.import.error.unableToReadField", + field.getHeaderName(), lineNumber), e); + } + } + + protected <T> T parseValue(ImportableColumn<O, T> field, int lineNumber, String value) { + try { + T parsedValue = field.parseValue(value); + return parsedValue; + } catch (Exception e) { + String message = t("csv.import.error.unableToParseValue", + value, field.getHeaderName(), lineNumber) + + "\n" + e.getMessage(); + throw new ImportRuntimeException(message, e); + } + } + + protected <T> void setValue(ImportableColumn<O, T> field, int lineNumber, O element, T parsedValue) { + try { + field.setValue(element, parsedValue); + } catch (Exception e) { + String message = t("csv.import.error.unableToSetValue", + parsedValue, + element.toString(), + lineNumber, field.getHeaderName()); + if (log.isErrorEnabled()) { + log.error(message); + } + throw new ImportRuntimeException(message, e); + } + } + + /** + * Read the next row from the reader and return {@code true} if line + * was successfully read. + * + * @return {@code true} if line was successfully read, says in fact there is + * something after this line. + * @throws ImportRuntimeException if could not read line + */ + protected boolean readRow() throws ImportRuntimeException { + try { + boolean hasNext = reader.readRecord(); + return hasNext; + } catch (IOException e) { + reader.close(); + throw new ImportRuntimeException(t("csv.import.error.unableToReadLine", 1), e); + } + } + + protected ImmutableList<String> getHeaders() throws ImportRuntimeException { + try { + boolean canReadHeaders = reader.readHeaders(); + if (!canReadHeaders) { + throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders")); + } + } catch (IOException e) { + throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders"), e); + } + + try { + String[] headers = reader.getHeaders(); + if (headers.length > 0) { + headers[0] = CsvReaders.removeBomCharacter(headers[0]); + } + return ImmutableList.copyOf(headers); + } catch (IOException e) { + throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders"), e); + } + } + + protected ImmutableList<ImportableColumn<O, ?>> getNonIgnoredHeaders(ImmutableSet<ImportableColumn<O, ?>> columns) { + ImmutableList.Builder<ImportableColumn<O, ?>> nonIgnoredHeaders = ImmutableList.builder(); + for (ImportableColumn<O, ?> field : columns) { + if (!field.isIgnored()) { + nonIgnoredHeaders.add(field); + } + } + return nonIgnoredHeaders.build(); + } + + protected List<ImportableColumn<O, ?>> getAllMandatoryHeaders(ImmutableSet<ImportableColumn<O, ?>> columns) { + List<ImportableColumn<O, ?>> allMandatoryHeaders = + new ArrayList<>(); + for (ImportableColumn<O, ?> field : columns) { + if (field.isMandatory()) { + allMandatoryHeaders.add(field); + } + } + return allMandatoryHeaders; + } + + protected void checkHeaderNamesAreAllKnown(ImmutableList<String> headers, ImmutableSet<ImportableColumn<O, ?>> columns) { + List<String> csvHeaders = new ArrayList<>(headers); + + for (ImportableColumn<O, ?> field : columns) { + csvHeaders.remove(field.getHeaderName()); + } + if (!csvHeaders.isEmpty()) { + List<String> validHeaderNames = new LinkedList<>(); + for (ImportableColumn<O, ?> importableColumn : columns) { + validHeaderNames.add(importableColumn.getHeaderName()); + } + String validationMessage = + t("csv.import.error.unrecognizedHeaders", + StringUtil.join(csvHeaders, ", ", true), + StringUtil.join(validHeaderNames, ", ", true)); + throw new ImportRuntimeException(validationMessage); + } + } + + protected void checkUniqueModelColumnNames(ImmutableSet<ImportableColumn<O, ?>> columns) { + Set<String> headerNames = new HashSet<>(); + Set<String> doubleHeaderNames = new HashSet<>(); + for (ImportableColumn<O, ?> importableColumn : columns) { + String headerName = importableColumn.getHeaderName(); + boolean alreadyUsed = !headerNames.add(headerName); + if (alreadyUsed) { + doubleHeaderNames.add(headerName); + } + } + if (!doubleHeaderNames.isEmpty()) { + String message = t("csv.import.error.duplicatedHeaders", + StringUtil.join(doubleHeaderNames, ", ", true)); + + throw new ImportRuntimeException(message); + } + } + + protected void checkAllMandatoryHeadersArePresent(ImmutableList<String> headers, ImmutableSet<ImportableColumn<O, ?>> columns) { + + List<String> csvHeaders = new ArrayList<>(headers); + + List<String> mandatoryHeadersNames = new ArrayList<>(); + for (ImportableColumn<O, ?> field : getAllMandatoryHeaders(columns)) { + mandatoryHeadersNames.add(field.getHeaderName()); + } + mandatoryHeadersNames.removeAll(csvHeaders); + + if (!mandatoryHeadersNames.isEmpty()) { + String validationMessage = + t("csv.import.error.missingMandatoryHeaders", + StringUtil.join(mandatoryHeadersNames, ", ", true)); + throw new ImportRuntimeException(validationMessage); + } + } + } + + /** + * Immutable importer configuration. + */ + protected final ImporterConfiguration<O> configuration; + + protected Importer(ImporterConfiguration<O> configuration) { + this.configuration = configuration; + } +} diff --git a/src/main/java/org/nuiton/csv/ImporterBuilder.java b/src/main/java/org/nuiton/csv/ImporterBuilder.java new file mode 100644 index 0000000..87d6ba2 --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterBuilder.java @@ -0,0 +1,208 @@ +package org.nuiton.csv; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterBuilder<O> { + + public static final String DEFAULT_CELL_SEPARATOR = ";"; + + public static final String DEFAULT_END_OF_LINE_SEPARATOR = "\n"; + + public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; + + public static final boolean DEFAULT_SAFETY_SWITCH = false; + + public static final boolean DEFAULT_STRICT_MODE = true; + + public static final boolean DEFAULT_IGNORE_UNKNOWN_HEADER = false; + + protected final ImporterConfiguration<O> configuration; + + protected final ImmutableSet.Builder<ImportableColumn<O, ?>> columnsBuilder = ImmutableSet.builder(); + + /** + * Set the {@link ImporterConfiguration#cellSeparator} and return the builder. + * + * @param cellSeparator new value for {@link ImporterConfiguration#cellSeparator}. + * @return the builder + */ + public ImporterBuilder<O> setCellSeparator(String cellSeparator) { + configuration.setCellSeparator(cellSeparator); + return this; + } + + /** + * Set the {@link ImporterConfiguration#endOfLineSeparator} and return the builder. + * + * @param endOfLineSeparator new value for {@link ImporterConfiguration#endOfLineSeparator}. + * @return the builder + */ + public ImporterBuilder<O> setEndOfLineSeparator(String endOfLineSeparator) { + configuration.setEndOfLineSeparator(endOfLineSeparator); + return this; + } + + /** + * Set the {@link ImporterConfiguration#charset} and return the builder. + * + * @param charset new value for {@link ImporterConfiguration#charset}. + * @return the builder + */ + public ImporterBuilder<O> setCharset(Charset charset) { + configuration.setCharset(charset); + return this; + } + + /** + * Set the {@link ImporterConfiguration#ignoreUnknownHeader} and return the builder. + * + * @param ignoreUnknownHeader new value for {@link ImporterConfiguration#ignoreUnknownHeader}. + * @return the builder + */ + public ImporterBuilder<O> setIgnoreUnknownHeader(boolean ignoreUnknownHeader) { + configuration.setIgnoreUnknownHeader(ignoreUnknownHeader); + return this; + } + + /** + * Set the {@link ImporterConfiguration#safetySwitch} and return the builder. + * + * @param safetySwitch new value for {@link ImporterConfiguration#safetySwitch}. + * @return the builder + */ + public ImporterBuilder<O> setSafetySwitch(boolean safetySwitch) { + configuration.setSafetySwitch(safetySwitch); + return this; + } + + /** + * Set the {@link ImporterConfiguration#strictMode} and return the builder. + * + * @param strictMode new value for {@link ImporterConfiguration#strictMode}. + * @return the builder + */ + public ImporterBuilder<O> setStrictMode(boolean strictMode) { + configuration.setStrictMode(strictMode); + return this; + } + + /** + * Set the {@link ImporterConfiguration#rowFactory} and return the builder. + * + * @param importerRowFactory new value for {@link ImporterConfiguration#rowFactory}. + * @return the builder + */ + public ImporterBuilder<O> setRowFactory(ImporterRowFactory<O> importerRowFactory) { + configuration.setRowFactory(importerRowFactory); + return this; + } + + /** + * Set the {@link ImporterConfiguration#columnsAdapter} and return the builder. + * + * @param columnsAdapter new value for {@link ImporterConfiguration#columnsAdapter}. + * @return the builder + */ + public ImporterBuilder<O> setColumnsAdapter(ImporterColumnsAdapter<O> columnsAdapter) { + configuration.setColumnsAdapter(columnsAdapter); + return this; + } + + /** + * Add the given columns. + * + * @param columns columns to add to the builder + * @return the builder + */ + public ImporterBuilder<O> addColumns(Iterable<ImportableColumn<O, Object>> columns) { + columnsBuilder.addAll(columns); + return this; + } + + public <T> ImporterBuilder<O> addColumn(ImportableColumn<O, T> newColumn) { + columnsBuilder.add(newColumn); + return this; + } + + public ImporterBuilder<O> addIgnoredColumn(String headerName) { + ImportableColumn<O, Object> newColumn = Column.newImportableColumn(headerName, null, null, true, false); + return addColumn(newColumn); + } + + public ImporterBuilder<O> addMandatoryColumn(String headerName, String propertyName) { + return addMandatoryColumn(headerName, propertyName, Common.STRING); + } + + public <T> ImporterBuilder<O> addMandatoryColumn(String headerName, String propertyName, ValueParser<T> valueParser) { + return addMandatoryColumn(headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + public <T> ImporterBuilder<O> addMandatoryColumn(String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, true); + return addColumn(newColumn); + } + + public ImporterBuilder<O> addOptionamColumn(String headerName, String propertyName) { + return addOptionamColumn(headerName, propertyName, Common.STRING); + } + + public <T> ImporterBuilder<O> addOptionamColumn(String headerName, String propertyName, ValueParser<T> valueParser) { + return addOptionamColumn(headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + public <T> ImporterBuilder<O> addOptionamColumn(String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, false); + return addColumn(newColumn); + } + + public Importer<O> build() { + configuration.setColumns(columnsBuilder.build()); + Preconditions.checkNotNull(configuration.getColumns(), "No columns defined."); + Preconditions.checkNotNull(configuration.getCellSeparator(), "No charset defined."); + Preconditions.checkNotNull(configuration.getCellSeparator(), "No cellSeparator defined."); + Preconditions.checkNotNull(configuration.getEndOfLineSeparator(), "No endOfLineSeparator defined."); + Preconditions.checkNotNull(configuration.getRowFactory(), "No rowFactory defined."); + + return new Importer<>(configuration); + } + + public ImporterBuilder(final Class<O> rowType) { + this.configuration = new ImporterConfiguration<>(rowType); + setCharset(DEFAULT_CHARSET); + setCellSeparator(DEFAULT_CELL_SEPARATOR); + setEndOfLineSeparator(DEFAULT_END_OF_LINE_SEPARATOR); + setStrictMode(DEFAULT_STRICT_MODE); + setSafetySwitch(DEFAULT_SAFETY_SWITCH); + setIgnoreUnknownHeader(DEFAULT_IGNORE_UNKNOWN_HEADER); + setRowFactory(new ImporterRowFactory<O>() { + + @Override + public O newInstance() throws IllegalAccessException, InstantiationException { + return rowType.newInstance(); + } + }); + } + + public ImporterBuilder(ImporterConfiguration<O> configuration) { + this.configuration = new ImporterConfiguration<>(configuration.rowType); + addColumns((Iterable) configuration.getColumns()); + setCharset(configuration.getCharset()); + setCellSeparator(configuration.getCellSeparator()); + setEndOfLineSeparator(configuration.getEndOfLineSeparator()); + setRowFactory(configuration.getRowFactory()); + setIgnoreUnknownHeader(configuration.isIgnoreUnknownHeader()); + setStrictMode(configuration.isStrictMode()); + setSafetySwitch(configuration.isSafetySwitch()); + setColumnsAdapter(configuration.getColumnsAdapter()); + } + +} diff --git a/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java b/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java new file mode 100644 index 0000000..b4a6a13 --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java @@ -0,0 +1,47 @@ +package org.nuiton.csv; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public abstract class ImporterColumnsAdapter<O> { + + public abstract void adapt(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, ImmutableSet<ImportableColumn<O, ?>> columns, ImmutableList<String> headers); + + protected void addIgnoredColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName) { + ImportableColumn<O, String> newColumn = Column.newImportableColumn(headerName, null, null, true, false); + columnBuilder.add(newColumn); + } + + protected void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName) { + addMandatoryColumn(columnBuilder, headerName, propertyName, Common.STRING); + } + + protected <T> void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName, ValueParser<T> valueParser) { + addMandatoryColumn(columnBuilder, headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + protected <T> void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, true); + columnBuilder.add(newColumn); + } + + protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName) { + addOptionalColumn(columnBuilder, headerName, propertyName, Common.STRING); + } + + protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName, ValueParser<T> valueParser) { + addOptionalColumn(columnBuilder, headerName, valueParser, new Common.BeanProperty<O, T>(propertyName)); + } + + protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) { + ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, false); + columnBuilder.add(newColumn); + + } + +} diff --git a/src/main/java/org/nuiton/csv/ImporterConfiguration.java b/src/main/java/org/nuiton/csv/ImporterConfiguration.java new file mode 100644 index 0000000..f8a12df --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterConfiguration.java @@ -0,0 +1,139 @@ +package org.nuiton.csv; + +import com.csvreader.CsvReader; +import com.google.common.collect.ImmutableSet; + +import java.nio.charset.Charset; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterConfiguration<O> { + + protected final Class<O> rowType; + + /** + * Columns import model. + */ + protected ImmutableSet<ImportableColumn<O, ?>> columns; + + /** + * Cell separator. + */ + protected String cellSeparator; + + /** + * Eof of line separator. + */ + protected String endOfLineSeparator; + + /** + * Charset used to perform the export. + */ + protected Charset charset; + + /** + * Flag to turn or not the safetySwitch (see {@link CsvReader#getSafetySwitch()}). + * + * By default, not used. + */ + protected boolean safetySwitch; + + /** + * Flag to use a strict mode (says import will failed at the first error), + * if setted to {@link false}, then errors for each rows will be stored in + * the current row and import will continue to the end. + * + * By default, used (strict mode). + */ + protected boolean strictMode; + + /** + * Flag to ignore header found in a import file and not declared in the import model. + * + * By default, not used (strict mode). + */ + protected boolean ignoreUnknownHeader; + + protected ImporterRowFactory<O> rowFactory; + + private ImporterColumnsAdapter<O> columnsAdapter; + + public ImporterConfiguration(Class<O> rowType) { + this.rowType = rowType; + } + + public String getCellSeparator() { + return cellSeparator; + } + + public String getEndOfLineSeparator() { + return endOfLineSeparator; + } + + public Charset getCharset() { + return charset; + } + + public boolean isIgnoreUnknownHeader() { + return ignoreUnknownHeader; + } + + public boolean isSafetySwitch() { + return safetySwitch; + } + + public boolean isStrictMode() { + return strictMode; + } + + public ImporterRowFactory<O> getRowFactory() { + return rowFactory; + } + + public ImmutableSet<ImportableColumn<O, ?>> getColumns() { + return columns; + } + + public ImporterColumnsAdapter<O> getColumnsAdapter() { + return columnsAdapter; + } + + public void setCellSeparator(String cellSeparator) { + this.cellSeparator = cellSeparator; + } + + public void setCharset(Charset charset) { + this.charset = charset; + } + + public void setColumns(ImmutableSet<ImportableColumn<O, ?>> columns) { + this.columns = columns; + } + + public void setEndOfLineSeparator(String endOfLineSeparator) { + this.endOfLineSeparator = endOfLineSeparator; + } + + public void setIgnoreUnknownHeader(boolean ignoreUnknownHeader) { + this.ignoreUnknownHeader = ignoreUnknownHeader; + } + + public void setSafetySwitch(boolean safetySwitch) { + this.safetySwitch = safetySwitch; + } + + public void setStrictMode(boolean strictMode) { + this.strictMode = strictMode; + } + + public void setRowFactory(ImporterRowFactory<O> rowFactory) { + this.rowFactory = rowFactory; + } + + public void setColumnsAdapter(ImporterColumnsAdapter<O> columnsAdapter) { + this.columnsAdapter = columnsAdapter; + } +} \ No newline at end of file diff --git a/src/main/java/org/nuiton/csv/ImporterRowFactory.java b/src/main/java/org/nuiton/csv/ImporterRowFactory.java new file mode 100644 index 0000000..dff6b5e --- /dev/null +++ b/src/main/java/org/nuiton/csv/ImporterRowFactory.java @@ -0,0 +1,12 @@ +package org.nuiton.csv; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public interface ImporterRowFactory<O> { + + O newInstance() throws IllegalAccessException, InstantiationException; + +} diff --git a/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java b/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java new file mode 100644 index 0000000..3792562 --- /dev/null +++ b/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java @@ -0,0 +1,79 @@ +package org.nuiton.csv; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.nuiton.util.DateUtil; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Set; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterExporterFixtures { + + public final Charset charset= StandardCharsets.UTF_8; + + protected final Iterable<Set<RowBean>> sets; + + protected final String cellSeparator = "~"; + + protected final Iterable<String> toImport; + + + public ImporterExporterFixtures() { + ImmutableSet.Builder<RowBean> builder = ImmutableSet.builder(); + + ImmutableList.Builder<Set<RowBean>> setBuilder = ImmutableList.builder(); + + setBuilder.add(Collections.<RowBean>emptySet()); + + builder.add(new RowBean(DateUtil.createDate(1, 12, 2011), "Batéman", 1, RowBeanEnum.ONE)); + setBuilder.add(builder.build()); + + builder.add(new RowBean(DateUtil.createDate(2, 12, 2011), "", 7, RowBeanEnum.TWO)); + setBuilder.add(builder.build()); + + builder.add((new RowBean(DateUtil.createDate(7, 12, 2011), "", 9, RowBeanEnum.ZERO))); + builder.add((new RowBean(DateUtil.createDate(18, 12, 2011), "", 18, RowBeanEnum.ONE))); + builder.add((new RowBean(DateUtil.createDate(23, 12, 2011), "", 4, RowBeanEnum.TWO))); + setBuilder.add(builder.build()); + + sets = setBuilder.build(); + + toImport = ImmutableList.of( + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n", + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" + + "01/12/2011~Batéman~1~ONE~jkfjk\n", + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" + + "01/12/2011~Batéman~1~ONE~jkfjk\n" + + "02/12/2011~~7~TWO~jkfjk\n", + "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" + + "01/12/2011~Batéman~1~ONE~jkfjk\n" + + "02/12/2011~~7~TWO~jkfjk\n" + + "07/12/2011~~9~ZERO~jkfjk\n" + + "18/12/2011~~18~ONE~jkfjk\n" + + "23/12/2011~~4~TWO~jkfjk\n" + ); + } + + public Iterable<Set<RowBean>> getSets() { + return sets; + } + + public String getCellSeparator() { + return cellSeparator; + } + + public Charset getCharset() { + return charset; + } + + public Iterable<String> getToImport() { + return toImport; + } +} diff --git a/src/test/java/org/nuiton/csv/ImporterTest.java b/src/test/java/org/nuiton/csv/ImporterTest.java new file mode 100644 index 0000000..00ad999 --- /dev/null +++ b/src/test/java/org/nuiton/csv/ImporterTest.java @@ -0,0 +1,149 @@ +package org.nuiton.csv; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.StringReader; +import java.util.Iterator; +import java.util.Set; + +/** + * Created on 19/12/15. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImporterTest { + + protected static ImporterExporterFixtures FIXTURES; + + protected static Importer<RowBean> IMPORTER; + + @BeforeClass + public static void setUp() throws Exception { + + FIXTURES = new ImporterExporterFixtures(); + + IMPORTER = Importer + .builder(RowBean.class) + .setCharset(FIXTURES.getCharset()) + .setCellSeparator(FIXTURES.getCellSeparator()) + .addMandatoryColumn("D~ATE", "date", Common.DAY) + .addIgnoredColumn("title") + .addMandatoryColumn("TITLEé", "title") + .addMandatoryColumn("NUMBER", "number", Common.INTEGER) + .addMandatoryColumn("rowBeanEnum", "rowBeanEnum", Common.newEnumByNameParserFormatter(RowBeanEnum.class)) + .build(); + + + } + + @Test + public void testFromString() throws Exception { + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(resultIterator.next())) { + assertImportResultEquals(expected, actual); + } + } + + } + + @Test + public void testFromInputStream() throws Exception { + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(new ByteArrayInputStream(resultIterator.next().getBytes()))) { + assertImportResultEquals(expected, actual); + } + } + + } + + @Test + public void testFromReader() throws Exception { + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(new StringReader(resultIterator.next()))) { + assertImportResultEquals(expected, actual); + } + } + + } + + @Test + public void testFromFile() throws Exception { + + File parentFile = new File(FileUtils.getTempDirectory(), getClass().getName() + "-testFromFile"); + FileUtils.forceMkdir(parentFile); + + int index = 0; + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + + File importFile = new File(parentFile, (index++) + ".csv"); + Files.write(resultIterator.next().getBytes(), importFile); + try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(importFile)) { + assertImportResultEquals(expected, actual); + } + + } + + } + + @Test + public void testFromStringWithColumnsAdapter() throws Exception { + + Importer<RowBean> importer = Importer + .builder(RowBean.class) + .setCharset(FIXTURES.getCharset()) + .setCellSeparator(FIXTURES.getCellSeparator()) + .addMandatoryColumn("D~ATE", "date", Common.DAY) + .setColumnsAdapter(new ImporterColumnsAdapter<RowBean>() { + + @Override + public void adapt(ImmutableSet.Builder<ImportableColumn<RowBean, ?>> columnBuilder, ImmutableSet<ImportableColumn<RowBean, ?>> importableColumns, ImmutableList<String> headers) { + columnBuilder.addAll(importableColumns); + addIgnoredColumn(columnBuilder, "title"); + addMandatoryColumn(columnBuilder, "TITLEé", "title"); + addMandatoryColumn(columnBuilder, "NUMBER", "number", Common.INTEGER); + addMandatoryColumn(columnBuilder, "rowBeanEnum", "rowBeanEnum", Common.newEnumByNameParserFormatter(RowBeanEnum.class)); + } + }) + .build(); + + Iterator<String> resultIterator = FIXTURES.getToImport().iterator(); + for (Set<RowBean> expected : FIXTURES.getSets()) { + try (Importer.ImporterAction<RowBean> actual = importer.from(resultIterator.next())) { + assertImportResultEquals(expected, actual); + } + } + + } + + protected void assertImportResultEquals(Iterable<RowBean> expected, Importer.ImporterAction<RowBean> actual) { + + Iterator<RowBean> expectedIterator = expected.iterator(); + + while (expectedIterator.hasNext()) { + + RowBean expectedRow = expectedIterator.next(); + RowBean actualRow = actual.next(); + Assert.assertEquals(expectedRow, actualRow); + + Assert.assertEquals(expectedIterator.hasNext(), actual.hasNext()); + + } + + } + +} \ No newline at end of file -- To stop receiving notification emails like this one, please contact nuiton.org SCM administrator <admin+scm@nuiton.org>.
participants (1)
-
nuiton.org scm