[Nuiton-csv-commits] 01/01: First flush of a new Importer API (See #3820)

19 Dec 2015

This is an automated email from the git hooks/post-receive script.

New commit to branch feature/3820_rethink_import_api in repository nuiton-csv.

See http://git.nuiton.org/nuiton-csv.git

commit e81761d9a35c9fac7b34a8731a4c2131cf75e28a
Author: Tony CHEMIT <chemit@codelutin.com>
Date:   Sat Dec 19 06:04:17 2015 +0100

    First flush of a new Importer API (See #3820)
---
 src/main/java/org/nuiton/csv/Importer.java         | 406 +++++++++++++++++++++
 src/main/java/org/nuiton/csv/ImporterBuilder.java  | 208 +++++++++++
 .../org/nuiton/csv/ImporterColumnsAdapter.java     |  47 +++
 .../java/org/nuiton/csv/ImporterConfiguration.java | 139 +++++++
 .../java/org/nuiton/csv/ImporterRowFactory.java    |  12 +
 .../org/nuiton/csv/ImporterExporterFixtures.java   |  79 ++++
 src/test/java/org/nuiton/csv/ImporterTest.java     | 149 ++++++++
 7 files changed, 1040 insertions(+)

diff --git a/src/main/java/org/nuiton/csv/Importer.java b/src/main/java/org/nuiton/csv/Importer.java
new file mode 100644
index 0000000..bdf6460
--- /dev/null
+++ b/src/main/java/org/nuiton/csv/Importer.java
@@ -0,0 +1,406 @@
+package org.nuiton.csv;
+
+import com.csvreader.CsvReader;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Files;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.nuiton.csv.ext.CsvReaders;
+import org.nuiton.util.StringUtil;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import static org.nuiton.i18n.I18n.t;
+
+/**
+ * Created on 19/12/15.
+ *
+ * @author Tony Chemit - chemit@codelutin.com
+ */
+public class Importer<O> {
+
+    /** Logger. */
+    private static final Log log = LogFactory.getLog(Importer.class);
+
+    /**
+     * Creates an importer from the given model, and using default configuration values.
+     *
+     * @param rowType type of data to import
+     * @param model   model of import
+     * @param <O>     type of data to import
+     * @return the importer ready to use
+     */
+    public static <O> Importer<O> of(Class<O> rowType, ImportModel<O> model) {
+        return builder(rowType, model).build();
+    }
+
+    /**
+     * Creates an importer from the given configuration.
+     *
+     * @param configuration import configuration
+     * @param <O>           type of data to import
+     * @return the importer ready to use
+     */
+    public static <O> Importer<O> of(ImporterConfiguration<O> configuration) {
+        return builder(configuration).build();
+    }
+
+    /**
+     * Creates a new importer builder from the given model.
+     *
+     * @param model   model of import
+     * @param rowType type of data to import
+     * @param <O>     type of data to import
+     * @return the importer builder
+     */
+    public static <O> ImporterBuilder<O> builder(Class<O> rowType, ImportModel<O> model) {
+        return Importer
+                .builder(rowType)
+                .addColumns(model.getColumnsForImport())
+                .setCellSeparator(String.valueOf(model.getSeparator()));
+    }
+
+    /**
+     * Creates a new importer builder from the given configuration.
+     *
+     * @param configuration configuration of import
+     * @param <O>           type of data to import
+     * @return the importer builder
+     */
+    public static <O> ImporterBuilder<O> builder(ImporterConfiguration<O> configuration) {
+        return new ImporterBuilder<>(configuration);
+    }
+
+    /**
+     * Creates a new empty importer builder.
+     *
+     * @param rowType type of data to import
+     * @param <O>     type of data to import
+     * @return the importer builder
+     */
+    public static <O> ImporterBuilder<O> builder(Class<O> rowType) {
+        return new ImporterBuilder<>(rowType);
+    }
+
+    public ImporterAction<O> from(InputStream inputStream) {
+        return new ImporterAction<>(configuration, inputStream);
+    }
+
+    public ImporterAction<O> from(Reader reader) {
+        return new ImporterAction<>(configuration, reader);
+    }
+
+    public ImporterAction<O> from(File file) throws FileNotFoundException {
+        return new ImporterAction<>(configuration, file);
+    }
+
+    public ImporterAction<O> from(String string) {
+        return new ImporterAction<>(configuration, string);
+    }
+
+    public static class ImporterAction<O> implements Iterator<O>, Closeable {
+
+        /**
+         * Importer configuration.
+         */
+        protected final ImporterConfiguration<O> configuration;
+
+        /** Csv reader (this is the input). */
+        protected final CsvReader reader;
+
+        protected final ImporterRowFactory<O> rowFactory;
+
+        // read first line since first line is header
+        protected boolean hasNext;
+
+        // to stock the current line number
+        protected int lineNumber;
+
+        protected ImmutableList<ImportableColumn<O, ?>> columns;
+
+        protected ImporterAction(ImporterConfiguration<O> configuration, InputStream inputStream) {
+            this(configuration, new CsvReader(inputStream, configuration.getCellSeparator().charAt(0), configuration.getCharset()));
+        }
+
+        protected ImporterAction(ImporterConfiguration<O> configuration, Reader reader) {
+            this(configuration, new CsvReader(reader, configuration.getCellSeparator().charAt(0)));
+        }
+
+        protected ImporterAction(ImporterConfiguration<O> configuration, File file) throws FileNotFoundException {
+            this(configuration, new CsvReader(Files.newReader(file, configuration.getCharset()), configuration.getCellSeparator().charAt(0)));
+        }
+
+        protected ImporterAction(ImporterConfiguration<O> configuration, String string) {
+            this(configuration, new CsvReader(new StringReader(string), configuration.getCellSeparator().charAt(0)));
+        }
+
+        protected ImporterAction(ImporterConfiguration<O> configuration, CsvReader csvReader) {
+            this.configuration = configuration;
+            this.reader = csvReader;
+            this.reader.setTrimWhitespace(true);
+            this.reader.setSafetySwitch(configuration.isSafetySwitch());
+            this.rowFactory = configuration.getRowFactory();
+
+            // obtains headers
+            ImmutableList<String> headers = getHeaders();
+
+            if (log.isTraceEnabled()) {
+                log.trace("headers of the CSV file are : " + headers);
+            }
+
+            ImmutableSet<ImportableColumn<O, ?>> configurationColumns = configuration.getColumns();
+
+            ImporterColumnsAdapter<O> columnsAdapter = configuration.getColumnsAdapter();
+            if (columnsAdapter != null) {
+
+                // hook to do some stuff from the model
+                ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder = ImmutableSet.builder();
+                columnsAdapter.adapt(columnBuilder, configurationColumns, headers);
+                configurationColumns = columnBuilder.build();
+
+            }
+
+            // check model columns name are unique
+            checkUniqueModelColumnNames(configurationColumns);
+
+            // check that given headers from csv file are all known
+            checkHeaderNamesAreAllKnown(headers, configurationColumns);
+
+            // check all mandatories column are on csv header
+            checkAllMandatoryHeadersArePresent(headers, configurationColumns);
+
+            this.columns = getNonIgnoredHeaders(configurationColumns);
+            this.hasNext = readRow();
+
+        }
+
+        @Override
+        public void close() throws IOException {
+            reader.close();
+        }
+
+        @Override
+        public boolean hasNext() {
+            return hasNext;
+        }
+
+        @Override
+        public O next() throws NoSuchElementException {
+
+            if (!hasNext) {
+                throw new NoSuchElementException();
+            }
+
+            lineNumber += 1;
+
+            O element = newRowInstance();
+
+            for (ImportableColumn<O, ?> field : columns) {
+
+                // read value from csv cell
+                String value = readValue(field, lineNumber);
+
+                // contravariance ftw
+                Object parsedValue = parseValue(field, lineNumber, value);
+
+                // set value to element
+                setValue((ImportableColumn) field, lineNumber, element, parsedValue);
+            }
+
+            // check if there is a next row to read
+            hasNext = readRow();
+
+            return element;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+
+        protected O newRowInstance() {
+            try {
+                return rowFactory.newInstance();
+            } catch (IllegalAccessException | InstantiationException e) {
+                throw new ImportRuntimeException("Could not instanciate new row", e);
+            }
+        }
+
+        protected <T> String readValue(ImportableColumn<O, T> field, int lineNumber) {
+            try {
+                String value = reader.get(field.getHeaderName());
+                return value;
+            } catch (Exception e) {
+                reader.close();
+                throw new ImportRuntimeException(
+                        t("csv.import.error.unableToReadField",
+                          field.getHeaderName(), lineNumber), e);
+            }
+        }
+
+        protected <T> T parseValue(ImportableColumn<O, T> field, int lineNumber, String value) {
+            try {
+                T parsedValue = field.parseValue(value);
+                return parsedValue;
+            } catch (Exception e) {
+                String message = t("csv.import.error.unableToParseValue",
+                                   value, field.getHeaderName(), lineNumber)
+                                 + "\n" + e.getMessage();
+                throw new ImportRuntimeException(message, e);
+            }
+        }
+
+        protected <T> void setValue(ImportableColumn<O, T> field, int lineNumber, O element, T parsedValue) {
+            try {
+                field.setValue(element, parsedValue);
+            } catch (Exception e) {
+                String message = t("csv.import.error.unableToSetValue",
+                                   parsedValue,
+                                   element.toString(),
+                                   lineNumber, field.getHeaderName());
+                if (log.isErrorEnabled()) {
+                    log.error(message);
+                }
+                throw new ImportRuntimeException(message, e);
+            }
+        }
+
+        /**
+         * Read the next row from the reader and return {@code true} if line
+         * was successfully read.
+         *
+         * @return {@code true} if line was successfully read, says in fact there is
+         * something after this line.
+         * @throws ImportRuntimeException if could not read line
+         */
+        protected boolean readRow() throws ImportRuntimeException {
+            try {
+                boolean hasNext = reader.readRecord();
+                return hasNext;
+            } catch (IOException e) {
+                reader.close();
+                throw new ImportRuntimeException(t("csv.import.error.unableToReadLine", 1), e);
+            }
+        }
+
+        protected ImmutableList<String> getHeaders() throws ImportRuntimeException {
+            try {
+                boolean canReadHeaders = reader.readHeaders();
+                if (!canReadHeaders) {
+                    throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders"));
+                }
+            } catch (IOException e) {
+                throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders"), e);
+            }
+
+            try {
+                String[] headers = reader.getHeaders();
+                if (headers.length > 0) {
+                    headers[0] = CsvReaders.removeBomCharacter(headers[0]);
+                }
+                return ImmutableList.copyOf(headers);
+            } catch (IOException e) {
+                throw new ImportRuntimeException(t("csv.import.error.unableToReadHeaders"), e);
+            }
+        }
+
+        protected ImmutableList<ImportableColumn<O, ?>> getNonIgnoredHeaders(ImmutableSet<ImportableColumn<O, ?>> columns) {
+            ImmutableList.Builder<ImportableColumn<O, ?>> nonIgnoredHeaders = ImmutableList.builder();
+            for (ImportableColumn<O, ?> field : columns) {
+                if (!field.isIgnored()) {
+                    nonIgnoredHeaders.add(field);
+                }
+            }
+            return nonIgnoredHeaders.build();
+        }
+
+        protected List<ImportableColumn<O, ?>> getAllMandatoryHeaders(ImmutableSet<ImportableColumn<O, ?>> columns) {
+            List<ImportableColumn<O, ?>> allMandatoryHeaders =
+                    new ArrayList<>();
+            for (ImportableColumn<O, ?> field : columns) {
+                if (field.isMandatory()) {
+                    allMandatoryHeaders.add(field);
+                }
+            }
+            return allMandatoryHeaders;
+        }
+
+        protected void checkHeaderNamesAreAllKnown(ImmutableList<String> headers, ImmutableSet<ImportableColumn<O, ?>> columns) {
+            List<String> csvHeaders = new ArrayList<>(headers);
+
+            for (ImportableColumn<O, ?> field : columns) {
+                csvHeaders.remove(field.getHeaderName());
+            }
+            if (!csvHeaders.isEmpty()) {
+                List<String> validHeaderNames = new LinkedList<>();
+                for (ImportableColumn<O, ?> importableColumn : columns) {
+                    validHeaderNames.add(importableColumn.getHeaderName());
+                }
+                String validationMessage =
+                        t("csv.import.error.unrecognizedHeaders",
+                          StringUtil.join(csvHeaders, ", ", true),
+                          StringUtil.join(validHeaderNames, ", ", true));
+                throw new ImportRuntimeException(validationMessage);
+            }
+        }
+
+        protected void checkUniqueModelColumnNames(ImmutableSet<ImportableColumn<O, ?>> columns) {
+            Set<String> headerNames = new HashSet<>();
+            Set<String> doubleHeaderNames = new HashSet<>();
+            for (ImportableColumn<O, ?> importableColumn : columns) {
+                String headerName = importableColumn.getHeaderName();
+                boolean alreadyUsed = !headerNames.add(headerName);
+                if (alreadyUsed) {
+                    doubleHeaderNames.add(headerName);
+                }
+            }
+            if (!doubleHeaderNames.isEmpty()) {
+                String message = t("csv.import.error.duplicatedHeaders",
+                                   StringUtil.join(doubleHeaderNames, ", ", true));
+
+                throw new ImportRuntimeException(message);
+            }
+        }
+
+        protected void checkAllMandatoryHeadersArePresent(ImmutableList<String> headers, ImmutableSet<ImportableColumn<O, ?>> columns) {
+
+            List<String> csvHeaders = new ArrayList<>(headers);
+
+            List<String> mandatoryHeadersNames = new ArrayList<>();
+            for (ImportableColumn<O, ?> field : getAllMandatoryHeaders(columns)) {
+                mandatoryHeadersNames.add(field.getHeaderName());
+            }
+            mandatoryHeadersNames.removeAll(csvHeaders);
+
+            if (!mandatoryHeadersNames.isEmpty()) {
+                String validationMessage =
+                        t("csv.import.error.missingMandatoryHeaders",
+                          StringUtil.join(mandatoryHeadersNames, ", ", true));
+                throw new ImportRuntimeException(validationMessage);
+            }
+        }
+    }
+
+    /**
+     * Immutable importer configuration.
+     */
+    protected final ImporterConfiguration<O> configuration;
+
+    protected Importer(ImporterConfiguration<O> configuration) {
+        this.configuration = configuration;
+    }
+}
diff --git a/src/main/java/org/nuiton/csv/ImporterBuilder.java b/src/main/java/org/nuiton/csv/ImporterBuilder.java
new file mode 100644
index 0000000..87d6ba2
--- /dev/null
+++ b/src/main/java/org/nuiton/csv/ImporterBuilder.java
@@ -0,0 +1,208 @@
+package org.nuiton.csv;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Created on 19/12/15.
+ *
+ * @author Tony Chemit - chemit@codelutin.com
+ */
+public class ImporterBuilder<O> {
+
+    public static final String DEFAULT_CELL_SEPARATOR = ";";
+
+    public static final String DEFAULT_END_OF_LINE_SEPARATOR = "\n";
+
+    public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+
+    public static final boolean DEFAULT_SAFETY_SWITCH = false;
+
+    public static final boolean DEFAULT_STRICT_MODE = true;
+
+    public static final boolean DEFAULT_IGNORE_UNKNOWN_HEADER = false;
+
+    protected final ImporterConfiguration<O> configuration;
+
+    protected final ImmutableSet.Builder<ImportableColumn<O, ?>> columnsBuilder = ImmutableSet.builder();
+
+    /**
+     * Set the {@link ImporterConfiguration#cellSeparator} and return the builder.
+     *
+     * @param cellSeparator new value for {@link ImporterConfiguration#cellSeparator}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setCellSeparator(String cellSeparator) {
+        configuration.setCellSeparator(cellSeparator);
+        return this;
+    }
+
+    /**
+     * Set the {@link ImporterConfiguration#endOfLineSeparator} and return the builder.
+     *
+     * @param endOfLineSeparator new value for {@link ImporterConfiguration#endOfLineSeparator}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setEndOfLineSeparator(String endOfLineSeparator) {
+        configuration.setEndOfLineSeparator(endOfLineSeparator);
+        return this;
+    }
+
+    /**
+     * Set the {@link ImporterConfiguration#charset} and return the builder.
+     *
+     * @param charset new value for {@link ImporterConfiguration#charset}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setCharset(Charset charset) {
+        configuration.setCharset(charset);
+        return this;
+    }
+
+    /**
+     * Set the {@link ImporterConfiguration#ignoreUnknownHeader} and return the builder.
+     *
+     * @param ignoreUnknownHeader new value for {@link ImporterConfiguration#ignoreUnknownHeader}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setIgnoreUnknownHeader(boolean ignoreUnknownHeader) {
+        configuration.setIgnoreUnknownHeader(ignoreUnknownHeader);
+        return this;
+    }
+
+    /**
+     * Set the {@link ImporterConfiguration#safetySwitch} and return the builder.
+     *
+     * @param safetySwitch new value for {@link ImporterConfiguration#safetySwitch}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setSafetySwitch(boolean safetySwitch) {
+        configuration.setSafetySwitch(safetySwitch);
+        return this;
+    }
+
+    /**
+     * Set the {@link ImporterConfiguration#strictMode} and return the builder.
+     *
+     * @param strictMode new value for {@link ImporterConfiguration#strictMode}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setStrictMode(boolean strictMode) {
+        configuration.setStrictMode(strictMode);
+        return this;
+    }
+
+    /**
+     * Set the {@link ImporterConfiguration#rowFactory} and return the builder.
+     *
+     * @param importerRowFactory new value for {@link ImporterConfiguration#rowFactory}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setRowFactory(ImporterRowFactory<O> importerRowFactory) {
+        configuration.setRowFactory(importerRowFactory);
+        return this;
+    }
+
+    /**
+     * Set the {@link ImporterConfiguration#columnsAdapter} and return the builder.
+     *
+     * @param columnsAdapter new value for {@link ImporterConfiguration#columnsAdapter}.
+     * @return the builder
+     */
+    public ImporterBuilder<O> setColumnsAdapter(ImporterColumnsAdapter<O> columnsAdapter) {
+        configuration.setColumnsAdapter(columnsAdapter);
+        return this;
+    }
+
+    /**
+     * Add the given columns.
+     *
+     * @param columns columns to add to the builder
+     * @return the builder
+     */
+    public ImporterBuilder<O> addColumns(Iterable<ImportableColumn<O, Object>> columns) {
+        columnsBuilder.addAll(columns);
+        return this;
+    }
+
+    public <T> ImporterBuilder<O> addColumn(ImportableColumn<O, T> newColumn) {
+        columnsBuilder.add(newColumn);
+        return this;
+    }
+
+    public ImporterBuilder<O> addIgnoredColumn(String headerName) {
+        ImportableColumn<O, Object> newColumn = Column.newImportableColumn(headerName, null, null, true, false);
+        return addColumn(newColumn);
+    }
+
+    public ImporterBuilder<O> addMandatoryColumn(String headerName, String propertyName) {
+        return addMandatoryColumn(headerName, propertyName, Common.STRING);
+    }
+
+    public <T> ImporterBuilder<O> addMandatoryColumn(String headerName, String propertyName, ValueParser<T> valueParser) {
+        return addMandatoryColumn(headerName, valueParser, new Common.BeanProperty<O, T>(propertyName));
+    }
+
+    public <T> ImporterBuilder<O> addMandatoryColumn(String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) {
+        ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, true);
+        return addColumn(newColumn);
+    }
+
+    public ImporterBuilder<O> addOptionamColumn(String headerName, String propertyName) {
+        return addOptionamColumn(headerName, propertyName, Common.STRING);
+    }
+
+    public <T> ImporterBuilder<O> addOptionamColumn(String headerName, String propertyName, ValueParser<T> valueParser) {
+        return addOptionamColumn(headerName, valueParser, new Common.BeanProperty<O, T>(propertyName));
+    }
+
+    public <T> ImporterBuilder<O> addOptionamColumn(String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) {
+        ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, false);
+        return addColumn(newColumn);
+    }
+
+    public Importer<O> build() {
+        configuration.setColumns(columnsBuilder.build());
+        Preconditions.checkNotNull(configuration.getColumns(), "No columns defined.");
+        Preconditions.checkNotNull(configuration.getCellSeparator(), "No charset defined.");
+        Preconditions.checkNotNull(configuration.getCellSeparator(), "No cellSeparator defined.");
+        Preconditions.checkNotNull(configuration.getEndOfLineSeparator(), "No endOfLineSeparator defined.");
+        Preconditions.checkNotNull(configuration.getRowFactory(), "No rowFactory defined.");
+
+        return new Importer<>(configuration);
+    }
+
+    public ImporterBuilder(final Class<O> rowType) {
+        this.configuration = new ImporterConfiguration<>(rowType);
+        setCharset(DEFAULT_CHARSET);
+        setCellSeparator(DEFAULT_CELL_SEPARATOR);
+        setEndOfLineSeparator(DEFAULT_END_OF_LINE_SEPARATOR);
+        setStrictMode(DEFAULT_STRICT_MODE);
+        setSafetySwitch(DEFAULT_SAFETY_SWITCH);
+        setIgnoreUnknownHeader(DEFAULT_IGNORE_UNKNOWN_HEADER);
+        setRowFactory(new ImporterRowFactory<O>() {
+
+            @Override
+            public O newInstance() throws IllegalAccessException, InstantiationException {
+                return rowType.newInstance();
+            }
+        });
+    }
+
+    public ImporterBuilder(ImporterConfiguration<O> configuration) {
+        this.configuration = new ImporterConfiguration<>(configuration.rowType);
+        addColumns((Iterable) configuration.getColumns());
+        setCharset(configuration.getCharset());
+        setCellSeparator(configuration.getCellSeparator());
+        setEndOfLineSeparator(configuration.getEndOfLineSeparator());
+        setRowFactory(configuration.getRowFactory());
+        setIgnoreUnknownHeader(configuration.isIgnoreUnknownHeader());
+        setStrictMode(configuration.isStrictMode());
+        setSafetySwitch(configuration.isSafetySwitch());
+        setColumnsAdapter(configuration.getColumnsAdapter());
+    }
+
+}
diff --git a/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java b/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java
new file mode 100644
index 0000000..b4a6a13
--- /dev/null
+++ b/src/main/java/org/nuiton/csv/ImporterColumnsAdapter.java
@@ -0,0 +1,47 @@
+package org.nuiton.csv;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Created on 19/12/15.
+ *
+ * @author Tony Chemit - chemit@codelutin.com
+ */
+public abstract class ImporterColumnsAdapter<O> {
+
+    public abstract void adapt(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, ImmutableSet<ImportableColumn<O, ?>> columns, ImmutableList<String> headers);
+
+    protected void addIgnoredColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName) {
+        ImportableColumn<O, String> newColumn = Column.newImportableColumn(headerName, null, null, true, false);
+        columnBuilder.add(newColumn);
+    }
+
+    protected void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName) {
+        addMandatoryColumn(columnBuilder, headerName, propertyName, Common.STRING);
+    }
+
+    protected <T> void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName, ValueParser<T> valueParser) {
+        addMandatoryColumn(columnBuilder, headerName, valueParser, new Common.BeanProperty<O, T>(propertyName));
+    }
+
+    protected <T> void addMandatoryColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) {
+        ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, true);
+        columnBuilder.add(newColumn);
+    }
+
+    protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName) {
+        addOptionalColumn(columnBuilder, headerName, propertyName, Common.STRING);
+    }
+
+    protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, String propertyName, ValueParser<T> valueParser) {
+        addOptionalColumn(columnBuilder, headerName, valueParser, new Common.BeanProperty<O, T>(propertyName));
+    }
+
+    protected <T> void addOptionalColumn(ImmutableSet.Builder<ImportableColumn<O, ?>> columnBuilder, String headerName, ValueParser<T> valueParser, ValueSetter<O, T> valueSetter) {
+        ImportableColumn<O, T> newColumn = Column.newImportableColumn(headerName, valueParser, valueSetter, false, false);
+        columnBuilder.add(newColumn);
+
+    }
+
+}
diff --git a/src/main/java/org/nuiton/csv/ImporterConfiguration.java b/src/main/java/org/nuiton/csv/ImporterConfiguration.java
new file mode 100644
index 0000000..f8a12df
--- /dev/null
+++ b/src/main/java/org/nuiton/csv/ImporterConfiguration.java
@@ -0,0 +1,139 @@
+package org.nuiton.csv;
+
+import com.csvreader.CsvReader;
+import com.google.common.collect.ImmutableSet;
+
+import java.nio.charset.Charset;
+
+/**
+ * Created on 19/12/15.
+ *
+ * @author Tony Chemit - chemit@codelutin.com
+ */
+public class ImporterConfiguration<O> {
+
+    protected final Class<O> rowType;
+
+    /**
+     * Columns import model.
+     */
+    protected ImmutableSet<ImportableColumn<O, ?>> columns;
+
+    /**
+     * Cell separator.
+     */
+    protected String cellSeparator;
+
+    /**
+     * Eof of line separator.
+     */
+    protected String endOfLineSeparator;
+
+    /**
+     * Charset used to perform the export.
+     */
+    protected Charset charset;
+
+    /**
+     * Flag to turn or not the safetySwitch (see {@link CsvReader#getSafetySwitch()}).
+     *
+     * By default, not used.
+     */
+    protected boolean safetySwitch;
+
+    /**
+     * Flag to use a strict mode (says import will failed at the first error),
+     * if setted to {@link false}, then errors for each rows will be stored in
+     * the current row and import will continue to the end.
+     *
+     * By default, used (strict mode).
+     */
+    protected boolean strictMode;
+
+    /**
+     * Flag to ignore header found in a import file and not declared in the import model.
+     *
+     * By default, not used (strict mode).
+     */
+    protected boolean ignoreUnknownHeader;
+
+    protected ImporterRowFactory<O> rowFactory;
+
+    private ImporterColumnsAdapter<O> columnsAdapter;
+
+    public ImporterConfiguration(Class<O> rowType) {
+        this.rowType = rowType;
+    }
+
+    public String getCellSeparator() {
+        return cellSeparator;
+    }
+
+    public String getEndOfLineSeparator() {
+        return endOfLineSeparator;
+    }
+
+    public Charset getCharset() {
+        return charset;
+    }
+
+    public boolean isIgnoreUnknownHeader() {
+        return ignoreUnknownHeader;
+    }
+
+    public boolean isSafetySwitch() {
+        return safetySwitch;
+    }
+
+    public boolean isStrictMode() {
+        return strictMode;
+    }
+
+    public ImporterRowFactory<O> getRowFactory() {
+        return rowFactory;
+    }
+
+    public ImmutableSet<ImportableColumn<O, ?>> getColumns() {
+        return columns;
+    }
+
+    public ImporterColumnsAdapter<O> getColumnsAdapter() {
+        return columnsAdapter;
+    }
+
+    public void setCellSeparator(String cellSeparator) {
+        this.cellSeparator = cellSeparator;
+    }
+
+    public void setCharset(Charset charset) {
+        this.charset = charset;
+    }
+
+    public void setColumns(ImmutableSet<ImportableColumn<O, ?>> columns) {
+        this.columns = columns;
+    }
+
+    public void setEndOfLineSeparator(String endOfLineSeparator) {
+        this.endOfLineSeparator = endOfLineSeparator;
+    }
+
+    public void setIgnoreUnknownHeader(boolean ignoreUnknownHeader) {
+        this.ignoreUnknownHeader = ignoreUnknownHeader;
+    }
+
+    public void setSafetySwitch(boolean safetySwitch) {
+        this.safetySwitch = safetySwitch;
+    }
+
+    public void setStrictMode(boolean strictMode) {
+        this.strictMode = strictMode;
+    }
+
+    public void setRowFactory(ImporterRowFactory<O> rowFactory) {
+        this.rowFactory = rowFactory;
+    }
+
+    public void setColumnsAdapter(ImporterColumnsAdapter<O> columnsAdapter) {
+        this.columnsAdapter = columnsAdapter;
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/org/nuiton/csv/ImporterRowFactory.java b/src/main/java/org/nuiton/csv/ImporterRowFactory.java
new file mode 100644
index 0000000..dff6b5e
--- /dev/null
+++ b/src/main/java/org/nuiton/csv/ImporterRowFactory.java
@@ -0,0 +1,12 @@
+package org.nuiton.csv;
+
+/**
+ * Created on 19/12/15.
+ *
+ * @author Tony Chemit - chemit@codelutin.com
+ */
+public interface ImporterRowFactory<O> {
+
+    O newInstance() throws IllegalAccessException, InstantiationException;
+
+}
diff --git a/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java b/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java
new file mode 100644
index 0000000..3792562
--- /dev/null
+++ b/src/test/java/org/nuiton/csv/ImporterExporterFixtures.java
@@ -0,0 +1,79 @@
+package org.nuiton.csv;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.nuiton.util.DateUtil;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * Created on 19/12/15.
+ *
+ * @author Tony Chemit - chemit@codelutin.com
+ */
+public class ImporterExporterFixtures {
+
+    public final Charset charset= StandardCharsets.UTF_8;
+
+    protected final Iterable<Set<RowBean>> sets;
+
+    protected final String cellSeparator = "~";
+
+    protected final Iterable<String> toImport;
+
+
+    public ImporterExporterFixtures() {
+        ImmutableSet.Builder<RowBean> builder = ImmutableSet.builder();
+
+        ImmutableList.Builder<Set<RowBean>> setBuilder = ImmutableList.builder();
+
+        setBuilder.add(Collections.<RowBean>emptySet());
+
+        builder.add(new RowBean(DateUtil.createDate(1, 12, 2011), "Batéman", 1, RowBeanEnum.ONE));
+        setBuilder.add(builder.build());
+
+        builder.add(new RowBean(DateUtil.createDate(2, 12, 2011), "", 7, RowBeanEnum.TWO));
+        setBuilder.add(builder.build());
+
+        builder.add((new RowBean(DateUtil.createDate(7, 12, 2011), "", 9, RowBeanEnum.ZERO)));
+        builder.add((new RowBean(DateUtil.createDate(18, 12, 2011), "", 18, RowBeanEnum.ONE)));
+        builder.add((new RowBean(DateUtil.createDate(23, 12, 2011), "", 4, RowBeanEnum.TWO)));
+        setBuilder.add(builder.build());
+
+        sets = setBuilder.build();
+
+        toImport = ImmutableList.of(
+                "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n",
+                "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" +
+                "01/12/2011~Batéman~1~ONE~jkfjk\n",
+                "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" +
+                "01/12/2011~Batéman~1~ONE~jkfjk\n"            +
+                "02/12/2011~~7~TWO~jkfjk\n",
+                "\"D~ATE\"~TITLEé~NUMBER~rowBeanEnum~title\n" +
+                "01/12/2011~Batéman~1~ONE~jkfjk\n"            +
+                "02/12/2011~~7~TWO~jkfjk\n"                   +
+                "07/12/2011~~9~ZERO~jkfjk\n"                  +
+                "18/12/2011~~18~ONE~jkfjk\n"                  +
+                "23/12/2011~~4~TWO~jkfjk\n"
+        );
+    }
+
+    public Iterable<Set<RowBean>> getSets() {
+        return sets;
+    }
+
+    public String getCellSeparator() {
+        return cellSeparator;
+    }
+
+    public Charset getCharset() {
+        return charset;
+    }
+
+    public Iterable<String> getToImport() {
+        return toImport;
+    }
+}
diff --git a/src/test/java/org/nuiton/csv/ImporterTest.java b/src/test/java/org/nuiton/csv/ImporterTest.java
new file mode 100644
index 0000000..00ad999
--- /dev/null
+++ b/src/test/java/org/nuiton/csv/ImporterTest.java
@@ -0,0 +1,149 @@
+package org.nuiton.csv;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * Created on 19/12/15.
+ *
+ * @author Tony Chemit - chemit@codelutin.com
+ */
+public class ImporterTest {
+
+    protected static ImporterExporterFixtures FIXTURES;
+
+    protected static Importer<RowBean> IMPORTER;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+
+        FIXTURES = new ImporterExporterFixtures();
+
+        IMPORTER = Importer
+                .builder(RowBean.class)
+                .setCharset(FIXTURES.getCharset())
+                .setCellSeparator(FIXTURES.getCellSeparator())
+                .addMandatoryColumn("D~ATE", "date", Common.DAY)
+                .addIgnoredColumn("title")
+                .addMandatoryColumn("TITLEé", "title")
+                .addMandatoryColumn("NUMBER", "number", Common.INTEGER)
+                .addMandatoryColumn("rowBeanEnum", "rowBeanEnum", Common.newEnumByNameParserFormatter(RowBeanEnum.class))
+                .build();
+
+
+    }
+
+    @Test
+    public void testFromString() throws Exception {
+
+        Iterator<String> resultIterator = FIXTURES.getToImport().iterator();
+        for (Set<RowBean> expected : FIXTURES.getSets()) {
+            try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(resultIterator.next())) {
+                assertImportResultEquals(expected, actual);
+            }
+        }
+
+    }
+
+    @Test
+    public void testFromInputStream() throws Exception {
+
+        Iterator<String> resultIterator = FIXTURES.getToImport().iterator();
+        for (Set<RowBean> expected : FIXTURES.getSets()) {
+            try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(new ByteArrayInputStream(resultIterator.next().getBytes()))) {
+                assertImportResultEquals(expected, actual);
+            }
+        }
+
+    }
+
+    @Test
+    public void testFromReader() throws Exception {
+
+        Iterator<String> resultIterator = FIXTURES.getToImport().iterator();
+        for (Set<RowBean> expected : FIXTURES.getSets()) {
+            try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(new StringReader(resultIterator.next()))) {
+                assertImportResultEquals(expected, actual);
+            }
+        }
+
+    }
+
+    @Test
+    public void testFromFile() throws Exception {
+
+        File parentFile = new File(FileUtils.getTempDirectory(), getClass().getName() + "-testFromFile");
+        FileUtils.forceMkdir(parentFile);
+
+        int index = 0;
+        Iterator<String> resultIterator = FIXTURES.getToImport().iterator();
+        for (Set<RowBean> expected : FIXTURES.getSets()) {
+
+            File importFile = new File(parentFile, (index++) + ".csv");
+            Files.write(resultIterator.next().getBytes(), importFile);
+            try (Importer.ImporterAction<RowBean> actual = IMPORTER.from(importFile)) {
+                assertImportResultEquals(expected, actual);
+            }
+
+        }
+
+    }
+
+    @Test
+    public void testFromStringWithColumnsAdapter() throws Exception {
+
+        Importer<RowBean> importer = Importer
+                .builder(RowBean.class)
+                .setCharset(FIXTURES.getCharset())
+                .setCellSeparator(FIXTURES.getCellSeparator())
+                .addMandatoryColumn("D~ATE", "date", Common.DAY)
+                .setColumnsAdapter(new ImporterColumnsAdapter<RowBean>() {
+
+                    @Override
+                    public void adapt(ImmutableSet.Builder<ImportableColumn<RowBean, ?>> columnBuilder, ImmutableSet<ImportableColumn<RowBean, ?>> importableColumns, ImmutableList<String> headers) {
+                        columnBuilder.addAll(importableColumns);
+                        addIgnoredColumn(columnBuilder, "title");
+                        addMandatoryColumn(columnBuilder, "TITLEé", "title");
+                        addMandatoryColumn(columnBuilder, "NUMBER", "number", Common.INTEGER);
+                        addMandatoryColumn(columnBuilder, "rowBeanEnum", "rowBeanEnum", Common.newEnumByNameParserFormatter(RowBeanEnum.class));
+                    }
+                })
+                .build();
+
+        Iterator<String> resultIterator = FIXTURES.getToImport().iterator();
+        for (Set<RowBean> expected : FIXTURES.getSets()) {
+            try (Importer.ImporterAction<RowBean> actual = importer.from(resultIterator.next())) {
+                assertImportResultEquals(expected, actual);
+            }
+        }
+
+    }
+
+    protected void assertImportResultEquals(Iterable<RowBean> expected, Importer.ImporterAction<RowBean> actual) {
+
+        Iterator<RowBean> expectedIterator = expected.iterator();
+
+        while (expectedIterator.hasNext()) {
+
+            RowBean expectedRow = expectedIterator.next();
+            RowBean actualRow = actual.next();
+            Assert.assertEquals(expectedRow, actualRow);
+
+            Assert.assertEquals(expectedIterator.hasNext(), actual.hasNext());
+
+        }
+
+    }
+
+}
\ No newline at end of file

-- 
To stop receiving notification emails like this one, please contact
nuiton.org SCM administrator <admin+scm@nuiton.org>.

    

[Nuiton-csv-commits] 01/01: First flush of a new Importer API (See #3820)

nuiton.org scm