Author: tchemit Date: 2011-12-15 12:02:33 +0100 (Thu, 15 Dec 2011) New Revision: 187 Url: http://forge.codelutin.com/repositories/revision/echobase/187 Log: - explode DbImportEXportService into two services - split decoratorService - try to improve the entity association model (need to be continued) Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbExportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationImportModel.java Removed: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DecoratorService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/CsvModelUtil.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/DbImportExportServiceTest.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java 2011-12-15 11:01:21 UTC (rev 186) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java 2011-12-15 11:02:33 UTC (rev 187) @@ -73,6 +73,8 @@ /** Logger. */ private static final Log log = LogFactory.getLog(DbEditorService.class); + public static final TimeLog timeLog = new TimeLog(DbEditorService.class); + protected DecoratorService decoratorService; protected DecoratorService getDecoratorService() { @@ -665,8 +667,6 @@ return Export.newExport(model, datas); } - public static final TimeLog timeLog = new TimeLog(DbEditorService.class); - protected Export<TopiaEntity> prepareExport(AssociationMeta associationMeta) { TableMeta tableMeta = getTableMeta(associationMeta.getSource()); Copied: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbExportService.java (from rev 184, trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java) =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbExportService.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbExportService.java 2011-12-15 11:02:33 UTC (rev 187) @@ -0,0 +1,129 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services; + +import com.google.common.collect.Lists; +import fr.ifremer.echobase.EchoBaseIOUtil; +import fr.ifremer.echobase.entities.EchoBaseEntityEnum; +import fr.ifremer.echobase.entities.EntitiesUtil; +import fr.ifremer.echobase.entities.meta.AssociationMeta; +import fr.ifremer.echobase.entities.meta.MetaFilenameAware; +import fr.ifremer.echobase.entities.meta.TableMeta; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.util.TimeLog; + +import java.io.File; +import java.util.List; + +/** + * Service to import / export a complete db. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class DbExportService extends EchoBaseServiceSupport { + + /** Logger. */ + private static final Log log = + LogFactory.getLog(DbExportService.class); + + public static final TimeLog timeLog = new TimeLog(DbEditorService.class); + + protected DbEditorService dbEditorService; + + /** + * Export the complete db into csv files stored in a zip archive file. + * + * @param fileName file name of the zip to create + * @return the zip archive file + */ + public File exportDb(String fileName) { + File tempDirectory = new File(FileUtils.getTempDirectory(), + "echobase-exportDb_" + System.nanoTime()); + tempDirectory.deleteOnExit(); + + File zipFile = new File(tempDirectory, fileName + ".zip"); + + DbEditorService service = getDbEditorService(); + File dir = new File(tempDirectory, "echobase"); + + MetaFilenameAware[] entries = getEntries(); + for (MetaFilenameAware entry : entries) { + + File entryFile = new File(dir, entry.getFilename()); + + if (entry instanceof AssociationMeta) { + AssociationMeta associationMeta = (AssociationMeta) entry; + service.exportDatas(associationMeta, entryFile); + } else { + TableMeta meta = (TableMeta) entry; + service.exportDatas(meta, entryFile); + } + } + + EchoBaseIOUtil.compressZipFile(zipFile, dir); + + return zipFile; + } + + public MetaFilenameAware[] getEntries() { + + List<MetaFilenameAware> entities = Lists.newArrayList(); + List<MetaFilenameAware> associations = Lists.newArrayList(); + + addEntries(entities, associations, EntitiesUtil.getReferenceTypesForCopy()); + addEntries(entities, associations, EntitiesUtil.getDataTypesforCopy()); + entities.addAll(associations); + return entities.toArray(new MetaFilenameAware[entities.size()]); + } + + protected TableMeta getTableMeta(EchoBaseEntityEnum tableName) { + return getDbEditorService().getTableMeta(tableName); + } + + protected void addEntries(List<MetaFilenameAware> entities, + List<MetaFilenameAware> associations, + EchoBaseEntityEnum[] types) { + for (EchoBaseEntityEnum type : types) { + + TableMeta tableMeta = getTableMeta(type); + entities.add(tableMeta); + +// for (AssociationMeta associationMeta : tableMeta.getAssociations()) { +// associations.add(associationMeta); +// } + associations.addAll(tableMeta.getAssociations()); + } + } + + protected DbEditorService getDbEditorService() { + if (dbEditorService == null) { + dbEditorService = newService(DbEditorService.class); + } + return dbEditorService; + } + +} Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbExportService.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Deleted: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java 2011-12-15 11:01:21 UTC (rev 186) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java 2011-12-15 11:02:33 UTC (rev 187) @@ -1,361 +0,0 @@ -/* - * #%L - * EchoBase :: Services - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2011 Ifremer, Codelutin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * #L% - */ -package fr.ifremer.echobase.services; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import fr.ifremer.echobase.EchoBaseFunctions; -import fr.ifremer.echobase.EchoBaseIOUtil; -import fr.ifremer.echobase.EchoBaseTechnicalException; -import fr.ifremer.echobase.entities.EchoBaseEntityEnum; -import fr.ifremer.echobase.entities.EchoBaseUser; -import fr.ifremer.echobase.entities.EntitiesUtil; -import fr.ifremer.echobase.entities.EntityModificationLog; -import fr.ifremer.echobase.entities.meta.AssociationMeta; -import fr.ifremer.echobase.entities.meta.MetaFilenameAware; -import fr.ifremer.echobase.entities.meta.TableMeta; -import fr.ifremer.echobase.services.models.EntityAssociationCsvModel; -import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.nuiton.topia.TopiaException; -import org.nuiton.topia.persistence.TopiaDAO; -import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.util.StringUtil; -import org.nuiton.util.csv.Import; -import org.nuiton.util.csv.ImportExportModel; -import org.nuiton.util.csv.ImportModel; - -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; - -/** - * Service to import / export a complete db. - * - * @author tchemit <chemit@codelutin.com> - * @since 0.2 - */ -public class DbImportExportService extends EchoBaseServiceSupport { - - /** Logger. */ - private static final Log log = - LogFactory.getLog(DbImportExportService.class); - - protected DbEditorService dbEditorService; - - /** - * Import a complete db from the given zip file. - * - * @param model model of the db import operation - * @param user user which perform the import - * @throws IOException if any io exception while import - * @throws TopiaException if any pb while topia treatment - */ - public void importDb(ImportDbConfiguration model, EchoBaseUser user) throws IOException, TopiaException { - - File file = model.getInput().getFile(); - - ZipFile zipFile = new ZipFile(file); - try { - MetaFilenameAware[] entries = getEntries(); - - List<String> missingEntries = Lists.newArrayList(); - - Map<MetaFilenameAware, ZipEntry> entriestoConsume = - Maps.newLinkedHashMap(); - - // check that all mandatories - for (MetaFilenameAware entry : entries) { - String filename = entry.getFilename(); - ZipEntry zipEntry = zipFile.getEntry("echobase/" + filename); - - if (zipEntry == null) { - missingEntries.add(filename); - } else { - entriestoConsume.put(entry, zipEntry); - } - } - - if (!missingEntries.isEmpty()) { - - if (log.isWarnEnabled()) { - log.warn("There is " + missingEntries.size() + "missing mandatory files " + - " in import " + file + "\n" + StringUtil.join(missingEntries, "\n", false)); - } - -// throw new EchoBaseTechnicalException( -// "There is some missing mandatory files " + -// missingEntries + " in import " + file); - } - - int size = entriestoConsume.size(); - model.setNbSteps(size + 1); - - // consume all found entries - for (Map.Entry<MetaFilenameAware, ZipEntry> entry : - entriestoConsume.entrySet()) { - - model.incrementsProgression(); - - MetaFilenameAware entryDef = entry.getKey(); - ZipEntry value = entry.getValue(); - CsvImportResult csvResult = CsvImportResult.newResult( - entryDef.getSource(), - value.getName(), - false - ); - Reader reader = - new InputStreamReader(zipFile.getInputStream(value)); - try { - importFile(entryDef, reader, csvResult); - } finally { - reader.close(); - } - } - - // add a log entry of import db - if (log.isInfoEnabled()) { - log.info("Import done with user " + user.getEmail()); - } - TopiaDAO<EntityModificationLog> dao = - getDAO(EntityModificationLog.class); - - dao.create( - EntityModificationLog.PROPERTY_ENTITY_TYPE, "import db", - EntityModificationLog.PROPERTY_ENTITY_ID, "Complete db", - EntityModificationLog.PROPERTY_MODIFICATION_USER, user.getEmail(), - EntityModificationLog.PROPERTY_MODIFICATION_DATE, newDate(), - EntityModificationLog.PROPERTY_MODIFICATION_TEXT, "import db from file " + file - ); - commitTransaction("Could not commit db import from file " + file); - } finally { - zipFile.close(); - } - } - - /** - * Export the complete db into csv files stored in a zip archive file. - * - * @param fileName file name of the zip to create - * @return the zip archive file - */ - public File exportDb(String fileName) { - File tempDirectory = new File(FileUtils.getTempDirectory(), - "echobase-exportDb_" + System.nanoTime()); - tempDirectory.deleteOnExit(); - - File zipFile = new File(tempDirectory, fileName + ".zip"); - - DbEditorService service = getDbEditorService(); - File dir = new File(tempDirectory, "echobase"); - - MetaFilenameAware[] entries = getEntries(); - for (MetaFilenameAware entry : entries) { - - File entryFile = new File(dir, entry.getFilename()); - - if (entry instanceof AssociationMeta) { - AssociationMeta associationMeta = (AssociationMeta) entry; - service.exportDatas(associationMeta, entryFile); - } else { - TableMeta meta = (TableMeta) entry; - service.exportDatas(meta, entryFile); - } - } - - EchoBaseIOUtil.compressZipFile(zipFile, dir); - - return zipFile; - } - - public void importFile(MetaFilenameAware entry, - Reader reader, - CsvImportResult csvResult) throws IOException { - - DbEditorService service = getDbEditorService(); - - if (entry instanceof AssociationMeta) { - - // load a association input - - AssociationMeta associationMeta = (AssociationMeta) entry; - - EchoBaseEntityEnum target = associationMeta.getTarget(); - List<TopiaEntity> targetEntities = getEntities(target); - Map<String, TopiaEntity> targetsById = - Maps.uniqueIndex(targetEntities, EchoBaseFunctions.TO_TOPIAID); - - ImportModel<TopiaEntity> model = - EntityAssociationCsvModel.newImportModel( - getConfiguration().getCsvSeparator(), - associationMeta, - targetsById - ); - - importEntityAssociation(associationMeta, model, csvResult, reader); - - } else { - - // normal entity table import - - TableMeta tableMeta = (TableMeta) entry; - - ImportExportModel<TopiaEntity> model = - service.buildForImport(tableMeta); - - importEntities(tableMeta, model, csvResult, reader); - } - } - - public MetaFilenameAware[] getEntries() { - - List<MetaFilenameAware> entities = Lists.newArrayList(); - List<MetaFilenameAware> associations = Lists.newArrayList(); - - addEntries(entities, associations, EntitiesUtil.getReferenceTypesForCopy()); - addEntries(entities, associations, EntitiesUtil.getDataTypesforCopy()); - entities.addAll(associations); - return entities.toArray(new MetaFilenameAware[entities.size()]); - } - - protected TableMeta getTableMeta(EchoBaseEntityEnum tableName) { - return getDbEditorService().getTableMeta(tableName); - } - - protected void addEntries(List<MetaFilenameAware> entities, - List<MetaFilenameAware> associations, - EchoBaseEntityEnum[] types) { - for (EchoBaseEntityEnum type : types) { - - TableMeta tableMeta = getTableMeta(type); - entities.add(tableMeta); - -// for (AssociationMeta associationMeta : tableMeta.getAssociations()) { -// associations.add(associationMeta); -// } - associations.addAll(tableMeta.getAssociations()); - } - } - - protected DbEditorService getDbEditorService() { - if (dbEditorService == null) { - dbEditorService = newService(DbEditorService.class); - } - return dbEditorService; - } - - protected void importEntities(TableMeta meta, - ImportExportModel<TopiaEntity> csvModel, - CsvImportResult result, - Reader reader) { - - if (log.isInfoEnabled()) { - log.info("Will import " + meta); - } - Import<TopiaEntity> importer = Import.newImport(csvModel, reader); - - try { - TopiaDAO<TopiaEntity> dao = getDAO(meta.getSource()); - - for (TopiaEntity entity : importer) { - - - TopiaEntity entityToSave = dao.create(TopiaEntity.TOPIA_ID, - entity.getTopiaId()); - - meta.copy(entity, entityToSave); - - result.incrementsNumberUpdated(); - } - } catch (Exception eee) { - throw new EchoBaseTechnicalException("Could not import entities of type " + meta.getSource(), eee); - } finally { - - importer.close(); - } - } - - protected <E extends TopiaEntity> void importEntityAssociation(AssociationMeta meta, - ImportModel<E> csvModel, - CsvImportResult result, - Reader reader) { - if (log.isInfoEnabled()) { - log.info("Will import " + meta); - } - try { - - EchoBaseEntityEnum sourceType = meta.getSource(); -// EchoBaseEntityEnum targetType = meta.getTarget(); - - Import<E> importer = Import.newImport(csvModel, reader); - TopiaDAO<E> sourceDAO = getDAO(sourceType); -// TopiaDAO<TopiaEntity> targetDAO = getDAO(targetType); - List<E> sourceEntities = getEntities(sourceType); - Map<String, E> sourcesById = - Maps.uniqueIndex(sourceEntities, EchoBaseFunctions.TO_TOPIAID); - try { - for (E row : importer) { - E entityToSave = sourcesById.get(row.getTopiaId()); - Collection<TopiaEntity> assoc = meta.getChilds(row); - meta.setChilds(entityToSave, assoc); -// sourceDAO.update(entityToSave); - -// E entityToSave = sourceDAO.findByTopiaId(row.getTopiaId()); -// -// // check entity exits ? -// if (entityToSave == null) { -// throw new EchoBaseTechnicalException( -// "Could not find entity with id " + row.getTopiaId()); -// } -// Collection<TopiaEntity> assoc = meta.getChilds(row); -// Collection<TopiaEntity> associationToSave = Lists.newArrayList(); -// for (TopiaEntity topiaEntity : assoc) { -// TopiaEntity byTopiaId = targetDAO.findByTopiaId(topiaEntity.getTopiaId()); -// if (byTopiaId == null) { -// throw new EchoBaseTechnicalException( -// "Could not find entity with id " + topiaEntity.getTopiaId()); -// } -// associationToSave.add(byTopiaId); -// result.incrementsNumberUpdated(); -// } -// meta.setChilds(entityToSave, associationToSave); - } - } finally { - - importer.close(); - } - - commitTransaction("Could not commit transaction"); - } catch (TopiaException eee) { - throw new EchoBaseTechnicalException(eee); - } - } -} Copied: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java (from rev 184, trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java) =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java 2011-12-15 11:02:33 UTC (rev 187) @@ -0,0 +1,427 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import fr.ifremer.echobase.EchoBaseFunctions; +import fr.ifremer.echobase.EchoBaseTechnicalException; +import fr.ifremer.echobase.entities.EchoBaseEntityEnum; +import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EntitiesUtil; +import fr.ifremer.echobase.entities.EntityModificationLog; +import fr.ifremer.echobase.entities.meta.AssociationMeta; +import fr.ifremer.echobase.entities.meta.MetaFilenameAware; +import fr.ifremer.echobase.entities.meta.TableMeta; +import fr.ifremer.echobase.services.models.EntityAssociationImportModel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.topia.TopiaException; +import org.nuiton.topia.persistence.TopiaDAO; +import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.StringUtil; +import org.nuiton.util.TimeLog; +import org.nuiton.util.csv.Import; +import org.nuiton.util.csv.ImportExportModel; +import org.nuiton.util.csv.ImportModel; +import org.nuiton.util.csv.ImportToMap; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Service to import a complete db. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class DbImportService extends EchoBaseServiceSupport { + + /** Logger. */ + private static final Log log = + LogFactory.getLog(DbImportService.class); + + public static final TimeLog timeLog = new TimeLog(DbImportService.class); + + protected DbEditorService dbEditorService; + + protected Map<MetaFilenameAware, ZipEntry> discoverEntries(File file, ZipFile zipFile) { + + List<String> missingEntries = Lists.newArrayList(); + + Map<MetaFilenameAware, ZipEntry> result = + Maps.newLinkedHashMap(); + + MetaFilenameAware[] entries = getEntries(); + + // check that all mandatories + for (MetaFilenameAware entry : entries) { + String filename = entry.getFilename(); + ZipEntry zipEntry = zipFile.getEntry("echobase/" + filename); + + if (zipEntry == null) { + missingEntries.add(filename); + } else { + result.put(entry, zipEntry); + } + } + if (!missingEntries.isEmpty()) { + + if (log.isWarnEnabled()) { + log.warn("There is " + missingEntries.size() + "missing mandatory files " + + " in import " + file + "\n" + StringUtil.join(missingEntries, "\n", false)); + } + +// throw new EchoBaseTechnicalException( +// "There is some missing mandatory files " + +// missingEntries + " in import " + file); + } + + return result; + } + + /** + * Import a complete db from the given zip file. + * + * @param model model of the db import operation + * @param user user which perform the import + * @throws IOException if any io exception while import + * @throws TopiaException if any pb while topia treatment + */ + public void importDb(ImportDbConfiguration model, + EchoBaseUser user) throws IOException, TopiaException { + + File file = model.getInput().getFile(); + + ZipFile zipFile = new ZipFile(file); + try { +// MetaFilenameAware[] entries = getEntries(); +// +// List<String> missingEntries = Lists.newArrayList(); + + Map<MetaFilenameAware, ZipEntry> entriestoConsume = discoverEntries( + file, zipFile); + +// // check that all mandatories +// for (MetaFilenameAware entry : entries) { +// String filename = entry.getFilename(); +// ZipEntry zipEntry = zipFile.getEntry("echobase/" + filename); +// +// if (zipEntry == null) { +// missingEntries.add(filename); +// } else { +// entriestoConsume.put(entry, zipEntry); +// } +// } +// +// if (!missingEntries.isEmpty()) { +// +// if (log.isWarnEnabled()) { +// log.warn("There is " + missingEntries.size() + "missing mandatory files " + +// " in import " + file + "\n" + StringUtil.join(missingEntries, "\n", false)); +// } +// +//// throw new EchoBaseTechnicalException( +//// "There is some missing mandatory files " + +//// missingEntries + " in import " + file); +// } + + int size = entriestoConsume.size(); + model.setNbSteps(size + 1); + + // consume all found entries + for (Map.Entry<MetaFilenameAware, ZipEntry> entry : + entriestoConsume.entrySet()) { + + model.incrementsProgression(); + + MetaFilenameAware entryDef = entry.getKey(); + ZipEntry value = entry.getValue(); + CsvImportResult csvResult = CsvImportResult.newResult( + entryDef.getSource(), + value.getName(), + false + ); + Reader reader = + new InputStreamReader(zipFile.getInputStream(value)); + try { + importFile(entryDef, reader, csvResult); + } finally { + reader.close(); + } + } + + // add a log entry of import db + if (log.isInfoEnabled()) { + log.info("Import done with user " + user.getEmail()); + } + TopiaDAO<EntityModificationLog> dao = + getDAO(EntityModificationLog.class); + + dao.create( + EntityModificationLog.PROPERTY_ENTITY_TYPE, "import db", + EntityModificationLog.PROPERTY_ENTITY_ID, "Complete db", + EntityModificationLog.PROPERTY_MODIFICATION_USER, user.getEmail(), + EntityModificationLog.PROPERTY_MODIFICATION_DATE, newDate(), + EntityModificationLog.PROPERTY_MODIFICATION_TEXT, "import db from file " + file.getName() + ); + commitTransaction("Could not commit db import from file " + file); + } finally { + zipFile.close(); + } + } + + static class MyImportToMap extends ImportToMap { + + public static MyImportToMap newImport(ImportModel<Map<String, Object>> model, + Reader reader) { + return new MyImportToMap(model, reader); + } + + protected MyImportToMap(ImportModel<Map<String, Object>> model, Reader reader) { + super(model, reader); + this.reader.setSafetySwitch(false); + } + + + } + + public <E extends TopiaEntity> void importFile(MetaFilenameAware entry, + Reader reader, + CsvImportResult csvResult) throws IOException { + + DbEditorService service = getDbEditorService(); + + long s0; + + s0 = TimeLog.getTime(); + + if (entry instanceof AssociationMeta) { + + // load a association input + + AssociationMeta meta = (AssociationMeta) entry; + + if (log.isInfoEnabled()) { + log.info("Will import " + meta); + } + + EchoBaseEntityEnum source = meta.getSource(); + EchoBaseEntityEnum target = meta.getTarget(); + timeLog.log(s0, "importFile::getTargetEntities"); +// List<TopiaEntity> targetEntities = getEntities(target); +// Map<String, TopiaEntity> targetsById = +// Maps.uniqueIndex(targetEntities, EchoBaseFunctions.TO_TOPIAID); + + ImportModel<Map<String, Object>> model = + EntityAssociationImportModel.newImportModel( + getConfiguration().getCsvSeparator(), + meta + ); + +// EchoBaseEntityEnum sourceType = meta.getSource(); + + ImportToMap importer = MyImportToMap.newImport(model, reader); + + s0 = TimeLog.getTime(); +// List<E> sourceEntities = getEntities(sourceType); + timeLog.log(s0, "importFile::getSourceEntities"); +// Map<String, E> sourcesById = +// Maps.uniqueIndex(sourceEntities, EchoBaseFunctions.TO_TOPIAID); + StringBuilder builder = new StringBuilder(); + try { + s0 = TimeLog.getTime(); + String updateString = "UPDATE " + target.getContract().getSimpleName() + " SET " + source.getContract().getSimpleName() + " = '%s' WHERE topiaId ='%s';"; + if (log.isInfoEnabled()) { + log.info("Will apply " + updateString); + } + int compt = 0; + for (Map<String, Object> row : importer) { + String topiaId = (String) row.get(TopiaEntity.TOPIA_ID); + String[] associations = (String[]) row.get("target"); + for (String association : associations) { + builder.append(String.format(updateString, topiaId, association)).append('\n'); + compt++; + if (compt == 10000) { + // flush it + getTransaction().executeSQL(builder.toString()); + builder = new StringBuilder(); + } + } + csvResult.incrementsNumberUpdated(); + } + + timeLog.log(s0, "importFile::import"); + } catch (TopiaException eee) { + throw new EchoBaseTechnicalException( + "Could not execute sql script", eee); + } finally { + + importer.close(); + } + + } else { + + // normal entity table import + + TableMeta meta = (TableMeta) entry; + + if (log.isInfoEnabled()) { + log.info("Will import " + meta); + } + + ImportModel<TopiaEntity> model = service.buildForImport(meta); + + Import<TopiaEntity> importer = Import.newImport(model, reader); + + try { + TopiaDAO<TopiaEntity> dao = getDAO(meta.getSource()); + s0 = TimeLog.getTime(); + for (TopiaEntity entity : importer) { + + TopiaEntity entityToSave = dao.create(TopiaEntity.TOPIA_ID, + entity.getTopiaId()); + + meta.copy(entity, entityToSave); + + csvResult.incrementsNumberUpdated(); + } + timeLog.log(s0, "importFile::import"); + } catch (Exception eee) { + throw new EchoBaseTechnicalException("Could not import entities of type " + meta.getSource(), eee); + } finally { + + importer.close(); + } + + } + } + + public MetaFilenameAware[] getEntries() { + + List<MetaFilenameAware> entities = Lists.newArrayList(); + List<MetaFilenameAware> associations = Lists.newArrayList(); + + addEntries(entities, associations, EntitiesUtil.getReferenceTypesForCopy()); + addEntries(entities, associations, EntitiesUtil.getDataTypesforCopy()); + entities.addAll(associations); + return entities.toArray(new MetaFilenameAware[entities.size()]); + } + + protected TableMeta getTableMeta(EchoBaseEntityEnum tableName) { + return getDbEditorService().getTableMeta(tableName); + } + + protected void addEntries(List<MetaFilenameAware> entities, + List<MetaFilenameAware> associations, + EchoBaseEntityEnum[] types) { + for (EchoBaseEntityEnum type : types) { + + TableMeta tableMeta = getTableMeta(type); + entities.add(tableMeta); + +// for (AssociationMeta associationMeta : tableMeta.getAssociations()) { +// associations.add(associationMeta); +// } + associations.addAll(tableMeta.getAssociations()); + } + } + + protected DbEditorService getDbEditorService() { + if (dbEditorService == null) { + dbEditorService = newService(DbEditorService.class); + } + return dbEditorService; + } + + protected void mportEntities(TableMeta meta, + ImportExportModel<TopiaEntity> csvModel, + CsvImportResult result, + Reader reader) { + + if (log.isInfoEnabled()) { + log.info("Will import " + meta); + } + Import<TopiaEntity> importer = Import.newImport(csvModel, reader); + + try { + TopiaDAO<TopiaEntity> dao = getDAO(meta.getSource()); + long s0 = TimeLog.getTime(); + for (TopiaEntity entity : importer) { + + TopiaEntity entityToSave = dao.create(TopiaEntity.TOPIA_ID, + entity.getTopiaId()); + + meta.copy(entity, entityToSave); + + result.incrementsNumberUpdated(); + } + timeLog.log(s0, "importEntities"); + } catch (Exception eee) { + throw new EchoBaseTechnicalException("Could not import entities of type " + meta.getSource(), eee); + } finally { + + importer.close(); + } + } + + protected <E extends TopiaEntity> void mportEntityAssociation(AssociationMeta meta, + ImportModel<E> csvModel, + CsvImportResult result, + Reader reader) { + if (log.isInfoEnabled()) { + log.info("Will import " + meta); + } + + EchoBaseEntityEnum sourceType = meta.getSource(); + + Import<E> importer = Import.newImport(csvModel, reader); + long s0; + s0 = TimeLog.getTime(); + List<E> sourceEntities = getEntities(sourceType); + timeLog.log(s0, "importEntityAssociation::getEntities"); + Map<String, E> sourcesById = + Maps.uniqueIndex(sourceEntities, EchoBaseFunctions.TO_TOPIAID); + try { + s0 = TimeLog.getTime(); + for (E row : importer) { + E entityToSave = sourcesById.get(row.getTopiaId()); + Collection<TopiaEntity> assoc = meta.getChilds(row); + meta.setChilds(entityToSave, assoc); + result.incrementsNumberUpdated(); + } + timeLog.log(s0, "importEntityAssociation::import"); + } finally { + + importer.close(); + } + } +} Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DecoratorService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DecoratorService.java 2011-12-15 11:01:21 UTC (rev 186) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DecoratorService.java 2011-12-15 11:02:33 UTC (rev 187) @@ -26,64 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import fr.ifremer.echobase.entities.EchoBaseUser; -import fr.ifremer.echobase.entities.ExportQuery; -import fr.ifremer.echobase.entities.data.Cell; -import fr.ifremer.echobase.entities.data.Data; -import fr.ifremer.echobase.entities.data.DataAcquisition; -import fr.ifremer.echobase.entities.data.DataProcessing; -import fr.ifremer.echobase.entities.data.Operation; -import fr.ifremer.echobase.entities.data.OperationGearMetadataValue; -import fr.ifremer.echobase.entities.data.OperationMetadataValue; -import fr.ifremer.echobase.entities.data.Result; -import fr.ifremer.echobase.entities.data.Sample; -import fr.ifremer.echobase.entities.data.SampleData; -import fr.ifremer.echobase.entities.data.Transect; -import fr.ifremer.echobase.entities.data.Transit; -import fr.ifremer.echobase.entities.data.Voyage; -import fr.ifremer.echobase.entities.references.AcousticInstrument; -import fr.ifremer.echobase.entities.references.AgeCategory; -import fr.ifremer.echobase.entities.references.AreaOfOperation; -import fr.ifremer.echobase.entities.references.Calibration; -import fr.ifremer.echobase.entities.references.Category; -import fr.ifremer.echobase.entities.references.CategoryMeaning; -import fr.ifremer.echobase.entities.references.CategoryRef; -import fr.ifremer.echobase.entities.references.CategoryType; -import fr.ifremer.echobase.entities.references.CellMethod; -import fr.ifremer.echobase.entities.references.CellType; -import fr.ifremer.echobase.entities.references.DataMetadata; -import fr.ifremer.echobase.entities.references.DataProtocol; -import fr.ifremer.echobase.entities.references.DataQuality; -import fr.ifremer.echobase.entities.references.DataType; -import fr.ifremer.echobase.entities.references.DepthStratum; -import fr.ifremer.echobase.entities.references.Echotype; -import fr.ifremer.echobase.entities.references.EchotypeCategory; -import fr.ifremer.echobase.entities.references.Gear; -import fr.ifremer.echobase.entities.references.GearCaracteristic; -import fr.ifremer.echobase.entities.references.GearMetadata; -import fr.ifremer.echobase.entities.references.GearMetadataValue; -import fr.ifremer.echobase.entities.references.GearType; -import fr.ifremer.echobase.entities.references.Impacte; -import fr.ifremer.echobase.entities.references.Localisation; -import fr.ifremer.echobase.entities.references.MeasureType; -import fr.ifremer.echobase.entities.references.MeasurementMetadata; -import fr.ifremer.echobase.entities.references.Mission; -import fr.ifremer.echobase.entities.references.Moment; -import fr.ifremer.echobase.entities.references.OperationMetadata; -import fr.ifremer.echobase.entities.references.OperationType; -import fr.ifremer.echobase.entities.references.ReferenceDatum; -import fr.ifremer.echobase.entities.references.ReferenceDatumType; -import fr.ifremer.echobase.entities.references.ReferencingMethod; -import fr.ifremer.echobase.entities.references.SampleDataType; -import fr.ifremer.echobase.entities.references.SampleType; -import fr.ifremer.echobase.entities.references.SexeCategory; -import fr.ifremer.echobase.entities.references.SizeCategory; -import fr.ifremer.echobase.entities.references.Species; -import fr.ifremer.echobase.entities.references.SpeciesCategory; -import fr.ifremer.echobase.entities.references.Station; -import fr.ifremer.echobase.entities.references.TSParameters; -import fr.ifremer.echobase.entities.references.Vessel; -import fr.ifremer.echobase.entities.references.VesselType; +import fr.ifremer.echobase.EchoBaseDecoratorProvider; import org.apache.commons.collections.CollectionUtils; import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.util.decorator.Decorator; @@ -168,190 +111,4 @@ return decorator; } - static class EchoBaseDecoratorProvider extends DecoratorMulti18nProvider { - - @Override - protected void loadDecorators(Locale locale) { - - // EchoBaseUser decorator - registerJXPathDecorator(locale, EchoBaseUser.class, "${email}$s"); - - // ExportQuery decorator - registerJXPathDecorator(locale, ExportQuery.class, "${name}$s - ${description}$s"); - - // AcousticInstrument decorator - registerJXPathDecorator(locale, AcousticInstrument.class, "${id}$s"); - - // AgeCategory decorator - registerJXPathDecorator(locale, AgeCategory.class, "${name}$s"); - - // AreaOfOperation decorator - registerJXPathDecorator(locale, AreaOfOperation.class, "${name}$s"); - - // Calibration decorator - registerJXPathDecorator(locale, Calibration.class, "${date}$s - acquisition ${aquisitionMethod}$s"); - - // Category decorator - registerJXPathDecorator(locale, Category.class, "${classCode}$s ${categoryValue}$s"); - - // CategoryMeaning decorator - registerJXPathDecorator(locale, CategoryMeaning.class, "${name}$s"); - - // CategoryRef decorator - registerJXPathDecorator(locale, CategoryRef.class, "${genusSpecies}$s"); - - // CategoryType decorator - registerJXPathDecorator(locale, CategoryType.class, "${name}$s"); - - // CellMethod decorator - registerJXPathDecorator(locale, CellMethod.class, "${name}$s"); - - // CellType decorator - registerJXPathDecorator(locale, CellType.class, "${name}$s"); - - // DataMetadata decorator - registerJXPathDecorator(locale, DataMetadata.class, "${name}$s"); - - // DataProtocol decorator - registerJXPathDecorator(locale, DataProtocol.class, "${description}$s [ ${startValidityDate}$s - ${endValidityDate}$s ]"); - - // DataQuality decorator - registerJXPathDecorator(locale, DataQuality.class, "${qualityDataFlagValues}$s - ${flagMeanings}$s"); - - // DataType decorator - registerJXPathDecorator(locale, DataType.class, "${name}$s"); - - // DepthStratum decorator - registerJXPathDecorator(locale, DepthStratum.class, "${id}$s - ${meaning}$s"); - - // Echotype decorator - registerJXPathDecorator(locale, Echotype.class, "${id}$s - ${name}$s"); - - // EchotypeCategory decorator - registerJXPathDecorator(locale, EchotypeCategory.class, "${name}$s"); - - // Gear decorator - registerJXPathDecorator(locale, Gear.class, "${name}$s [ ${startValidityDate}$s - ${endValidityDate}$s ]"); - - // GearCaracteristic decorator - registerJXPathDecorator(locale, GearCaracteristic.class, "${name}$s"); - - // GearMetadata decorator - registerJXPathDecorator(locale, GearMetadata.class, "${name}$s"); - - // GearMetadataValue decorator - registerJXPathDecorator(locale, GearMetadataValue.class, "${dataValue}$s"); - - // GearType decorator - registerJXPathDecorator(locale, GearType.class, "${name}$s"); - - // Impacte decorator - registerJXPathDecorator(locale, Impacte.class, "${measurementUnit}$s - ${species.genusSpecies}"); - - // Localisation decorator - registerJXPathDecorator(locale, Localisation.class, "Latitude <${minLatitude}$s - ${maxLatitude}$s>, Longitude <${minLongitude}$s - ${maxLongitude}$s>"); - - // MeasurementMetadata decorator - registerJXPathDecorator(locale, MeasurementMetadata.class, "${name}$s"); - - // MeasureType decorator - registerJXPathDecorator(locale, MeasureType.class, "${name}$s"); - - // Mission decorator - registerJXPathDecorator(locale, Mission.class, "${name}$s"); - - // Moment decorator - registerJXPathDecorator(locale, Moment.class, "${name}$s"); - - // OperationMetadata decorator - registerJXPathDecorator(locale, OperationMetadata.class, "${name}$s"); - - // OperationType decorator - registerJXPathDecorator(locale, OperationType.class, "${name}$s"); - - // ReferenceDatum decorator - registerJXPathDecorator(locale, ReferenceDatum.class, "${id}$s"); - - // ReferenceDatumType decorator - registerJXPathDecorator(locale, ReferenceDatumType.class, "${name}$s"); - - // ReferencingMethod decorator - registerJXPathDecorator(locale, ReferencingMethod.class, "${name}$s"); - - // SampleDataType decorator - registerJXPathDecorator(locale, SampleDataType.class, "${name}$s"); - - // SampleType decorator - registerJXPathDecorator(locale, SampleType.class, "${name}$s - level ${level}$s"); - - // SexeCategory decorator - registerJXPathDecorator(locale, SexeCategory.class, "${name}$s"); - - // SizeCategory decorator - registerJXPathDecorator(locale, SizeCategory.class, "${name}$s"); - - // Species decorator - registerJXPathDecorator(locale, Species.class, "${genusSpecies}$s"); - - //TODO - // SpeciesCategory decorator - registerJXPathDecorator(locale, SpeciesCategory.class, "${species.genusSpecies}$s - ..."); - - // Station decorator - registerJXPathDecorator(locale, Station.class, "${startLatitude}$s - ${startLongitude}$s"); - - //TODO - // TSParameters decorator - registerJXPathDecorator(locale, TSParameters.class, "${}$s"); - - // Vessel decorator - registerJXPathDecorator(locale, Vessel.class, "${name}$s"); - - // VesselType decorator - registerJXPathDecorator(locale, VesselType.class, "${name}$s"); - - // Cell decorator - registerJXPathDecorator(locale, Cell.class, "${name}$s"); - - // Data decorator - registerJXPathDecorator(locale, Data.class, "${dataValue}$s"); - - //TODO - // DataAcquisition decorator - registerJXPathDecorator(locale, DataAcquisition.class, ""); - - //TODO - // DataProcessing decorator - registerJXPathDecorator(locale, DataProcessing.class, ""); - - // Operation decorator - registerJXPathDecorator(locale, Operation.class, "${startDate}$s - ${endDate}$s"); - - // OperationGearMetadataValue decorator - registerJXPathDecorator(locale, OperationGearMetadataValue.class, "${dataValue}"); - - // OperationMetadataValue decorator - registerJXPathDecorator(locale, OperationMetadataValue.class, "${dataValue}$s - moment ${moment.name}"); - - // Result decorator - registerJXPathDecorator(locale, Result.class, "${resultvalue}$s"); - - // Sample decorator - registerJXPathDecorator(locale, Sample.class, "${resultvalue}$s"); - - // SampleData decorator - registerJXPathDecorator(locale, SampleData.class, "${resultvalue}$s"); - - // Transect decorator - registerJXPathDecorator(locale, Transect.class, "${title}$s - Vessel ${vessel}$s"); - - // Transit decorator - registerJXPathDecorator(locale, Transit.class, "${startTime}$s - ${endTime}$s"); - - // Voyage decorator - registerJXPathDecorator(locale, Voyage.class, "${voyageName}$s"); - - - } - } } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/CsvModelUtil.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/CsvModelUtil.java 2011-12-15 11:01:21 UTC (rev 186) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/CsvModelUtil.java 2011-12-15 11:02:33 UTC (rev 187) @@ -35,6 +35,7 @@ import org.nuiton.util.StringUtil; import org.nuiton.util.csv.Common; import org.nuiton.util.csv.ValueFormatter; +import org.nuiton.util.csv.ValueParser; import org.nuiton.util.csv.ValueParserFormatter; import java.text.ParseException; @@ -124,6 +125,17 @@ } } + public static final AssociationValueParser ASSOCIATION_VALUE_PARSER = new AssociationValueParser(); + + public static class AssociationValueParser implements ValueParser<String[]> { + + @Override + public String[] parse(String value) throws ParseException { + String[] ids = value.split("\\|"); + return ids; + } + } + public static class ForeignKeyValue<E extends TopiaEntity> implements ValueParserFormatter<E> { protected final Class<E> entityType; Copied: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationImportModel.java (from rev 184, trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationCsvModel.java) =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationImportModel.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationImportModel.java 2011-12-15 11:02:33 UTC (rev 187) @@ -0,0 +1,102 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services.models; + +import fr.ifremer.echobase.entities.meta.AssociationMeta; +import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.csv.ImportModel; +import org.nuiton.util.csv.ImportableColumn; +import org.nuiton.util.csv.ModelBuilder; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * A model to import / export associations of entities into csv files. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class EntityAssociationImportModel implements ImportModel<Map<String, Object>> { + + protected final char separator; + + protected final AssociationMeta meta; + + protected ModelBuilder<Map<String, Object>> modelBuilder; + + public static ImportModel<Map<String, Object>> newImportModel(char separator, + AssociationMeta meta + ) { + EntityAssociationImportModel model = new EntityAssociationImportModel( + separator, meta); + + // topiaId <-> topiaId + model.modelBuilder.newMandatoryColumn( + TopiaEntity.TOPIA_ID, + CsvModelUtil.<Map<String, Object>, String>newMapProperty(TopiaEntity.TOPIA_ID) + ); + + // add association -> target +// Class<TopiaEntity> target = +// (Class<TopiaEntity>) meta.getTarget().getImplementation(); + + model.modelBuilder.newMandatoryColumn( + meta.getName(), + CsvModelUtil.ASSOCIATION_VALUE_PARSER, + CsvModelUtil.<Map<String, Object>, String[]>newMapProperty("target") + ); + + return model; + } + + @Override + public char getSeparator() { + return separator; + } + + @Override + public Iterable<ImportableColumn<Map<String, Object>, Object>> getColumnsForImport() { + return (Collection) + modelBuilder.getColumnsForImport(); + } + + @Override + public void pushCsvHeaderNames(List<String> headerNames) { + } + + @Override + public Map<String, Object> newEmptyInstance() { + return null; + } + + protected EntityAssociationImportModel(char separator, AssociationMeta meta) { + this.separator = separator; + this.meta = meta; + modelBuilder = new ModelBuilder<Map<String, Object>>(); + } + + +} \ No newline at end of file Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/DbImportExportServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/DbImportExportServiceTest.java 2011-12-15 11:01:21 UTC (rev 186) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/DbImportExportServiceTest.java 2011-12-15 11:02:33 UTC (rev 187) @@ -47,7 +47,7 @@ import java.util.Set; /** - * Test the {@link DbImportExportService}. + * Test the {@link DbExportService}. * * @author tchemit <chemit@codelutin.com> * @since 0.2