This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository observe. See http://git.codelutin.com/observe.git commit 9c7e8d75426d168644069d5a2ccfc6612e54a737 Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed Aug 26 15:46:14 2015 +0200 Replication des référentiels et données ok \o/ (see #7476) --- .../ird/observe/entities/longline/TdrTopiaDao.java | 82 +++++++++ .../src/main/xmi/observe-common.properties | 6 +- .../DataSourceDumpProducerServiceTopia.java | 183 ++++++++++++++++++--- .../DataSourceDumpProducerServiceTopiaTest.java | 3 +- 4 files changed, 242 insertions(+), 32 deletions(-) diff --git a/observe-entities/src/main/java/fr/ird/observe/entities/longline/TdrTopiaDao.java b/observe-entities/src/main/java/fr/ird/observe/entities/longline/TdrTopiaDao.java new file mode 100644 index 0000000..b8e0036 --- /dev/null +++ b/observe-entities/src/main/java/fr/ird/observe/entities/longline/TdrTopiaDao.java @@ -0,0 +1,82 @@ +package fr.ird.observe.entities.longline; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import org.nuiton.topia.persistence.support.TopiaSqlQuery; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class TdrTopiaDao extends AbstractTdrTopiaDao<Tdr> { + + + public Multimap<String, String> getTdrIdsBySeineIds(TripLongline tripLongline) { + + Set<String> setLonglineIds = TripLonglines.getSetIdsWithTdr(tripLongline); + + Multimap<String, String> result = ArrayListMultimap.create(); + + GetTdrIdsQuery sqlQuery = new GetTdrIdsQuery(); + + for (String setLonglineId : setLonglineIds) { + + sqlQuery.setSetId(setLonglineId); + + List<String> tdrIds = topiaSqlSupport.findMultipleResult(sqlQuery); + + result.putAll(setLonglineId, tdrIds); + + } + return result; + + } + + public void applyTdrAssociationFix(Multimap<String, String> tdrAssociation) { + + String request = "\nUPDATE OBSERVE_LONGLINE.TDR SET SET = '%s' WHERE topiaid = '%s';"; + + StringBuilder builder = new StringBuilder(); + + if (tdrAssociation != null) { + for (Map.Entry<String, String> entry : tdrAssociation.entries()) { + String setLonglineId = entry.getKey(); + String tdrId = entry.getValue(); + builder.append(String.format(request, setLonglineId, tdrId)); + } + } + + topiaSqlSupport.executeSql(builder.toString()); + + } + + private static class GetTdrIdsQuery extends TopiaSqlQuery<String> { + + protected String setId; + + @Override + public PreparedStatement prepareQuery(Connection connection) throws SQLException { + String sql = "SELECT t.topiaId " + + "FROM OBSERVE_LONGLINE.TDR t " + + "WHERE t.SET = ?"; + PreparedStatement ps = connection.prepareStatement(sql); + ps.setString(1, setId); + return ps; + } + + @Override + public String prepareResult(ResultSet set) throws SQLException { + String result = set.getString(1); + return result; + } + + public void setSetId(String setId) { + this.setId = setId; + } + } + +} diff --git a/observe-entities/src/main/xmi/observe-common.properties b/observe-entities/src/main/xmi/observe-common.properties index 301c0e4..6ccdc4b 100644 --- a/observe-entities/src/main/xmi/observe-common.properties +++ b/observe-entities/src/main/xmi/observe-common.properties @@ -40,9 +40,9 @@ package.fr.ird.observe.entities.tagvalue.dbSchema=OBSERVE_COMMON ### Champ Commentaire en text ################################################# ############################################################################### -fr.ird.observe.entities.CommentableEntity.attribute.comment.tagValue.hibernateAttributeType.String=text -fr.ird.observe.entities.referentiel.Vessel.attribute.comment.tagValue.hibernateAttributeType.String=text -fr.ird.observe.entities.referentiel.Program.attribute.comment.tagValue.hibernateAttributeType.String=text +fr.ird.observe.entities.CommentableEntity.attribute.comment.tagValue.hibernateAttributeType=text +fr.ird.observe.entities.referentiel.Vessel.attribute.comment.tagValue.hibernateAttributeType=text +fr.ird.observe.entities.referentiel.Program.attribute.comment.tagValue.hibernateAttributeType=text ############################################################################### ### Champ Numeric (utilisation du type sql numeric) ########################### diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java index 62d3df0..0913aac 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java @@ -1,8 +1,10 @@ package fr.ird.observe.services.service; +import com.google.common.collect.Multimap; import fr.ird.observe.ObserveTopiaApplicationContext; import fr.ird.observe.ObserveTopiaPersistenceContext; import fr.ird.observe.entities.Entities; +import fr.ird.observe.entities.longline.TripLongline; import fr.ird.observe.services.ObserveServiceTopia; import fr.ird.observe.services.ObserveTopiaApplicationContextFactory; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; @@ -11,6 +13,7 @@ import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportExcept import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.topia.replication.TopiaReplicationService; import org.nuiton.topia.replication.model.ReplicationModel; import org.nuiton.util.StringUtil; @@ -19,6 +22,10 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; /** * Created on 23/08/15. @@ -43,53 +50,95 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl log.info(String.format("Referential temporary database created in %s", StringUtil.convertTime(t0, System.nanoTime()))); } + ObserveTopiaApplicationContext sourceTopiaApplicationContext = serviceContext.getTopiaApplicationContext(); ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(temporaryDataSourceConfiguration); - try { - t0 = now().getTime(); + replicateReferential(sourceTopiaApplicationContext, temporaryTopiaApplicationContext); - TopiaReplicationService service = temporaryTopiaApplicationContext.getReplicationService(); - ReplicationModel model = service.prepareForAll(Entities.REFERENCE_ENTITIES); + Path dumpFile = exportToFile(temporaryDataSourceConfiguration, temporaryTopiaApplicationContext); - ObserveTopiaApplicationContext sourceTopiaApplicationContext = serviceContext.getTopiaApplicationContext(); - service.doReplicate(model, sourceTopiaApplicationContext); + byte[] content = getBytes(dumpFile); + return content; - if (log.isInfoEnabled()) { - log.info(String.format("Referential replication done in %s", StringUtil.convertTime(t0, System.nanoTime()))); - } + } + + protected void replicateReferential(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { + long t0 = now().getTime(); + + TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); + ReplicationModel model = service.prepareForAll(Entities.REFERENCE_ENTITIES); + + try { + service.doReplicate(model, temporaryTopiaApplicationContext); } catch (Exception e) { + //TODO Avoir une exception concrete throw new RuntimeException("Could not replicate referantial", e); } - Path dumpFile = exportToFile(temporaryDataSourceConfiguration, temporaryTopiaApplicationContext); - try { - byte[] content = Files.readAllBytes(dumpFile); - return content; - } catch (IOException e) { - //TODO Avoir une exception concrete - throw new RuntimeException("Could not read file " + dumpFile + " content", e); + if (log.isInfoEnabled()) { + log.info(String.format("Referential replication done in %s", StringUtil.convertTime(t0, System.nanoTime()))); } } - protected Path exportToFile(ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration, - ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { + @Override + public byte[] getDataDump(String... importDataIds) { - Path dumpFile = temporaryDataSourceConfiguration.getDirectory().toPath().resolve("dump.sql"); + Set<String> ids = new LinkedHashSet<>(); + + if (importDataIds.length == 0) { + + // cas limite où on exporte toutes les données + List<String> tripSeineIds = getTopiaPersistenceContext().getTripSeineDao().findAllIds(); + ids.addAll(tripSeineIds); + List<String> tripLonglineIds = getTopiaPersistenceContext().getTripLonglineDao().findAllIds(); + ids.addAll(tripLonglineIds); + } else { + + ids.addAll(Arrays.asList(importDataIds)); + } + + long t0 = now().getTime(); + + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = createTemporaryDatabase("referentialDump"); if (log.isInfoEnabled()) { - log.info("Export sql to file: " + dumpFile); + log.info(String.format("Data temporary database created in %s", StringUtil.convertTime(t0, System.nanoTime()))); } - try (ObserveTopiaPersistenceContext temporaryPersistenceContext = temporaryTopiaApplicationContext.newPersistenceContext()) { - String scriptSqlQuery = String.format(EXPORT_SQL_STATEMENT, dumpFile); + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(temporaryDataSourceConfiguration); + ObserveTopiaApplicationContext sourceTopiaApplicationContext = serviceContext.getTopiaApplicationContext(); - temporaryPersistenceContext.getSqlSupport().executeSql(scriptSqlQuery); + replicateReferential(sourceTopiaApplicationContext, temporaryTopiaApplicationContext); + try { + + t0 = now().getTime(); + + for (String id : ids) { + long t1 = now().getTime(); + + replicateOneData(sourceTopiaApplicationContext, temporaryTopiaApplicationContext, id); + + if (log.isInfoEnabled()) { + log.info(String.format("Data replication [%s] done in %s", id, StringUtil.convertTime(t1, System.nanoTime()))); + } + } + + if (log.isInfoEnabled()) { + log.info(String.format("Data replication for %s data done in %s", ids.size(), StringUtil.convertTime(t0, System.nanoTime()))); + } + + } catch (Exception e) { + throw new RuntimeException("Could not replicate data", e); } - return dumpFile; + + Path dumpFile = exportToFile(temporaryDataSourceConfiguration, temporaryTopiaApplicationContext); + + byte[] content = getBytes(dumpFile); + return content; } @@ -118,8 +167,88 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl } - @Override - public byte[] getDataDump(String... importDataIds) { - return new byte[0]; + protected Path exportToFile(ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration, + ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { + + Path dumpFile = temporaryDataSourceConfiguration.getDirectory().toPath().resolve("dump.sql"); + + if (log.isInfoEnabled()) { + log.info("Export sql to file: " + dumpFile); + } + try (ObserveTopiaPersistenceContext temporaryPersistenceContext = temporaryTopiaApplicationContext.newPersistenceContext()) { + + String scriptSqlQuery = String.format(EXPORT_SQL_STATEMENT, dumpFile); + + temporaryPersistenceContext.getSqlSupport().executeSql(scriptSqlQuery); + + } + return dumpFile; + + } + + protected byte[] getBytes(Path dumpFile) { + try { + byte[] content = Files.readAllBytes(dumpFile); + return content; + } catch (IOException e) { + //TODO Avoir une exception concrete + throw new RuntimeException("Could not read file " + dumpFile + " content", e); + } } + + /** + * Duplication de l'unique donnée observateur depuis ce service vers le service donné. + * + * La duplication utilise une transaction dédiée afin de ne pas saturer le serveur + * et aussi une meilleure maitrise du rollback en cas d'une erreur. + * + * Voir http://forge.codelutin.com/issues/4837 + * + * @param id l'id de la donnée à répliquer + */ + protected void replicateOneData(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext, String id) { + + TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); + + ReplicationModel model = service.prepare(Entities.DATA_ENTITIES, false, id); + + // To fix missing tdr associations (see https://forge.codelutin.com/issues/6611) + Multimap<String, String> tdrAssociation = null; + + try (ObserveTopiaPersistenceContext sourcePersistenceContext = sourceTopiaApplicationContext.newPersistenceContext()) { + + TopiaEntity e = getTopiaPersistenceContext().findByTopiaId(id); + + if (e instanceof TripLongline) { + + // Grab tdr missing associations (see https://forge.codelutin.com/issues/6611) + if (log.isInfoEnabled()) { + log.info("Should keep SetLongline - Tdr association ids for: " + id); + } + + tdrAssociation = sourcePersistenceContext.getTdrDao().getTdrIdsBySeineIds((TripLongline) e); + + } + + } + + // do the replicate + try { + service.doReplicate(model, temporaryTopiaApplicationContext); + } catch (Exception e) { + throw new RuntimeException("Could not replicate data: " + id, e); + } + + if (tdrAssociation != null) { + + // Apply back tdr missing associations (see https://forge.codelutin.com/issues/6611) + try (ObserveTopiaPersistenceContext observeTopiaPersistenceContext = temporaryTopiaApplicationContext.newPersistenceContext()) { + + observeTopiaPersistenceContext.getTdrDao().applyTdrAssociationFix(tdrAssociation); + + } + + } + } + } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java index 2ab540c..4eb5c36 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java @@ -23,7 +23,7 @@ public class DataSourceDumpProducerServiceTopiaTest extends AbstractServiceTopia @Override public String getScriptName() { - return "referentiel"; + return "dataForTestSeine"; } @Before @@ -47,6 +47,5 @@ public class DataSourceDumpProducerServiceTopiaTest extends AbstractServiceTopia byte[] referentialDump = service.getDataDump(); Assert.assertNotNull(referentialDump); - } } \ No newline at end of file -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@list.forge.codelutin.com>.