This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository tutti. See http://git.codelutin.com/tutti.git commit 87cbe4e32bd581aabc64f64326c76062b18d7aba Author: Tony CHEMIT <chemit@codelutin.com> Date: Tue Jan 5 08:50:47 2016 +0100 Utilisation du nouveau moteur de requete sql de topia --- observe-entities/pom.xml | 5 + .../observe/ObserveTopiaApplicationContext.java | 342 ++++++++++++++++----- .../observe/ObserveTopiaConfigurationFactory.java | 4 +- .../java/fr/ird/observe/entities/Entities.java | 16 +- .../fr/ird/observe/entities/SqlDatabaseType.java | 26 -- .../replication/ObserveReplicationTables.java | 169 ---------- .../ObserveTopiaApplicationContextTest.java | 19 ++ .../services/service/SqlScriptProducerRequest.java | 16 +- observe-services-topia/pom.xml | 4 + .../service/SqlScriptProducerServiceTopia.java | 115 ++++--- .../replication/ReplicateReferentialsTest.java | 22 +- .../entities/replication/ReplicateTestSupport.java | 70 ++--- .../replication/ReplicateTripLonglineTest.java | 62 ++-- .../replication/ReplicateTripSeineTest.java | 63 ++-- .../service/SqlScriptProducerServiceTopiaTest.java | 4 +- .../src/test/resources/log4j.properties | 2 +- pom.xml | 6 + 17 files changed, 491 insertions(+), 454 deletions(-) diff --git a/observe-entities/pom.xml b/observe-entities/pom.xml index 5f9e32d..48faece 100644 --- a/observe-entities/pom.xml +++ b/observe-entities/pom.xml @@ -101,6 +101,11 @@ <groupId>org.nuiton.topia</groupId> <artifactId>topia-service-replication</artifactId> </dependency> + <dependency> + <groupId>org.nuiton.topia</groupId> + <artifactId>topia-service-sql-batch</artifactId> + <scope>compile</scope> + </dependency> <dependency> <groupId>org.hibernate</groupId> diff --git a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java index 5e4e31e..a3f229c 100644 --- a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java +++ b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java @@ -22,22 +22,22 @@ package fr.ird.observe; * #L% */ +import com.google.common.collect.ImmutableSet; import fr.ird.observe.entities.Entities; -import fr.ird.observe.entities.SqlDatabaseType; import fr.ird.observe.entities.migration.ObserveMigrationEngine; -import fr.ird.observe.entities.replication.ObserveReplicationTables; -import org.apache.commons.io.output.WriterOutputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hibernate.HibernateException; import org.hibernate.cfg.Configuration; -import org.hibernate.cfg.Environment; import org.hibernate.tool.hbm2ddl.SchemaExport; import org.nuiton.topia.persistence.TopiaException; import org.nuiton.topia.persistence.jdbc.JdbcH2Helper; import org.nuiton.topia.persistence.jdbc.JdbcHelper; +import org.nuiton.topia.persistence.metadata.TopiaMetadataEntity; import org.nuiton.topia.replication.TopiaReplicationService; -import org.nuiton.topia.replication.sql.TopiaSqlReplicationService; +import org.nuiton.topia.service.sql.batch.TopiaSqlBatchService; +import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; +import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTablesFactory; import org.nuiton.util.GZUtil; import java.io.BufferedInputStream; @@ -45,11 +45,11 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.Writer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.LinkedHashSet; import java.util.Objects; -import java.util.Properties; +import java.util.Set; import java.util.UUID; import java.util.zip.GZIPInputStream; @@ -64,14 +64,14 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat public static final String REPLICATION_SERVICE_NAME = "replication"; - public static final String SQL_REPLICATION_SERVICE_NAME = "sqlReplication"; + public static final String SQL_BATCH_SERVICE_NAME = "sqlBatch"; public static final String DB_VERSION = "db.version"; private static final String CREATE_SCHEMA_SCRIPT - = "CREATE SCHEMA OBSERVE_COMMON IF NOT EXISTS;\n" - + "CREATE SCHEMA OBSERVE_LONGLINE IF NOT EXISTS;\n" - + "CREATE SCHEMA OBSERVE_SEINE IF NOT EXISTS;\n"; + = "CREATE SCHEMA OBSERVE_COMMON;\n" + + "CREATE SCHEMA OBSERVE_LONGLINE;\n" + + "CREATE SCHEMA OBSERVE_SEINE;\n"; /** * Mise à {@code true} quand au moins une connection a été effectuée. @@ -79,6 +79,10 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat protected boolean open; protected final String authenticationToken; + protected TopiaSqlTablesFactory topiaSqlTablesFactory; + protected TopiaSqlTables tripSeineTables; + protected TopiaSqlTables tripLonglineTables; + protected TopiaSqlTables referentialTables; public ObserveTopiaApplicationContext(ObserveTopiaConfiguration topiaConfiguration) { super(topiaConfiguration); @@ -86,6 +90,12 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat } @Override + protected void init() { + super.init(); + topiaSqlTablesFactory = new TopiaSqlTablesFactory(getMetadataModel(), this); + } + + @Override public ObserveTopiaConfiguration getConfiguration() { return (ObserveTopiaConfiguration) super.getConfiguration(); } @@ -106,8 +116,8 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat return getServices(TopiaReplicationService.class).get(REPLICATION_SERVICE_NAME); } - public TopiaSqlReplicationService getSqlReplicationService() { - return getServices(TopiaSqlReplicationService.class).get(SQL_REPLICATION_SERVICE_NAME); + public TopiaSqlBatchService getSqlBatchService() { + return getServices(TopiaSqlBatchService.class).get(SQL_BATCH_SERVICE_NAME); } @Override @@ -207,85 +217,265 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat } - public void replicateAllTrips(Writer writer) { - - TopiaSqlReplicationService sqlReplicationService = getSqlReplicationService(); - sqlReplicationService - .newReplicatetoSqlActionBuilder(writer) - .setTables(ObserveReplicationTables.getTripSeineTables(false)) - .build().run(); + public TopiaSqlTables getTripSeineTables() { - sqlReplicationService - .newReplicatetoSqlActionBuilder(writer) - .setTables(ObserveReplicationTables.getTripLonglineTables(false)) - .build().run(); + if (tripSeineTables == null) { + log.info("Loading tripSeineTables."); + tripSeineTables = topiaSqlTablesFactory.newReplicateEntityTables( + new TripReplicateTablesPredicate(), + ObserveEntityEnum.TripSeine); + } + return tripSeineTables; + + /* + return TopiaSqlTables.builder(filterTable, ObserveEntityEnum.TripSeine) // → TripSeine + .addAndEnterJoinTable(ObserveEntityEnum.GearUseFeaturesSeine) // → GearUseFeaturesSeine + .addJoinTable(ObserveEntityEnum.GearUseFeaturesMeasurementSeine) + .backToParent() // ← TripSeine + .addAndEnterJoinTable(ObserveEntityEnum.Route) // → Route + .addAndEnterJoinTable(ObserveEntityEnum.ActivitySeine) // → ActivitySeine + .addAndEnterReverseJoinTable(ObserveEntityEnum.SetSeine) // → SetSeine + .addJoinTable(ObserveEntityEnum.SchoolEstimate) + .addJoinTable(ObserveEntityEnum.NonTargetCatch) + .addJoinTable(ObserveEntityEnum.TargetCatch) + .addAndEnterJoinTable(ObserveEntityEnum.TargetSample) // → TargetSample + .addJoinTable(ObserveEntityEnum.TargetLength) + .backToParent() // ← SetSeine + .addAndEnterJoinTable(ObserveEntityEnum.NonTargetSample) // → NonTargetSample + .addJoinTable(ObserveEntityEnum.NonTargetLength) + .backToParent() // ← SetSeine + .backToParent() // ← ActivitySeine + .addAssociationTable(ObserveEntityEnum.ObservedSystem.name(), true) + .addAndEnterJoinTable(ObserveEntityEnum.FloatingObject) // → FloatingObject + .addJoinTable(ObserveEntityEnum.ObjectObservedSpecies) + .addJoinTable(ObserveEntityEnum.ObjectSchoolEstimate) + .addJoinTable(ObserveEntityEnum.TransmittingBuoy) + .build(); + */ } - public void replicateTrips(Writer writer, Iterable<String> tripIds) { - - TopiaSqlReplicationService sqlReplicationService = getSqlReplicationService(); - - for (String tripId : tripIds) { + public TopiaSqlTables getTripLonglineTables() { - if (Entities.isSeineId(tripId)) { - - sqlReplicationService - .newReplicatetoSqlActionBuilder(writer) - .setTables(ObserveReplicationTables.getTripSeineTables(true)) - .setArg(tripId) - .build().run(); - } else { - - sqlReplicationService - .newReplicatetoSqlActionBuilder(writer) - .setTables(ObserveReplicationTables.getTripLonglineTables(true)) - .setArg(tripId) - .build().run(); - - } + if (tripLonglineTables == null) { + log.info("Loading tripLonglineTables."); + tripLonglineTables = topiaSqlTablesFactory.newReplicateEntityTables( + new TripReplicateTablesPredicate(), + ObserveEntityEnum.TripLongline); } - + return tripLonglineTables; + + /* + return TopiaSqlTables.builder(filterTable, ObserveEntityEnum.TripLongline) // → TripLongline + .addAndEnterJoinTable(ObserveEntityEnum.GearUseFeaturesLongline) // → GearUseFeaturesLongline + .addJoinTable(ObserveEntityEnum.GearUseFeaturesMeasurementLongline) + .backToParent() // ← TripLongline + .addAndEnterJoinTable(ObserveEntityEnum.ActivityLongline) // → ActivityLongline + .addAndEnterReverseJoinTable(ObserveEntityEnum.SetLongline) // → SetLongline + .addJoinTable(ObserveEntityEnum.BaitsComposition) + .addJoinTable(ObserveEntityEnum.FloatlinesComposition) + .addJoinTable(ObserveEntityEnum.HooksComposition) + .addJoinTable(ObserveEntityEnum.BranchlinesComposition) + .addAssociationTable(ObserveEntityEnum.MitigationType.name(), true) + .addAndEnterJoinTable(ObserveEntityEnum.Section) // → Section + .addAndEnterJoinTable(ObserveEntityEnum.Basket) // → Basket + .addJoinTable(ObserveEntityEnum.Branchline) + .backToParent() // ← Section + .backToParent() // ← SetLongline + .addAndEnterJoinTable(ObserveEntityEnum.CatchLongline) // → CatchLongline + .addJoinTable(ObserveEntityEnum.SizeMeasure) + .addJoinTable(ObserveEntityEnum.WeightMeasure) + .addAssociationTable("Predator", true) + .backToParent() // ← SetLongline + .addAndEnterJoinTable(ObserveEntityEnum.Tdr) // → Tdr + .addJoinTable(ObserveEntityEnum.TdrRecord) + .addAssociationTable(ObserveEntityEnum.Species.name(), true) + .backToParent() // ← SetLongline + .backToParent() // ← ActivityLongline + .addJoinTable(ObserveEntityEnum.Encounter) + .addJoinTable(ObserveEntityEnum.SensorUsed) + .build(); + */ } - public void replicateReferentials(Writer writer) { - - TopiaSqlReplicationService sqlReplicationService = getSqlReplicationService(); - sqlReplicationService - .newReplicatetoSqlActionBuilder(writer) - .setTables(ObserveReplicationTables.getReferentialTables()) - .build().run(); - + public TopiaSqlTables getReferentialTables() { + + if (referentialTables == null) { + log.info("Loading referentialTables."); + referentialTables = topiaSqlTablesFactory.newReplicateEntityTables( + new TripReplicateTablesPredicate(), + Entities.REFERENCE_ENTITIES); +// ObserveEntityEnum.VesselSizeCategory, +// ObserveEntityEnum.Country, +// ObserveEntityEnum.Harbour, +// ObserveEntityEnum.VesselType, +// ObserveEntityEnum.Vessel, +// ObserveEntityEnum.Ocean, +// ObserveEntityEnum.SpeciesGroup, +// ObserveEntityEnum.Species, +// ObserveEntityEnum.Sex, +// ObserveEntityEnum.FpaZone, +// ObserveEntityEnum.SpeciesList, +// ObserveEntityEnum.Person, +// ObserveEntityEnum.Organism, +// ObserveEntityEnum.LengthWeightParameter, +// ObserveEntityEnum.Program, +// ObserveEntityEnum.GearCaracteristicType, +// ObserveEntityEnum.GearCaracteristic, +// ObserveEntityEnum.Gear, +// ObserveEntityEnum.LastUpdateDate, +// ObserveEntityEnum.VesselActivitySeine, +// ObserveEntityEnum.SurroundingActivity, +// ObserveEntityEnum.ReasonForNullSet, +// ObserveEntityEnum.ReasonForNoFishing, +// ObserveEntityEnum.SpeciesFate, +// ObserveEntityEnum.ObjectFate, +// ObserveEntityEnum.WeightCategory, +// ObserveEntityEnum.DetectionMode, +// ObserveEntityEnum.TransmittingBuoyOperation, +// ObserveEntityEnum.ObjectOperation, +// ObserveEntityEnum.ReasonForDiscard, +// ObserveEntityEnum.SpeciesStatus, +// ObserveEntityEnum.ObservedSystem, +// ObserveEntityEnum.TransmittingBuoyType, +// ObserveEntityEnum.ObjectType, +// ObserveEntityEnum.Wind, +// ObserveEntityEnum.BaitHaulingStatus, +// ObserveEntityEnum.BaitSettingStatus, +// ObserveEntityEnum.BaitType, +// ObserveEntityEnum.CatchFateLongline, +// ObserveEntityEnum.EncounterType, +// ObserveEntityEnum.Healthness, +// ObserveEntityEnum.HookPosition, +// ObserveEntityEnum.HookSize, +// ObserveEntityEnum.HookType, +// ObserveEntityEnum.ItemVerticalPosition, +// ObserveEntityEnum.ItemHorizontalPosition, +// ObserveEntityEnum.LightsticksColor, +// ObserveEntityEnum.LightsticksType, +// ObserveEntityEnum.LineType, +// ObserveEntityEnum.MaturityStatus, +// ObserveEntityEnum.MitigationType, +// ObserveEntityEnum.SensorBrand, +// ObserveEntityEnum.SensorDataFormat, +// ObserveEntityEnum.SensorType, +// ObserveEntityEnum.SettingShape, +// ObserveEntityEnum.SizeMeasureType, +// ObserveEntityEnum.StomacFullness, +// ObserveEntityEnum.TripType, +// ObserveEntityEnum.VesselActivityLongline, +// ObserveEntityEnum.WeightMeasureType); + } + return referentialTables; + + /* + + return TopiaSqlTables.builder() + .addMainTable(ObserveEntityEnum.VesselSizeCategory) // → VesselSizeCategory + .addMainTable(ObserveEntityEnum.Country) // → Country + .addMainTable(ObserveEntityEnum.Harbour) // → Harbour + .addMainTable(ObserveEntityEnum.VesselType) // → VesselType + .addMainTable(ObserveEntityEnum.Vessel) // → Vessel + .addMainTable(ObserveEntityEnum.Ocean) // → Ocean + .addMainTable(ObserveEntityEnum.SpeciesGroup) // → SpeciesGroup + .addMainTable(ObserveEntityEnum.Species) // → Species + .addAssociationTable(ObserveEntityEnum.Ocean.name(), false) + .addMainTable(ObserveEntityEnum.Sex) // → Sex + .addMainTable(ObserveEntityEnum.FpaZone) // → FpaZone + .addMainTable(ObserveEntityEnum.SpeciesList) // → SpeciesList + .addAssociationTable(ObserveEntityEnum.Species.name(), false) + .addMainTable(ObserveEntityEnum.Person) // → Person + .addMainTable(ObserveEntityEnum.Organism) // → Organism + .addMainTable(ObserveEntityEnum.LengthWeightParameter) // → LengthWeightParameter + .addMainTable(ObserveEntityEnum.Program) // → Program + .addMainTable(ObserveEntityEnum.GearCaracteristicType) // → GearCaracteristicType + .addMainTable(ObserveEntityEnum.GearCaracteristic) // → GearCaracteristic + .addMainTable(ObserveEntityEnum.Gear) // → Gear + .addAssociationTable(ObserveEntityEnum.GearCaracteristic.name(), false) + .addMainTable(ObserveEntityEnum.LastUpdateDate) // → LastUpdateDate + .addMainTable(ObserveEntityEnum.VesselActivitySeine) // → VesselActivitySeine + .addMainTable(ObserveEntityEnum.SurroundingActivity) // → SurroundingActivity + .addMainTable(ObserveEntityEnum.ReasonForNullSet) // → ReasonForNullSet + .addMainTable(ObserveEntityEnum.ReasonForNoFishing) // → ReasonForNoFishing + .addMainTable(ObserveEntityEnum.SpeciesFate) // → SpeciesFate + .addMainTable(ObserveEntityEnum.ObjectFate) // → ObjectFate + .addMainTable(ObserveEntityEnum.WeightCategory) // → WeightCategory + .addMainTable(ObserveEntityEnum.DetectionMode) // → DetectionMode + .addMainTable(ObserveEntityEnum.TransmittingBuoyOperation) // → TransmittingBuoyOperation + .addMainTable(ObserveEntityEnum.ObjectOperation) // → ObjectOperation + .addMainTable(ObserveEntityEnum.ReasonForDiscard) // → ReasonForDiscard + .addMainTable(ObserveEntityEnum.SpeciesStatus) // → SpeciesStatus + .addMainTable(ObserveEntityEnum.ObservedSystem) // → ObservedSystem + .addMainTable(ObserveEntityEnum.TransmittingBuoyType) // → TransmittingBuoyType + .addMainTable(ObserveEntityEnum.ObjectType) // → ObjectType + .addMainTable(ObserveEntityEnum.Wind) // → Wind + .addMainTable(ObserveEntityEnum.BaitHaulingStatus) // → BaitHaulingStatus + .addMainTable(ObserveEntityEnum.BaitSettingStatus) // → BaitSettingStatus + .addMainTable(ObserveEntityEnum.BaitType) // → BaitType + .addMainTable(ObserveEntityEnum.CatchFateLongline) // → CatchFateLongline + .addMainTable(ObserveEntityEnum.EncounterType) // → EncounterType + .addMainTable(ObserveEntityEnum.Healthness) // → Healthness + .addMainTable(ObserveEntityEnum.HookPosition) // → HookPosition + .addMainTable(ObserveEntityEnum.HookSize) // → HookSize + .addMainTable(ObserveEntityEnum.HookType) // → HookType + .addMainTable(ObserveEntityEnum.ItemVerticalPosition) // → ItemVerticalPosition + .addMainTable(ObserveEntityEnum.ItemHorizontalPosition) // → ItemHorizontalPosition + .addMainTable(ObserveEntityEnum.LightsticksColor) // → LightsticksColor + .addMainTable(ObserveEntityEnum.LightsticksType) // → LightsticksType + .addMainTable(ObserveEntityEnum.LineType) // → LineType + .addMainTable(ObserveEntityEnum.MaturityStatus) // → MaturityStatus + .addMainTable(ObserveEntityEnum.MitigationType) // → MitigationType + .addMainTable(ObserveEntityEnum.SensorBrand) // → SensorBrand + .addMainTable(ObserveEntityEnum.SensorDataFormat) // → SensorDataFormat + .addMainTable(ObserveEntityEnum.SensorType) // → SensorType + .addMainTable(ObserveEntityEnum.SettingShape) // → SettingShape + .addMainTable(ObserveEntityEnum.SizeMeasureType) // → SizeMeasureType + .addMainTable(ObserveEntityEnum.StomacFullness) // → StomacFullness + .addMainTable(ObserveEntityEnum.TripType) // → TripType + .addMainTable(ObserveEntityEnum.VesselActivityLongline) // → VesselActivityLongline + .addMainTable(ObserveEntityEnum.WeightMeasureType) // → WeightMeasureType + + .build(); +*/ } - public void replicateSchema(SqlDatabaseType databaseType, File temporaryDirectory, Writer writer) throws IOException { - - writer.append(CREATE_SCHEMA_SCRIPT); - - try { - - File sqlScriptFile = new File(temporaryDirectory, "replicateSchema_" + System.nanoTime() + ".sql"); - - Configuration hibernateConfiguration = getHibernateProvider().getHibernateConfiguration(); + private static class TripReplicateTablesPredicate implements TopiaSqlTablesFactory.TopiaSqlTablesPredicate { + + protected final Set<TopiaMetadataEntity> dones = new LinkedHashSet<>(); + protected final Set<String> sectionsHolders = ImmutableSet.of( + ObserveEntityEnum.CatchLongline.name(), + ObserveEntityEnum.Tdr.name() + ); + protected final Set<String> sections = ImmutableSet.of( + ObserveEntityEnum.Branchline.name(), + ObserveEntityEnum.Basket.name(), + ObserveEntityEnum.Section.name() + ); + + @Override + public boolean acceptEntity(TopiaMetadataEntity metadataEntity) { + return dones.add(metadataEntity); + } - Properties properties = new Properties(); + @Override + public boolean acceptAssociation(TopiaMetadataEntity metadataEntity, String propertyName, TopiaMetadataEntity propertyType) { - properties.put(Environment.DIALECT, databaseType.getDialectType().getName()); + if (sectionsHolders.contains(propertyType.getType()) && sections.contains(metadataEntity.getType())) { + return false; + } - new SchemaExport(hibernateConfiguration, properties) - .setOutputFile(sqlScriptFile.getAbsolutePath()) - .setDelimiter(";") - .create(false, false); + return true; + } - WriterOutputStream out = new WriterOutputStream(writer); - Files.copy(sqlScriptFile.toPath(), out); - out.flush(); - Files.delete(sqlScriptFile.toPath()); + @Override + public boolean acceptReversedAssociation(TopiaMetadataEntity metadataEntity, String propertyName, TopiaMetadataEntity propertyType) { + return true; + } - } catch (HibernateException eee) { - throw new TopiaException(String.format("Could not create schema for reason: %s", eee.getMessage()), eee); + @Override + public boolean acceptNmAssociation(TopiaMetadataEntity metadataEntity, String propertyName, TopiaMetadataEntity propertyType) { + return true; } } - } diff --git a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaConfigurationFactory.java b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaConfigurationFactory.java index 3bc774d..19ec009 100644 --- a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaConfigurationFactory.java +++ b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaConfigurationFactory.java @@ -30,7 +30,7 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.topia.persistence.jdbc.JdbcConfiguration; import org.nuiton.topia.persistence.jdbc.JdbcConfigurationBuilder; import org.nuiton.topia.replication.TopiaReplicationServiceImpl; -import org.nuiton.topia.replication.sql.TopiaSqlReplicationServiceImpl; +import org.nuiton.topia.service.sql.batch.TopiaSqlBatchServiceImpl; import java.io.File; @@ -153,7 +153,7 @@ public class ObserveTopiaConfigurationFactory { ); topiaConfiguration.addDeclaredService(ObserveTopiaApplicationContext.MIGRATION_SERVICE_NAME, ObserveMigrationEngine.class, migrationServiceConfiguration); topiaConfiguration.addDeclaredService(ObserveTopiaApplicationContext.REPLICATION_SERVICE_NAME, TopiaReplicationServiceImpl.class); - topiaConfiguration.addDeclaredService(ObserveTopiaApplicationContext.SQL_REPLICATION_SERVICE_NAME, TopiaSqlReplicationServiceImpl .class); + topiaConfiguration.addDeclaredService(ObserveTopiaApplicationContext.SQL_BATCH_SERVICE_NAME, TopiaSqlBatchServiceImpl.class); return topiaConfiguration; diff --git a/observe-entities/src/main/java/fr/ird/observe/entities/Entities.java b/observe-entities/src/main/java/fr/ird/observe/entities/Entities.java index 0084e9b..c4df77e 100644 --- a/observe-entities/src/main/java/fr/ird/observe/entities/Entities.java +++ b/observe-entities/src/main/java/fr/ird/observe/entities/Entities.java @@ -60,12 +60,12 @@ public class Entities { ObserveEntityEnum.VesselType, ObserveEntityEnum.Vessel, ObserveEntityEnum.SpeciesGroup, + ObserveEntityEnum.Ocean, ObserveEntityEnum.Species, ObserveEntityEnum.Sex, ObserveEntityEnum.FpaZone, ObserveEntityEnum.SpeciesList, ObserveEntityEnum.Person, - ObserveEntityEnum.Ocean, ObserveEntityEnum.Organism, ObserveEntityEnum.LengthWeightParameter, ObserveEntityEnum.Program, @@ -141,12 +141,12 @@ public class Entities { ObserveEntityEnum.VesselType, ObserveEntityEnum.Vessel, ObserveEntityEnum.SpeciesGroup, + ObserveEntityEnum.Ocean, ObserveEntityEnum.Species, ObserveEntityEnum.Sex, ObserveEntityEnum.FpaZone, ObserveEntityEnum.SpeciesList, ObserveEntityEnum.Person, - ObserveEntityEnum.Ocean, ObserveEntityEnum.Organism, ObserveEntityEnum.LengthWeightParameter, ObserveEntityEnum.Program, @@ -301,6 +301,18 @@ public class Entities { }; public static final ObserveEntityEnum[] ALL_ENTITIES = ObserveEntityEnum.values(); + public static final Predicate<String> IS_SEINE_ID = new Predicate<String>() { + @Override + public boolean apply(String input) { + return Entities.isSeineId(input); + } + }; + public static final Predicate<String> IS_LONGLINE_ID = new Predicate<String>() { + @Override + public boolean apply(String input) { + return Entities.isLonglineId(input); + } + }; /** * @param klass le type d'entite a tester diff --git a/observe-entities/src/main/java/fr/ird/observe/entities/SqlDatabaseType.java b/observe-entities/src/main/java/fr/ird/observe/entities/SqlDatabaseType.java deleted file mode 100644 index bf03604..0000000 --- a/observe-entities/src/main/java/fr/ird/observe/entities/SqlDatabaseType.java +++ /dev/null @@ -1,26 +0,0 @@ -package fr.ird.observe.entities; - -import org.hibernate.dialect.Dialect; -import org.hibernate.dialect.H2Dialect; -import org.hibernate.dialect.PostgreSQL9Dialect; - -/** - * Created on 31/12/15. - * - * @author Tony Chemit - chemit@codelutin.com - */ -public enum SqlDatabaseType { - - h2(H2Dialect.class), - postgres(PostgreSQL9Dialect.class); - - private final Class<? extends Dialect> dialectType; - - SqlDatabaseType(Class<? extends Dialect> dialectType) { - this.dialectType = dialectType; - } - - public Class<? extends Dialect> getDialectType() { - return dialectType; - } -} diff --git a/observe-entities/src/main/java/fr/ird/observe/entities/replication/ObserveReplicationTables.java b/observe-entities/src/main/java/fr/ird/observe/entities/replication/ObserveReplicationTables.java deleted file mode 100644 index 68243fa..0000000 --- a/observe-entities/src/main/java/fr/ird/observe/entities/replication/ObserveReplicationTables.java +++ /dev/null @@ -1,169 +0,0 @@ -package fr.ird.observe.entities.replication; - -import fr.ird.observe.ObserveEntityEnum; -import org.nuiton.topia.replication.sql.action.ReplicateActionBuilderSupport; -import org.nuiton.topia.replication.sql.table.TopiaReplicationTables; -import org.nuiton.topia.replication.sql.table.TopiaReplicationTablesBuilder; - -/** - * Created on 30/12/15. - * - * @author Tony Chemit - chemit@codelutin.com - */ -public class ObserveReplicationTables { - - protected static <B extends ReplicateActionBuilderSupport> void addReferentialTables(B actionBuilder) { - actionBuilder.setTables(getReferentialTables()); - } - - protected static <B extends ReplicateActionBuilderSupport> void addTripLongline(B actionBuilder, String tripId) { - actionBuilder.setTables(getTripLonglineTables(tripId != null)).setArg(tripId); - - } - - protected static <B extends ReplicateActionBuilderSupport> void addTripSeine(B actionBuilder, String tripId) { - actionBuilder.setTables(getTripSeineTables(tripId != null)).setArg(tripId); - } - - public static TopiaReplicationTables getTripSeineTables(boolean withId) { - - return TopiaReplicationTablesBuilder - .builder(withId) - .addTopTable(ObserveEntityEnum.TripSeine) - .addTopJoinTable(ObserveEntityEnum.TripSeine, ObserveEntityEnum.GearUseFeaturesSeine) - .addJoinTable(ObserveEntityEnum.GearUseFeaturesSeine, ObserveEntityEnum.GearUseFeaturesMeasurementSeine) - .addTopJoinTable(ObserveEntityEnum.TripSeine, ObserveEntityEnum.Route) - .orderPlusOne() - .addJoinTable(ObserveEntityEnum.Route, ObserveEntityEnum.ActivitySeine) - .orderMinusTwo() - .addInversedJoinTable(ObserveEntityEnum.ActivitySeine, ObserveEntityEnum.SetSeine) - .orderPlusOne() - .addAssociationTable(ObserveEntityEnum.ActivitySeine, ObserveEntityEnum.ObservedSystem.name(), true) - .addJoinTable(ObserveEntityEnum.ActivitySeine, ObserveEntityEnum.FloatingObject) - .addJoinTable(ObserveEntityEnum.SetSeine, ObserveEntityEnum.SchoolEstimate) - .addJoinTable(ObserveEntityEnum.SetSeine, ObserveEntityEnum.NonTargetCatch) - .addJoinTable(ObserveEntityEnum.SetSeine, ObserveEntityEnum.NonTargetSample) - .addJoinTable(ObserveEntityEnum.NonTargetSample, ObserveEntityEnum.NonTargetLength) - .addJoinTable(ObserveEntityEnum.SetSeine, ObserveEntityEnum.TargetCatch) - .addJoinTable(ObserveEntityEnum.SetSeine, ObserveEntityEnum.TargetSample) - .addJoinTable(ObserveEntityEnum.TargetSample, ObserveEntityEnum.TargetLength) - .addJoinTable(ObserveEntityEnum.FloatingObject, ObserveEntityEnum.ObjectObservedSpecies) - .addJoinTable(ObserveEntityEnum.FloatingObject, ObserveEntityEnum.ObjectSchoolEstimate) - .addJoinTable(ObserveEntityEnum.FloatingObject, ObserveEntityEnum.TransmittingBuoy) - .build(); - - } - - public static TopiaReplicationTables getTripLonglineTablesForAll() { - return getTripLonglineTables(false); - } - - public static TopiaReplicationTables getTripLonglineTablesForOne() { - return getTripLonglineTables(true); - } - - public static TopiaReplicationTables getTripLonglineTables(boolean withId) { - - return TopiaReplicationTablesBuilder - .builder(withId) - .addTopTable(ObserveEntityEnum.TripLongline) - .addTopJoinTable(ObserveEntityEnum.TripLongline, ObserveEntityEnum.GearUseFeaturesLongline) - .addJoinTable(ObserveEntityEnum.GearUseFeaturesLongline, ObserveEntityEnum.GearUseFeaturesMeasurementLongline) - .orderPlusOne() - .addJoinTable(ObserveEntityEnum.TripLongline, ObserveEntityEnum.ActivityLongline) - .orderMinusTwo() - .addInversedJoinTable(ObserveEntityEnum.ActivityLongline, ObserveEntityEnum.SetLongline) - .orderPlusOne() - .addJoinTable(ObserveEntityEnum.ActivityLongline, ObserveEntityEnum.Encounter) - .addJoinTable(ObserveEntityEnum.ActivityLongline, ObserveEntityEnum.SensorUsed) - .addJoinTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.Tdr) - .addJoinTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.BaitsComposition) - .addJoinTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.FloatlinesComposition) - .addJoinTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.HooksComposition) - .addJoinTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.BranchlinesComposition) - .addJoinTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.Section) - .addJoinTable(ObserveEntityEnum.Section, ObserveEntityEnum.Basket) - .addJoinTable(ObserveEntityEnum.Basket, ObserveEntityEnum.Branchline) - .addJoinTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.CatchLongline) - .addAssociationTable(ObserveEntityEnum.SetLongline, ObserveEntityEnum.MitigationType.name(), true) - .addJoinTable(ObserveEntityEnum.CatchLongline, ObserveEntityEnum.SizeMeasure) - .addJoinTable(ObserveEntityEnum.CatchLongline, ObserveEntityEnum.WeightMeasure) - .addAssociationTable(ObserveEntityEnum.CatchLongline, "Predator", true) - .addJoinTable(ObserveEntityEnum.Tdr, ObserveEntityEnum.TdrRecord) - .addAssociationTable(ObserveEntityEnum.Tdr, ObserveEntityEnum.Species.name(), true) - .build(); - - } - - public static TopiaReplicationTables getReferentialTables() { - - return TopiaReplicationTablesBuilder - .builder(false) - .addTopTable(ObserveEntityEnum.VesselSizeCategory) - .addTopTable(ObserveEntityEnum.Country) - .addTopTable(ObserveEntityEnum.Harbour) - .addTopTable(ObserveEntityEnum.VesselType) - .addTopTable(ObserveEntityEnum.Vessel) - .addTopTable(ObserveEntityEnum.Ocean) - .addTopTable(ObserveEntityEnum.SpeciesGroup) - .addTopTable(ObserveEntityEnum.Species) - .addAssociationTable(ObserveEntityEnum.Species, ObserveEntityEnum.Ocean.name(), false) - .addTopTable(ObserveEntityEnum.Sex) - .addTopTable(ObserveEntityEnum.FpaZone) - .addTopTable(ObserveEntityEnum.SpeciesList) - .addAssociationTable(ObserveEntityEnum.SpeciesList, ObserveEntityEnum.Species.name(), false) - .addTopTable(ObserveEntityEnum.Person) - .addTopTable(ObserveEntityEnum.Organism) - .addTopTable(ObserveEntityEnum.LengthWeightParameter) - .addTopTable(ObserveEntityEnum.Program) - .addTopTable(ObserveEntityEnum.GearCaracteristicType) - .addTopTable(ObserveEntityEnum.GearCaracteristic) - .addTopTable(ObserveEntityEnum.Gear) - .addAssociationTable(ObserveEntityEnum.Gear, ObserveEntityEnum.GearCaracteristic.name(), false) - .addTopTable(ObserveEntityEnum.LastUpdateDate) - .addTopTable(ObserveEntityEnum.VesselActivitySeine) - .addTopTable(ObserveEntityEnum.SurroundingActivity) - .addTopTable(ObserveEntityEnum.ReasonForNullSet) - .addTopTable(ObserveEntityEnum.ReasonForNoFishing) - .addTopTable(ObserveEntityEnum.SpeciesFate) - .addTopTable(ObserveEntityEnum.ObjectFate) - .addTopTable(ObserveEntityEnum.WeightCategory) - .addTopTable(ObserveEntityEnum.DetectionMode) - .addTopTable(ObserveEntityEnum.TransmittingBuoyOperation) - .addTopTable(ObserveEntityEnum.ObjectOperation) - .addTopTable(ObserveEntityEnum.ReasonForDiscard) - .addTopTable(ObserveEntityEnum.SpeciesStatus) - .addTopTable(ObserveEntityEnum.ObservedSystem) - .addTopTable(ObserveEntityEnum.TransmittingBuoyType) - .addTopTable(ObserveEntityEnum.ObjectType) - .addTopTable(ObserveEntityEnum.Wind) - .addTopTable(ObserveEntityEnum.BaitHaulingStatus) - .addTopTable(ObserveEntityEnum.BaitSettingStatus) - .addTopTable(ObserveEntityEnum.BaitType) - .addTopTable(ObserveEntityEnum.CatchFateLongline) - .addTopTable(ObserveEntityEnum.EncounterType) - .addTopTable(ObserveEntityEnum.Healthness) - .addTopTable(ObserveEntityEnum.HookPosition) - .addTopTable(ObserveEntityEnum.HookSize) - .addTopTable(ObserveEntityEnum.HookType) - .addTopTable(ObserveEntityEnum.ItemVerticalPosition) - .addTopTable(ObserveEntityEnum.ItemHorizontalPosition) - .addTopTable(ObserveEntityEnum.LightsticksColor) - .addTopTable(ObserveEntityEnum.LightsticksType) - .addTopTable(ObserveEntityEnum.LineType) - .addTopTable(ObserveEntityEnum.MaturityStatus) - .addTopTable(ObserveEntityEnum.MitigationType) - .addTopTable(ObserveEntityEnum.SensorBrand) - .addTopTable(ObserveEntityEnum.SensorDataFormat) - .addTopTable(ObserveEntityEnum.SensorType) - .addTopTable(ObserveEntityEnum.SettingShape) - .addTopTable(ObserveEntityEnum.SizeMeasureType) - .addTopTable(ObserveEntityEnum.StomacFullness) - .addTopTable(ObserveEntityEnum.TripType) - .addTopTable(ObserveEntityEnum.VesselActivityLongline) - .addTopTable(ObserveEntityEnum.WeightMeasureType) - - .build(); - - } -} diff --git a/observe-entities/src/test/java/fr/ird/observe/ObserveTopiaApplicationContextTest.java b/observe-entities/src/test/java/fr/ird/observe/ObserveTopiaApplicationContextTest.java index cef34aa..f3a47b7 100644 --- a/observe-entities/src/test/java/fr/ird/observe/ObserveTopiaApplicationContextTest.java +++ b/observe-entities/src/test/java/fr/ird/observe/ObserveTopiaApplicationContextTest.java @@ -30,6 +30,8 @@ import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import org.nuiton.topia.persistence.metadata.TopiaMetadataModel; +import org.nuiton.topia.persistence.metadata.TopiaMetadataModelVisitor; import java.io.File; @@ -86,5 +88,22 @@ public class ObserveTopiaApplicationContextTest { } + TopiaMetadataModel metadataModel = topiaApplicationContext.getMetadataModel(); + + { + TopiaMetadataModelVisitor.PrintVisitor printVisitor = new TopiaMetadataModelVisitor.PrintVisitor(true, "\n"); + + metadataModel.getEntity(ObserveEntityEnum.TripSeine.name()).accept(printVisitor, metadataModel); + + log.info("TripSeine shell:\n" + printVisitor.toString()); + } + + { + TopiaMetadataModelVisitor.PrintVisitor printVisitor = new TopiaMetadataModelVisitor.PrintVisitor(true, "\n"); + + metadataModel.getEntity(ObserveEntityEnum.TripLongline.name()).accept(printVisitor, metadataModel); + + log.info("TripLongline shell:\n" + printVisitor.toString()); + } } } \ No newline at end of file diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerRequest.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerRequest.java index 20c2add..517005c 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerRequest.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerRequest.java @@ -9,18 +9,18 @@ import com.google.common.collect.ImmutableSet; */ public class SqlScriptProducerRequest { - protected final boolean h2; + protected final boolean postgres; protected boolean addSchema; protected boolean addReferential; protected boolean addData; protected ImmutableSet<String> dataIds; public static SqlScriptProducerRequest forH2() { - return new SqlScriptProducerRequest(true); + return new SqlScriptProducerRequest(false); } public static SqlScriptProducerRequest forPostgres() { - return new SqlScriptProducerRequest(false); + return new SqlScriptProducerRequest(true); } public SqlScriptProducerRequest addSchema() { @@ -52,8 +52,8 @@ public class SqlScriptProducerRequest { return addReferential; } - protected SqlScriptProducerRequest(boolean h2) { - this.h2 = h2; + protected SqlScriptProducerRequest(boolean postgres) { + this.postgres = postgres; } public boolean isAddData() { @@ -65,6 +65,10 @@ public class SqlScriptProducerRequest { } public boolean isH2() { - return h2; + return !postgres; + } + + public boolean isPostgres() { + return postgres; } } diff --git a/observe-services-topia/pom.xml b/observe-services-topia/pom.xml index 89366ed..b0d84d5 100644 --- a/observe-services-topia/pom.xml +++ b/observe-services-topia/pom.xml @@ -109,6 +109,10 @@ <groupId>org.nuiton.topia</groupId> <artifactId>topia-service-replication</artifactId> </dependency> + <dependency> + <groupId>org.nuiton.topia</groupId> + <artifactId>topia-service-sql-batch</artifactId> + </dependency> <dependency> <groupId>org.apache.struts.xwork</groupId> diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java index ac8c592..24c0ace 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java @@ -1,14 +1,20 @@ package fr.ird.observe.services.service; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import fr.ird.observe.ObserveTopiaApplicationContext; -import fr.ird.observe.entities.SqlDatabaseType; +import fr.ird.observe.entities.Entities; import fr.ird.observe.services.ObserveServiceTopia; +import org.nuiton.topia.service.sql.batch.SqlRequests; +import org.nuiton.topia.service.sql.batch.TopiaSqlBatchService; +import org.nuiton.topia.service.sql.batch.actions.TopiaSqlTableSelectArgument; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; +import java.util.Set; import java.util.zip.GZIPOutputStream; /** @@ -25,24 +31,11 @@ public class SqlScriptProducerServiceTopia extends ObserveServiceTopia implement try (Writer writer = new OutputStreamWriter(new GZIPOutputStream(out))) { - if (request.isAddSchema()) { - addSchema(request.isH2(), writer); - } - - if (request.isAddReferential()) { - addReferentialSqlScript(writer); - } + TopiaSqlBatchService sqlBatchService = serviceContext.getTopiaApplicationContext().getSqlBatchService(); + SqlRequests.Builder builder = sqlBatchService.requestBuilder().to(writer); - if (request.isAddData()) { - - ImmutableSet<String> dataIds = request.getDataIds(); - if (dataIds == null) { - addAllTrips(writer); - } else { - addTrips(writer, dataIds); - } - - } + SqlRequests sqlRequests = sqlRequests(request, builder); + sqlBatchService.execute(sqlRequests); } @@ -54,29 +47,81 @@ public class SqlScriptProducerServiceTopia extends ObserveServiceTopia implement } - public void addSchema(boolean h2, Writer writer) { + protected SqlRequests sqlRequests(SqlScriptProducerRequest request, SqlRequests.Builder builder) { + + if (request.isAddSchema()) { + + SqlRequests.CreateSchemaRequestBuilder createSchemaRequestBuilder = builder + .createSchemaBuilder() + .setAddSchema(true) + .setTemporaryPath(serviceContext.getTemporaryDirectoryRoot().toPath()); + if (request.isH2()) { + createSchemaRequestBuilder.forH2(); + } else { + createSchemaRequestBuilder.forPostgres(); + } + createSchemaRequestBuilder.flush(); - try { - getObserveTopiaApplicationContext().replicateSchema(h2 ? SqlDatabaseType.h2 : SqlDatabaseType.postgres, serviceContext.getTemporaryDirectoryRoot(), writer); - } catch (IOException e) { - throw new RuntimeException("Could not add schema sql script", e); } - } - public void addReferentialSqlScript(Writer writer) { - getObserveTopiaApplicationContext().replicateReferentials(writer); - } + ObserveTopiaApplicationContext topiaApplicationContext = serviceContext.getTopiaApplicationContext(); - public void addAllTrips(Writer writer) { - getObserveTopiaApplicationContext().replicateAllTrips(writer); - } + if (request.isAddReferential()) { - public void addTrips(Writer writer, ImmutableSet<String> tripIds) { - getObserveTopiaApplicationContext().replicateTrips(writer, tripIds); - } + builder.replicateTablesBuilder() + .setTables(topiaApplicationContext.getReferentialTables()) + .flush(); + + } + + if (request.isAddData()) { + + ImmutableSet<String> tripIds = request.getDataIds(); + if (tripIds == null) { + + builder.replicateTablesBuilder() + .setTables(topiaApplicationContext.getTripSeineTables()) + .replicateTablesBuilder() + .setTables(topiaApplicationContext.getTripLonglineTables()) + .flush(); + + } else { + + { + + Set<String> tripIds1 = Sets.newHashSet(Iterables.filter(tripIds, Entities.IS_SEINE_ID)); + + if (!tripIds1.isEmpty()) { + + builder.replicateTablesBuilder() + .setTables(topiaApplicationContext.getTripSeineTables()) + .setSelectArgument(TopiaSqlTableSelectArgument.of(tripIds1)) + .flush(); + + } + + } + + { + + Set<String> tripIds1 = Sets.newHashSet(Iterables.filter(tripIds, Entities.IS_LONGLINE_ID)); + + if (!tripIds1.isEmpty()) { + + builder.replicateTablesBuilder() + .setTables(topiaApplicationContext.getTripLonglineTables()) + .setSelectArgument(TopiaSqlTableSelectArgument.of(tripIds1)) + .flush(); + + } + + } + + } + + } - protected ObserveTopiaApplicationContext getObserveTopiaApplicationContext() { - return serviceContext.getTopiaApplicationContext(); + return builder.build(); } } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateReferentialsTest.java b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateReferentialsTest.java index 64e2ec3..3a93b7c 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateReferentialsTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateReferentialsTest.java @@ -4,8 +4,8 @@ import fr.ird.observe.test.DatabaseName; import fr.ird.observe.test.ObserveFixtures; import fr.ird.observe.test.spi.DatabaseNameConfiguration; import org.junit.Test; -import org.nuiton.topia.replication.sql.action.todb.ReplicateToDbActionBuilder; -import org.nuiton.topia.replication.sql.action.tosql.ReplicateToSqlActionBuilder; +import org.nuiton.topia.service.sql.batch.actions.ReplicateTablesRequest; +import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; /** * Created on 29/12/15. @@ -16,20 +16,16 @@ import org.nuiton.topia.replication.sql.action.tosql.ReplicateToSqlActionBuilder public class ReplicateReferentialsTest extends ReplicateTestSupport { @Test - public void testReplicateToDb() throws Exception { + public void testReplicate() throws Exception { - ReplicateToDbActionBuilder actionBuilder = createReplicateToDbActionBuilder(DatabaseName.empty); - ObserveReplicationTables.addReferentialTables(actionBuilder); - testReplicateToDb0(actionBuilder, ObserveFixtures.REFERENTIAL_TABLES_COUNT); + TopiaSqlTables tables = dataSourceResource.getTopiaApplicationContext().getReferentialTables(); - } - - @Test - public void testReplicateToSql() throws Exception { + ReplicateTablesRequest request + = createReplicateTablesRequest(DatabaseName.empty) + .setTables(tables) + .build(); - ReplicateToSqlActionBuilder actionBuilder = createReplicateToSqlActionBuilder(); - ObserveReplicationTables.addReferentialTables(actionBuilder); - testReplicateToSql0(actionBuilder); + testReplicate0(request, ObserveFixtures.REFERENTIAL_TABLES_COUNT); } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTestSupport.java b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTestSupport.java index 6ce7987..25ec614 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTestSupport.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTestSupport.java @@ -13,14 +13,9 @@ import fr.ird.observe.test.DatabaseName; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.Assert; -import org.junit.Before; -import org.nuiton.topia.replication.sql.TopiaSqlReplicationService; -import org.nuiton.topia.replication.sql.action.todb.ReplicateToDbAction; -import org.nuiton.topia.replication.sql.action.todb.ReplicateToDbActionBuilder; -import org.nuiton.topia.replication.sql.action.todb.ReplicateToDbActionRequest; -import org.nuiton.topia.replication.sql.action.tosql.ReplicateToSqlActionBuilder; -import org.nuiton.topia.replication.sql.table.TopiaReplicationTable; -import org.nuiton.topia.replication.sql.table.TopiaReplicationTables; +import org.nuiton.topia.service.sql.batch.actions.ReplicateTablesRequest; +import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTable; +import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; import org.nuiton.util.StringUtil; import java.io.File; @@ -36,47 +31,36 @@ import java.util.Map; */ public class ReplicateTestSupport extends AbstractServiceTopiaTest { - /** Logger. */ + /** + * Logger. + */ private static final Log log = LogFactory.getLog(ReplicateTestSupport.class); - protected TopiaSqlReplicationService sqlReplicationService; - protected File scriptFile; - @Before - public void setUp() throws Exception { - - ObserveTopiaApplicationContext sourceTopiaApplicationContext = dataSourceResource.getTopiaApplicationContext(); - - sqlReplicationService = sourceTopiaApplicationContext.getSqlReplicationService(); - - } - - protected ReplicateToDbActionBuilder createReplicateToDbActionBuilder(DatabaseName databaseName) throws DataSourceCreateWithNoReferentialImportException, IOException, IncompatibleDataSourceCreateConfigurationException { + protected ReplicateTablesRequest.Builder createReplicateTablesRequest(DatabaseName databaseName) throws DataSourceCreateWithNoReferentialImportException, IOException, IncompatibleDataSourceCreateConfigurationException { File targetDatabaseDirectory = new File(dataSourceResource.getTestDirectory(), "targetReplication"); ObserveDataSourceConfigurationTopiaH2 targetTopiaConfiguration = applicationContextResource.createDataSource(dataSourceResource.getDbVersion(), databaseName.name(), targetDatabaseDirectory); ObserveTopiaApplicationContext targetTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(targetTopiaConfiguration); - return sqlReplicationService.newReplicateToDbActionBuilder(targetTopiaApplicationContext); - } - - protected ReplicateToSqlActionBuilder createReplicateToSqlActionBuilder() throws IOException { - scriptFile = new File(dataSourceResource.getTestDirectory(), "script.sql"); Writer writer = Files.newWriter(scriptFile, StandardCharsets.UTF_8); - return sqlReplicationService.newReplicatetoSqlActionBuilder(writer); - + return dataSourceResource.getTopiaApplicationContext() + .getSqlBatchService() + .replicateTablesRequestBuilder() + .to(targetTopiaApplicationContext) + .to(writer); } - protected static void assertReplicateTripResults(ReplicateToDbActionRequest replicationRequest, ImmutableMap<String, Long> expectedResults) { + protected static void assertReplicateTripResults(ReplicateTablesRequest replicationRequest, ImmutableMap<String, Long> expectedResults) { - TopiaReplicationTables tables = replicationRequest.getTables(); + TopiaSqlTables tables = replicationRequest.getTables(); try (ObserveTopiaPersistenceContext persistenceContext = (ObserveTopiaPersistenceContext) replicationRequest.getTargetTopiaApplicationContext().newPersistenceContext()) { - for (TopiaReplicationTable table : tables) { + for (TopiaSqlTable table : tables) { String fullyTableName = table.getFullyTableName(); @@ -103,25 +87,21 @@ public class ReplicateTestSupport extends AbstractServiceTopiaTest { } - protected void testReplicateToSql0(ReplicateToSqlActionBuilder actionBuilder) { + protected void testReplicate0(ReplicateTablesRequest request, ImmutableMap<String, Long> expectedTablesCount) { - actionBuilder - .build() - .run(); + TopiaSqlTables tables = request.getTables(); + log.info(tables); - if (log.isInfoEnabled()) { - log.info(String.format("Replicate to script (length: %s) :\n%s", StringUtil.convertMemory(scriptFile.length()), scriptFile)); - } - - } + dataSourceResource.getTopiaApplicationContext() + .getSqlBatchService() + .execute(request); - protected void testReplicateToDb0(ReplicateToDbActionBuilder actionBuilder, ImmutableMap<String, Long> expectedTablesCount) { - ReplicateToDbAction action = actionBuilder.build(); + assertReplicateTripResults(request, expectedTablesCount); - action.run(); - - assertReplicateTripResults(action.getRequest(), expectedTablesCount); + if (log.isInfoEnabled()) { + log.info(String.format("Replicate to script (length: %s) :\n%s", StringUtil.convertMemory(scriptFile.length()), scriptFile)); + } } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripLonglineTest.java b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripLonglineTest.java index 663248d..4f7f20a 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripLonglineTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripLonglineTest.java @@ -7,8 +7,9 @@ import fr.ird.observe.test.DatabaseName; import fr.ird.observe.test.ObserveFixtures; import fr.ird.observe.test.spi.DatabaseNameConfiguration; import org.junit.Test; -import org.nuiton.topia.replication.sql.action.todb.ReplicateToDbActionBuilder; -import org.nuiton.topia.replication.sql.action.tosql.ReplicateToSqlActionBuilder; +import org.nuiton.topia.service.sql.batch.actions.ReplicateTablesRequest; +import org.nuiton.topia.service.sql.batch.actions.TopiaSqlTableSelectArgument; +import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; import java.io.IOException; @@ -21,58 +22,43 @@ import java.io.IOException; public class ReplicateTripLonglineTest extends ReplicateTestSupport { @Test - public void testReplicateAllTripLonglineToDb() throws Exception { - testReplicateToDb(null, ObserveFixtures.ALL_TRIP_LONGLINE_COUNT); + public void testReplicateAllTripLongline() throws Exception { + testReplicate(ObserveFixtures.ALL_TRIP_LONGLINE_COUNT); } @Test - public void testReplicateTripLonline1ToDb() throws Exception { - testReplicateToDb(ObserveFixtures.TRIP_LONGLINE_ID_1, ObserveFixtures.TRIP_LONGLINE_1_TABLES_COUNT); + public void testReplicateAllTripLongline2() throws Exception { + testReplicate(ObserveFixtures.ALL_TRIP_LONGLINE_COUNT, + ObserveFixtures.TRIP_LONGLINE_ID_1, + ObserveFixtures.TRIP_LONGLINE_ID_2, + ObserveFixtures.TRIP_LONGLINE_ID_3); } @Test - public void testReplicateTripLonline2ToDb() throws Exception { - testReplicateToDb(ObserveFixtures.TRIP_LONGLINE_ID_2, ObserveFixtures.TRIP_LONGLINE_2_TABLES_COUNT); + public void testReplicateTripLonline1() throws Exception { + testReplicate(ObserveFixtures.TRIP_LONGLINE_1_TABLES_COUNT, ObserveFixtures.TRIP_LONGLINE_ID_1); } @Test - public void testReplicateTripLonline3ToDb() throws Exception { - testReplicateToDb(ObserveFixtures.TRIP_LONGLINE_ID_3, ObserveFixtures.TRIP_LONGLINE_3_TABLES_COUNT); + public void testReplicateTripLonline2() throws Exception { + testReplicate(ObserveFixtures.TRIP_LONGLINE_2_TABLES_COUNT, ObserveFixtures.TRIP_LONGLINE_ID_2); } @Test - public void testReplicateAllTripLonglineToSql() throws Exception { - testReplicateToSql(null); + public void testReplicateTripLonline3() throws Exception { + testReplicate(ObserveFixtures.TRIP_LONGLINE_3_TABLES_COUNT, ObserveFixtures.TRIP_LONGLINE_ID_3); } - @Test - public void testReplicateTripLonline1ToSql() throws Exception { - testReplicateToSql(ObserveFixtures.TRIP_LONGLINE_ID_1); - } - - @Test - public void testReplicateTripLonline2ToSql() throws Exception { - testReplicateToSql(ObserveFixtures.TRIP_LONGLINE_ID_2); - } - - @Test - public void testReplicateTripLonline3ToSql() throws Exception { - testReplicateToSql(ObserveFixtures.TRIP_LONGLINE_ID_3); - } - - protected void testReplicateToSql(String tripId) throws IOException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException { - - ReplicateToSqlActionBuilder actionBuilder = createReplicateToSqlActionBuilder(); - ObserveReplicationTables.addTripLongline(actionBuilder, tripId); - testReplicateToSql0(actionBuilder); - - } + protected void testReplicate(ImmutableMap<String, Long> expectedTablesCount, String... tripIds) throws DataSourceCreateWithNoReferentialImportException, IOException, IncompatibleDataSourceCreateConfigurationException { - protected void testReplicateToDb(String tripId, ImmutableMap<String, Long> expectedTablesCount) throws DataSourceCreateWithNoReferentialImportException, IOException, IncompatibleDataSourceCreateConfigurationException { + TopiaSqlTables tables = dataSourceResource.getTopiaApplicationContext().getTripLonglineTables(); + ReplicateTablesRequest request + = createReplicateTablesRequest(DatabaseName.referentiel) + .setSelectArgument(TopiaSqlTableSelectArgument.of(tripIds)) + .setTables(tables) + .build(); - ReplicateToDbActionBuilder actionBuilder = createReplicateToDbActionBuilder(DatabaseName.referentiel); - ObserveReplicationTables.addTripLongline(actionBuilder, tripId); - testReplicateToDb0(actionBuilder, expectedTablesCount); + testReplicate0(request, expectedTablesCount); } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripSeineTest.java b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripSeineTest.java index e5e0eeb..abfb9db 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripSeineTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/entities/replication/ReplicateTripSeineTest.java @@ -5,10 +5,9 @@ import fr.ird.observe.test.DatabaseName; import fr.ird.observe.test.ObserveFixtures; import fr.ird.observe.test.spi.DatabaseNameConfiguration; import org.junit.Test; -import org.nuiton.topia.replication.sql.action.todb.ReplicateToDbActionBuilder; -import org.nuiton.topia.replication.sql.action.tosql.ReplicateToSqlActionBuilder; - -import java.io.IOException; +import org.nuiton.topia.service.sql.batch.actions.ReplicateTablesRequest; +import org.nuiton.topia.service.sql.batch.actions.TopiaSqlTableSelectArgument; +import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; /** * Created on 29/12/15. @@ -19,58 +18,44 @@ import java.io.IOException; public class ReplicateTripSeineTest extends ReplicateTestSupport { @Test - public void testReplicateToDbAllTripSeine() throws Exception { - testReplicateToDb(null, ObserveFixtures.ALL_TRIP_SEINE_COUNT); - } - - @Test - public void testReplicateTripSeine1ToDb() throws Exception { - testReplicateToDb(ObserveFixtures.TRIP_SEINE_ID_1, ObserveFixtures.TRIP_SEINE_1_TABLES_COUNT); - } - - @Test - public void testReplicateTripSeine2ToDb() throws Exception { - testReplicateToDb(ObserveFixtures.TRIP_SEINE_ID_2, ObserveFixtures.TRIP_SEINE_2_TABLES_COUNT); - } - - @Test - public void testReplicateTripSeine3ToDb() throws Exception { - testReplicateToDb(ObserveFixtures.TRIP_SEINE_ID_3, ObserveFixtures.TRIP_SEINE_3_TABLES_COUNT); + public void testReplicateAllTripSeine() throws Exception { + testReplicate(ObserveFixtures.ALL_TRIP_SEINE_COUNT); } @Test - public void testReplicateToSqlScriptAllTripSeine() throws Exception { - testReplicateToSql(null); + public void testReplicateTripSeine1() throws Exception { + testReplicate(ObserveFixtures.TRIP_SEINE_1_TABLES_COUNT, ObserveFixtures.TRIP_SEINE_ID_1); } @Test - public void testReplicateTripSeine1ToSql() throws Exception { - testReplicateToSql(ObserveFixtures.TRIP_SEINE_ID_1); + public void testReplicateAllTripSeine2() throws Exception { + testReplicate(ObserveFixtures.ALL_TRIP_SEINE_COUNT, + ObserveFixtures.TRIP_SEINE_ID_1, + ObserveFixtures.TRIP_SEINE_ID_2, + ObserveFixtures.TRIP_SEINE_ID_3); } @Test - public void testReplicateTripSeine2ToSql() throws Exception { - testReplicateToSql(ObserveFixtures.TRIP_SEINE_ID_2); + public void testReplicateTripSeine2() throws Exception { + testReplicate(ObserveFixtures.TRIP_SEINE_2_TABLES_COUNT, ObserveFixtures.TRIP_SEINE_ID_2); } @Test - public void testReplicateTripSeine3ToSql() throws Exception { - testReplicateToSql(ObserveFixtures.TRIP_SEINE_ID_3); + public void testReplicateTripSeine3() throws Exception { + testReplicate(ObserveFixtures.TRIP_SEINE_3_TABLES_COUNT, ObserveFixtures.TRIP_SEINE_ID_3); } - protected void testReplicateToSql(String tripId) throws IOException { - - ReplicateToSqlActionBuilder actionBuilder = createReplicateToSqlActionBuilder(); - ObserveReplicationTables.addTripSeine(actionBuilder, tripId); - testReplicateToSql0(actionBuilder); + protected void testReplicate(ImmutableMap<String, Long> expectedTablesCount, String... tripIds) throws Exception { - } + TopiaSqlTables tables = dataSourceResource.getTopiaApplicationContext().getTripSeineTables(); - protected void testReplicateToDb(String tripId, ImmutableMap<String, Long> expectedTablesCount) throws Exception { + ReplicateTablesRequest request + = createReplicateTablesRequest(DatabaseName.referentiel) + .setSelectArgument(TopiaSqlTableSelectArgument.of(tripIds)) + .setTables(tables) + .build(); - ReplicateToDbActionBuilder actionBuilder = createReplicateToDbActionBuilder(DatabaseName.referentiel); - ObserveReplicationTables.addTripSeine(actionBuilder, tripId); - testReplicateToDb0(actionBuilder, expectedTablesCount); + testReplicate0(request, expectedTablesCount); } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopiaTest.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopiaTest.java index 7989941..6c81da2 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopiaTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopiaTest.java @@ -50,7 +50,7 @@ public class SqlScriptProducerServiceTopiaTest extends AbstractServiceTopiaTest @Test public void testProduceTripSeine() throws Exception { - produce(SqlScriptProducerRequest.forPostgres().addSchema().addReferential().addAllData()); +// produce(SqlScriptProducerRequest.forPostgres().addSchema().addReferential().addAllData()); produce(SqlScriptProducerRequest.forPostgres().addSchema().addReferential().addDataIds(ObserveFixtures.TRIP_SEINE_IDS)); } @@ -60,7 +60,7 @@ public class SqlScriptProducerServiceTopiaTest extends AbstractServiceTopiaTest public void testProduceTripLongline() throws Exception { produce(SqlScriptProducerRequest.forH2().addSchema().addReferential().addAllData()); - produce(SqlScriptProducerRequest.forH2().addSchema().addReferential().addDataIds(ObserveFixtures.TRIP_LONGLINE_IDS)); +// produce(SqlScriptProducerRequest.forH2().addSchema().addReferential().addDataIds(ObserveFixtures.TRIP_LONGLINE_IDS)); } diff --git a/observe-services-topia/src/test/resources/log4j.properties b/observe-services-topia/src/test/resources/log4j.properties index fb23e8d..bfb1f61 100644 --- a/observe-services-topia/src/test/resources/log4j.properties +++ b/observe-services-topia/src/test/resources/log4j.properties @@ -30,5 +30,5 @@ log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%c:%L) %M - %m%n #log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) %M - %m%n log4j.logger.fr.ird.observe=INFO -log4j.logger.org.nuiton.topia.replication.sql=DEBUG +log4j.logger.org.nuiton.topia.service.sql=DEBUG log4j.logger.org.nuiton.topia.persistence.util.TopiaEntityHelper=INFO diff --git a/pom.xml b/pom.xml index 0eff8c7..7b24e45 100644 --- a/pom.xml +++ b/pom.xml @@ -450,6 +450,12 @@ </dependency> <dependency> <groupId>org.nuiton.topia</groupId> + <artifactId>topia-service-sql-batch</artifactId> + <version>${topiaVersion}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.nuiton.topia</groupId> <artifactId>topia-service-replication</artifactId> <version>${topiaVersion}</version> <scope>test</scope> -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.