This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository observe. See https://gitlab.nuiton.org/codelutin/observe.git commit ade1cba6bd469c0ec11a2f94ec6a54eb39f7505a Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Sep 8 14:56:36 2016 +0200 Mieux gérer les associations pour la réplication --- .../service/TripManagementServiceTopiaTest.java | 39 +++++++++++++++--- services-topia/src/test/resources/log4j.properties | 1 + .../topia/service/sql/batch/SqlRequests.java | 12 +++--- .../sql/batch/actions/AbstractTablesAction.java | 47 +++++++++++++--------- .../sql/batch/actions/ReplicateTablesRequest.java | 4 -- .../sql/batch/actions/UpdateTablesAction.java | 4 +- .../sql/batch/actions/UpdateTablesRequest.java | 12 ++++++ .../service/sql/batch/tables/TopiaSqlTable.java | 16 +++++--- .../service/sql/batch/tables/TopiaSqlTables.java | 15 ++++--- 9 files changed, 101 insertions(+), 49 deletions(-) diff --git a/services-topia/src/test/java/fr/ird/observe/services/topia/service/TripManagementServiceTopiaTest.java b/services-topia/src/test/java/fr/ird/observe/services/topia/service/TripManagementServiceTopiaTest.java index 5842fd4..51517e0 100644 --- a/services-topia/src/test/java/fr/ird/observe/services/topia/service/TripManagementServiceTopiaTest.java +++ b/services-topia/src/test/java/fr/ird/observe/services/topia/service/TripManagementServiceTopiaTest.java @@ -4,8 +4,14 @@ import fr.ird.observe.services.configuration.ObserveDataSourceConnection; import fr.ird.observe.services.configuration.topia.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.services.dto.DataReferenceSet; import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; +import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; +import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; import fr.ird.observe.services.dto.seine.TripSeineDto; +import fr.ird.observe.services.service.BabModelVersionException; import fr.ird.observe.services.service.DataSourceService; +import fr.ird.observe.services.service.DatabaseConnexionNotAuthorizedException; +import fr.ird.observe.services.service.DatabaseNotFoundException; +import fr.ird.observe.services.service.longline.TripLonglineService; import fr.ird.observe.services.service.seine.TripSeineService; import fr.ird.observe.services.service.trip.DeleteTripRequest; import fr.ird.observe.services.service.trip.DeleteTripResult; @@ -22,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; /** @@ -47,7 +54,7 @@ public class TripManagementServiceTopiaTest extends AbstractServiceTopiaTest { } @Test - public void exportTrip() throws Exception { + public void exportTrip() { ExportTripRequest request = new ExportTripRequest(false, ObserveFixtures.PROGRAM_ID, ObserveFixtures.TRIP_SEINE_ID_1); ExportTripResult result = tripManagementService.exportTrip(request); @@ -57,7 +64,7 @@ public class TripManagementServiceTopiaTest extends AbstractServiceTopiaTest { @CopyDatabaseConfiguration @Test - public void deleteTrip() throws Exception { + public void deleteTrip() { DataReferenceSet<TripSeineDto> allTripSeineBefore = tripSeineService.getAllTripSeine(); @@ -70,9 +77,26 @@ public class TripManagementServiceTopiaTest extends AbstractServiceTopiaTest { } @Test - public void importTrip() throws Exception { + public void importTripSeine() throws DatabaseConnexionNotAuthorizedException, DataSourceCreateWithNoReferentialImportException, BabModelVersionException, IncompatibleDataSourceCreateConfigurationException, DatabaseNotFoundException { ExportTripRequest request = new ExportTripRequest(false, ObserveFixtures.PROGRAM_ID, ObserveFixtures.TRIP_SEINE_ID_1); + importTrip(request, true); + + } + + // FIXME Le test met 6 minutes, comprendre pourquoi l'import est aussi long (environ 18000 branchlines) + @Ignore + @DatabaseNameConfiguration(DatabaseName.dataForTestLongline) + @Test + public void importTripLongline() throws DatabaseConnexionNotAuthorizedException, DataSourceCreateWithNoReferentialImportException, BabModelVersionException, IncompatibleDataSourceCreateConfigurationException, DatabaseNotFoundException { + + ExportTripRequest request = new ExportTripRequest(false, ObserveFixtures.PROGRAM_ID, ObserveFixtures.TRIP_LONGLINE_ID_1); + importTrip(request, false); + + } + + private void importTrip(ExportTripRequest request, boolean forSeine) throws DatabaseConnexionNotAuthorizedException, DataSourceCreateWithNoReferentialImportException, BabModelVersionException, IncompatibleDataSourceCreateConfigurationException, DatabaseNotFoundException { + ExportTripResult result = tripManagementService.exportTrip(request); ObserveDataSourceConfigurationTopiaH2 dataSourceConfiguration = topiaTestMethodResource.createDataSourceConfigurationH2(getClass(), "importTripTarget"); @@ -87,13 +111,16 @@ public class TripManagementServiceTopiaTest extends AbstractServiceTopiaTest { TripManagementService tripManagementService = TOPIA_TEST_CLASS_RESOURCE.newService(dataSourceConnection, TripManagementService.class); TripSeineService tripSeineService = TOPIA_TEST_CLASS_RESOURCE.newService(dataSourceConnection, TripSeineService.class); + TripLonglineService tripLonglineService = TOPIA_TEST_CLASS_RESOURCE.newService(dataSourceConnection, TripLonglineService.class); - DataReferenceSet<TripSeineDto> allTripSeineBefore = tripSeineService.getAllTripSeine(); + int allTripSeineBefore = forSeine ? tripSeineService.getAllTripSeine().sizeReference() : + tripLonglineService.getAllTripLongline().sizeReference(); tripManagementService.importTrip(new ImportTripRequest(result)); - DataReferenceSet<TripSeineDto> allTripSeineAfter = tripSeineService.getAllTripSeine(); - Assert.assertTrue(allTripSeineAfter.sizeReference() == allTripSeineBefore.sizeReference() + 1); + int allTripSeineAfter = forSeine ? tripSeineService.getAllTripSeine().sizeReference() : + tripLonglineService.getAllTripLongline().sizeReference(); + Assert.assertTrue(allTripSeineAfter == allTripSeineBefore + 1); } diff --git a/services-topia/src/test/resources/log4j.properties b/services-topia/src/test/resources/log4j.properties index 38e120f..189f688 100644 --- a/services-topia/src/test/resources/log4j.properties +++ b/services-topia/src/test/resources/log4j.properties @@ -30,5 +30,6 @@ log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%c:%L) %M - %m%n #log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) %M - %m%n log4j.logger.fr.ird.observe=INFO +#log4j.logger.fr.ird.observe.RunScriptTopiaSqlWork=DEBUG log4j.logger.org.nuiton.topia.service.sql=DEBUG log4j.logger.org.nuiton.topia.persistence.util.TopiaEntityHelper=INFO diff --git a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/SqlRequests.java b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/SqlRequests.java index 5bf84a9..a67c967 100644 --- a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/SqlRequests.java +++ b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/SqlRequests.java @@ -93,7 +93,7 @@ public class SqlRequests implements Iterable<AbstractSqlRequest> { ReplicateTablesRequestBuilder replicateTablesBuilder(TopiaMetadataModel metadataModel); - UpdateTablesRequestBuilder updateTablesBuilder(); + UpdateTablesRequestBuilder updateTablesBuilder(TopiaMetadataModel metadataModel); DeleteTablesRequestBuilder deleteTablesBuilder(); @@ -169,8 +169,10 @@ public class SqlRequests implements Iterable<AbstractSqlRequest> { } @Override - public UpdateTablesRequestBuilder updateTablesBuilder() { - return new UpdateTablesRequestBuilder(this, initBuilder(new UpdateTablesRequest.Builder())); + public UpdateTablesRequestBuilder updateTablesBuilder(TopiaMetadataModel metadataModel) { + UpdateTablesRequest.Builder builder = new UpdateTablesRequest.Builder(); + builder.setTopiaMetaModel(metadataModel); + return new UpdateTablesRequestBuilder(this, initBuilder(builder)); } @Override @@ -291,8 +293,8 @@ public class SqlRequests implements Iterable<AbstractSqlRequest> { } @Override - public UpdateTablesRequestBuilder updateTablesBuilder() { - return flush().updateTablesBuilder(); + public UpdateTablesRequestBuilder updateTablesBuilder(TopiaMetadataModel metadataModel) { + return flush().updateTablesBuilder(metadataModel); } @Override diff --git a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java index 3e3f701..cbf7aa8 100644 --- a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java +++ b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; /** * Support to create action. @@ -112,16 +113,40 @@ public abstract class AbstractTablesAction<R extends AbstractTablesRequest> exte List<String> builder = new ArrayList<>(columnCount); for (int i = 1; i <= columnCount; i++) { String columnName = readResultTatMetaData.getColumnName(i); - if (authorizedColumnNames ==null || authorizedColumnNames.contains(columnName.toLowerCase())) { + if (authorizedColumnNames == null || authorizedColumnNames.contains(columnName.toLowerCase())) { builder.add(columnName.toLowerCase()); } } return builder; } - protected class ReadSqlWork implements TopiaSqlWork { + protected List<String> getColumnNames(TopiaMetadataEntity metadataEntity, TopiaSqlTable table, ResultSet readResultSet) throws SQLException { + + Set<String> authorizedColumnNames; + if (table.isAssociationTable()) { + authorizedColumnNames = new TreeSet<>(); + + String entityType = table.getAssociationMetadataEntity().getType(); + TopiaMetadataEntity entity = table.getAssociationMetadataEntity(); + authorizedColumnNames.addAll(entity.getManyToManyAssociations().entrySet().stream() + .filter(entry -> entry.getValue().equals(metadataEntity.getType())) + .map(entry -> entry.getKey().toLowerCase()) + .collect(Collectors.toSet())); + authorizedColumnNames.add(table.getJoinColumnName()); + } else { + authorizedColumnNames = metadataEntity.getAllDbColumnNames(); + } + ResultSetMetaData readResultSetMetaData = readResultSet.getMetaData(); + int columnCount = readResultSetMetaData.getColumnCount(); + + return getColumnNames(readResultSetMetaData, columnCount, authorizedColumnNames); + + } + + protected class ReadSqlWork implements TopiaSqlWork { private final R request; + private final TopiaSqlTable table; public ReadSqlWork(R request, TopiaSqlTable table) { @@ -175,22 +200,4 @@ public abstract class AbstractTablesAction<R extends AbstractTablesRequest> exte } - protected List<String> getColumnNames(TopiaMetadataEntity metadataEntity, TopiaSqlTable table, ResultSet readResultSet ) throws SQLException { - - Set<String> authorizedColumnNames; - if (table.isAssociationTable()) { - authorizedColumnNames = new TreeSet<>(); - authorizedColumnNames.add(metadataEntity.getDbTableName().toLowerCase()); - authorizedColumnNames.add(table.getJoinColumnName().toLowerCase()); - } else { - authorizedColumnNames = metadataEntity.getAllDbColumnNames(); - } - - ResultSetMetaData readResultSetMetaData = readResultSet.getMetaData(); - int columnCount = readResultSetMetaData.getColumnCount(); - - return getColumnNames(readResultSetMetaData, columnCount, authorizedColumnNames); - - } - } diff --git a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesRequest.java b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesRequest.java index ef87b93..a2dde2c 100644 --- a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesRequest.java +++ b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesRequest.java @@ -42,10 +42,6 @@ public class ReplicateTablesRequest extends AbstractTablesRequest { return topiaMetaModel; } - public void setTopiaMetaModel(TopiaMetadataModel topiaMetaModel) { - this.topiaMetaModel = topiaMetaModel; - } - public static class Builder extends AbstractTablesRequestBuilder<Builder, ReplicateTablesRequest> { public Builder() { diff --git a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesAction.java b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesAction.java index c6fea87..3a520d0 100644 --- a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesAction.java +++ b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesAction.java @@ -184,9 +184,7 @@ public class UpdateTablesAction extends AbstractTablesAction<UpdateTablesRequest StringBuilder columnNamesBuilder = new StringBuilder(); - columnNames.forEach(columnName -> { - columnNamesBuilder.append(", SET ").append(columnName).append(" = ?"); - }); + columnNames.forEach(columnName -> columnNamesBuilder.append(", SET ").append(columnName).append(" = ?")); String sql = String.format(UPDATE_STATEMENT, table.getSchemaName(), diff --git a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesRequest.java b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesRequest.java index 16708a4..7ae1b0b 100644 --- a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesRequest.java +++ b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/UpdateTablesRequest.java @@ -22,6 +22,8 @@ package org.nuiton.topia.service.sql.batch.actions; * #L% */ +import org.nuiton.topia.persistence.metadata.TopiaMetadataModel; + /** * Created on 01/01/16. * @@ -30,6 +32,12 @@ package org.nuiton.topia.service.sql.batch.actions; */ public class UpdateTablesRequest extends AbstractTablesRequest { + private TopiaMetadataModel topiaMetaModel; + + public TopiaMetadataModel getTopiaMetaModel() { + return topiaMetaModel; + } + public static Builder builder() { return new Builder(); } @@ -40,6 +48,10 @@ public class UpdateTablesRequest extends AbstractTablesRequest { super(new UpdateTablesRequest()); } + public Builder setTopiaMetaModel(TopiaMetadataModel topiaMetadataModel) { + request.topiaMetaModel = topiaMetadataModel; + return this; + } } } diff --git a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java index 22c3528..7c1a6fa 100644 --- a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java +++ b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java @@ -39,6 +39,7 @@ import java.util.Objects; public class TopiaSqlTable { protected final TopiaMetadataEntity metadataEntity; + protected final TopiaMetadataEntity associationMetadataEntity; /** * Table schema name. @@ -81,17 +82,18 @@ public class TopiaSqlTable { protected final ImmutableSet<String> joinClauses; public TopiaSqlTable(TopiaMetadataEntity metadataEntity, + TopiaMetadataEntity associationMetadataEntity, String schemaName, String tableName, String fromClause, String whereClauseAlias, ImmutableSet<String> joinClauses, - boolean associationTable, String joinColumnName) { this.metadataEntity = metadataEntity; + this.associationMetadataEntity = associationMetadataEntity; this.schemaName = schemaName.toLowerCase(); this.tableName = tableName.toLowerCase(); - this.associationTable = associationTable; + this.associationTable = associationMetadataEntity != null; this.fullyTableName = this.schemaName + "." + this.tableName; this.fromClause = fromClause; this.whereClauseAlias = whereClauseAlias; @@ -103,6 +105,10 @@ public class TopiaSqlTable { return metadataEntity; } + public TopiaMetadataEntity getAssociationMetadataEntity() { + return associationMetadataEntity; + } + public String getSchemaName() { return schemaName; } @@ -165,8 +171,8 @@ public class TopiaSqlTable { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("fullyTableName", fullyTableName) - .add("associationTable", associationTable) - .toString(); + .add("fullyTableName", fullyTableName) + .add("associationTable", associationTable) + .toString(); } } diff --git a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java index fcad0a8..836e250 100644 --- a/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java +++ b/topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java @@ -181,11 +181,12 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { String whereClauseAlias, String fromClause, ImmutableSet<String> joinClauses) { - return registerTable(entityEnum, schemaName, tableName, whereClauseAlias, fromClause, joinClauses, false, null); + return registerTable(entityEnum, null, schemaName, tableName, whereClauseAlias, fromClause, joinClauses, null); } protected Builder registerAssociationTable(TopiaEntityEnum entityEnum, + TopiaMetadataEntity associationEntityEnum, String schemaName, String tableName, String whereClauseAlias, @@ -193,31 +194,32 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { ImmutableSet<String> joinClauses, String joinColumnName) { - return registerTable(entityEnum, schemaName, tableName, whereClauseAlias, fromClause, joinClauses, true, joinColumnName); + return registerTable(entityEnum, associationEntityEnum, schemaName, tableName, whereClauseAlias, fromClause, joinClauses, joinColumnName); } private Builder registerTable(TopiaEntityEnum entityEnum, + TopiaMetadataEntity associationMetadataEntity, String schemaName, String tableName, String whereClauseAlias, String fromClause, ImmutableSet<String> joinClauses, - boolean associationTable, String joinColumnName) { //TODO check that this table is not already registred - TopiaMetadataEntity metadataEntity = metadataModel.getEntity(entityEnum.name()); + TopiaMetadataEntity metadataEntity = metadataModel.getEntity(entityEnum.name()); + boolean associationTable = associationMetadataEntity != null; TopiaSqlTable table = new TopiaSqlTable( metadataEntity, + associationMetadataEntity, schemaName, tableName, fromClause, whereClauseAlias, joinClauses, - associationTable, joinColumnName); if (log.isInfoEnabled()) { @@ -348,7 +350,7 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { } @Override - public BuilderStepOnTable addAssociationTable(TopiaEntityEnum entityEnum,String associationName) { + public BuilderStepOnTable addAssociationTable(TopiaEntityEnum entityEnum, String associationName) { TopiaSqlTable parentTable = getTable(); @@ -375,6 +377,7 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { String joinColumnName = parentTable.getTableName(); registerAssociationTable(entityEnum, + parentTable.getMetadataEntity(), schemaName, tableName, whereClauseAlias, -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.