Tony CHEMIT pushed to branch develop at ultreiaio / ird-observe Commits: d0cee04c by Tony Chemit at 2020-12-22T19:35:16+01:00 Add referential replication order more easy from topia application context - See #1691 - - - - - 17341bb8 by Tony Chemit at 2020-12-22T19:37:31+01:00 Add referential replication order more easy from topia application context - Closes #1691 - - - - - 1acf1864 by Tony Chemit at 2020-12-22T19:37:38+01:00 fix typo - - - - - 8 changed files: - models/persistence/src/main/java/fr/ird/observe/entities/ObserveTopiaApplicationContext.java - models/persistence/src/main/java/fr/ird/observe/entities/ObserveTopiaEntitySqlModelSupportImpl.java - server/core/src/main/filtered-resources/mapping - services/api/src/main/java/fr/ird/observe/services/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeEngine.java - services/api/src/main/java/fr/ird/observe/services/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalService.java - services/api/src/main/java/fr/ird/observe/services/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeResult.java - services/local-impl/src/main/java/fr/ird/observe/services/local/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalServiceLocal.java - services/local-impl/src/main/java/fr/ird/observe/services/local/service/actions/synchro/referential/sql/ReplaceSqlStatementGenerator.java Changes: ===================================== models/persistence/src/main/java/fr/ird/observe/entities/ObserveTopiaApplicationContext.java ===================================== @@ -22,6 +22,8 @@ package fr.ird.observe.entities; * #L% */ +import com.google.common.collect.ImmutableSet; +import fr.ird.observe.dto.referential.ReferentialDto; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hibernate.tool.hbm2ddl.SchemaExport; @@ -165,6 +167,10 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat return getTopiaEntitySqlModelSupport().getSqlModel(); } + public ImmutableSet<Class<? extends ReferentialDto>> referentialForReplicationDto() { + return getTopiaEntitySqlModelSupport().referentialForReplicationDto(); + } + @Override public boolean equals(Object o) { if (this == o) return true; ===================================== models/persistence/src/main/java/fr/ird/observe/entities/ObserveTopiaEntitySqlModelSupportImpl.java ===================================== @@ -22,7 +22,12 @@ package fr.ird.observe.entities; * #L% */ +import com.google.common.collect.ImmutableSet; +import fr.ird.observe.dto.referential.ReferentialDto; import fr.ird.observe.entities.referential.ReferentialEntity; +import fr.ird.observe.spi.PersistenceBusinessProject; +import fr.ird.observe.spi.context.ReferentialDtoEntityContext; +import io.ultreia.java4all.lang.Objects2; import org.nuiton.topia.persistence.metadata.sql.TopiaEntitySqlDescriptor; import org.nuiton.topia.persistence.metadata.sql.TopiaEntitySqlDescriptors; @@ -34,6 +39,8 @@ import org.nuiton.topia.persistence.metadata.sql.TopiaEntitySqlDescriptors; */ public class ObserveTopiaEntitySqlModelSupportImpl extends ObserveTopiaEntitySqlModelSupport { + private ImmutableSet<Class<? extends ReferentialDto>> referentialForReplication; + public TopiaEntitySqlDescriptors dataPsTripForReplication() { return getSqlModel().getReplicationOrderByEntryPointDescriptors(fr.ird.observe.entities.data.ps.common.Trip.class.getName()); } @@ -64,4 +71,21 @@ public class ObserveTopiaEntitySqlModelSupportImpl extends ObserveTopiaEntitySql return getSqlModel().getDescriptor(fqn); } + @SuppressWarnings({"rawtypes", "unchecked"}) + public ImmutableSet<Class<? extends ReferentialDto>> referentialForReplicationDto() { + if (referentialForReplication == null) { + ImmutableSet.Builder<Class<? extends ReferentialDto>> builder = ImmutableSet.builder(); + for (TopiaEntitySqlDescriptor descriptor : getSqlModel().getReplicationOrderWithStandaloneDescriptors()) { + String entityName = descriptor.getTable().getEntityName(); + Class<? extends ReferentialEntity> entityType = Objects2.forName(entityName); + ReferentialDtoEntityContext spi = PersistenceBusinessProject.fromReferentialEntity(entityType); + if (spi != null) { + Class<? extends ReferentialDto> dtoType = spi.toDtoType(); + builder.add(dtoType); + } + } + referentialForReplication = builder.build(); + } + return referentialForReplication; + } } ===================================== server/core/src/main/filtered-resources/mapping ===================================== @@ -81,6 +81,7 @@ POST /api/v1/actions/synchro/referential/legacy/UnidirectionalReferentialSynch GET /api/v1/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalService/filterIdsUsedInLocalSource v1.actions.synchro.referential.legacy.UnidirectionalReferentialSynchronizeLocalServiceRestApi.filterIdsUsedInLocalSource GET /api/v1/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalService/generateSqlRequests v1.actions.synchro.referential.legacy.UnidirectionalReferentialSynchronizeLocalServiceRestApi.generateSqlRequests GET /api/v1/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalService/getLocalSourceReferentialToDelete v1.actions.synchro.referential.legacy.UnidirectionalReferentialSynchronizeLocalServiceRestApi.getLocalSourceReferentialToDelete +GET /api/v1/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalService/referentialReplicationOrder v1.actions.synchro.referential.legacy.UnidirectionalReferentialSynchronizeLocalServiceRestApi.referentialReplicationOrder POST /api/v1/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalService/updateLastUpdateDates v1.actions.synchro.referential.legacy.UnidirectionalReferentialSynchronizeLocalServiceRestApi.updateLastUpdateDates POST /api/v1/actions/synchro/referential/ng/ReferentialSynchronizeService/executeSqlListRequest v1.actions.synchro.referential.ng.ReferentialSynchronizeServiceRestApi.executeSqlListRequest POST /api/v1/actions/synchro/referential/ng/ReferentialSynchronizeService/produceSqlListRequest v1.actions.synchro.referential.ng.ReferentialSynchronizeServiceRestApi.produceSqlListRequest ===================================== services/api/src/main/java/fr/ird/observe/services/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeEngine.java ===================================== @@ -78,6 +78,8 @@ public class UnidirectionalReferentialSynchronizeEngine { .addAll(rightDiffStates.keySet()) .build(); + ImmutableSet<Class<? extends ReferentialDto>> replicationOrder = localService.referentialReplicationOrder(); + ImmutableSet.Builder<UnidirectionalReferentialSynchronizeRequest.Builder<?>> referentialSynchronizeRequestBuilders = ImmutableSet.builder(); UnidirectionalReferentialSynchronizeCallbackRequests callbackRequests = new UnidirectionalReferentialSynchronizeCallbackRequests(); @@ -86,7 +88,7 @@ public class UnidirectionalReferentialSynchronizeEngine { ObserveBusinessProject businessProject = ObserveBusinessProject.get(); - for (Class<? extends ReferentialDto> dtoType : businessProject.getReferentialTypes()) { + for (Class<? extends ReferentialDto> dtoType : replicationOrder) { if (types.contains(dtoType)) { @SuppressWarnings("rawtypes") DtoReferenceDefinition referentialDefinition = businessProject.getOptionalReferenceDefinition(dtoType).orElseThrow(IllegalStateException::new); @@ -141,7 +143,7 @@ public class UnidirectionalReferentialSynchronizeEngine { result.flushRequest(referentialSynchronizeRequest); - Set<String> generatedSqlRequests = localService.generateSqlRequests(referentialSynchronizeRequest); + List<String> generatedSqlRequests = localService.generateSqlRequests(referentialSynchronizeRequest); for (String sqlStatement : generatedSqlRequests) { if (sqlStatement.startsWith("INSERT")) { ===================================== services/api/src/main/java/fr/ird/observe/services/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalService.java ===================================== @@ -22,6 +22,7 @@ package fr.ird.observe.services.service.actions.synchro.referential.legacy; * #L% */ +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import fr.ird.observe.dto.reference.ReferentialDtoReference; import fr.ird.observe.dto.reference.ReferentialDtoReferenceSet; @@ -43,6 +44,14 @@ import io.ultreia.java4all.http.spi.Post; */ public interface UnidirectionalReferentialSynchronizeLocalService extends ObserveService { + /** + * @return the replication order to insert data + * @since 8.0.2 + */ + @MethodCredential(Permission.READ_REFERENTIAL) + @Get + ImmutableSet<Class<? extends ReferentialDto>> referentialReplicationOrder(); + /** * Pour un référentiel d'un type donné (son nom est donné), détecte les référentiels dont * les identifiants sont passés en paramètres qui sont réellement utilisés dans la source locale. @@ -73,16 +82,16 @@ public interface UnidirectionalReferentialSynchronizeLocalService extends Observ * * @param <D> type des référentiels à traiter * @param request la demande des actions à produire pour un référentiel donné - * @return l'ensemble des requètes sql à appliquer. + * @return l'ensemble des requêtes sql à appliquer. */ @MethodCredential(Permission.READ_REFERENTIAL) @Get - <D extends ReferentialDto> ImmutableSet<String> generateSqlRequests(UnidirectionalReferentialSynchronizeRequest<D> request); + <D extends ReferentialDto> ImmutableList<String> generateSqlRequests(UnidirectionalReferentialSynchronizeRequest<D> request); /** - * Pour appliquer les requètes sql de mise à jour du réferentiel. + * Pour appliquer les requêtes sql de mise à jour du référentiel. * - * @param sqlRequests les requètes sql à appliquer + * @param sqlRequests les requêtes sql à appliquer */ @MethodCredential(Permission.READ_REFERENTIAL) //FIXME::Security Il faut introduire une nouvelle permission EditReferentialPermission et ici bien utiliser WriteReferentialPermission ===================================== services/api/src/main/java/fr/ird/observe/services/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeResult.java ===================================== @@ -23,14 +23,12 @@ package fr.ird.observe.services.service.actions.synchro.referential.legacy; */ import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; import fr.ird.observe.dto.ObserveDto; import fr.ird.observe.dto.referential.ReferentialDto; import org.apache.commons.lang3.tuple.Pair; -import java.util.Collection; import java.util.LinkedHashSet; -import java.util.Map; +import java.util.List; import java.util.Set; /** @@ -44,10 +42,10 @@ import java.util.Set; public class UnidirectionalReferentialSynchronizeResult implements ObserveDto { private final Set<Class<? extends ReferentialDto>> referentialNames; - private final Multimap<Class<? extends ReferentialDto>, String> referentialAdded; - private final Multimap<Class<? extends ReferentialDto>, String> referentialUpdated; - private final Multimap<Class<? extends ReferentialDto>, Pair<String, String>> referentialReplaced; - private final Multimap<Class<? extends ReferentialDto>, String> referentialRemoved; + private final ArrayListMultimap<Class<? extends ReferentialDto>, String> referentialAdded; + private final ArrayListMultimap<Class<? extends ReferentialDto>, String> referentialUpdated; + private final ArrayListMultimap<Class<? extends ReferentialDto>, Pair<String, String>> referentialReplaced; + private final ArrayListMultimap<Class<? extends ReferentialDto>, String> referentialRemoved; public UnidirectionalReferentialSynchronizeResult() { this.referentialNames = new LinkedHashSet<>(); @@ -65,58 +63,36 @@ public class UnidirectionalReferentialSynchronizeResult implements ObserveDto { return referentialNames; } - public Collection<String> getReferentialAdded(Class<? extends ReferentialDto> referentialName) { + public List<String> getReferentialAdded(Class<? extends ReferentialDto> referentialName) { return referentialAdded.get(referentialName); } - public Collection<String> getReferentialUpdated(Class<? extends ReferentialDto> referentialName) { + public List<String> getReferentialUpdated(Class<? extends ReferentialDto> referentialName) { return referentialUpdated.get(referentialName); } - public Collection<Pair<String, String>> getReferentialReplaced(Class<? extends ReferentialDto> referentialName) { + public List<Pair<String, String>> getReferentialReplaced(Class<? extends ReferentialDto> referentialName) { return referentialReplaced.get(referentialName); } - public Collection<String> getReferentialRemoved(Class<? extends ReferentialDto> referentialName) { + public List<String> getReferentialRemoved(Class<? extends ReferentialDto> referentialName) { return referentialRemoved.get(referentialName); } void flushRequest(UnidirectionalReferentialSynchronizeRequest<?> referentialSynchronizeRequest) { - Class<? extends ReferentialDto> referentialName = referentialSynchronizeRequest.getReferentialName(); - if (referentialSynchronizeRequest.withReferentialToAdd()) { - - for (ReferentialDto referentialDto : referentialSynchronizeRequest.getReferentialToAdd()) { - addReferentialAdded(referentialName, referentialDto.getId()); - } - + referentialSynchronizeRequest.getReferentialToAdd().forEach(referentialDto -> addReferentialAdded(referentialName, referentialDto.getId())); } - if (referentialSynchronizeRequest.withReferentialToUpdate()) { - - for (ReferentialDto referentialDto : referentialSynchronizeRequest.getReferentialToUpdate()) { - addReferentialUpdated(referentialName, referentialDto.getId()); - } - + referentialSynchronizeRequest.getReferentialToUpdate().forEach(referentialDto -> addReferentialUpdated(referentialName, referentialDto.getId())); } - if (referentialSynchronizeRequest.withReferentialToRemove()) { - - for (String id : referentialSynchronizeRequest.getReferentialToRemove()) { - addReferentialRemoved(referentialName, id); - } - + referentialSynchronizeRequest.getReferentialToRemove().forEach(id -> addReferentialRemoved(referentialName, id)); } - if (referentialSynchronizeRequest.withReferentialToReplace()) { - - for (Map.Entry<String, String> entry : referentialSynchronizeRequest.getReferentialToReplace().entrySet()) { - addReferentialReplaced(referentialName, entry.getKey(), entry.getValue()); - } - + referentialSynchronizeRequest.getReferentialToReplace().forEach((key, value) -> addReferentialReplaced(referentialName, key, value)); } - } private void addReferentialAdded(Class<? extends ReferentialDto> referentialName, String id) { ===================================== services/local-impl/src/main/java/fr/ird/observe/services/local/service/actions/synchro/referential/legacy/UnidirectionalReferentialSynchronizeLocalServiceLocal.java ===================================== @@ -32,7 +32,6 @@ import fr.ird.observe.entities.ObserveEntityEnum; import fr.ird.observe.entities.referential.ReferentialEntity; import fr.ird.observe.services.local.ObserveServiceContextLocal; import fr.ird.observe.services.local.service.ObserveServiceLocal; -import fr.ird.observe.services.local.service.actions.synchro.referential.sql.ApplySqlRequestWork; import fr.ird.observe.services.local.service.actions.synchro.referential.sql.DeleteSqlStatementGenerator; import fr.ird.observe.services.local.service.actions.synchro.referential.sql.InsertSqlStatementGenerator; import fr.ird.observe.services.local.service.actions.synchro.referential.sql.ReplaceSqlStatementGenerator; @@ -45,12 +44,18 @@ import fr.ird.observe.services.service.usage.DtoUsageCountResult; import fr.ird.observe.services.service.usage.UsageService; import fr.ird.observe.spi.ObservePersistenceBusinessProject; import fr.ird.observe.spi.context.ReferentialDtoEntityContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.nuiton.topia.persistence.TopiaDao; import org.nuiton.topia.persistence.metadata.TopiaMetadataEntity; import org.nuiton.topia.persistence.metadata.TopiaMetadataModel; -import org.nuiton.topia.persistence.support.TopiaSqlWork; +import org.nuiton.topia.persistence.script.SqlScriptReader; +import org.nuiton.topia.persistence.script.TopiaSqlScript; +import java.io.IOException; +import java.nio.file.Path; import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -62,6 +67,7 @@ import java.util.Set; * @since 5.0 */ public class UnidirectionalReferentialSynchronizeLocalServiceLocal extends ObserveServiceLocal implements UnidirectionalReferentialSynchronizeLocalService { + private static final Logger log = LogManager.getLogger(UnidirectionalReferentialSynchronizeLocalServiceLocal.class); private LastUpdateDateService lastUpdateDateService; private UsageService usageService; @@ -73,6 +79,11 @@ public class UnidirectionalReferentialSynchronizeLocalServiceLocal extends Obser usageService = serviceContext.newService(UsageService.class); } + @Override + public ImmutableSet<Class<? extends ReferentialDto>> referentialReplicationOrder() { + return serviceContext.getTopiaApplicationContext().referentialForReplicationDto(); + } + @Override public <D extends ReferentialDto> ImmutableSet<String> filterIdsUsedInLocalSource(Class<D> dtoType, ImmutableSet<String> ids) { Set<String> result = new LinkedHashSet<>(); @@ -89,13 +100,12 @@ public class UnidirectionalReferentialSynchronizeLocalServiceLocal extends Obser public <D extends ReferentialDto, R extends ReferentialDtoReference> ReferentialDtoReferenceSet<R> getLocalSourceReferentialToDelete(Class<D> dtoType, ImmutableSet<String> ids) { ReferentialDtoEntityContext<D, R, ?, ?> modelContext = ObservePersistenceBusinessProject.fromReferentialDto(dtoType); Class<? extends ReferentialEntity<D, R>> entityType = modelContext.toEntityType(); - Class<R> referenceType = modelContext.toReferenceType(); - return getLocalSourceReferentialToDelete0(entityType, referenceType, ids); + return getLocalSourceReferentialToDelete0(entityType, ids); } @Override - public <D extends ReferentialDto> ImmutableSet<String> generateSqlRequests(UnidirectionalReferentialSynchronizeRequest<D> request) { - Set<String> result = new LinkedHashSet<>(); + public <D extends ReferentialDto> ImmutableList<String> generateSqlRequests(UnidirectionalReferentialSynchronizeRequest<D> request) { + List<String> result = new LinkedList<>(); Class<D> dtoType = request.getReferentialName(); Class<ReferentialEntity<D, ?>> entityType = ObservePersistenceBusinessProject.fromReferentialDtoWeak(dtoType).toEntityType(); ObserveEntityEnum entityEnum = ObserveEntityEnum.valueOf(entityType); @@ -137,13 +147,23 @@ public class UnidirectionalReferentialSynchronizeLocalServiceLocal extends Obser result.addAll(sql); } } - return ImmutableSet.copyOf(result); + return ImmutableList.copyOf(result); } @Override public void applySqlRequests(ImmutableSet<String> sqlRequests) { - TopiaSqlWork applySqlWork = new ApplySqlRequestWork(sqlRequests); - getTopiaPersistenceContext().getSqlSupport().doSqlWork(applySqlWork); + + Path scriptPath = serviceContext.getTemporaryDirectoryRoot().toPath().resolve("apply-UnidirectionalReferentialSynchronize-" + serviceContext.now().getTime() + "-sql"); + + TopiaSqlScript topiaSqlScript = TopiaSqlScript.of(scriptPath); + topiaSqlScript.setLocation(() -> SqlScriptReader.of(sqlRequests)); + try { + topiaSqlScript.copy(scriptPath); + } catch (IOException e) { + throw new IllegalStateException("Can't write script at: " + scriptPath, e); + } + log.info(String.format("Will execute script (with %d statement(s): %s", sqlRequests.size(), scriptPath)); + getTopiaPersistenceContext().executeSqlScript(topiaSqlScript); } @Override @@ -157,7 +177,7 @@ public class UnidirectionalReferentialSynchronizeLocalServiceLocal extends Obser return usageCount.getCount().values().stream().reduce(Long::sum).orElse(0L); } - private <E extends ReferentialEntity<D, R>, D extends ReferentialDto, R extends ReferentialDtoReference> ReferentialDtoReferenceSet<R> getLocalSourceReferentialToDelete0(Class<E> entityType, Class<R> referenceType, ImmutableSet<String> ids) { + private <E extends ReferentialEntity<D, R>, D extends ReferentialDto, R extends ReferentialDtoReference> ReferentialDtoReferenceSet<R> getLocalSourceReferentialToDelete0(Class<E> entityType, ImmutableSet<String> ids) { TopiaDao<E> dao = getTopiaPersistenceContext().getDao(entityType); List<E> entities = dao.forTopiaIdIn(ids).findAll(); ReferentialDtoEntityContext<D, R, E, ?> spi = ObservePersistenceBusinessProject.fromReferentialEntity(entityType); ===================================== services/local-impl/src/main/java/fr/ird/observe/services/local/service/actions/synchro/referential/sql/ReplaceSqlStatementGenerator.java ===================================== @@ -70,7 +70,7 @@ public class ReplaceSqlStatementGenerator { for (TopiaMetadataAssociation replacementStruct : associations) { Class<? extends TopiaEntity> entityType = ObserveEntityEnum.valueOf(replacementStruct.getOwner().getType()).getContract(); if (ReferentialEntity.class.isAssignableFrom(entityType)) { - // do not update referentials associations (see https://gitlab.com/ultreiaio/ird-observe/issues/1065) + // do not update referential associations (see https://gitlab.com/ultreiaio/ird-observe/issues/1065) continue; } String sql = SqlStatements.generateAssociationUpdateStatement(replacementStruct, sourceId, replacementId); View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/compare/5ffad0061b1ace8710dd1c0a0... -- View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/compare/5ffad0061b1ace8710dd1c0a0... You're receiving this email because of your account on gitlab.com.