Tony CHEMIT pushed to branch develop at ultreiaio / ird-observe Commits: a864fb66 by Tony Chemit at 2022-03-16T18:47:49+01:00 Impossible de créer une base centrale vierge, quelque soit la méthode appliquée - See ultreiaio/ird-observe#2214 - - - - - 11 changed files: - client/core/src/main/java/fr/ird/observe/client/datasource/api/ObserveSwingDataSource.java - client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/DataSourceHelper.java - client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/StorageBackupUILauncher.java - client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/StorageUIModel.java - client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/launchers/CreateLauncher.java - + core/persistence/java/src/main/java/fr/ird/observe/entities/DataSourceCreateHelper.java - core/persistence/java/src/main/java/fr/ird/observe/entities/ObserveTopiaApplicationContext.java - core/services/client/src/main/java/fr/ird/observe/services/client/AnonymousServiceClient.java - core/services/local/src/main/java/fr/ird/observe/services/local/service/AnonymousServiceLocalSupport.java - server/core/src/main/filtered-resources/META-INF/mapping-api-client.wm - server/core/src/main/java/fr/ird/observe/server/controller/AnonymousServiceRestApi.java Changes: ===================================== client/core/src/main/java/fr/ird/observe/client/datasource/api/ObserveSwingDataSource.java ===================================== @@ -40,6 +40,7 @@ import fr.ird.observe.dto.db.DatabaseConnexionNotAuthorizedException; import fr.ird.observe.dto.db.DatabaseNotFoundException; import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; import fr.ird.observe.dto.db.configuration.DataSourceCreateWithNoReferentialImportException; +import fr.ird.observe.dto.db.configuration.DataSourceImportConfigurationDto; import fr.ird.observe.dto.db.configuration.IncompatibleDataSourceCreateConfigurationException; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfiguration; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfigurationAndConnection; @@ -241,23 +242,26 @@ public class ObserveSwingDataSource extends AbstractJavaBean implements ObserveS fireOpened(); } - public void createFrom(ObserveSwingDataSource localSource) + public void create(DataSourceCreateConfigurationDto createDto) throws IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException, DatabaseNotFoundException, DatabaseConnexionNotAuthorizedException, BabModelVersionException { - DataSourceCreateConfigurationDto createDto = new DataSourceCreateConfigurationDto(); - createDto.setLeaveOpenSource(true); - createDto.setImportReferentialDataSourceConfiguration(localSource.getConfiguration()); - create(createDto); + checkIsNotOpen(); + fireNewMessage(t("observe.ui.datasource.storage.message.creating", getLabel())); + fireOpening(); + AnonymousService dataSourceService = getAnonymousService(); + setConnection(dataSourceService.create(getConfiguration(), createDto)); + fireNewMessage(t("observe.ui.datasource.storage.message.created", getLabel())); + fireOpened(); } - public void create(DataSourceCreateConfigurationDto createDto) + public void createAndImport(DataSourceImportConfigurationDto createDto) throws IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException, DatabaseNotFoundException, DatabaseConnexionNotAuthorizedException, BabModelVersionException { checkIsNotOpen(); fireNewMessage(t("observe.ui.datasource.storage.message.creating", getLabel())); fireOpening(); AnonymousService dataSourceService = getAnonymousService(); - setConnection(dataSourceService.create(getConfiguration(), createDto)); + setConnection(dataSourceService.createAndImport(getConfiguration(), createDto)); fireNewMessage(t("observe.ui.datasource.storage.message.created", getLabel())); fireOpened(); } ===================================== client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/DataSourceHelper.java ===================================== @@ -259,6 +259,31 @@ public class DataSourceHelper { throw new IllegalStateException("Can't come here"); } } + if (model.getChooseDb().doImportData()) { + + String dbLabel = I18n.n("observe.ui.datasource.storage.label.data.import.db"); + switch (model.getChooseDb().getDataImportMode()) { + case IMPORT_EXTERNAL_DUMP: + File dumpFile = ObstunaAdminAction.CREATE.equals(model.getAdminAction()) ? model.getDataSourceModel().getDumpFile() : model.getDumpFile(); + importReferenceConfig.setImportDatabase(TopiaSqlScript.of(dumpFile.toURI())); + break; + case IMPORT_REMOTE_STORAGE: { + // import referentiel from a remote db + ObserveDataSourceConfigurationTopiaPG dataSourceConfig = toPGStorageConfig(model.getDataSourceModel(), dbLabel); + dataSourceConfig.setCanMigrate(false); + importReferenceConfig.setImportDataConfiguration(dataSourceConfig, model.getSelectDataModel().getSelectedDataIds()); + } + break; + case IMPORT_SERVER_STORAGE: { + // import referentiel from a server db + ObserveDataSourceConfigurationRest dataSourceConfig = toRestStorageConfig(model.getDataSourceModel(), dbLabel); + importReferenceConfig.setImportDataConfiguration(dataSourceConfig, model.getSelectDataModel().getSelectedDataIds()); + break; + } + default: + throw new IllegalStateException("Can't come here"); + } + } return importReferenceConfig; } @@ -311,6 +336,7 @@ public class DataSourceHelper { return importDataSource; } + static ObserveDataSourceConfigurationTopiaH2 toH2StorageConfig(StorageUIModel model, String label) { ObserveDataSourceConfigurationTopiaH2 h2Config = model.getLocalConfig().getConfiguration(); return ObserveDataSourceConfigurationTopiaH2.create( ===================================== client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/StorageBackupUILauncher.java ===================================== @@ -33,6 +33,7 @@ import fr.ird.observe.dto.db.BabModelVersionException; import fr.ird.observe.dto.db.DatabaseConnexionNotAuthorizedException; import fr.ird.observe.dto.db.DatabaseNotFoundException; import fr.ird.observe.dto.db.configuration.DataSourceConnectMode; +import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; import fr.ird.observe.dto.db.configuration.DataSourceCreateMode; import fr.ird.observe.dto.db.configuration.DataSourceCreateWithNoReferentialImportException; import fr.ird.observe.dto.db.configuration.DataSourceInitMode; @@ -137,7 +138,10 @@ public class StorageBackupUILauncher extends StorageUILauncher { try (ObserveSwingDataSource targetSource = dataSourcesManager.newTemporaryH2DataSource("Backup-" + backupFile.getName())) { // create temporary source - targetSource.createFrom(incomingSource); + DataSourceCreateConfigurationDto createDto = new DataSourceCreateConfigurationDto(); + createDto.setLeaveOpenSource(true); + createDto.setImportReferentialDataSourceConfiguration(incomingSource.getConfiguration()); + targetSource.create(createDto); progressModel.increments(); // insert selected data ===================================== client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/StorageUIModel.java ===================================== @@ -671,7 +671,9 @@ public class StorageUIModel extends WizardModel<StorageStep> implements WithClie } break; case CONFIG_REFERENTIAL: - + if (chooseDb.getReferentielImportMode() == null) { + return false; + } switch (chooseDb.getReferentielImportMode()) { case EMPTY: @@ -703,7 +705,9 @@ public class StorageUIModel extends WizardModel<StorageStep> implements WithClie break; case CONFIG_DATA: - + if (chooseDb.getDataImportMode() == null) { + return false; + } switch (chooseDb.getDataImportMode()) { case EMPTY: ===================================== client/datasource/editor/api/src/main/java/fr/ird/observe/client/datasource/editor/api/wizard/launchers/CreateLauncher.java ===================================== @@ -22,26 +22,28 @@ package fr.ird.observe.client.datasource.editor.api.wizard.launchers; * #L% */ +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import fr.ird.observe.client.datasource.api.ObserveSwingDataSource; import fr.ird.observe.client.datasource.editor.api.wizard.DataSourceHelper; import fr.ird.observe.client.datasource.editor.api.wizard.ObstunaAdminAction; import fr.ird.observe.client.datasource.editor.api.wizard.StorageUI; import fr.ird.observe.client.datasource.editor.api.wizard.StorageUIModel; -import fr.ird.observe.dto.db.BabModelVersionException; -import fr.ird.observe.dto.db.DatabaseConnexionNotAuthorizedException; -import fr.ird.observe.dto.db.DatabaseNotFoundException; import fr.ird.observe.dto.db.configuration.DataSourceConnectMode; import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; -import fr.ird.observe.dto.db.configuration.DataSourceCreateWithNoReferentialImportException; -import fr.ird.observe.dto.db.configuration.IncompatibleDataSourceCreateConfigurationException; +import fr.ird.observe.dto.db.configuration.DataSourceImportConfigurationDto; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfiguration; import fr.ird.observe.dto.db.configuration.ObserveDataSourceInformation; -import fr.ird.observe.navigation.tree.selection.SelectionTreeModel; +import io.ultreia.java4all.util.Version; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.nuiton.jaxx.runtime.JAXXContext; +import org.nuiton.topia.persistence.script.TopiaSqlScript; +import org.nuiton.topia.service.sql.request.CreateDatabaseRequest; import java.awt.Window; +import java.util.Collection; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -54,6 +56,7 @@ import java.util.Set; public class CreateLauncher extends RemoteUILauncher { private static final Logger log = LogManager.getLogger(CreateLauncher.class); DataSourceCreateConfigurationDto createConfigurationDto; + DataSourceImportConfigurationDto importConfiguration; public CreateLauncher(JAXXContext context, Window frame, DataSourceConnectMode connectMode) { super(ObstunaAdminAction.CREATE, context, frame, connectMode); @@ -67,26 +70,82 @@ public class CreateLauncher extends RemoteUILauncher { //FIXME This logic should be done in StorageModel not here, like this! model.getChooseDb().setCanMigrate(true); model.getRemoteConfig().setCanCreateDatabase(true); + model.getServerConfig().setCanCreateDatabase(true); super.init(ui); } @Override - protected ObserveSwingDataSource initTask(StorageUIModel model) throws Exception { + protected ObserveSwingDataSource initTask(StorageUIModel model) { ObserveSwingDataSource dataSource = getDataSource(model); createConfigurationDto = DataSourceHelper.toImportReferentielSourceConfig(model); + boolean importReferential = createConfigurationDto.isImportReferential(); + boolean importData = createConfigurationDto.isImportData(); + if (!importReferential && !importData) { + importConfiguration = null; + return dataSource; + } + Version version = dataSource.getConfiguration().getModelVersion(); + importConfiguration = new DataSourceImportConfigurationDto(); + importConfiguration.setMigrate(createConfigurationDto.isMigrate()); + ObserveDataSourceConfiguration importReferentialConfig = createConfigurationDto.getImportReferentialDataSourceConfiguration(); + ObserveDataSourceConfiguration importDataConfig = createConfigurationDto.getImportDataDataSourceConfiguration(); + boolean importStandaloneReferential = importReferential; + if (importReferentialConfig != null) { - log.info(String.format("Use referentiel import data source %s", importReferentialConfig.getLabel())); - checkImportDbVersion(model, importReferentialConfig); + log.info(String.format("Will use referentiel import data source %s", importReferentialConfig.getLabel())); } - // data import - ObserveSwingDataSource importDataConfig = DataSourceHelper.toImportDataSourceConfig(model); if (importDataConfig != null) { - log.info(String.format("Use data import data source %s", importDataConfig.getLabel())); - checkImportDbVersion(model, importDataConfig.getConfiguration()); - SelectionTreeModel dataModel = model.getSelectDataModel(); - Set<String> importDataIds = dataModel.getSelectedDataIds(); - createConfigurationDto.setImportDataConfiguration(importDataConfig.getConfiguration(), importDataIds); + log.info(String.format("Will use data import data source %s", importDataConfig.getLabel())); + importStandaloneReferential = importReferentialConfig != null && !importReferentialConfig.equals(importDataConfig); + } + + boolean referentialImported = false; + if (importStandaloneReferential) { + try (ObserveSwingDataSource importDataSource = Objects.requireNonNull(ui.getModel().getDataSourcesManager().newDataSource(importReferentialConfig))) { + log.info(String.format("Prepare standalone referential dump from %s", importDataSource.getLabel())); + ObserveSwingDataSource.doOpenSource(importDataSource); + model.checkImportDbVersion(importDataSource); + CreateDatabaseRequest request = CreateDatabaseRequest.builder(true, version).addStandaloneTables().build(); + TopiaSqlScript createSqlScript = importDataSource.getDataSourceService().produceCreateSqlScript(request); + importConfiguration.setReferentialDump(createSqlScript); + } + referentialImported = true; + } + + if (importData) { + CreateDatabaseRequest.Builder requestBuilder = CreateDatabaseRequest.builder(true, version); + if (!referentialImported) { + log.info("Add referential to data dump"); + requestBuilder.addStandaloneTables(); + } + Set<String> importDataIds = createConfigurationDto.getImportDataIds(); + Multimap<String, String> idsByType = ArrayListMultimap.create(); + for (String importDataId : importDataIds) { + //FIXME + String className = importDataId.contains(".ll.") ? "fr.ird.observe.entities.data.ll.common.Trip" : "fr.ird.observe.entities.data.ps.common.Trip"; + idsByType.put(className, importDataId); + } + for (Map.Entry<String, Collection<String>> entry : idsByType.asMap().entrySet()) { + String type = entry.getKey(); + Set<String> dataIds = Set.copyOf(entry.getValue()); + log.info(String.format("Add %d data of type: %s", dataIds.size(), type)); + requestBuilder.dataIdsToAdd(type, dataIds); + } + try (ObserveSwingDataSource importDataSource = Objects.requireNonNull(ui.getModel().getDataSourcesManager().newDataSource(importDataConfig))) { + log.info(String.format("Prepare data dump from %s", importDataSource.getLabel())); + ObserveSwingDataSource.doOpenSource(importDataSource); + model.checkImportDbVersion(importDataSource); + CreateDatabaseRequest request = requestBuilder.build(); + TopiaSqlScript dump = importDataSource.getDataSourceService().produceCreateSqlScript(request); + if (referentialImported) { + // already a referential dump, so just data dump + importConfiguration.setDataDump(dump); + } else { + // referential dump not existing, will use this one + importConfiguration.setReferentialDump(dump); + } + } } return dataSource; } @@ -95,7 +154,11 @@ public class CreateLauncher extends RemoteUILauncher { protected void execute(ObserveSwingDataSource dataSource, ObserveDataSourceInformation dataSourceInformation) throws Exception { log.info("Create db..."); try { - dataSource.create(createConfigurationDto); + if (importConfiguration != null) { + dataSource.createAndImport(importConfiguration); + } else { + dataSource.create(createConfigurationDto); + } log.info(String.format("Open [%s] and create it.", dataSource.getLabel())); } finally { if (dataSource.isOpen()) { @@ -104,12 +167,5 @@ public class CreateLauncher extends RemoteUILauncher { } } - void checkImportDbVersion(StorageUIModel model, ObserveDataSourceConfiguration dataSourceConfig) throws DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException, BabModelVersionException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException { - try (ObserveSwingDataSource importDataSource = Objects.requireNonNull(model.getDataSourcesManager().newDataSource(dataSourceConfig))) { - importDataSource.open(); - model.checkImportDbVersion(importDataSource); - } - } - } ===================================== core/persistence/java/src/main/java/fr/ird/observe/entities/DataSourceCreateHelper.java ===================================== @@ -0,0 +1,179 @@ +package fr.ird.observe.entities; + +/*- + * #%L + * ObServe Core :: Persistence :: Java + * %% + * Copyright (C) 2008 - 2022 IRD, Code Lutin, Ultreia.io + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import fr.ird.observe.dto.db.BabModelVersionException; +import fr.ird.observe.dto.db.DatabaseConnexionNotAuthorizedException; +import fr.ird.observe.dto.db.DatabaseNotFoundException; +import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; +import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfiguration; +import fr.ird.observe.dto.db.configuration.topia.ObserveDataSourceConfigurationTopiaH2; +import io.ultreia.java4all.util.Version; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.topia.persistence.TopiaIdFactory; +import org.nuiton.topia.persistence.script.TopiaSqlScript; +import org.nuiton.topia.service.migration.TopiaMigrationService; +import org.nuiton.topia.service.sql.request.CreateDatabaseRequest; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +/** + * Created on 16/03/2022. + * + * @author Tony Chemit - dev@tchemit.fr + * @since 6.0.7 + */ +public class DataSourceCreateHelper { + + private static final Logger log = LogManager.getLogger(DataSourceCreateHelper.class); + + public static void create(ToolkitTopiaApplicationContextSupport<?> applicationContext, ObserveDataSourceConfiguration dataSourceConfiguration, DataSourceCreateConfigurationDto dataSourceCreateConfiguration) throws BabModelVersionException, DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException { + log.debug(String.format("Create topia application context: %s", applicationContext)); + Path temporaryDirectory = dataSourceConfiguration.getTemporaryDirectory(); + if (dataSourceCreateConfiguration.isImportDatabase()) { + log.info("Create new database from a script."); + TopiaSqlScript importDatabase = dataSourceCreateConfiguration.getImportDatabase(); + boolean migrate = dataSourceCreateConfiguration.isMigrate(); + TopiaMigrationService migrationService = (TopiaMigrationService) applicationContext.getMigrationService(); + if (dataSourceConfiguration.isLocal()) { + applicationContext.executeSqlStatements(importDatabase); + if (migrate) { + migrationService.createSchemaIfNotExist(); + migrationService.runSchemaMigration(); + } + } else { // base postgres + // do import in temporary h2 database + File tmpDir; + try { + tmpDir = Files.createTempDirectory(temporaryDirectory, "obstuna").toFile(); + } catch (IOException e) { + throw new IllegalStateException("could not create temporary directory ", e); + } + ObserveDataSourceConfigurationTopiaH2 temporaryConfiguration = applicationContext.createTemporaryConfiguration(tmpDir, dataSourceConfiguration.getModelVersion()); + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.createTopiaApplicationContext(temporaryConfiguration, false); + temporaryTopiaApplicationContext.executeSqlStatements(importDatabase); + if (migrate) { + temporaryTopiaApplicationContext.getMigrationService().createSchemaIfNotExist(); + temporaryTopiaApplicationContext.getMigrationService().runSchemaMigration(); + } + CreateDatabaseRequest request = CreateDatabaseRequest.builder(true, dataSourceConfiguration.getModelVersion()).addSchema().addStandaloneTables().addAllData().build(); + TopiaSqlScript dump = applicationContext.getSqlService().consume(request); + applicationContext.executeSqlStatements(dump); + migrationService.createSchemaIfNotExist(); + migrationService.runSchemaMigration(); + temporaryTopiaApplicationContext.close(); + File databaseFile = temporaryConfiguration.getDatabaseFile(); + if (!databaseFile.delete()) { + throw new IllegalStateException("could not delete " + databaseFile); + } + } + } else { + boolean importReferential = dataSourceCreateConfiguration.isImportReferential(); + boolean importData = dataSourceCreateConfiguration.isImportData(); + // si le referentiel n'est pas importé on aliment la table lastUpdateDate + if (!importReferential) { + applicationContext.insertLastUpdateDate(); + } + boolean importStandaloneReferential = importReferential; + if (importReferential && importData) { + // Si on est sur la même source de données pour les deux imports, on fait tout sur la même base temporaire + ObserveDataSourceConfiguration importReferentialDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); + ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); + importStandaloneReferential = !importReferentialDataSourceConfiguration.equals(importDataSourceConfiguration); + } + boolean referentialImported = false; + if (importStandaloneReferential) { + log.info("Import referential."); + ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); + if (importDataSourceConfiguration.getTemporaryDirectory() == null) { + importDataSourceConfiguration.setTemporaryDirectory(temporaryDirectory); + } + Version dbVersion = Version.valueOf(applicationContext.getModelVersion()); + CreateDatabaseRequest request = CreateDatabaseRequest.builder(!dataSourceConfiguration.isLocal(), dbVersion) + .addStandaloneTables().build(); + TopiaSqlScript referentialDump; + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = null; + try { + temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.createTopiaApplicationContext(importDataSourceConfiguration, false); + referentialDump = temporaryTopiaApplicationContext.getSqlService().consume(request); + applicationContext.executeSqlStatements(referentialDump); + } finally { + if (!dataSourceCreateConfiguration.isLeaveOpenSource() && temporaryTopiaApplicationContext != null) { + temporaryTopiaApplicationContext.close(); + } + } + referentialImported = true; + } + + if (importData) { + ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); + if (importDataSourceConfiguration.getTemporaryDirectory() == null) { + importDataSourceConfiguration.setTemporaryDirectory(temporaryDirectory); + } + Version dbVersion = Version.valueOf(applicationContext.getModelVersion()); + CreateDatabaseRequest.Builder requestBuilder = CreateDatabaseRequest.builder(!dataSourceConfiguration.isLocal(), dbVersion); + if (!referentialImported) { + log.info("Get referential."); + requestBuilder.addStandaloneTables(); + } + + // Récupération du dump qui contient les données + Set<String> importDataIds = dataSourceCreateConfiguration.getImportDataIds(); + TopiaIdFactory idFactory = applicationContext.getTopiaIdFactory(); + Multimap<String, String> idsByType = ArrayListMultimap.create(); + for (String importDataId : importDataIds) { + Class<TopiaEntity> className = idFactory.getClassName(importDataId); + idsByType.put(className.getName(), importDataId); + } + log.info(String.format("Get data: %s", importDataIds)); + for (Map.Entry<String, Collection<String>> entry : idsByType.asMap().entrySet()) { + requestBuilder.dataIdsToAdd(entry.getKey(), Set.copyOf(entry.getValue())); + } + TopiaSqlScript dataDump; + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = null; + CreateDatabaseRequest request = requestBuilder.build(); + try { + temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.createTopiaApplicationContext(importDataSourceConfiguration, false); + dataDump = temporaryTopiaApplicationContext.getSqlService().consume(request); + } finally { + if (!dataSourceCreateConfiguration.isLeaveOpenSource() && temporaryTopiaApplicationContext != null) { + temporaryTopiaApplicationContext.close(); + } + } + log.info(String.format("Import data%s", request.isAddStandaloneTables() ? " and referential." : ".")); + applicationContext.executeSqlStatements(dataDump); + } + } + } + +} ===================================== core/persistence/java/src/main/java/fr/ird/observe/entities/ObserveTopiaApplicationContext.java ===================================== @@ -22,14 +22,12 @@ package fr.ird.observe.entities; * #L% */ -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; import fr.ird.observe.dto.db.BabModelVersionException; import fr.ird.observe.dto.db.DatabaseConnexionNotAuthorizedException; import fr.ird.observe.dto.db.DatabaseNotFoundException; import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; +import fr.ird.observe.dto.db.configuration.DataSourceImportConfigurationDto; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfiguration; -import fr.ird.observe.dto.db.configuration.topia.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.dto.referential.ReferentialDto; import fr.ird.observe.entities.data.DataGroupByEntity; import fr.ird.observe.spi.module.BusinessProject; @@ -37,16 +35,12 @@ import fr.ird.observe.spi.referential.differential.DifferentialMetaModel; import fr.ird.observe.spi.referential.synchro.OneSideSqlResultBuilder; import fr.ird.observe.spi.script.ToolkitEntitySqlScriptModel; import fr.ird.observe.spi.service.ServiceContext; -import io.ultreia.java4all.util.Version; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.nuiton.topia.persistence.TagValues; import org.nuiton.topia.persistence.TopiaConfigurationExtension; import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.topia.persistence.TopiaIdFactory; import org.nuiton.topia.persistence.TopiaPersistenceContextConstructorParameterSupport; -import org.nuiton.topia.persistence.script.TopiaSqlScript; -import org.nuiton.topia.service.migration.TopiaMigrationService; import org.nuiton.topia.service.sql.blob.TopiaEntitySqlBlobModel; import org.nuiton.topia.service.sql.metadata.TopiaMetadataModel; import org.nuiton.topia.service.sql.metadata.TopiaMetadataModelPaths; @@ -54,19 +48,11 @@ import org.nuiton.topia.service.sql.model.TopiaEntitySqlModel; import org.nuiton.topia.service.sql.plan.copy.TopiaEntitySqlCopyPlanModel; import org.nuiton.topia.service.sql.plan.delete.TopiaEntitySqlDeletePlanModel; import org.nuiton.topia.service.sql.plan.replicate.TopiaEntitySqlReplicatePlanModel; -import org.nuiton.topia.service.sql.request.CreateDatabaseRequest; import org.nuiton.topia.service.sql.usage.TopiaEntitySqlUsageModel; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; -import java.util.Collection; import java.util.Date; -import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.function.Consumer; public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicationContext { @@ -187,131 +173,16 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat return ObserveTopiaEntitySqlModelResource.get().getScriptModel(); } + @Deprecated public void create(ObserveDataSourceConfiguration dataSourceConfiguration, DataSourceCreateConfigurationDto dataSourceCreateConfiguration) throws BabModelVersionException, DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException { - log.debug(String.format("Create topia application context: %s", this)); - if (dataSourceCreateConfiguration.isImportDatabase()) { - log.info("Create new database from a script."); - TopiaSqlScript importDatabase = dataSourceCreateConfiguration.getImportDatabase(); - boolean migrate = dataSourceCreateConfiguration.isMigrate(); - TopiaMigrationService migrationService = getMigrationService(); - if (dataSourceConfiguration.isLocal()) { - executeSqlStatements(importDatabase); - if (migrate) { - migrationService.createSchemaIfNotExist(); - migrationService.runSchemaMigration(); - } - } else { // base postgres - // do import in temporary h2 database - File tmpDir; - try { - Path temporaryDirectory = dataSourceConfiguration.getTemporaryDirectory(); - tmpDir = Files.createTempDirectory(temporaryDirectory, "obstuna").toFile(); - } catch (IOException e) { - throw new IllegalStateException("could not create temporary directory ", e); - } - ObserveDataSourceConfigurationTopiaH2 temporaryConfiguration = createTemporaryConfiguration(tmpDir, dataSourceConfiguration.getModelVersion()); - ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.createTopiaApplicationContext(temporaryConfiguration, false); - temporaryTopiaApplicationContext.executeSqlStatements(importDatabase); - if (migrate) { - temporaryTopiaApplicationContext.getMigrationService().createSchemaIfNotExist(); - temporaryTopiaApplicationContext.getMigrationService().runSchemaMigration(); - } - CreateDatabaseRequest request = CreateDatabaseRequest.builder(true, dataSourceConfiguration.getModelVersion()).addSchema().addStandaloneTables().addAllData().build(); - TopiaSqlScript dump = getSqlService().consume(request); - executeSqlStatements(dump); - migrationService.createSchemaIfNotExist(); - migrationService.runSchemaMigration(); - temporaryTopiaApplicationContext.close(); - File databaseFile = temporaryConfiguration.getDatabaseFile(); - if (!databaseFile.delete()) { - throw new IllegalStateException("could not delete " + databaseFile); - } - } - } else { - boolean importReferential = dataSourceCreateConfiguration.isImportReferential(); - boolean importData = dataSourceCreateConfiguration.isImportData(); - // si le referentiel n'est pas importé on aliment la table lastUpdateDate - if (!importReferential) { - insertLastUpdateDate(); - } - boolean importStandaloneReferential = importReferential; - if (importReferential && importData) { - // Si on est sur la même source de données pour les deux imports, on fait tout sur la même base temporaire - ObserveDataSourceConfiguration importReferentialDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); - ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); - importStandaloneReferential = !importReferentialDataSourceConfiguration.equals(importDataSourceConfiguration); - } - boolean referentialImported = false; - if (importStandaloneReferential) { - log.info("Import referential."); - ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); - Version dbVersion = Version.valueOf(getModelVersion()); - CreateDatabaseRequest request = CreateDatabaseRequest.builder(!dataSourceConfiguration.isLocal(), dbVersion) - .addStandaloneTables().build(); - TopiaSqlScript referentialDump; - ObserveTopiaApplicationContext temporaryTopiaApplicationContext = null; - try { - temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.createTopiaApplicationContext(importDataSourceConfiguration, false); -// ObserveDataSourceConnection importDataSourceConnection = serviceContext().newService(importDataSourceConfiguration, AnonymousService.class).open(importDataSourceConfiguration); -// DataSourceService dumpProducerService = serviceContext().newService(importDataSourceConnection, DataSourceService.class); - referentialDump = temporaryTopiaApplicationContext.getSqlService().consume(request); - executeSqlStatements(referentialDump); - } finally { - if (!dataSourceCreateConfiguration.isLeaveOpenSource() && temporaryTopiaApplicationContext != null) { - temporaryTopiaApplicationContext.close(); - } - } - referentialImported = true; - } - - if (importData) { - ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); - Version dbVersion = Version.valueOf(getModelVersion()); - CreateDatabaseRequest.Builder requestBuilder = CreateDatabaseRequest.builder(!dataSourceConfiguration.isLocal(), dbVersion); - if (!referentialImported) { - log.info("Get referential."); - requestBuilder.addStandaloneTables(); - } - - // Récupération du dump qui contient les données - Set<String> importDataIds = dataSourceCreateConfiguration.getImportDataIds(); - TopiaIdFactory idFactory = getTopiaIdFactory(); - Multimap<String, String> idsByType = ArrayListMultimap.create(); - for (String importDataId : importDataIds) { - Class<TopiaEntity> className = idFactory.getClassName(importDataId); - idsByType.put(className.getName(), importDataId); - } - log.info(String.format("Get data: %s", importDataIds)); - for (Map.Entry<String, Collection<String>> entry : idsByType.asMap().entrySet()) { - requestBuilder.dataIdsToAdd(entry.getKey(), Set.copyOf(entry.getValue())); - } - TopiaSqlScript dataDump; - ObserveTopiaApplicationContext temporaryTopiaApplicationContext = null; - CreateDatabaseRequest request = requestBuilder.build(); - try { - temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.createTopiaApplicationContext(importDataSourceConfiguration, false); -// ObserveDataSourceConnection importDataSourceConnection = serviceContext().newService(importDataSourceConfiguration, AnonymousService.class).open(importDataSourceConfiguration); -// DataSourceService dumpProducerService = serviceContext().newService(importDataSourceConnection, DataSourceService.class); -// dataDump = dumpProducerService.produceCreateSqlScript(request); - dataDump = temporaryTopiaApplicationContext.getSqlService().consume(request); - } finally { - if (!dataSourceCreateConfiguration.isLeaveOpenSource() && temporaryTopiaApplicationContext != null) { - temporaryTopiaApplicationContext.close(); -// serviceContext().newService(importDataSourceConfiguration, DataSourceService.class).close(); - } - } - log.info(String.format("Import data%s", request.isAddStandaloneTables() ? " and referential." : ".")); - executeSqlStatements(dataDump); - } - } + DataSourceCreateHelper.create(this, dataSourceConfiguration, dataSourceCreateConfiguration); } - private ObserveDataSourceConfigurationTopiaH2 createTemporaryConfiguration(File tmpDirectory, Version version) { - File dbDirectory = new File(tmpDirectory, "obstuna" + UUID.randomUUID()); - ObserveDataSourceConfigurationTopiaH2 result = ObserveDataSourceConfigurationTopiaH2.create( - "obtunaTmp", dbDirectory, "obstuna", "sa", "sa".toCharArray(), true, true, version); - result.setTemporaryDirectory(tmpDirectory.toPath()); - return result; + public void createAndImport(DataSourceImportConfigurationDto dataSourceCreateConfiguration) throws BabModelVersionException, DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException { + log.debug(String.format("Create topia application context: %s", this)); + executeSqlStatements(dataSourceCreateConfiguration.getReferentialDump()); + if (dataSourceCreateConfiguration.isImportData()) { + executeSqlStatements(dataSourceCreateConfiguration.getDataDump()); + } } - } ===================================== core/services/client/src/main/java/fr/ird/observe/services/client/AnonymousServiceClient.java ===================================== @@ -27,6 +27,7 @@ import fr.ird.observe.dto.db.DatabaseConnexionNotAuthorizedException; import fr.ird.observe.dto.db.DatabaseNotFoundException; import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; import fr.ird.observe.dto.db.configuration.DataSourceCreateWithNoReferentialImportException; +import fr.ird.observe.dto.db.configuration.DataSourceImportConfigurationDto; import fr.ird.observe.dto.db.configuration.IncompatibleDataSourceCreateConfigurationException; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfiguration; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConnection; @@ -44,6 +45,16 @@ public class AnonymousServiceClient extends GeneratedAnonymousServiceClient { return dataSourceConnection; } + @Override + public ObserveDataSourceConnection createAndImport(ObserveDataSourceConfiguration config, DataSourceImportConfigurationDto createConfiguration) throws IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException, BabModelVersionException, DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException { + ObserveServiceInitializer initializer = getServiceContext().getInitializer(); + initializer.setConfiguration(config); + ObserveDataSourceConnection dataSourceConnection = super.createAndImport(config, createConfiguration); + initializer.setConnection(dataSourceConnection); + initializer.setConfiguration(null); + return dataSourceConnection; + } + @Override public ObserveDataSourceConnection open(ObserveDataSourceConfiguration config) throws DatabaseNotFoundException, DatabaseConnexionNotAuthorizedException, BabModelVersionException { ObserveServiceInitializer initializer = getServiceContext().getInitializer(); ===================================== core/services/local/src/main/java/fr/ird/observe/services/local/service/AnonymousServiceLocalSupport.java ===================================== @@ -30,6 +30,7 @@ import fr.ird.observe.dto.db.DatabaseNotFoundException; import fr.ird.observe.dto.db.DatabaseVersionIsTooHighException; import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; import fr.ird.observe.dto.db.configuration.DataSourceCreateWithNoReferentialImportException; +import fr.ird.observe.dto.db.configuration.DataSourceImportConfigurationDto; import fr.ird.observe.dto.db.configuration.IncompatibleDataSourceCreateConfigurationException; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfiguration; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConnection; @@ -124,6 +125,22 @@ public class AnonymousServiceLocalSupport extends ObserveServiceLocal implements return config.toConnection(topiaApplicationContext.getAuthenticationToken(), dataSourceInformation); } + @Override + public ObserveDataSourceConnection createAndImport(ObserveDataSourceConfiguration config, DataSourceImportConfigurationDto createConfiguration) throws IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException, BabModelVersionException, DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException { + createConfiguration.validateConfiguration(getApplicationLocale()); + setTemporaryDirectory(config); + boolean initSchema = true; + ObserveTopiaApplicationContext topiaApplicationContext = ObserveTopiaApplicationContextFactory.createTopiaApplicationContext(config, initSchema); + try { + topiaApplicationContext.createAndImport(createConfiguration); + } catch (Exception e) { + close(config, topiaApplicationContext); + throw e; + } + ObserveDataSourceInformation dataSourceInformation = getDataSourceInformation(config, topiaApplicationContext.getConfiguration(), false); + return config.toConnection(topiaApplicationContext.getAuthenticationToken(), dataSourceInformation); + } + @Override public ObserveDataSourceConnection open(ObserveDataSourceConfiguration config) throws DatabaseNotFoundException, DatabaseConnexionNotAuthorizedException, BabModelVersionException { setTemporaryDirectory(config); ===================================== server/core/src/main/filtered-resources/META-INF/mapping-api-client.wm ===================================== @@ -26,6 +26,7 @@ GET /AnonymousService/applySecurity Anon GET /AnonymousService/checkCanConnect AnonymousServiceRestApi.checkCanConnect GET /AnonymousService/checkCanConnectOrBeEmpty AnonymousServiceRestApi.checkCanConnectOrBeEmpty GET /AnonymousService/create AnonymousServiceRestApi.create +POST /AnonymousService/createAndImport AnonymousServiceRestApi.createAndImport GET /AnonymousService/generateHomeId AnonymousServiceRestApi.generateHomeId GET /AnonymousService/getModelVersion AnonymousServiceRestApi.getModelVersion GET /AnonymousService/getServerVersion AnonymousServiceRestApi.getServerVersion ===================================== server/core/src/main/java/fr/ird/observe/server/controller/AnonymousServiceRestApi.java ===================================== @@ -28,6 +28,7 @@ import fr.ird.observe.dto.db.DatabaseConnexionNotAuthorizedException; import fr.ird.observe.dto.db.DatabaseNotFoundException; import fr.ird.observe.dto.db.configuration.DataSourceCreateConfigurationDto; import fr.ird.observe.dto.db.configuration.DataSourceCreateWithNoReferentialImportException; +import fr.ird.observe.dto.db.configuration.DataSourceImportConfigurationDto; import fr.ird.observe.dto.db.configuration.IncompatibleDataSourceCreateConfigurationException; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConfiguration; import fr.ird.observe.dto.db.configuration.ObserveDataSourceConnection; @@ -49,6 +50,12 @@ public class AnonymousServiceRestApi extends GeneratedAnonymousServiceRestApi { private static final Logger log = LogManager.getLogger(AnonymousServiceRestApi.class); + @Override + public ObserveDataSourceInformation checkCanConnectOrBeEmpty(ObserveDataSourceConfiguration config) throws DatabaseNotFoundException, DatabaseConnexionNotAuthorizedException, BabModelVersionException { + ObserveWebUserSession session = getRequestContext().getConfigurationSession(config); + return getService().checkCanConnectOrBeEmpty(session.getConfiguration()); + } + @Override public ObserveDataSourceInformation checkCanConnect(ObserveDataSourceConfiguration config) throws DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException, BabModelVersionException { ObserveWebUserSession session = getRequestContext().getConfigurationSession(config); @@ -65,6 +72,14 @@ public class AnonymousServiceRestApi extends GeneratedAnonymousServiceRestApi { return authenticatedSession.toRestConnection(config); } + @Override + public ObserveDataSourceConnection createAndImport(ObserveDataSourceConfiguration config, DataSourceImportConfigurationDto createConfiguration) throws IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException, BabModelVersionException, DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException { + ObserveWebUserSession configurationSession = getRequestContext().getConfigurationSession(config); + ObserveDataSourceConnection connection = getService().createAndImport(configurationSession.getConfiguration(), createConfiguration); + ObserveWebUserSession authenticatedSession = getSecurityApplicationContext().registerAuthenticatedSession(configurationSession, connection); + return authenticatedSession.toRestConnection(config); + } + @Override public ObserveDataSourceConnection open(ObserveDataSourceConfiguration config) throws DatabaseNotFoundException, DatabaseConnexionNotAuthorizedException, BabModelVersionException { ObserveWebUserSession configurationSession = getRequestContext().getConfigurationSession(config); View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/commit/a864fb664be6b4f57a7edf8a1a... -- View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/commit/a864fb664be6b4f57a7edf8a1a... You're receiving this email because of your account on gitlab.com.