Tony CHEMIT pushed to branch develop at ultreiaio / ird-observe

Commits:

5 changed files:

Changes:

  • common-persistence/pom.xml
    ... ... @@ -47,6 +47,10 @@
    47 47 
           <groupId>org.nuiton.topia</groupId>
    48 48 
           <artifactId>topia-persistence</artifactId>
    49 49 
         </dependency>
    50 
    +    <dependency>
    51 
    +      <groupId>io.ultreia.java4all.topia</groupId>
    52 
    +      <artifactId>persistence</artifactId>
    53 
    +    </dependency>
    50 54 
         <dependency>
    51 55 
           <groupId>io.ultreia.java4all.topia</groupId>
    52 56 
           <artifactId>service-migration</artifactId>

  • common-persistence/src/main/java/fr/ird/observe/entities/BlobIdsIterator.java
    1 
    +package fr.ird.observe.entities;
    2 
    +
    3 
    +/*-
    4 
    + * #%L
    5 
    + * ObServe Toolkit :: Common Persistence
    6 
    + * %%
    7 
    + * Copyright (C) 2008 - 2020 IRD, Code Lutin, Ultreia.io
    8 
    + * %%
    9 
    + * This program is free software: you can redistribute it and/or modify
    10 
    + * it under the terms of the GNU General Public License as
    11 
    + * published by the Free Software Foundation, either version 3 of the
    12 
    + * License, or (at your option) any later version.
    13 
    + *
    14 
    + * This program is distributed in the hope that it will be useful,
    15 
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    16 
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    17 
    + * GNU General Public License for more details.
    18 
    + *
    19 
    + * You should have received a copy of the GNU General Public
    20 
    + * License along with this program.  If not, see
    21 
    + * <http://www.gnu.org/licenses/gpl-3.0.html>.
    22 
    + * #L%
    23 
    + */
    24 
    +
    25 
    +import org.nuiton.topia.persistence.TopiaApplicationContext;
    26 
    +import org.nuiton.topia.persistence.jdbc.JdbcPostgresHelper;
    27 
    +import org.nuiton.topia.persistence.metadata.TopiaMetadataEntity;
    28 
    +import org.nuiton.topia.persistence.metadata.TopiaMetadataModel;
    29 
    +
    30 
    +import java.io.BufferedReader;
    31 
    +import java.io.BufferedWriter;
    32 
    +import java.io.Closeable;
    33 
    +import java.io.IOException;
    34 
    +import java.nio.file.Files;
    35 
    +import java.nio.file.Path;
    36 
    +import java.sql.PreparedStatement;
    37 
    +import java.sql.ResultSet;
    38 
    +import java.sql.SQLException;
    39 
    +import java.util.Iterator;
    40 
    +import java.util.Objects;
    41 
    +
    42 
    +public class BlobIdsIterator implements Iterator<String>, Closeable {
    43 
    +
    44 
    +    private final TopiaMetadataModel model;
    45 
    +
    46 
    +    private final Path cachePath;
    47 
    +    private final JdbcPostgresHelper jdbcHelper;
    48 
    +    private FileIterator iterator;
    49 
    +
    50 
    +    public BlobIdsIterator(TopiaMetadataModel model, Path cachePath, TopiaApplicationContext<?> applicationContext) {
    51 
    +        this.model = Objects.requireNonNull(model);
    52 
    +        this.cachePath = Objects.requireNonNull(cachePath);
    53 
    +        this.jdbcHelper = new JdbcPostgresHelper(Objects.requireNonNull(applicationContext).getConfiguration());
    54 
    +    }
    55 
    +
    56 
    +    @Override
    57 
    +    public boolean hasNext() {
    58 
    +        return iterator().hasNext();
    59 
    +    }
    60 
    +
    61 
    +    @Override
    62 
    +    public String next() {
    63 
    +        return iterator().next();
    64 
    +    }
    65 
    +
    66 
    +    public FileIterator iterator() {
    67 
    +        if (iterator == null) {
    68 
    +            iterator = new FileIterator(cachePath, model, jdbcHelper);
    69 
    +        }
    70 
    +        return iterator;
    71 
    +    }
    72 
    +
    73 
    +    @Override
    74 
    +    public void close() throws IOException {
    75 
    +        if (iterator != null) {
    76 
    +            iterator.close();
    77 
    +            iterator = null;
    78 
    +        }
    79 
    +    }
    80 
    +
    81 
    +    private static class FileIterator implements Iterator<String>, Closeable {
    82 
    +        private final Iterator<String> iterator;
    83 
    +        private final BufferedReader bufferedReader;
    84 
    +
    85 
    +        public FileIterator(Path cachePath, TopiaMetadataModel model, JdbcPostgresHelper jdbcHelper) {
    86 
    +            if (Files.notExists(cachePath)) {
    87 
    +                try {
    88 
    +                    createCache(jdbcHelper, model, cachePath);
    89 
    +                } catch (IOException e) {
    90 
    +                    throw new IllegalStateException("can't create cache at " + cachePath, e);
    91 
    +                }
    92 
    +            }
    93 
    +            try {
    94 
    +                bufferedReader = Files.newBufferedReader(Objects.requireNonNull(cachePath));
    95 
    +            } catch (IOException e) {
    96 
    +                throw new IllegalStateException("can't create reader on " + cachePath, e);
    97 
    +            }
    98 
    +            iterator = bufferedReader.lines().iterator();
    99 
    +        }
    100 
    +
    101 
    +        @Override
    102 
    +        public boolean hasNext() {
    103 
    +            return iterator.hasNext();
    104 
    +        }
    105 
    +
    106 
    +        @Override
    107 
    +        public String next() {
    108 
    +            return iterator.next();
    109 
    +        }
    110 
    +
    111 
    +        @Override
    112 
    +        public void close() throws IOException {
    113 
    +            bufferedReader.close();
    114 
    +        }
    115 
    +
    116 
    +        private void createCache(JdbcPostgresHelper jdbcHelper, TopiaMetadataModel model, Path cachePath) throws IOException {
    117 
    +            if (Files.notExists(cachePath.getParent())) {
    118 
    +                Files.createDirectory(cachePath.getParent());
    119 
    +            }
    120 
    +            try (BufferedWriter writer = Files.newBufferedWriter(cachePath)) {
    121 
    +                for (TopiaMetadataEntity metadataEntity : model) {
    122 
    +                    if (metadataEntity.withBlob()) {
    123 
    +                        for (String blobProperty : metadataEntity.getBlobProperties()) {
    124 
    +                            fillCache(jdbcHelper, metadataEntity.getDbSchemaName(), metadataEntity.getDbTableName(), metadataEntity.getDbColumnName(blobProperty), writer);
    125 
    +                        }
    126 
    +                    }
    127 
    +                }
    128 
    +                writer.flush();
    129 
    +            }
    130 
    +        }
    131 
    +
    132 
    +        private void fillCache(JdbcPostgresHelper jdbcHelper, String dbSchemaName, String dbTableName, String dbColumnName, BufferedWriter writer) {
    133 
    +            String sql = String.format("SELECT %1$s FROM %2$s.%3$s WHERE %1$s IS NOT NULL ORDER BY topiaId ASC", dbColumnName, dbSchemaName, dbTableName);
    134 
    +            jdbcHelper.consume(c -> {
    135 
    +                try (PreparedStatement preparedStatement = c.prepareStatement(sql)) {
    136 
    +                    try (ResultSet resultSet = preparedStatement.executeQuery()) {
    137 
    +                        while (resultSet.next()) {
    138 
    +                            writer.write(resultSet.getString(1));
    139 
    +                            writer.newLine();
    140 
    +                        }
    141 
    +                    }
    142 
    +                } catch (SQLException | IOException e) {
    143 
    +                    throw new IllegalStateException(e);
    144 
    +                }
    145 
    +            });
    146 
    +        }
    147 
    +    }
    148 
    +}

  • persistence/src/main/java/fr/ird/observe/entities/ObserveTopiaApplicationContext.java
    ... ... @@ -43,6 +43,7 @@ import org.nuiton.topia.service.script.table.TopiaSqlTable;
    43 43 
     import org.nuiton.topia.service.script.table.TopiaSqlTables;
    44 44 
     import org.nuiton.topia.service.script.table.TopiaSqlTablesFactory;
    45 45
    46 
    +import java.nio.file.Path;
    46 47 
     import java.util.EnumSet;
    47 48 
     import java.util.LinkedHashSet;
    48 49 
     import java.util.Objects;
    ... ... @@ -330,6 +331,10 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat
    330 331
    331 332 
         }
    332 333
    334 
    +    public BlobIdsIterator newBlobIdsIterator(Path blobIdsPath) {
    335 
    +        return new BlobIdsIterator(getMetadataModel(), blobIdsPath, this);
    336 
    +    }
    337 
    +
    333 338 
         private static class TripReplicateTablesPredicate implements TopiaSqlTablesFactory.TopiaSqlTablesPredicate {
    334 339
    335 340 
             private final Set<TopiaMetadataEntity> done = new LinkedHashSet<>();

  • services-local/src/main/java/fr/ird/observe/services/local/ObserveSecurityHelper.java
    ... ... @@ -25,9 +25,9 @@ package fr.ird.observe.services.local;
    25 25 
     import com.google.common.collect.ImmutableSet;
    26 26 
     import fr.ird.observe.dto.ObserveDbRole;
    27 27 
     import fr.ird.observe.dto.db.ObserveDbUserDto;
    28 
    +import fr.ird.observe.entities.BlobIdsIterator;
    28 29 
     import fr.ird.observe.entities.Entities;
    29 30 
     import fr.ird.observe.entities.ObserveTopiaApplicationContext;
    30 
    -import fr.ird.observe.entities.ObserveTopiaConfiguration;
    31 31 
     import org.apache.commons.lang3.tuple.Pair;
    32 32 
     import org.apache.logging.log4j.LogManager;
    33 33 
     import org.apache.logging.log4j.Logger;
    ... ... @@ -49,6 +49,7 @@ import java.util.Collections;
    49 49 
     import java.util.HashSet;
    50 50 
     import java.util.LinkedHashSet;
    51 51 
     import java.util.List;
    52 
    +import java.util.Objects;
    52 53 
     import java.util.Set;
    53 54 
     import java.util.function.Function;
    54 55 
     import java.util.stream.Collectors;
    ... ... @@ -62,7 +63,11 @@ public class ObserveSecurityHelper {
    62 63 
         public static final String OBSERVE_SEINE_COMMON_SCHEMA_NAME = "ps_common";
    63 64 
         private static final Function<String, String> ESCAPE_STRING = input -> "\"" + input + "\"";
    64 65 
         private static final String REVOKE_ON_TABLE_ALL_PATTERN = "REVOKE ALL ON %s.%s FROM %s CASCADE;";
    66 
    +    private static final String REVOKE_ON_LARGE_OBJECT_ALL_PATTERN = "REVOKE ALL ON LARGE OBJECT %s FROM %s CASCADE;";
    65 67 
         private static final String SET_ON_TABLE_OWNER_PATTERN = "ALTER TABLE %s.%s OWNER TO %s;";
    68 
    +    private static final String SET_ON_LARGE_OBJECT_OWNER_PATTERN = "ALTER LARGE OBJECT %s OWNER TO %s;";
    69 
    +    private static final String GRANT_ON_LARGE_OBJECT_READ_PATTERN = "GRANT SELECT ON LARGE OBJECT %s TO %s;";
    70 
    +    private static final String GRANT_ON_LARGE_OBJECT_ALL_PATTERN = "GRANT ALL ON LARGE OBJECT %s TO %s;";
    66 71 
         private static final String GRANT_ON_TABLE_READ_PATTERN = "GRANT SELECT ON %s.%s TO %s;";
    67 72 
         private static final String GRANT_ON_TABLE_ALL_PATTERN = "GRANT ALL ON %s.%s TO %s;";
    68 73 
         private static final String GRANT_ON_FUNCTION_PATTERN = "GRANT EXECUTE ON FUNCTION %s TO %s;";
    ... ... @@ -82,11 +87,13 @@ public class ObserveSecurityHelper {
    82 87 
         private static final Logger log = LogManager.getLogger(ObserveSecurityHelper.class);
    83 88 
         private final JdbcPostgresHelper jdbcHelper;
    84 89 
         private final Path temporaryDirectory;
    90 
    +    private final ObserveTopiaApplicationContext applicationContext;
    85 91
    86 92
    87 
    -    public ObserveSecurityHelper(ObserveTopiaConfiguration jdbcConfiguration) {
    88 
    -        this.jdbcHelper = new JdbcPostgresHelper(jdbcConfiguration);
    89 
    -        this.temporaryDirectory = jdbcConfiguration.getTemporaryDirectory();
    93 
    +    public ObserveSecurityHelper(ObserveTopiaApplicationContext applicationContext) {
    94 
    +        this.applicationContext = Objects.requireNonNull(applicationContext);
    95 
    +        this.jdbcHelper = new JdbcPostgresHelper(applicationContext.getConfiguration());
    96 
    +        this.temporaryDirectory = applicationContext.getConfiguration().getTemporaryDirectory();
    90 97 
         }
    91 98
    92 99 
         public void applySecurity(Set<ObserveDbUserDto> users) {
    ... ... @@ -99,10 +106,11 @@ public class ObserveSecurityHelper {
    99 106 
             } catch (IOException e) {
    100 107 
                 throw new IllegalStateException("Can't create temporary path", e);
    101 108 
             }
    109 
    +        Path blobIdsPath = scriptPath.getParent().resolve(scriptPath.toFile().getName().replace(".sql",".blob-ids"));
    102 110
    103 111 
             try {
    104 112 
                 try (SqlScriptWriter sqlScriptWriter = SqlScriptWriter.of(scriptPath)) {
    105 
    -                createSecurityScript(users, sqlScriptWriter);
    113 
    +                createSecurityScript(users, sqlScriptWriter, blobIdsPath);
    106 114 
                     log.info(String.format("Generate security script %d statements(s) at %s", sqlScriptWriter.getStatementCount(), scriptPath));
    107 115 
                 }
    108 116 
                 jdbcHelper.consume(SqlScriptConsumer.of(scriptPath));
    ... ... @@ -111,7 +119,7 @@ public class ObserveSecurityHelper {
    111 119 
             }
    112 120 
         }
    113 121
    114 
    -    private void createSecurityScript(Set<ObserveDbUserDto> users, SqlScriptWriter sqlScriptWriter) {
    122 
    +    private void createSecurityScript(Set<ObserveDbUserDto> users, SqlScriptWriter sqlScriptWriter, Path blobIdsPath) throws IOException {
    115 123
    116 124 
             Set<String> schemas = ImmutableSet.<String>builder().add(SCHEMA_PUBLIC).addAll(ObserveTopiaApplicationContext.newModelSupport().getMetadataModel().getSchemaNames()).build();
    117 125
    ... ... @@ -151,6 +159,7 @@ public class ObserveSecurityHelper {
    151 159 
             Set<String> referentialEscapedNames = escapedNames(referentialNames);
    152 160 
             Set<String> unusedEscapedNames = escapedNames(unusedNames);
    153 161
    162 
    +        BlobIdsIterator blobIdsIterator = applicationContext.newBlobIdsIterator(blobIdsPath);
    154 163
    155 164 
             // suppression de tous les droits
    156 165 
             {
    ... ... @@ -165,6 +174,7 @@ public class ObserveSecurityHelper {
    165 174 
                 addOnTablesForRole(REVOKE_ON_TABLE_ALL_PATTERN, sqlScriptWriter, tables, roles);
    166 175 
                 addOnSchemaForRole(REVOKE_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, schemas, roles);
    167 176 
                 addOnFunctionForRole(REVOKE_ON_FUNCTIONS_PATTERN, sqlScriptWriter, allPostgisFunctions, roles);
    177 
    +            addOnLargeObjectForRole(REVOKE_ON_LARGE_OBJECT_ALL_PATTERN, sqlScriptWriter, blobIdsIterator, roles);
    168 178
    169 179 
             }
    170 180
    ... ... @@ -172,6 +182,8 @@ public class ObserveSecurityHelper {
    172 182 
             addOnTablesForRole(SET_ON_TABLE_OWNER_PATTERN, sqlScriptWriter, tables, administratorEscapedName);
    173 183 
             addOnSchemaForRole(GRANT_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, schemas, administratorEscapedName);
    174 184 
             addOnSchemaForRole(GRANT_ON_FUNCTION_PATTERN, sqlScriptWriter, allPostgisFunctions, administratorEscapedName);
    185 
    +        addOnLargeObjectForRole(SET_ON_LARGE_OBJECT_OWNER_PATTERN, sqlScriptWriter, blobIdsIterator, administratorEscapedName);
    186 
    +        addOnLargeObjectForRole(GRANT_ON_LARGE_OBJECT_ALL_PATTERN, sqlScriptWriter, blobIdsIterator, administratorEscapedName);
    175 187
    176 188 
             // ajout administrateurs
    177 189 
             if (!technicalEscapedNames.isEmpty()) {
    ... ... @@ -179,6 +191,7 @@ public class ObserveSecurityHelper {
    179 191 
                 addOnTablesForRole(GRANT_ON_TABLE_ALL_PATTERN, sqlScriptWriter, tables, roles);
    180 192 
                 addOnSchemaForRole(GRANT_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, schemas, roles);
    181 193 
                 addOnSchemaForRole(GRANT_ON_FUNCTION_PATTERN, sqlScriptWriter, allPostgisFunctions, roles);
    194 
    +            addOnLargeObjectForRole(GRANT_ON_LARGE_OBJECT_ALL_PATTERN, sqlScriptWriter, blobIdsIterator, roles);
    182 195 
             }
    183 196
    184 197 
             // ajout utilisateur
    ... ... @@ -187,6 +200,7 @@ public class ObserveSecurityHelper {
    187 200 
                 addOnTablesForRole(GRANT_ON_TABLE_READ_PATTERN, sqlScriptWriter, tables, roles);
    188 201 
                 addOnSchemaForRole(GRANT_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, schemas, roles);
    189 202 
                 addOnSchemaForRole(GRANT_ON_FUNCTION_PATTERN, sqlScriptWriter, allPostgisFunctions, roles);
    203 
    +            addOnLargeObjectForRole(GRANT_ON_LARGE_OBJECT_READ_PATTERN, sqlScriptWriter, blobIdsIterator, roles);
    190 204 
             }
    191 205
    192 206 
             // ajout referentiel
    ... ... @@ -273,6 +287,18 @@ public class ObserveSecurityHelper {
    273 287 
             }
    274 288 
         }
    275 289
    290 
    +    private void addOnLargeObjectForRole(String pattern, SqlScriptWriter builder, BlobIdsIterator blobIdsIterator, String role) throws IOException {
    291 
    +        try {
    292 
    +            while (blobIdsIterator.hasNext()) {
    293 
    +                String blobId = blobIdsIterator.next();
    294 
    +                builder.writeSql(String.format(pattern, blobId, role));
    295 
    +            }
    296 
    +        } finally {
    297 
    +            blobIdsIterator.close();
    298 
    +        }
    299 
    +
    300 
    +    }
    301 
    +
    276 302 
         private void addOnFunctionForRole(String pattern, SqlScriptWriter builder, Set<String> functions, String role) {
    277 303 
             for (String t : functions) {
    278 304 
                 builder.writeSql(String.format(pattern, t, role));

  • services-local/src/main/java/fr/ird/observe/services/local/service/DataSourceServiceLocal.java
    ... ... @@ -516,10 +516,8 @@ public class DataSourceServiceLocal extends ObserveServiceLocal implements DataS
    516 516 
             // pas de securité  pour les bases autres que postgresql
    517 517 
             if (dataSourceConfiguration instanceof ObserveDataSourceConfigurationTopiaPG) {
    518 518 
                 ObserveDataSourceConfigurationTopiaPG sourceConfiguration = (ObserveDataSourceConfigurationTopiaPG) dataSourceConfiguration;
    519 
    -            ObserveTopiaConfiguration topiaConfiguration;
    520 
    -            try (ObserveTopiaApplicationContext optionalTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(sourceConfiguration)) {
    521 
    -                topiaConfiguration = optionalTopiaApplicationContext.getConfiguration();
    522 
    -                new ObserveSecurityHelper(topiaConfiguration).applySecurity(users);
    519 
    +            try (ObserveTopiaApplicationContext topiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(sourceConfiguration)) {
    520 
    +                new ObserveSecurityHelper(topiaApplicationContext).applySecurity(users);
    523 521 
                 }
    524 522 
             }
    525 523 
         }

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.