r512 - in trunk/echobase-services/src/main: java/fr/ifremer/echobase/services/importdata java/fr/ifremer/echobase/services/importdata/csv resources/i18n - Echobase-commits

10 Apr 2012

Author: tchemit
Date: 2012-04-10 16:33:12 +0200 (Tue, 10 Apr 2012)
New Revision: 512

Url: http://forge.codelutin.com/repositories/revision/echobase/512

Log:
improve import progression

Modified:
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesImportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsEsduCellImportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsMapCellImportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsRegionCellImportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsVoyageImportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeAndSpeciesCategoryImportModel.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeImportModel.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndAgeCategoryImportModel.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndSizeCategoryImportModel.java
   trunk/echobase-services/src/main/resources/i18n/echobase-services_en_GB.properties
   trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java
===================================================================

--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -262,6 +262,27 @@
         }
     }
 
+    protected String getImportMessage(InputFile inputFile) {
+        String message = l_(getLocale(), "echobase.importLabel.withFile",
+                            getImportLabel(), inputFile.getFileName());
+        return message;
+    }
+
+    protected final void doFlushTransaction(int rowNumber, InputFile inputFile) {
+        if (rowNumber % 1000 == 0) {
+            // flush each 1000 imported rows
+            String message = getImportMessage(inputFile);
+            try {
+                long s0 = TimeLog.getTime();
+                flushTransaction();
+                TIME_LOG.log(s0, message);
+            } catch (TopiaException e) {
+                throw new EchoBaseTechnicalException(
+                        "Could not flush transaction " + message, e);
+            }
+        }
+    }
+
     protected Map<String, SpeciesCategory> speciesCategories = Maps.newTreeMap();
 
     protected SpeciesCategory getSpeciesCategory(

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -150,8 +150,7 @@
         InputFile inputFile = configuration.getMoviesFile();
 
         if (log.isInfoEnabled()) {
-            log.info("Starts import of acoustic datas from file " +
-                     inputFile.getFileName());
+            log.info("Starts " + getImportMessage(inputFile));
         }
 
         CsvFileImportResult importResult = new CsvFileImportResult(
@@ -199,6 +198,8 @@
             configuration.incrementsProgression();
             for (AcousticImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
+
                 configuration.incrementsProgression();
 
                 if (dataProcessingId == null) {
@@ -210,7 +211,7 @@
                 }
 
                 Date startDate = row.getCellDateStart();
-                Date endDate  = row.getCellDateEnd();
+                Date endDate = row.getCellDateEnd();
 
                 // get transect to use
                 Transit transit = voyage.getTransit(startDate);

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesImportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesImportService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesImportService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -194,9 +194,11 @@
             Import<TotalSampleImportRow> importer =
                     EchoBaseImport.newImport(csvModel, reader);
 
+            int rowNumber=0;
             configuration.incrementsProgression();
             for (TotalSampleImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
                 Operation operation = row.getOperation();
 
@@ -361,8 +363,10 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber=0;
             for (SubSampleImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
                 Operation operation = row.getOperation();
 
@@ -477,8 +481,10 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber=0;
             for (BiometrySampleImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
                 Operation operation = row.getOperation();
 

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -161,9 +161,11 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber=0;
             for (OperationImportRow row : importer) {
 
                 configuration.incrementsProgression();
+                doFlushTransaction(++rowNumber, inputFile);
                 Vessel vessel = row.getVessel();
 
                 Operation operation = row.getOperation();
@@ -224,9 +226,11 @@
             Import<OperationMetadataValueImportRow> importer =
                     EchoBaseImport.newImport(csvModel, reader);
 
+            int rowNumber=0;
             configuration.incrementsProgression();
             for (OperationMetadataValueImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
                 Operation operation = row.getOperation();
 
@@ -281,9 +285,11 @@
             Import<GearMetadataValueImportRow> importer =
                     EchoBaseImport.newImport(csvModel, reader);
 
+            int rowNumber=0;
             configuration.incrementsProgression();
             for (GearMetadataValueImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
                 Operation operation = row.getOperation();
 

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsEsduCellImportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsEsduCellImportService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsEsduCellImportService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -242,8 +242,11 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber = 0;
+
             for (EsduResultByEchotypeImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Cell cell = row.getCell();
@@ -320,8 +323,10 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber = 0;
             for (EsduResultByEchotypeAndSpeciesCategoryImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Cell cell = row.getCell();
@@ -401,8 +406,10 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber = 0;
             for (EsduResultBySpeciesAndSizeCategoryImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Cell cell = row.getCell();
@@ -502,8 +509,10 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber = 0;
             for (EsduResultBySpeciesAndAgeCategoryImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Cell cell = row.getCell();

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsMapCellImportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsMapCellImportService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsMapCellImportService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -191,8 +191,10 @@
 
             configuration.incrementsProgression();
 
+            int rowNumber=0;
             for (MapCellImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Cell cell = create(dao, row.getCell());

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsRegionCellImportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsRegionCellImportService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsRegionCellImportService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -166,9 +166,13 @@
             Cell cell = null;
 
             configuration.incrementsProgression();
+            int rowNumber = 0;
             for (RegionCellImportRow row : importer) {
                 configuration.incrementsProgression();
 
+                rowNumber++;
+                doFlushTransaction(rowNumber, inputFile);
+
                 if (cell == null || !row.getName().equals(cell.getName())) {
 
                     cell = create(dao,
@@ -240,8 +244,11 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber=0;
             for (RegionCellAssociationImportRow row : importer) {
 
+                rowNumber++;
+                doFlushTransaction(rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Cell regionCell = row.getRegionCell();
@@ -328,8 +335,11 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber=0;
             for (RegionCellResultImportRow row : importer) {
 
+                rowNumber++;
+                doFlushTransaction(rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Cell cell = row.getCell();

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsVoyageImportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsVoyageImportService.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsVoyageImportService.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -182,8 +182,10 @@
                     EchoBaseImport.newImport(csvModel, reader);
 
             configuration.incrementsProgression();
+            int rowNumber = 0;
             for (LengthWeightKeyImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
                 Voyage voyage = row.getVoyage();
 
@@ -249,8 +251,10 @@
 
             configuration.incrementsProgression();
 
+            int rowNumber = 0;
             for (LengthAgeKeyImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Voyage voyage = row.getVoyage();
@@ -304,8 +308,10 @@
 
             configuration.incrementsProgression();
 
+            int rowNumber = 0;
             for (EchotypeImportRow row : importer) {
 
+                doFlushTransaction(++rowNumber, inputFile);
                 configuration.incrementsProgression();
 
                 Voyage voyage = row.getVoyage();

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeAndSpeciesCategoryImportModel.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeAndSpeciesCategoryImportModel.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeAndSpeciesCategoryImportModel.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -50,7 +50,8 @@
             Species.PROPERTY_BARACOUDA_CODE,
             EsduResultByEchotypeAndSpeciesCategoryImportRow.PROPERTY_SIZE_CATEGORY,
             EsduResultByEchotypeAndSpeciesCategoryImportRow.PROPERTY_ECHOTYPE,
-            EsduResultByEchotypeAndSpeciesCategoryImportRow.PROPERTY_DATA_QUALITY
+            EsduResultByEchotypeAndSpeciesCategoryImportRow.PROPERTY_DATA_QUALITY,
+            EsduResultByEchotypeAndSpeciesCategoryImportRow.PROPERTY_VOYAGE
     };
 
     public EsduResultByEchotypeAndSpeciesCategoryImportModel(

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeImportModel.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeImportModel.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultByEchotypeImportModel.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -46,7 +46,8 @@
     public static final String[] COLUMN_NAMES_TO_EXCLUDE = {
             CELL_NAME,
             EsduResultByEchotypeImportRow.PROPERTY_ECHOTYPE,
-            EsduResultByEchotypeImportRow.PROPERTY_DATA_QUALITY
+            EsduResultByEchotypeImportRow.PROPERTY_DATA_QUALITY,
+            EsduResultByEchotypeImportRow.PROPERTY_VOYAGE
     };
 
     public EsduResultByEchotypeImportModel(

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndAgeCategoryImportModel.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndAgeCategoryImportModel.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndAgeCategoryImportModel.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -48,7 +48,8 @@
             Species.PROPERTY_BARACOUDA_CODE,
             EsduResultBySpeciesAndAgeCategoryImportRow.PROPERTY_AGE_CATEGORY,
             EsduResultBySpeciesAndAgeCategoryImportRow.PROPERTY_AGE_CATEGORY_MEANING,
-            EsduResultBySpeciesAndAgeCategoryImportRow.PROPERTY_DATA_QUALITY
+            EsduResultBySpeciesAndAgeCategoryImportRow.PROPERTY_DATA_QUALITY,
+            EsduResultBySpeciesAndAgeCategoryImportRow.PROPERTY_VOYAGE
     };
 
     public EsduResultBySpeciesAndAgeCategoryImportModel(

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndSizeCategoryImportModel.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndSizeCategoryImportModel.java	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/EsduResultBySpeciesAndSizeCategoryImportModel.java	2012-04-10 14:33:12 UTC (rev 512)
@@ -48,7 +48,8 @@
             Species.PROPERTY_BARACOUDA_CODE,
             EsduResultBySpeciesAndSizeCategoryImportRow.PROPERTY_SIZE_CATEGORY,
             EsduResultBySpeciesAndSizeCategoryImportRow.PROPERTY_SIZE_CATEGORY_MEANING,
-            EsduResultBySpeciesAndSizeCategoryImportRow.PROPERTY_DATA_QUALITY
+            EsduResultBySpeciesAndSizeCategoryImportRow.PROPERTY_DATA_QUALITY,
+            EsduResultBySpeciesAndSizeCategoryImportRow.PROPERTY_VOYAGE
     };
 
     public EsduResultBySpeciesAndSizeCategoryImportModel(

Modified: trunk/echobase-services/src/main/resources/i18n/echobase-services_en_GB.properties
===================================================================
--- trunk/echobase-services/src/main/resources/i18n/echobase-services_en_GB.properties	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/resources/i18n/echobase-services_en_GB.properties	2012-04-10 14:33:12 UTC (rev 512)
@@ -43,6 +43,7 @@
 echobase.importError.no.voyage.imported=
 echobase.importError.transect.notfound=
 echobase.importError.transit.notfound=
+echobase.importLabel.withFile=%s with file %s
 echobase.info.modifiedProperty=Property '%s', old value\: '%s', new value\: '%s'
 echobase.info.onePropertyModified=One property was modified\:
 echobase.info.somePropertiesModified=%s properties were modified\:

Modified: trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties
===================================================================
--- trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties	2012-04-10 08:41:08 UTC (rev 511)
+++ trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties	2012-04-10 14:33:12 UTC (rev 512)
@@ -43,6 +43,7 @@
 echobase.importError.no.voyage.imported=Aucun voyage importé
 echobase.importError.transect.notfound=Transit non trouvé pour la campagne %s et la date %s et le navire %s (ligne %s)
 echobase.importError.transit.notfound=Transit non trouvé pour la campagne %s et la date %s (ligne %s)
+echobase.importLabel.withFile=%s avec le fichier %s
 echobase.info.modifiedProperty=Propriété '%s', ancienne valeur \: '%s', nouvelle valeur \: '%s'
 echobase.info.newEntityCreated=Une nouvelle entité a été créée %s
 echobase.info.onePropertyModified=Une propriété a été modifiée \:

    

r512 - in trunk/echobase-services/src/main: java/fr/ifremer/echobase/services/importdata java/fr/ifremer/echobase/services/importdata/csv resources/i18n

tchemit＠users.forge.codelutin.com

tags

participants (1)