Author: tchemit Date: 2012-09-11 18:59:23 +0200 (Tue, 11 Sep 2012) New Revision: 650 Url: http://forge.codelutin.com/repositories/revision/echobase/650 Log: improve importDb progression + make it works even for small jvm Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/AbstractEchobaseActionConfiguration.java trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/ProgressionModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/AbstractImportDbStrategy.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/EchoBaseImportStrategy.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ImportDbServiceTest.java Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java 2012-09-11 16:55:09 UTC (rev 649) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java 2012-09-11 16:59:23 UTC (rev 650) @@ -37,9 +37,12 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.util.Collection; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; /** * Usefull methods on io. @@ -110,7 +113,6 @@ } finally { reader.close(); } - } } finally { writer.close(); @@ -118,13 +120,48 @@ } public static int countLines(File file) { + InputStream inputStream = null; + try { + inputStream = FileUtils.openInputStream(file); + int result = countLines(inputStream); + inputStream.close(); + return result; + } catch (IOException e) { + throw new EchoBaseTechnicalException("Could not countLines", e); + } finally { + if (inputStream != null) { + IOUtils.closeQuietly(inputStream); + } + } + } + + public static long countLines(ZipFile zipFile, + Iterable<ZipEntry> entries) throws IOException { + long result = 0l; + for (ZipEntry entry : entries) { + InputStream inputStream = zipFile.getInputStream(entry); + try { + int i = countLines(inputStream); + if (log.isDebugEnabled()) { + log.debug("entry: " + entry.getName() + " - " + i); + } + result += i; + inputStream.close(); + } finally { + if (inputStream != null) { + IOUtils.closeQuietly(inputStream); + } + } + } + return result; + } + + public static int countLines(InputStream file) { int result = 0; LineNumberReader reader = null; try { - reader = new LineNumberReader( - new InputStreamReader(FileUtils.openInputStream(file)) - ); + reader = new LineNumberReader(new InputStreamReader(file)); while (reader.readLine() != null) { } result = reader.getLineNumber(); Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/AbstractEchobaseActionConfiguration.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/AbstractEchobaseActionConfiguration.java 2012-09-11 16:55:09 UTC (rev 649) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/AbstractEchobaseActionConfiguration.java 2012-09-11 16:59:23 UTC (rev 650) @@ -168,7 +168,6 @@ return endTime; } - @Override public void setEndTime(long endTime) { this.endTime = endTime; Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/ProgressionModel.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/ProgressionModel.java 2012-09-11 16:55:09 UTC (rev 649) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/services/ProgressionModel.java 2012-09-11 16:59:23 UTC (rev 650) @@ -22,6 +22,8 @@ */ package fr.ifremer.echobase.services; +import org.nuiton.topia.persistence.csv.CsvProgressionModel; + import java.io.Serializable; /** @@ -30,18 +32,8 @@ * @author tchemit <chemit@codelutin.com> * @since 0.3 */ -public interface ProgressionModel extends Serializable { +public interface ProgressionModel extends CsvProgressionModel, Serializable { - long getNbSteps(); - - float getProgression(); - - void setProgression(float progression); - - void incrementsProgression(); - - void setNbSteps(long nbSteps); - long getStartTime(); void setStartTime(long startTime); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/AbstractImportDbStrategy.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/AbstractImportDbStrategy.java 2012-09-11 16:55:09 UTC (rev 649) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/AbstractImportDbStrategy.java 2012-09-11 16:59:23 UTC (rev 650) @@ -29,6 +29,7 @@ import fr.ifremer.echobase.entities.EntityModificationLog; import fr.ifremer.echobase.entities.ImportLog; import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.io.EchoBaseIOUtil; import fr.ifremer.echobase.persistence.EchoBaseDbMeta; import fr.ifremer.echobase.services.EchoBaseServiceSupport; import fr.ifremer.echobase.services.importdata.ImportException; @@ -96,20 +97,24 @@ validateAssociationEntries(dbMeta, associations); if (model.isComputeSteps()) { - int size = tables.size() + associations.size(); - model.setNbSteps(size + missingEntries.size()); - } - for (String missingEntry : missingEntries) { + long size = EchoBaseIOUtil.countLines(zipFile, tables.values()) + + EchoBaseIOUtil.countLines(zipFile, associations.values()) - + tables.size() - + associations.size(); - // skip thoses not found entries - model.incrementsProgression(); - if (log.isInfoEnabled()) { - log.info("Skip not found entry " + missingEntry); + log.info("NB Steps: " + size); } + model.setNbSteps(size); } + for (String missingEntry : missingEntries) { + if (log.isDebugEnabled()) { + log.debug("Skip not found entry " + missingEntry); + } + } + ImportStrategy<EchoBaseEntityEnum> strategy = new EchoBaseImportStrategy( EchoBaseImportModelFactory.newFactory(this), getTransaction(), @@ -176,15 +181,15 @@ for (Map.Entry<TableMeta<EchoBaseEntityEnum>, ZipEntry> entry : entriestoConsume.entrySet()) { - model.incrementsProgression(); - TableMeta<EchoBaseEntityEnum> entryDef = entry.getKey(); ZipEntry value = entry.getValue(); - CsvImportResult<EchoBaseEntityEnum> csvResult = CsvImportResult.newResult( - entryDef.getSource(), - value.getName(), - false - ); + CsvImportResult<EchoBaseEntityEnum> csvResult = + CsvImportResult.newResult( + entryDef.getSource(), + value.getName(), + false, + model + ); Reader reader = IOUtils.toBufferedReader( new InputStreamReader(zipFile.getInputStream(value))); @@ -239,18 +244,19 @@ for (Map.Entry<AssociationMeta<EchoBaseEntityEnum>, ZipEntry> entry : associations.entrySet()) { - model.incrementsProgression(); - AssociationMeta<EchoBaseEntityEnum> entryDef = entry.getKey(); ZipEntry value = entry.getValue(); - CsvImportResult<EchoBaseEntityEnum> csvResult = CsvImportResult.newResult( - entryDef.getSource(), - value.getName(), - false - ); + CsvImportResult<EchoBaseEntityEnum> csvResult = + CsvImportResult.newResult( + entryDef.getSource(), + value.getName(), + false, + model + ); - Reader reader = IOUtils.toBufferedReader(new InputStreamReader(zipFile.getInputStream(value))); + Reader reader = IOUtils.toBufferedReader( + new InputStreamReader(zipFile.getInputStream(value))); try { long s0 = TimeLog.getTime(); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/EchoBaseImportStrategy.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/EchoBaseImportStrategy.java 2012-09-11 16:55:09 UTC (rev 649) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdb/strategy/EchoBaseImportStrategy.java 2012-09-11 16:59:23 UTC (rev 650) @@ -74,7 +74,7 @@ CsvImportResult<EchoBaseEntityEnum> csvResult) throws TopiaException { TopiaDAO<E> dao = persistenceHelper.getDAO(tx, meta.getSource()); - TopiaCsvImports.importAllEntities(dao, meta, importer, csvResult); + TopiaCsvImports.importAllEntities(dao, meta, importer, csvResult, nbRowBuffer); } @Override Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java 2012-09-11 16:55:09 UTC (rev 649) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java 2012-09-11 16:59:23 UTC (rev 650) @@ -26,6 +26,7 @@ import fr.ifremer.echobase.services.exportdb.ExportDbConfiguration; import fr.ifremer.echobase.services.exportdb.ExportDbMode; import fr.ifremer.echobase.services.exportdb.ExportDbService; +import junit.framework.Assert; import org.junit.Test; import org.nuiton.util.FileUtil; @@ -55,10 +56,12 @@ File workingDirectory = new File(getTestDir(), "work-dir"); FileUtil.createDirectoryIfNecessary(workingDirectory); conf.setWorkingDirectory(workingDirectory); - conf.setFileName("echobase"); + conf.setFileName("echobase-referential"); conf.setComputeSteps(true); conf.setExportDbMode(ExportDbMode.REFERENTIAL); service.doExport(conf); + Assert.assertNotNull(conf.getExportFile()); + Assert.assertTrue(conf.getExportFile().exists()); } @Test @@ -73,10 +76,12 @@ File workingDirectory = new File(getTestDir(), "work-dir"); FileUtil.createDirectoryIfNecessary(workingDirectory); conf.setWorkingDirectory(workingDirectory); - conf.setFileName("echobase"); + conf.setFileName("echobase-data"); conf.setComputeSteps(true); conf.setExportDbMode(ExportDbMode.DATA); service.doExport(conf); + Assert.assertNotNull(conf.getExportFile()); + Assert.assertTrue(conf.getExportFile().exists()); } @Test @@ -91,10 +96,12 @@ File workingDirectory = new File(getTestDir(), "work-dir"); FileUtil.createDirectoryIfNecessary(workingDirectory); conf.setWorkingDirectory(workingDirectory); - conf.setFileName("echobase"); + conf.setFileName("echobase-referentialAndData"); conf.setComputeSteps(true); conf.setExportDbMode(ExportDbMode.REFERENTIAL_AND_DATA); service.doExport(conf); + Assert.assertNotNull(conf.getExportFile()); + Assert.assertTrue(conf.getExportFile().exists()); } @Test @@ -107,26 +114,12 @@ File workingDirectory = new File(getTestDir(), "work-dir"); FileUtil.createDirectoryIfNecessary(workingDirectory); conf.setWorkingDirectory(workingDirectory); - conf.setFileName("echobase"); + conf.setFileName("echobase-all"); conf.setComputeSteps(true); conf.setExportDbMode(ExportDbMode.ALL); service.doExport(conf); + Assert.assertNotNull(conf.getExportFile()); + Assert.assertTrue(conf.getExportFile().exists()); } -// -// @Test -// public void exportPartialDb() throws IOException { -// -// ExportDbService service = getService(ExportDbService.class); -// -// ExportDbConfiguration conf = new ExportDbConfiguration(); -// -// conf.setVoyageIds(getVoyageId()); -// -// File workingDirectory = new File(getTestDir(), "work-dir"); -// FileUtil.createDirectoryIfNecessary(workingDirectory); -// conf.setWorkingDirectory(workingDirectory); -// conf.setFileName("echobase"); -// -// service.exportPartialDb(conf, true); -// } + } Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ImportDbServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ImportDbServiceTest.java 2012-09-11 16:55:09 UTC (rev 649) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ImportDbServiceTest.java 2012-09-11 16:59:23 UTC (rev 650) @@ -71,7 +71,7 @@ service.doImport(conf, user); - Assert.assertEquals(100f, conf.getProgression(), 1); + Assert.assertTrue(conf.getProgression() > 94); } }