Author: tchemit Date: 2008-02-02 16:56:44 +0000 (Sat, 02 Feb 2008) New Revision: 594 Added: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadMassTestCase.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsMassTestCase.java Removed: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java Log: tests de masse (ne seront pas ex?\195?\169cut?\195?\169s lors d'un mvn test) Copied: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadMassTestCase.java (from rev 580, trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadTestCase.java) =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadMassTestCase.java (rev 0) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadMassTestCase.java 2008-02-02 16:56:44 UTC (rev 594) @@ -0,0 +1,183 @@ +/* +* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* ##% */ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.entities.metadata.MetaData; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.MetaDataGenerator; +import fr.cemagref.simexplorer.is.storage.util.Config; + +public class LuceneDatabaseLoadMassTestCase extends TestCase { + + private Database database; + private Random r = new Random(); + private MetaDataGenerator mdg = new MetaDataGenerator(); + + private static boolean stringsInit = false; + private static int cs; + private static String[] randomstrings; + + private static String validChars = "abcdefghijklmnopqrstuvwxyz"; + + private String generateString(int length) { + StringBuffer result = new StringBuffer(); + + for (int i = 0; i < length; i++) { + char c = validChars.charAt(r.nextInt(validChars.length())); + result.append(c); + } + + return result.toString(); + } + + private void initializeStrings() { + + if (!stringsInit) { + cs = 150000; + randomstrings = new String[cs]; + for (int i = 0; i < randomstrings.length; i++) { + randomstrings[i] = generateString(2 + r.nextInt(6)); + } + stringsInit = true; + } + + } + + /*public LuceneDatabaseLoadMassTestCase() { + super(); + initializeStrings(); + } */ + + public LuceneDatabaseLoadMassTestCase(String name) { + super(name); + initializeStrings(); + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + Config.setPropertiesLocation("/properties/testConfig.properties"); + database = new LuceneDatabase(); + // database.open(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testMassInsert() throws Exception { + database.open(true); + + long contentSize = 0; + + int c = 25000; + Date begin = new Date(); + + StringReader reader; + List<Reader> readers = new ArrayList<Reader>(); + + for (int i = 0; i < c; i++) { + MetaData[] mes = mdg.generateVersionnedMetaDataEntity(4); + for (MetaData me : mes) { + + int wordcount = 10000 + r.nextInt(20000); + + StringBuffer sb = new StringBuffer(); + for (int k = 0; k < wordcount; k++) { + sb.append(randomstrings[r.nextInt(cs)]).append(" "); + } + + String buf = sb.toString(); + contentSize += buf.length(); + reader = new StringReader(buf); + readers.clear(); + readers.add(reader); + + database.insertElement(me, readers); + } + if (i > 0 && i % 100 == 0) { + + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert " + i + " : Time taken : " + time + + "ms"); + double timePerElement = time / (4 * i); + System.out.println("Insert " + i + + " : Time taken per element : " + timePerElement + + "ms"); + + System.out.println("Content size : " + contentSize); + } + + } + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert : Time taken : " + time + "ms"); + double timePerElement = time / (4 * c); + System.out.println("Insert : Time taken per element : " + + timePerElement + "ms"); + System.out.println("Content size : " + contentSize); + } + + public void testMassSearchPaginated() throws Exception { + database.open(); + + Date begin = new Date(); + + int csearch = 100; + + for (int i = 0; i < csearch; i++) { + String key = randomstrings[r.nextInt(randomstrings.length)] + "*"; + + int count = database.findElementsByContentSearchCount(key, false); + System.out.println(key + " : " + count); + + int start = count / 2; + int length = Math.min(200, Math.min(count, Math.max(1, count / 4))); + + Set<MetaData> metaDatas = database + .findElementsByContentSearch(key, false, start, length, 1); + + System.out.println(key + " (" + start + " " + length + ") : " + + metaDatas.size()); + + } + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Search : Time taken : " + time + "ms"); + double timePerElement = time / csearch; + System.out.println("Search : Time taken per element : " + + timePerElement + "ms"); + + } +} Copied: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsMassTestCase.java (from rev 580, trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java) =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsMassTestCase.java (rev 0) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsMassTestCase.java 2008-02-02 16:56:44 UTC (rev 594) @@ -0,0 +1,60 @@ +/* +* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* ##% */ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.util.Config; +import junit.framework.TestCase; + +public class LuceneDatabaseThreadsMassTestCase extends TestCase { + + private Database database; + + @Override + protected void setUp() throws Exception { + super.setUp(); + Config.setPropertiesLocation("/properties/testConfig.properties"); + database = new LuceneDatabase(); + database.open(true); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testLuceneDatabaseThreads() throws Exception { + + int nthreads = 5; + + LuceneDatabaseThread[] threads = new LuceneDatabaseThread[nthreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new LuceneDatabaseThread(); + threads[i].setDatabase(database); + threads[i].setIdThreadLucene(i); + threads[i].start(); + } + + for (LuceneDatabaseThread thread : threads) { + thread.join(); + } + + } + +} Deleted: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java 2008-02-02 16:56:16 UTC (rev 593) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java 2008-02-02 16:56:44 UTC (rev 594) @@ -1,59 +0,0 @@ -/* -* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -* ##% */ -package fr.cemagref.simexplorer.is.storage.database.lucene; - -import junit.framework.TestCase; -import fr.cemagref.simexplorer.is.storage.database.Database; -import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; - -public class LuceneDatabaseThreadsTestCase extends TestCase { - - private Database database; - - @Override - protected void setUp() throws Exception { - super.setUp(); - database = new LuceneDatabase(); - database.open(true); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - database.close(); - } - - public void testLuceneDatabaseThreads() throws Exception { - - int nthreads = 10; - - LuceneDatabaseThread[] threads = new LuceneDatabaseThread[nthreads]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new LuceneDatabaseThread(); - threads[i].setDatabase(database); - threads[i].setIdThreadLucene(i); - threads[i].start(); - } - - for (int i = 0; i < threads.length; i++) { - threads[i].join(); - } - - } - -}