Author: glandais Date: 2007-12-17 11:00:55 +0000 (Mon, 17 Dec 2007) New Revision: 71 Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java Log: Test unitaires/preformance Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java 2007-12-17 11:00:36 UTC (rev 70) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java 2007-12-17 11:00:55 UTC (rev 71) @@ -20,7 +20,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Result; import fr.cemagref.simexplorer.is.storage.entities.data.Structure; import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; -import fr.cemagref.simexplorer.is.storage.factories.DataEntityFactory; +import fr.cemagref.simexplorer.is.storage.factories.BaseEntityFactory; public class ElementGenerator { @@ -52,7 +52,7 @@ Set<T> elements = new HashSet<T>(); int i = 1 + r.nextInt(2); for (int j = 0; j < i; j++) { - T element = DataEntityFactory.getFactory(clazz).createInstance(); + T element = BaseEntityFactory.getFactory(clazz).createInstance(); if (element instanceof LoggableElement) { update((LoggableElement) element); } Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,163 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; + +public class LuceneDatabaseLoadTestCase extends TestCase { + + private Database database; + private Random r = new Random(); + private MetaDataGenerator mdg = new MetaDataGenerator(); + + private static boolean stringsInit = false; + private static int cs; + private static String[] randomstrings; + + private static String validChars = "abcdefghijklmnopqrstuvwxyz"; + + private String generateString(int length) { + StringBuffer result = new StringBuffer(); + + for (int i = 0; i < length; i++) { + char c = validChars.charAt(r.nextInt(validChars.length())); + result.append(c); + } + + return result.toString(); + } + + private void initializeStrings() { + + if (!stringsInit) { + cs = 150000; + randomstrings = new String[cs]; + for (int i = 0; i < randomstrings.length; i++) { + randomstrings[i] = generateString(2 + r.nextInt(6)); + } + stringsInit = true; + } + + } + + public LuceneDatabaseLoadTestCase() { + super(); + initializeStrings(); + } + + public LuceneDatabaseLoadTestCase(String name) { + super(name); + initializeStrings(); + } + + protected void setUp() throws Exception { + super.setUp(); + + database = new LuceneDatabase(); + // database.open(); + } + + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testMassInsert() throws Exception { + database.open(true); + + long contentSize = 0; + + int c = 25000; + Date begin = new Date(); + + StringReader reader = null; + List<Reader> readers = new ArrayList<Reader>(); + + for (int i = 0; i < c; i++) { + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); + for (int j = 0; j < mes.length; j++) { + + int wordcount = 10000 + r.nextInt(20000); + + StringBuffer sb = new StringBuffer(); + for (int k = 0; k < wordcount; k++) { + sb.append(randomstrings[r.nextInt(cs)]).append(" "); + } + + String buf = sb.toString(); + contentSize += buf.length(); + reader = new StringReader(buf); + readers.clear(); + readers.add(reader); + + database.insertElement(mes[j], readers); + } + if (i > 0 && i % 100 == 0) { + + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert " + i + " : Time taken : " + time + + "ms"); + double timePerElement = time / (4 * i); + System.out.println("Insert " + i + + " : Time taken per element : " + timePerElement + + "ms"); + + System.out.println("Content size : " + contentSize); + } + + } + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert : Time taken : " + time + "ms"); + double timePerElement = time / (4 * c); + System.out.println("Insert : Time taken per element : " + + timePerElement + "ms"); + System.out.println("Content size : " + contentSize); + } + + public void testMassSearchPaginated() throws Exception { + database.open(); + + Date begin = new Date(); + + int csearch = 100; + + for (int i = 0; i < csearch; i++) { + String key = randomstrings[r.nextInt(randomstrings.length)] + "*"; + + int count = database.findElementsByContentSearchCount(key); + System.out.println(key + " : " + count); + + int start = count / 2; + int length = Math.min(200, Math.min(count, Math.max(1, count / 4))); + + Set<MetaDataEntity> metaDatas = database + .findElementsByContentSearch(key, start, length); + + System.out.println(key + " (" + start + " " + length + ") : " + + metaDatas.size()); + + } + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Search : Time taken : " + time + "ms"); + double timePerElement = time / csearch; + System.out.println("Search : Time taken per element : " + + timePerElement + "ms"); + + } +} Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2007-12-17 11:00:36 UTC (rev 70) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2007-12-17 11:00:55 UTC (rev 71) @@ -22,13 +22,14 @@ private Database database; // private ElementGenerator elementGenerator; + private MetaDataGenerator mdg = new MetaDataGenerator(); private Random r = new Random(); protected void setUp() throws Exception { super.setUp(); database = new LuceneDatabase(); - database.open(); + database.open(true); // elementGenerator = new ElementGenerator(); } @@ -37,107 +38,18 @@ database.close(); } - private MetaDataEntity generateMetaDataEntity(String uuid, Version version, - MetaDataEntity parentVersion, MetaDataEntity parentData) { - MetaDataEntity me = generateMetaDataEntity(uuid, version, parentVersion); - me.setParentData(parentData); - return me; - } - - private MetaDataEntity generateMetaDataEntity(String uuid, Version version, - MetaDataEntity parentVersion) { - MetaDataEntity me = generateMetaDataEntity(uuid, version); - me.setParentVersion(parentVersion); - return me; - } - - private MetaDataEntity generateMetaDataEntity(String uuid, Version version) { - MetaDataEntity me = new MetaDataEntity(); - - me.setUuid(uuid); - me.setName(uuid); - me.setType("LuceneDatabaseTestCase"); - me.setDescription(uuid + " " + version.toString()); - me.setVersion(version.toString()); - me.setCreationDate(new Date()); - me.setHash(UUID.randomUUID().toString()); - - Map<String, String> descriptors = new HashMap<String, String>(); - descriptors.put("overallId", uuid); - int i = 5 + r.nextInt(10); - for (int j = 0; j < i; j++) { - descriptors.put(UUID.randomUUID().toString(), UUID.randomUUID() - .toString()); - } - me.setDescriptors(descriptors); - me.setAttachments(new HashMap<String, String>()); - me.setParentData(null); - me.setParentVersion(null); - - return me; - } - - private MetaDataEntity generateMetaDataEntity() { - return generateMetaDataEntity(UUID.randomUUID().toString(), - new Version("1.0")); - } - - private MetaDataEntity[] generateVersionnedMetaDataEntity() { - String uuid = UUID.randomUUID().toString(); - - MetaDataEntity[] mes = new MetaDataEntity[4]; - mes[0] = generateMetaDataEntity(uuid, new Version("1.0")); - mes[1] = generateMetaDataEntity(uuid, new Version("1.1"), mes[0]); - mes[2] = generateMetaDataEntity(uuid, new Version("1.2"), mes[1]); - mes[3] = generateMetaDataEntity(uuid, new Version("2.0"), mes[2]); - - return mes; - } - public void testInsertElementGetElement() throws Exception { - MetaDataEntity me = generateMetaDataEntity(); + MetaDataEntity me = mdg.generateMetaDataEntity(); database.insertElement(me, new ArrayList<Reader>()); + database.commit(); MetaDataEntity me2 = database.getElement(me.getUuid(), me.getVersion()); assertEquals(me.getDescription(), me2.getDescription()); database.insertElement(me, new ArrayList<Reader>()); - } - - public void testInsertElementGetElementExtra() throws Exception { - int c = 10; - Date begin = new Date(); - List<String> ids = new ArrayList<String>(); - for (int i = 0; i < c; i++) { - MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); - for (int j = 0; j < mes.length; j++) { - database.insertElement(mes[j], new ArrayList<Reader>(), false); - } - ids.add(mes[0].getUuid()); - } database.commit(); - - Date end = new Date(); - long time = end.getTime() - begin.getTime(); - System.out.println("Insert : Time taken : " + time + "ms"); - double timePerElement = time / (4 * c); - System.out.println("Insert : Time taken per element : " - + timePerElement + "ms"); - begin = new Date(); - - for (String uuid : ids) { - MetaDataEntity me = database.getElementLatestVersion(uuid); - assertEquals(uuid, me.getUuid()); - } - - end = new Date(); - time = end.getTime() - begin.getTime(); - System.out.println("Search : Time taken : " + time + "ms"); - timePerElement = time / (4 * c); - System.out.println("Search : Time taken per element : " - + timePerElement + "ms"); } public void testIndexAddFindElementsByContentSearch() throws Exception { - MetaDataEntity me = generateMetaDataEntity(); + MetaDataEntity me = mdg.generateMetaDataEntity(); String keyphrase = Long.toHexString(r.nextLong()); @@ -148,6 +60,7 @@ readers.add(reader); database.insertElement(me, readers); + database.commit(); Set<MetaDataEntity> mes = database .findElementsByContentSearch(keyphrase); @@ -156,10 +69,12 @@ } public void testGetVersions() throws Exception { - MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); for (int i = 0; i < mes.length; i++) { database.insertElement(mes[i], new ArrayList<Reader>()); } + database.commit(); + List<Version> versions = database.getVersions(mes[0].getUuid()); assertEquals(mes.length, versions.size()); Collections.sort(versions); @@ -170,19 +85,23 @@ } public void testFindElementsById() throws Exception { - MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); for (int i = 0; i < mes.length; i++) { database.insertElement(mes[i], new ArrayList<Reader>()); } + database.commit(); + Set<MetaDataEntity> elements = database.findElementsById(mes[0] .getUuid()); assertEquals(elements.size(), mes.length); } public void testFindElementsByProperties() throws Exception { - MetaDataEntity me = generateMetaDataEntity(); + MetaDataEntity me = mdg.generateMetaDataEntity(); database.insertElement(me, new ArrayList<Reader>()); + database.commit(); + } public void testDeleteElements() { Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,100 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.Random; + +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; + +public class LuceneDatabaseThread extends Thread { + + private Database database; + private int idThreadLucene; + + private MetaDataGenerator mdg = new MetaDataGenerator(); + + /** + * @return the database + */ + public Database getDatabase() { + return database; + } + + /** + * @param database + * the database to set + */ + public void setDatabase(Database database) { + this.database = database; + } + + @Override + public void run() { + int npass = 10; + String[] ids = new String[npass]; + for (int i = 0; i < npass; i++) { + try { + ids[i] = insertElements(i); + } catch (Exception e) { + System.err.println(idThreadLucene + " - insert : " + e); + } + } + + for (int i = 0; i < npass; i++) { + try { + searchElements(ids[i]); + } catch (Exception e) { + System.err.println(idThreadLucene + " - search : " + e); + } + } + + } + + private void searchElements(String id) throws Exception { + MetaDataEntity mde = database.getElementLatestVersion(id); + if (mde == null) { + System.err.println(idThreadLucene + " - " + id); + } + } + + private String insertElements(int ipass) throws Exception { + + String id = null; + + int ncycles = 5 + (new Random()).nextInt(20); + + for (int j = 0; j < ncycles; j++) { + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); + for (int i = 0; i < mes.length; i++) { + mes[i].setParentDataUuid(Integer.toString(idThreadLucene)); + database.insertElement(mes[i], new ArrayList<Reader>()); + /* + System.out.println(idThreadLucene + " - " + ipass + " - " + j + + " - " + i); + */ + id = mes[i].getUuid(); + } + } + System.out.print(idThreadLucene + " "); + database.commit(); + //sleep(1000); + return id; + } + + /** + * @return the idThreadLucene + */ + public int getIdThreadLucene() { + return idThreadLucene; + } + + /** + * @param idThreadLucene + * the idThreadLucene to set + */ + public void setIdThreadLucene(int idThreadLucene) { + this.idThreadLucene = idThreadLucene; + } + +} Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,40 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; + +public class LuceneDatabaseThreadsTestCase extends TestCase { + + private Database database; + + protected void setUp() throws Exception { + super.setUp(); + database = new LuceneDatabase(); + database.open(true); + } + + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testLuceneDatabaseThreads() throws Exception { + + int nthreads = 10; + + LuceneDatabaseThread[] threads = new LuceneDatabaseThread[nthreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new LuceneDatabaseThread(); + threads[i].setDatabase(database); + threads[i].setIdThreadLucene(i); + threads[i].start(); + } + + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + + } + +} Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,61 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +public class LuceneSimpleTest extends TestCase { + + public void testLuceneSimple() throws Exception { + + IndexWriter writer; + Searcher searcher; + Analyzer analyser; + Query query; + + /* + * String indexDir = "./index/"; searcher = new IndexSearcher(indexDir); + * query = new TermQuery(new Term("simexplorer.type", + * "LuceneDatabaseTestCase")); Filter filter = new QueryFilter(query); + * System.out.println(searcher.search(query, null, 10).totalHits); + */ + + String indexDir = "./indextest/"; + analyser = new SimpleAnalyzer(); + Directory dir = FSDirectory.getDirectory(indexDir); + writer = new IndexWriter(dir, true, analyser, true); + + searcher = new IndexSearcher(indexDir); + + Document doc = new Document(); + + doc.add(new Field("field1", "abcd", Field.Store.YES, + Field.Index.UN_TOKENIZED)); + doc.add(new Field("field2", "dcba", Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + writer.addDocument(doc); + + query = new TermQuery(new Term("field1", "abcd")); + + System.out.println(searcher.search(query).length()); + writer.flush(); + System.out.println(searcher.search(query).length()); + writer.close(); + System.out.println(searcher.search(query).length()); + searcher = new IndexSearcher(indexDir); + System.out.println(searcher.search(query).length()); + + } +} Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,91 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; +import fr.cemagref.simexplorer.is.storage.entities.metadata.Version; + +public class MetaDataGenerator { + + Random r = new Random(); + + /* + * private static boolean descriptorFieldsInit = false; private static + * String[] descriptorFields; + * + * private static void generateDescriptors() { descriptorFields = new + * String[10]; for (int i = 0; i < descriptorFields.length; i++) { + * descriptorFields[i] = "descriptor" + i; } } + */ + + public MetaDataEntity generateMetaDataEntity(String uuid, Version version, + MetaDataEntity parentVersion, MetaDataEntity parentData) { + MetaDataEntity me = generateMetaDataEntity(uuid, version, parentVersion); + if (parentData != null) { + me.setParentDataUuid(parentData.getUuid()); + me.setParentDataVersion(parentData.getVersion().toString()); + } + return me; + } + + public MetaDataEntity generateMetaDataEntity(String uuid, Version version, + MetaDataEntity parentVersion) { + MetaDataEntity me = generateMetaDataEntity(uuid, version); + + if (parentVersion != null) { + me.setParentVersionUuid(parentVersion.getUuid()); + me.setParentVersionVersion(parentVersion.getVersion().toString()); + } + + return me; + } + + public MetaDataEntity generateMetaDataEntity(String uuid, Version version) { + MetaDataEntity me = new MetaDataEntity(); + + me.setUuid(uuid); + me.setName(uuid); + me.setType("LuceneDatabaseTestCase"); + me.setDescription(uuid + " " + version.toString()); + me.setVersion(version.toString()); + me.setCreationDate(new Date()); + me.setHash(UUID.randomUUID().toString()); + + Map<String, String> descriptors = new HashMap<String, String>(); + descriptors.put("overallId", uuid); + int i = 5 + r.nextInt(10); + for (int j = 0; j < i; j++) { + descriptors.put("descriptor" + j, UUID.randomUUID().toString()); + } + me.setDescriptors(descriptors); + me.setAttachments(new HashMap<String, String>()); + me.setParentDataUuid(null); + me.setParentDataVersion(null); + me.setParentVersionUuid(null); + me.setParentVersionVersion(null); + + return me; + } + + public MetaDataEntity generateMetaDataEntity() { + return generateMetaDataEntity(UUID.randomUUID().toString(), + new Version("1.0")); + } + + public MetaDataEntity[] generateVersionnedMetaDataEntity() { + String uuid = UUID.randomUUID().toString(); + + MetaDataEntity[] mes = new MetaDataEntity[4]; + mes[0] = generateMetaDataEntity(uuid, new Version("1.0")); + mes[1] = generateMetaDataEntity(uuid, new Version("1.1"), mes[0]); + mes[2] = generateMetaDataEntity(uuid, new Version("1.2"), mes[1]); + mes[3] = generateMetaDataEntity(uuid, new Version("2.0"), mes[2]); + + return mes; + } + +}