- Simexplorer-si-commits - mailman.cloud.codelutin.com

r72 - trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler
by glandais＠users.labs.libre-entreprise.org Dec. 17, 2007

Dec. 17, 2007

Author: glandais Date: 2007-12-17 11:01:12 +0000 (Mon, 17 Dec 2007) New Revision: 72 Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java Log: throws inutile Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java 2007-12-17 11:00:55 UTC (rev 71) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java 2007-12-17 11:01:12 UTC (rev 72) @@ -33,7 +33,7 @@ * @return Instance of file * @throws Exception */ - private File getFile(MetaDataEntity entity, String field) throws Exception { + private File getFile(MetaDataEntity entity, String field) { String resultPath = baseFolder; // Create file path

1 0

r71 - trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test
by glandais＠users.labs.libre-entreprise.org Dec. 17, 2007

Dec. 17, 2007

Author: glandais Date: 2007-12-17 11:00:55 +0000 (Mon, 17 Dec 2007) New Revision: 71 Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java Log: Test unitaires/preformance Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java 2007-12-17 11:00:36 UTC (rev 70) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/ElementGenerator.java 2007-12-17 11:00:55 UTC (rev 71) @@ -20,7 +20,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Result; import fr.cemagref.simexplorer.is.storage.entities.data.Structure; import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; -import fr.cemagref.simexplorer.is.storage.factories.DataEntityFactory; +import fr.cemagref.simexplorer.is.storage.factories.BaseEntityFactory; public class ElementGenerator { @@ -52,7 +52,7 @@ Set<T> elements = new HashSet<T>(); int i = 1 + r.nextInt(2); for (int j = 0; j < i; j++) { - T element = DataEntityFactory.getFactory(clazz).createInstance(); + T element = BaseEntityFactory.getFactory(clazz).createInstance(); if (element instanceof LoggableElement) { update((LoggableElement) element); } Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,163 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; + +public class LuceneDatabaseLoadTestCase extends TestCase { + + private Database database; + private Random r = new Random(); + private MetaDataGenerator mdg = new MetaDataGenerator(); + + private static boolean stringsInit = false; + private static int cs; + private static String[] randomstrings; + + private static String validChars = "abcdefghijklmnopqrstuvwxyz"; + + private String generateString(int length) { + StringBuffer result = new StringBuffer(); + + for (int i = 0; i < length; i++) { + char c = validChars.charAt(r.nextInt(validChars.length())); + result.append(c); + } + + return result.toString(); + } + + private void initializeStrings() { + + if (!stringsInit) { + cs = 150000; + randomstrings = new String[cs]; + for (int i = 0; i < randomstrings.length; i++) { + randomstrings[i] = generateString(2 + r.nextInt(6)); + } + stringsInit = true; + } + + } + + public LuceneDatabaseLoadTestCase() { + super(); + initializeStrings(); + } + + public LuceneDatabaseLoadTestCase(String name) { + super(name); + initializeStrings(); + } + + protected void setUp() throws Exception { + super.setUp(); + + database = new LuceneDatabase(); + // database.open(); + } + + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testMassInsert() throws Exception { + database.open(true); + + long contentSize = 0; + + int c = 25000; + Date begin = new Date(); + + StringReader reader = null; + List<Reader> readers = new ArrayList<Reader>(); + + for (int i = 0; i < c; i++) { + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); + for (int j = 0; j < mes.length; j++) { + + int wordcount = 10000 + r.nextInt(20000); + + StringBuffer sb = new StringBuffer(); + for (int k = 0; k < wordcount; k++) { + sb.append(randomstrings[r.nextInt(cs)]).append(" "); + } + + String buf = sb.toString(); + contentSize += buf.length(); + reader = new StringReader(buf); + readers.clear(); + readers.add(reader); + + database.insertElement(mes[j], readers); + } + if (i > 0 && i % 100 == 0) { + + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert " + i + " : Time taken : " + time + + "ms"); + double timePerElement = time / (4 * i); + System.out.println("Insert " + i + + " : Time taken per element : " + timePerElement + + "ms"); + + System.out.println("Content size : " + contentSize); + } + + } + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert : Time taken : " + time + "ms"); + double timePerElement = time / (4 * c); + System.out.println("Insert : Time taken per element : " + + timePerElement + "ms"); + System.out.println("Content size : " + contentSize); + } + + public void testMassSearchPaginated() throws Exception { + database.open(); + + Date begin = new Date(); + + int csearch = 100; + + for (int i = 0; i < csearch; i++) { + String key = randomstrings[r.nextInt(randomstrings.length)] + "*"; + + int count = database.findElementsByContentSearchCount(key); + System.out.println(key + " : " + count); + + int start = count / 2; + int length = Math.min(200, Math.min(count, Math.max(1, count / 4))); + + Set<MetaDataEntity> metaDatas = database + .findElementsByContentSearch(key, start, length); + + System.out.println(key + " (" + start + " " + length + ") : " + + metaDatas.size()); + + } + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Search : Time taken : " + time + "ms"); + double timePerElement = time / csearch; + System.out.println("Search : Time taken per element : " + + timePerElement + "ms"); + + } +} Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2007-12-17 11:00:36 UTC (rev 70) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2007-12-17 11:00:55 UTC (rev 71) @@ -22,13 +22,14 @@ private Database database; // private ElementGenerator elementGenerator; + private MetaDataGenerator mdg = new MetaDataGenerator(); private Random r = new Random(); protected void setUp() throws Exception { super.setUp(); database = new LuceneDatabase(); - database.open(); + database.open(true); // elementGenerator = new ElementGenerator(); } @@ -37,107 +38,18 @@ database.close(); } - private MetaDataEntity generateMetaDataEntity(String uuid, Version version, - MetaDataEntity parentVersion, MetaDataEntity parentData) { - MetaDataEntity me = generateMetaDataEntity(uuid, version, parentVersion); - me.setParentData(parentData); - return me; - } - - private MetaDataEntity generateMetaDataEntity(String uuid, Version version, - MetaDataEntity parentVersion) { - MetaDataEntity me = generateMetaDataEntity(uuid, version); - me.setParentVersion(parentVersion); - return me; - } - - private MetaDataEntity generateMetaDataEntity(String uuid, Version version) { - MetaDataEntity me = new MetaDataEntity(); - - me.setUuid(uuid); - me.setName(uuid); - me.setType("LuceneDatabaseTestCase"); - me.setDescription(uuid + " " + version.toString()); - me.setVersion(version.toString()); - me.setCreationDate(new Date()); - me.setHash(UUID.randomUUID().toString()); - - Map<String, String> descriptors = new HashMap<String, String>(); - descriptors.put("overallId", uuid); - int i = 5 + r.nextInt(10); - for (int j = 0; j < i; j++) { - descriptors.put(UUID.randomUUID().toString(), UUID.randomUUID() - .toString()); - } - me.setDescriptors(descriptors); - me.setAttachments(new HashMap<String, String>()); - me.setParentData(null); - me.setParentVersion(null); - - return me; - } - - private MetaDataEntity generateMetaDataEntity() { - return generateMetaDataEntity(UUID.randomUUID().toString(), - new Version("1.0")); - } - - private MetaDataEntity[] generateVersionnedMetaDataEntity() { - String uuid = UUID.randomUUID().toString(); - - MetaDataEntity[] mes = new MetaDataEntity[4]; - mes[0] = generateMetaDataEntity(uuid, new Version("1.0")); - mes[1] = generateMetaDataEntity(uuid, new Version("1.1"), mes[0]); - mes[2] = generateMetaDataEntity(uuid, new Version("1.2"), mes[1]); - mes[3] = generateMetaDataEntity(uuid, new Version("2.0"), mes[2]); - - return mes; - } - public void testInsertElementGetElement() throws Exception { - MetaDataEntity me = generateMetaDataEntity(); + MetaDataEntity me = mdg.generateMetaDataEntity(); database.insertElement(me, new ArrayList<Reader>()); + database.commit(); MetaDataEntity me2 = database.getElement(me.getUuid(), me.getVersion()); assertEquals(me.getDescription(), me2.getDescription()); database.insertElement(me, new ArrayList<Reader>()); - } - - public void testInsertElementGetElementExtra() throws Exception { - int c = 10; - Date begin = new Date(); - List<String> ids = new ArrayList<String>(); - for (int i = 0; i < c; i++) { - MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); - for (int j = 0; j < mes.length; j++) { - database.insertElement(mes[j], new ArrayList<Reader>(), false); - } - ids.add(mes[0].getUuid()); - } database.commit(); - - Date end = new Date(); - long time = end.getTime() - begin.getTime(); - System.out.println("Insert : Time taken : " + time + "ms"); - double timePerElement = time / (4 * c); - System.out.println("Insert : Time taken per element : " - + timePerElement + "ms"); - begin = new Date(); - - for (String uuid : ids) { - MetaDataEntity me = database.getElementLatestVersion(uuid); - assertEquals(uuid, me.getUuid()); - } - - end = new Date(); - time = end.getTime() - begin.getTime(); - System.out.println("Search : Time taken : " + time + "ms"); - timePerElement = time / (4 * c); - System.out.println("Search : Time taken per element : " - + timePerElement + "ms"); } public void testIndexAddFindElementsByContentSearch() throws Exception { - MetaDataEntity me = generateMetaDataEntity(); + MetaDataEntity me = mdg.generateMetaDataEntity(); String keyphrase = Long.toHexString(r.nextLong()); @@ -148,6 +60,7 @@ readers.add(reader); database.insertElement(me, readers); + database.commit(); Set<MetaDataEntity> mes = database .findElementsByContentSearch(keyphrase); @@ -156,10 +69,12 @@ } public void testGetVersions() throws Exception { - MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); for (int i = 0; i < mes.length; i++) { database.insertElement(mes[i], new ArrayList<Reader>()); } + database.commit(); + List<Version> versions = database.getVersions(mes[0].getUuid()); assertEquals(mes.length, versions.size()); Collections.sort(versions); @@ -170,19 +85,23 @@ } public void testFindElementsById() throws Exception { - MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); for (int i = 0; i < mes.length; i++) { database.insertElement(mes[i], new ArrayList<Reader>()); } + database.commit(); + Set<MetaDataEntity> elements = database.findElementsById(mes[0] .getUuid()); assertEquals(elements.size(), mes.length); } public void testFindElementsByProperties() throws Exception { - MetaDataEntity me = generateMetaDataEntity(); + MetaDataEntity me = mdg.generateMetaDataEntity(); database.insertElement(me, new ArrayList<Reader>()); + database.commit(); + } public void testDeleteElements() { Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,100 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.Random; + +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; + +public class LuceneDatabaseThread extends Thread { + + private Database database; + private int idThreadLucene; + + private MetaDataGenerator mdg = new MetaDataGenerator(); + + /** + * @return the database + */ + public Database getDatabase() { + return database; + } + + /** + * @param database + * the database to set + */ + public void setDatabase(Database database) { + this.database = database; + } + + @Override + public void run() { + int npass = 10; + String[] ids = new String[npass]; + for (int i = 0; i < npass; i++) { + try { + ids[i] = insertElements(i); + } catch (Exception e) { + System.err.println(idThreadLucene + " - insert : " + e); + } + } + + for (int i = 0; i < npass; i++) { + try { + searchElements(ids[i]); + } catch (Exception e) { + System.err.println(idThreadLucene + " - search : " + e); + } + } + + } + + private void searchElements(String id) throws Exception { + MetaDataEntity mde = database.getElementLatestVersion(id); + if (mde == null) { + System.err.println(idThreadLucene + " - " + id); + } + } + + private String insertElements(int ipass) throws Exception { + + String id = null; + + int ncycles = 5 + (new Random()).nextInt(20); + + for (int j = 0; j < ncycles; j++) { + MetaDataEntity[] mes = mdg.generateVersionnedMetaDataEntity(); + for (int i = 0; i < mes.length; i++) { + mes[i].setParentDataUuid(Integer.toString(idThreadLucene)); + database.insertElement(mes[i], new ArrayList<Reader>()); + /* + System.out.println(idThreadLucene + " - " + ipass + " - " + j + + " - " + i); + */ + id = mes[i].getUuid(); + } + } + System.out.print(idThreadLucene + " "); + database.commit(); + //sleep(1000); + return id; + } + + /** + * @return the idThreadLucene + */ + public int getIdThreadLucene() { + return idThreadLucene; + } + + /** + * @param idThreadLucene + * the idThreadLucene to set + */ + public void setIdThreadLucene(int idThreadLucene) { + this.idThreadLucene = idThreadLucene; + } + +} Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,40 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; + +public class LuceneDatabaseThreadsTestCase extends TestCase { + + private Database database; + + protected void setUp() throws Exception { + super.setUp(); + database = new LuceneDatabase(); + database.open(true); + } + + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testLuceneDatabaseThreads() throws Exception { + + int nthreads = 10; + + LuceneDatabaseThread[] threads = new LuceneDatabaseThread[nthreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new LuceneDatabaseThread(); + threads[i].setDatabase(database); + threads[i].setIdThreadLucene(i); + threads[i].start(); + } + + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + + } + +} Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,61 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +public class LuceneSimpleTest extends TestCase { + + public void testLuceneSimple() throws Exception { + + IndexWriter writer; + Searcher searcher; + Analyzer analyser; + Query query; + + /* + * String indexDir = "./index/"; searcher = new IndexSearcher(indexDir); + * query = new TermQuery(new Term("simexplorer.type", + * "LuceneDatabaseTestCase")); Filter filter = new QueryFilter(query); + * System.out.println(searcher.search(query, null, 10).totalHits); + */ + + String indexDir = "./indextest/"; + analyser = new SimpleAnalyzer(); + Directory dir = FSDirectory.getDirectory(indexDir); + writer = new IndexWriter(dir, true, analyser, true); + + searcher = new IndexSearcher(indexDir); + + Document doc = new Document(); + + doc.add(new Field("field1", "abcd", Field.Store.YES, + Field.Index.UN_TOKENIZED)); + doc.add(new Field("field2", "dcba", Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + writer.addDocument(doc); + + query = new TermQuery(new Term("field1", "abcd")); + + System.out.println(searcher.search(query).length()); + writer.flush(); + System.out.println(searcher.search(query).length()); + writer.close(); + System.out.println(searcher.search(query).length()); + searcher = new IndexSearcher(indexDir); + System.out.println(searcher.search(query).length()); + + } +} Added: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java (rev 0) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/MetaDataGenerator.java 2007-12-17 11:00:55 UTC (rev 71) @@ -0,0 +1,91 @@ +package fr.cemagref.simexplorer.is.storage.test; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; +import fr.cemagref.simexplorer.is.storage.entities.metadata.Version; + +public class MetaDataGenerator { + + Random r = new Random(); + + /* + * private static boolean descriptorFieldsInit = false; private static + * String[] descriptorFields; + * + * private static void generateDescriptors() { descriptorFields = new + * String[10]; for (int i = 0; i < descriptorFields.length; i++) { + * descriptorFields[i] = "descriptor" + i; } } + */ + + public MetaDataEntity generateMetaDataEntity(String uuid, Version version, + MetaDataEntity parentVersion, MetaDataEntity parentData) { + MetaDataEntity me = generateMetaDataEntity(uuid, version, parentVersion); + if (parentData != null) { + me.setParentDataUuid(parentData.getUuid()); + me.setParentDataVersion(parentData.getVersion().toString()); + } + return me; + } + + public MetaDataEntity generateMetaDataEntity(String uuid, Version version, + MetaDataEntity parentVersion) { + MetaDataEntity me = generateMetaDataEntity(uuid, version); + + if (parentVersion != null) { + me.setParentVersionUuid(parentVersion.getUuid()); + me.setParentVersionVersion(parentVersion.getVersion().toString()); + } + + return me; + } + + public MetaDataEntity generateMetaDataEntity(String uuid, Version version) { + MetaDataEntity me = new MetaDataEntity(); + + me.setUuid(uuid); + me.setName(uuid); + me.setType("LuceneDatabaseTestCase"); + me.setDescription(uuid + " " + version.toString()); + me.setVersion(version.toString()); + me.setCreationDate(new Date()); + me.setHash(UUID.randomUUID().toString()); + + Map<String, String> descriptors = new HashMap<String, String>(); + descriptors.put("overallId", uuid); + int i = 5 + r.nextInt(10); + for (int j = 0; j < i; j++) { + descriptors.put("descriptor" + j, UUID.randomUUID().toString()); + } + me.setDescriptors(descriptors); + me.setAttachments(new HashMap<String, String>()); + me.setParentDataUuid(null); + me.setParentDataVersion(null); + me.setParentVersionUuid(null); + me.setParentVersionVersion(null); + + return me; + } + + public MetaDataEntity generateMetaDataEntity() { + return generateMetaDataEntity(UUID.randomUUID().toString(), + new Version("1.0")); + } + + public MetaDataEntity[] generateVersionnedMetaDataEntity() { + String uuid = UUID.randomUUID().toString(); + + MetaDataEntity[] mes = new MetaDataEntity[4]; + mes[0] = generateMetaDataEntity(uuid, new Version("1.0")); + mes[1] = generateMetaDataEntity(uuid, new Version("1.1"), mes[0]); + mes[2] = generateMetaDataEntity(uuid, new Version("1.2"), mes[1]); + mes[3] = generateMetaDataEntity(uuid, new Version("2.0"), mes[2]); + + return mes; + } + +}

1 0

r70 - in trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage: service xml
by glandais＠users.labs.libre-entreprise.org Dec. 17, 2007

Dec. 17, 2007

Author: glandais Date: 2007-12-17 11:00:36 +0000 (Mon, 17 Dec 2007) New Revision: 70 Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java Log: Aspect service Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java 2007-12-17 10:59:58 UTC (rev 69) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java 2007-12-17 11:00:36 UTC (rev 70) @@ -1,18 +1,41 @@ package fr.cemagref.simexplorer.is.storage.service; import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.w3c.dom.Document; +import org.w3c.dom.Element; import fr.cemagref.simexplorer.is.storage.engine.StorageEngine; -import fr.cemagref.simexplorer.is.storage.entities.data.LoggableElement; import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; +import fr.cemagref.simexplorer.is.storage.factories.BaseEntityFactory; +import fr.cemagref.simexplorer.is.storage.factories.MetaDataEntityFactory; import fr.cemagref.simexplorer.is.storage.xml.MetaDataParser; +import fr.cemagref.simexplorer.is.storage.xml.Parser; public class StorageService { private StorageEngine storageEngine; + // FIXME centralized + private static String KEY_METADATA = "metadata"; + private static String KEY_XML = "xml"; + private static String KEY_ATTACHMENT = "attachment"; + private static String KEY_TYPE_EA = "EA"; + private static String KEY_COMPONENT = "C"; + private static String KEY_ED = "ED"; + private static String KEY_RESULT = "R"; + private static String KEY_FILE = "FILE"; + + private static String FILE_XML = "data.xml"; + private static String FILE_DATA_PREFIX = "attachment"; + public StorageService() { super(); storageEngine = new StorageEngine(); @@ -26,19 +49,167 @@ storageEngine.close(); } - public LoggableElement saveElement(InputStream xmlFile, - InputStream attachment) throws Exception { + public MetaDataEntity saveElement(InputStream zipStream) throws Exception { + String xmlFile = null; + Map<String, String> attachments = null; + + ZipInputStream zis = new ZipInputStream(zipStream); + + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (!entry.isDirectory()) { + String entryName = entry.getName(); + if (entryName.equals(FILE_XML)) { + xmlFile = storageEngine.storeTempData(zis); + } else { + if (entryName.startsWith(FILE_DATA_PREFIX)) { + String fileName = entryName.replace(FILE_DATA_PREFIX + + "/", ""); + String idFile = storageEngine.storeTempData(zis); + attachments.put(fileName, idFile); + } + } + } + } + + return saveElement(xmlFile, attachments); + } + + public MetaDataEntity saveElement(InputStream xmlFile, + Map<String, InputStream> attachments) throws Exception { + // Store temporary data String idxml = storageEngine.storeTempData(xmlFile); - String idattachment = null; - if (attachment != null) { - idattachment = storageEngine.storeTempData(attachment); + Map<String, String> idsattachment = new HashMap<String, String>(); + for (Map.Entry<String, InputStream> entry : attachments.entrySet()) { + String idattachment = storageEngine.storeTempData(entry.getValue()); + idsattachment.put(entry.getKey(), idattachment); } + return saveElement(idxml, idsattachment); + } + public MetaDataEntity saveElement(String idxml, + Map<String, String> idsattachment) throws Exception { + // Load xml metadata MetaDataEntity metaData = MetaDataParser.parse(storageEngine .retrieveTempData(idxml)); - LoggableElement result = null; - return result; + String uuid = metaData.getUuid(); + + // If element is an EA, save inner Components and Data + + if (metaData.getType().equals(KEY_TYPE_EA)) { + + List<String> components = new ArrayList<String>(); + List<String[]> explorationDatas = new ArrayList<String[]>(); + + // Retrieve elements + extractChildren(idxml, idsattachment, components, explorationDatas); + + // For each exploration data + for (String[] explorationData : explorationDatas) { + // where is stored xml + String idxmlED = explorationData[0]; + + // where is stored result + Map<String, String> attachmentsED = new HashMap<String, String>(); + if (explorationData.length > 1) { + attachmentsED.put(explorationData[1], idsattachment + .get(explorationData[1])); + } + // recursive save + saveElement(idxmlED, attachmentsED); + + } + + // For each component + for (String idComponent : components) { + saveElement(idComponent, new HashMap<String, String>()); + } + + // Don't save any file for an EA + idsattachment = new HashMap<String, String>(); + + } + + // Check existing version in storage + MetaDataEntity previousVersion = storageEngine.getElementVersion(uuid, + metaData.getVersion()); + /* + * MetaDataEntity parentData = storageEngine.getElementVersion(metaData + * .getParentData().getUuid(), metaData.getParentData() .getVersion()); + * MetaDataEntity parentVersion = + * storageEngine.getElementVersion(metaData + * .getParentVersion().getUuid(), metaData.getParentVersion() + * .getVersion()); + */ + + // Version rules + if (previousVersion != null) { + /* + * metaData.setParentVersion(storageEngine.getElementVersion(uuid, + * previousVersion.getVersion())); + */ + metaData.setVersion(previousVersion.getVersion().incVersion(0) + .toString()); + } + + // Prepare saving + Map<String, InputStream> attachments = new HashMap<String, InputStream>(); + + attachments.put(KEY_XML, storageEngine.retrieveTempData(idxml)); + for (Map.Entry<String, String> entry : idsattachment.entrySet()) { + attachments.put(KEY_ATTACHMENT + "." + entry.getKey(), + storageEngine.retrieveTempData(entry.getValue())); + } + storageEngine.saveElement(metaData, attachments); + + return metaData; } + private void extractChildren(String idxml, + Map<String, String> idsattachment, List<String> components, + List<String[]> explorationDatas) throws Exception { + MetaDataEntityFactory<MetaDataEntity> elementFactory = (MetaDataEntityFactory<MetaDataEntity>) BaseEntityFactory + .getFactory("MetaDataEntity"); + + Document document = BaseEntityFactory.getBuilder().parse( + storageEngine.retrieveTempData(idxml)); + + Element rootElement = (Element) document.getFirstChild(); + + // Components + Set<Element> componentElements = elementFactory.getElementsByTagName( + rootElement, KEY_COMPONENT); + for (Element element : componentElements) { + components.add(storageEngine.storeTempData(Parser + .serializeElement(element))); + } + + Set<Element> applicationDataElements = elementFactory + .getElementsByTagName(rootElement, KEY_ED); + for (Element elementAD : applicationDataElements) { + + Element element = elementFactory.getElementByTagName(elementAD, + KEY_RESULT); + String result = elementFactory.getProperty(element, KEY_FILE); + + String[] explorationDataArray = null; + if (result != null) { + explorationDataArray = new String[2]; + explorationDataArray[1] = result; + } else { + explorationDataArray = new String[1]; + } + explorationDataArray[0] = storageEngine.storeTempData(Parser + .serializeElement(elementAD)); + + } + + } + + /* + * public MetaDataEntity loadElement(String idxml, Map<String, String> + * idsattachment) throws Exception { } + */ + } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java 2007-12-17 10:59:58 UTC (rev 69) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java 2007-12-17 11:00:36 UTC (rev 70) @@ -5,26 +5,50 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; +import fr.cemagref.simexplorer.is.storage.entities.BaseEntity; +import fr.cemagref.simexplorer.is.storage.entities.data.Code; import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; +import fr.cemagref.simexplorer.is.storage.factories.BaseEntityFactory; +import fr.cemagref.simexplorer.is.storage.factories.MetaDataEntityFactory; public class MetaDataParser extends Parser { private static String KEY_METADATA = "metadata"; - private static String KEY_UUID = "metadata"; + private static String KEY_UUID = "uuid"; private static String KEY_VERSION = "version"; - + + /* + * @Override public static MetaDataEntity parse(InputStream xmlStream) + * throws Exception { MetaDataEntity mde = new MetaDataEntity(); + * + * Document document = getBuilder().parse(xmlStream); Element rootElement = + * (Element) document.getFirstChild(); Element metadataElement = + * getElementByTagName(rootElement, KEY_METADATA); + * + * mde.setUuid(getProperty(metadataElement, KEY_UUID)); + * mde.setVersion(getProperty(metadataElement, KEY_VERSION)); + * + * return mde; } + */ + public static MetaDataEntity parse(InputStream xmlStream) throws Exception { - MetaDataEntity mde = new MetaDataEntity(); + MetaDataEntityFactory<MetaDataEntity> elementFactory = (MetaDataEntityFactory<MetaDataEntity>) BaseEntityFactory + .getFactory("MetaDataEntity"); - Document document = getBuilder().parse(xmlStream); + Document document = elementFactory.getBuilder().parse(xmlStream); Element rootElement = (Element) document.getFirstChild(); - Element metadataElement = getElementByTagName(rootElement, KEY_METADATA); + Element metadataElement = elementFactory.getElementByTagName( + rootElement, KEY_METADATA); - mde.setUuid(getProperty(metadataElement, KEY_UUID)); - mde.setVersion(getProperty(metadataElement, KEY_VERSION)); - - return mde; + MetaDataEntity metaData = elementFactory + .loadXMLElement(metadataElement); + return metaData; } - + + public static void save(Document document, Element xmlElement, + MetaDataEntity element) throws Exception { + + } + } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java 2007-12-17 10:59:58 UTC (rev 69) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java 2007-12-17 11:00:36 UTC (rev 70) @@ -1,10 +1,17 @@ package fr.cemagref.simexplorer.is.storage.xml; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.util.HashSet; import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -26,7 +33,7 @@ return documentBuilder; } - protected static String getProperty(Element xmlElement, String tagName) { + public static String getProperty(Element xmlElement, String tagName) { Element element = getElementByTagName(xmlElement, tagName); if (element != null) { return element.getFirstChild().getNodeValue(); @@ -35,7 +42,7 @@ } } - protected static void setProperty(Document document, Element xmlElement, + public static void setProperty(Document document, Element xmlElement, String tagName, Object value) { if (value != null) { Element xmlProperty = document.createElement(tagName); @@ -45,7 +52,7 @@ } } - protected static Element getElementByTagName(Element xmlElement, String tagName) { + public static Element getElementByTagName(Element xmlElement, String tagName) { Set<Element> elements = getElementsByTagName(xmlElement, tagName); if (elements.size() > 0) { return elements.iterator().next(); @@ -54,7 +61,7 @@ } } - protected static Set<Element> getElementsByTagName(Element xmlElement, + public static Set<Element> getElementsByTagName(Element xmlElement, String tagName) { Set<Element> elements = new HashSet<Element>(); @@ -69,4 +76,57 @@ return elements; } + public static InputStream serializeElement(Element element) + throws Exception { + org.w3c.dom.Document xmlDocument = getBuilder().newDocument(); + // Propriétés du DOM + xmlDocument.setXmlVersion("1.0"); + xmlDocument.setXmlStandalone(true); + // Création de l'arborescence du DOM + Element racine = xmlDocument.createElement(element.getClass() + .getSimpleName()); + + racine.appendChild(element); + + xmlDocument.appendChild(racine); + DOMSource domSource = new DOMSource(xmlDocument); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + StreamResult result = new StreamResult(os); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.transform(domSource, result); + + ByteArrayInputStream inputStream = new ByteArrayInputStream(os + .toByteArray()); + + return inputStream; + } + + /* + * public static BaseEntity parse(InputStream xmlStream) throws Exception { + * return null; } + * + * public static void save(Document document, Element xmlElement, + * MetaDataEntity element) throws Exception { } /* public static void + * save(BaseEntity element, OutputStream os) throws Exception { + * org.w3c.dom.Document xmlDocument = getBuilder().newDocument(); // + * Propriétés du DOM xmlDocument.setXmlVersion("1.0"); + * xmlDocument.setXmlStandalone(true); // Création de l'arborescence du DOM + * Element racine = xmlDocument.createElement(element.getClass() + * .getSimpleName()); + * + * save(xmlDocument, racine, element); + * + * xmlDocument.appendChild(racine); DOMSource domSource = new + * DOMSource(xmlDocument); StreamResult result = new StreamResult(os); + * TransformerFactory tf = TransformerFactory.newInstance(); Transformer + * transformer = tf.newTransformer(); transformer.transform(domSource, + * result); } + * + * public static String save(BaseEntity element) throws Exception { + * ByteArrayOutputStream writer = new ByteArrayOutputStream(); save(element, + * writer); return writer.toString(); } + */ + }

1 0

r69 - trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories
by glandais＠users.labs.libre-entreprise.org Dec. 17, 2007

Dec. 17, 2007

Author: glandais Date: 2007-12-17 10:59:58 +0000 (Mon, 17 Dec 2007) New Revision: 69 Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/BaseEntityFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/MetaDataEntityFactory.java Removed: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/DataEntityFactory.java Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/CodeFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ComponentFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantValueFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/LoggableElementFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/RepositoryFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ResultFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/StructureFactory.java Log: Factory de base = BaseEntityFactory Initialisation de MetaDataEntityFactory Copied: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/BaseEntityFactory.java (from rev 62, trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/DataEntityFactory.java) =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/BaseEntityFactory.java (rev 0) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/BaseEntityFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -0,0 +1,187 @@ +package fr.cemagref.simexplorer.is.storage.factories; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; + +import fr.cemagref.simexplorer.is.storage.entities.BaseEntity; +import fr.cemagref.simexplorer.is.storage.entities.data.DataEntity; + +public abstract class BaseEntityFactory<E extends BaseEntity> { + + private static DocumentBuilder documentBuilder = null; + + private static Map<String, BaseEntityFactory<? extends DataEntity>> factories = new HashMap<String, BaseEntityFactory<? extends DataEntity>>(); + + private final static String PACKAGE_DAO = "fr.cemagref.simexplorer.is.storage.factories"; + + public static synchronized DocumentBuilder getBuilder() throws Exception { + if (documentBuilder == null) { + DocumentBuilderFactory factory = DocumentBuilderFactory + .newInstance(); + documentBuilder = factory.newDocumentBuilder(); + return documentBuilder; + } + return documentBuilder; + } + + // @SuppressWarnings("unchecked") + public static <T extends DataEntity> BaseEntityFactory<T> getFactory( + Class<T> entityClass) throws Exception { + // FIXME Raw types and unchecked casts + BaseEntityFactory<T> result = (BaseEntityFactory<T>) factories + .get(entityClass.getSimpleName()); + if (result == null) { + String elementFactoryClassName = PACKAGE_DAO + "." + + entityClass.getSimpleName() + "Factory"; + + Class<? extends BaseEntityFactory<T>> factoryClass = (Class<? extends BaseEntityFactory<T>>) Class + .forName(elementFactoryClassName).asSubclass( + BaseEntityFactory.class); + + result = factoryClass.newInstance(); + factories.put(entityClass.getSimpleName(), result); + } + return result; + } + + public static BaseEntityFactory<? extends DataEntity> getFactory( + String entityClassName) throws Exception { + Class<? extends DataEntity> entityClass = Class + .forName(entityClassName).asSubclass(DataEntity.class); + return getFactory(entityClass); + } + + public abstract E createInstance(); + + public Element getElementByTagName(Element xmlElement, String tagName) { + Set<Element> elements = getElementsByTagName(xmlElement, tagName); + if (elements.size() > 0) { + return elements.iterator().next(); + } else { + return null; + } + } + + public Set<Element> getElementsByTagName(Element xmlElement, + String tagName) { + Set<Element> elements = new HashSet<Element>(); + + NodeList nodes = xmlElement.getChildNodes(); + for (int i = 0; i < nodes.getLength(); i++) { + Node node = nodes.item(i); + if (node instanceof Element + && tagName.equals(((Element) node).getTagName())) { + elements.add((Element) node); + } + } + return elements; + } + + public String getElementXML(E element) throws Exception { + ByteArrayOutputStream writer = new ByteArrayOutputStream(); + getElementXML(element, writer); + return writer.toString(); + } + + public void getElementXML(E element, OutputStream os) throws Exception { + org.w3c.dom.Document xmlDocument = getBuilder().newDocument(); + // Propriétés du DOM + xmlDocument.setXmlVersion("1.0"); + xmlDocument.setXmlStandalone(true); + // Création de l'arborescence du DOM + Element racine = xmlDocument.createElement(element.getClass() + .getSimpleName()); + + saveXMLElement(xmlDocument, racine, element); + + xmlDocument.appendChild(racine); + DOMSource domSource = new DOMSource(xmlDocument); + StreamResult result = new StreamResult(os); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.transform(domSource, result); + } + + public String getProperty(Element xmlElement, String tagName) { + Element element = getElementByTagName(xmlElement, tagName); + if (element != null) { + return element.getFirstChild().getNodeValue(); + } else { + return null; + } + } + + public <T extends DataEntity> Set<T> loadCollection(String tagSetName, + String tagName, Class<T> clazz, Element xmlElement, + DataEntity parentData) throws Exception { + Set<T> entities = new HashSet<T>(); + + Element xmlCollection = getElementByTagName(xmlElement, tagSetName); + if (xmlCollection != null) { + BaseEntityFactory<T> elementFactory = getFactory(clazz); + + Set<Element> list = getElementsByTagName(xmlCollection, tagName); + for (Element element : list) { + T entity = elementFactory.loadXMLElement(element); + entity.setParentData(parentData); + entities.add(entity); + } + } + + return entities; + } + + public E loadXMLElement(Element xmlElement) throws Exception { + E element = createInstance(); + return element; + } + + public <T extends DataEntity> void saveCollection(Set<T> entities, + String tagSetName, String tagName, Class<T> clazz, + Document document, Element xmlElement) throws Exception { + if (entities != null && entities.size() > 0) { + Element xmlCollection = document.createElement(tagSetName); + BaseEntityFactory<T> elementFactory = getFactory(clazz); + for (T entity : entities) { + Element childElement = document.createElement(tagName); + // TODO + // if (entity instanceof LoggableElement) { + // elementFactory.saveXMLElementByReference + elementFactory.saveXMLElement(document, childElement, entity); + xmlCollection.appendChild(childElement); + } + xmlElement.appendChild(xmlCollection); + } + } + + public abstract void saveXMLElement(Document document, Element xmlElement, + E element) throws Exception; + + protected void setProperty(Document document, Element xmlElement, + String tagName, Object value) { + if (value != null) { + Element xmlProperty = document.createElement(tagName); + Text xmlDescriptionText = document.createTextNode(value.toString()); + xmlProperty.appendChild(xmlDescriptionText); + xmlElement.appendChild(xmlProperty); + } + } + +} Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/CodeFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/CodeFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/CodeFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -5,7 +5,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Code; -public class CodeFactory<E extends Code> extends DataEntityFactory<Code> { +public class CodeFactory<E extends Code> extends BaseEntityFactory<Code> { public static final String KEY_LANGUAGE = "language"; public static final String KEY_CODE = "code"; Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ComponentFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ComponentFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ComponentFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -10,7 +10,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Structure; public class ComponentFactory<E extends Component> extends - DataEntityFactory<Component> { + BaseEntityFactory<Component> { public static final String KEY_CONSTANTS = "constants"; public static final String KEY_CONSTANT_NODE = "constant"; Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -6,7 +6,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Constant; public class ConstantFactory<E extends Constant> extends - DataEntityFactory<Constant> { + BaseEntityFactory<Constant> { public static final String KEY_NAME = "name"; public static final String KEY_TYPE = "type"; Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantValueFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantValueFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ConstantValueFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -7,7 +7,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.ConstantValue; public class ConstantValueFactory<E extends ConstantValue> extends - DataEntityFactory<ConstantValue> { + BaseEntityFactory<ConstantValue> { public static final String KEY_CONSTANT = "constant"; public static final String KEY_VALUE = "value"; Deleted: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/DataEntityFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/DataEntityFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/DataEntityFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -1,186 +0,0 @@ -package fr.cemagref.simexplorer.is.storage.factories; - -import java.io.ByteArrayOutputStream; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; - -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.w3c.dom.Text; - -import fr.cemagref.simexplorer.is.storage.entities.data.DataEntity; - -public abstract class DataEntityFactory<E extends DataEntity> { - - private static DocumentBuilder documentBuilder = null; - - private static Map<String, DataEntityFactory<? extends DataEntity>> factories = new HashMap<String, DataEntityFactory<? extends DataEntity>>(); - - private final static String PACKAGE_DAO = "fr.cemagref.simexplorer.is.storage.factories"; - - public static synchronized DocumentBuilder getBuilder() throws Exception { - if (documentBuilder == null) { - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - documentBuilder = factory.newDocumentBuilder(); - return documentBuilder; - } - return documentBuilder; - } - - // @SuppressWarnings("unchecked") - public static <T extends DataEntity> DataEntityFactory<T> getFactory( - Class<T> entityClass) throws Exception { - // FIXME Raw types and unchecked casts - DataEntityFactory<T> result = (DataEntityFactory<T>) factories - .get(entityClass.getSimpleName()); - if (result == null) { - String elementFactoryClassName = PACKAGE_DAO + "." - + entityClass.getSimpleName() + "Factory"; - - Class<? extends DataEntityFactory<T>> factoryClass = (Class<? extends DataEntityFactory<T>>) Class - .forName(elementFactoryClassName).asSubclass( - DataEntityFactory.class); - - result = factoryClass.newInstance(); - factories.put(entityClass.getSimpleName(), result); - } - return result; - } - - public static DataEntityFactory<? extends DataEntity> getFactory( - String entityClassName) throws Exception { - Class<? extends DataEntity> entityClass = Class - .forName(entityClassName).asSubclass(DataEntity.class); - return getFactory(entityClass); - } - - public abstract E createInstance(); - - protected Element getElementByTagName(Element xmlElement, String tagName) { - Set<Element> elements = getElementsByTagName(xmlElement, tagName); - if (elements.size() > 0) { - return elements.iterator().next(); - } else { - return null; - } - } - - protected Set<Element> getElementsByTagName(Element xmlElement, - String tagName) { - Set<Element> elements = new HashSet<Element>(); - - NodeList nodes = xmlElement.getChildNodes(); - for (int i = 0; i < nodes.getLength(); i++) { - Node node = nodes.item(i); - if (node instanceof Element - && tagName.equals(((Element) node).getTagName())) { - elements.add((Element) node); - } - } - return elements; - } - - public String getElementXML(E element) throws Exception { - ByteArrayOutputStream writer = new ByteArrayOutputStream(); - getElementXML(element, writer); - return writer.toString(); - } - - public void getElementXML(E element, OutputStream os) throws Exception { - org.w3c.dom.Document xmlDocument = getBuilder().newDocument(); - // Propriétés du DOM - xmlDocument.setXmlVersion("1.0"); - xmlDocument.setXmlStandalone(true); - // Création de l'arborescence du DOM - Element racine = xmlDocument.createElement(element.getClass() - .getSimpleName()); - - saveXMLElement(xmlDocument, racine, element); - - xmlDocument.appendChild(racine); - DOMSource domSource = new DOMSource(xmlDocument); - StreamResult result = new StreamResult(os); - TransformerFactory tf = TransformerFactory.newInstance(); - Transformer transformer = tf.newTransformer(); - transformer.transform(domSource, result); - } - - protected String getProperty(Element xmlElement, String tagName) { - Element element = getElementByTagName(xmlElement, tagName); - if (element != null) { - return element.getFirstChild().getNodeValue(); - } else { - return null; - } - } - - public <T extends DataEntity> Set<T> loadCollection(String tagSetName, - String tagName, Class<T> clazz, Element xmlElement, - DataEntity parentData) throws Exception { - Set<T> entities = new HashSet<T>(); - - Element xmlCollection = getElementByTagName(xmlElement, tagSetName); - if (xmlCollection != null) { - DataEntityFactory<T> elementFactory = getFactory(clazz); - - Set<Element> list = getElementsByTagName(xmlCollection, tagName); - for (Element element : list) { - T entity = elementFactory.loadXMLElement(element); - entity.setParentData(parentData); - entities.add(entity); - } - } - - return entities; - } - - public E loadXMLElement(Element xmlElement) throws Exception { - E element = createInstance(); - return element; - } - - public <T extends DataEntity> void saveCollection(Set<T> entities, - String tagSetName, String tagName, Class<T> clazz, - Document document, Element xmlElement) throws Exception { - if (entities != null && entities.size() > 0) { - Element xmlCollection = document.createElement(tagSetName); - DataEntityFactory<T> elementFactory = getFactory(clazz); - for (T entity : entities) { - Element childElement = document.createElement(tagName); - // TODO - // if (entity instanceof LoggableElement) { - // elementFactory.saveXMLElementByReference - elementFactory.saveXMLElement(document, childElement, entity); - xmlCollection.appendChild(childElement); - } - xmlElement.appendChild(xmlCollection); - } - } - - public abstract void saveXMLElement(Document document, Element xmlElement, - E element) throws Exception; - - protected void setProperty(Document document, Element xmlElement, - String tagName, Object value) { - if (value != null) { - Element xmlProperty = document.createElement(tagName); - Text xmlDescriptionText = document.createTextNode(value.toString()); - xmlProperty.appendChild(xmlDescriptionText); - xmlElement.appendChild(xmlProperty); - } - } - -} Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/LoggableElementFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/LoggableElementFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/LoggableElementFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -6,7 +6,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.LoggableElement; public abstract class LoggableElementFactory<E extends LoggableElement> extends - DataEntityFactory<E> implements DatabaseConstants { + BaseEntityFactory<E> implements DatabaseConstants { private static final String KEY_DESCRIPTION = "description"; Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/MetaDataEntityFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/MetaDataEntityFactory.java (rev 0) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/MetaDataEntityFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -0,0 +1,39 @@ +package fr.cemagref.simexplorer.is.storage.factories; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; + +public class MetaDataEntityFactory<E extends MetaDataEntity> extends + BaseEntityFactory<MetaDataEntity> { + + private static String KEY_UUID = "uuid"; + private static String KEY_VERSION = "version"; + + @Override + public MetaDataEntity createInstance() { + return new MetaDataEntity(); + } + + /* + * (non-Javadoc) + * + * @see fr.cemagref.simexplorer.is.storage.factories.BaseEntityFactory#loadXMLElement(org.w3c.dom.Element) + */ + @Override + public MetaDataEntity loadXMLElement(Element xmlElement) throws Exception { + MetaDataEntity entity = super.loadXMLElement(xmlElement); + entity.setUuid(getProperty(xmlElement, KEY_UUID)); + entity.setVersion(getProperty(xmlElement, KEY_VERSION)); + return entity; + } + + @Override + public void saveXMLElement(Document document, Element xmlElement, + MetaDataEntity element) throws Exception { + // TODO Auto-generated method stub + + } + +} Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/RepositoryFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/RepositoryFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/RepositoryFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -6,7 +6,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Repository; public class RepositoryFactory<E extends Repository> extends - DataEntityFactory<Repository> { + BaseEntityFactory<Repository> { @Override public Repository createInstance() { Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ResultFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ResultFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/ResultFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -5,7 +5,7 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Result; -public class ResultFactory<E extends Result> extends DataEntityFactory<Result> { +public class ResultFactory<E extends Result> extends BaseEntityFactory<Result> { @Override public Result createInstance() { Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/StructureFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/StructureFactory.java 2007-12-17 10:58:32 UTC (rev 68) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/factories/StructureFactory.java 2007-12-17 10:59:58 UTC (rev 69) @@ -3,6 +3,6 @@ import fr.cemagref.simexplorer.is.storage.entities.data.Structure; public abstract class StructureFactory<E extends Structure> extends - DataEntityFactory<Structure> { + BaseEntityFactory<Structure> { }

1 0

r68 - trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata
by glandais＠users.labs.libre-entreprise.org Dec. 17, 2007

Dec. 17, 2007

Author: glandais Date: 2007-12-17 10:58:32 +0000 (Mon, 17 Dec 2007) New Revision: 68 Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java Log: R?\195?\169f?\195?\169rences aux parents par id/version au lieu d'instance (performances) Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java 2007-12-17 10:57:51 UTC (rev 67) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java 2007-12-17 10:58:32 UTC (rev 68) @@ -20,8 +20,10 @@ private String hash; private Map<String, String> descriptors; private Map<String, String> attachments; - private MetaDataEntity parentData; - private MetaDataEntity parentVersion; + private String parentDataUuid; + private String parentDataVersion; + private String parentVersionUuid; + private String parentVersionVersion; /** * @return the uuid @@ -144,47 +146,73 @@ } /** - * @return the parentData + * @return the attachments */ - public MetaDataEntity getParentData() { - return parentData; + public Map<String, String> getAttachments() { + return attachments; } /** - * @param parentData - * the parentData to set + * @param attachments the attachments to set */ - public void setParentData(MetaDataEntity parentData) { - this.parentData = parentData; + public void setAttachments(Map<String, String> attachments) { + this.attachments = attachments; } /** - * @return the parentVersion + * @return the parentDataUuid */ - public MetaDataEntity getParentVersion() { - return parentVersion; + public String getParentDataUuid() { + return parentDataUuid; } /** - * @param parentVersion - * the parentVersion to set + * @param parentDataUuid the parentDataUuid to set */ - public void setParentVersion(MetaDataEntity parentVersion) { - this.parentVersion = parentVersion; + public void setParentDataUuid(String parentDataUuid) { + this.parentDataUuid = parentDataUuid; } /** - * @return the attachments + * @return the parentDataVersion */ - public Map<String, String> getAttachments() { - return attachments; + public String getParentDataVersion() { + return parentDataVersion; } /** - * @param attachments the attachments to set + * @param parentDataVersion the parentDataVersion to set */ - public void setAttachments(Map<String, String> attachments) { - this.attachments = attachments; + public void setParentDataVersion(String parentDataVersion) { + this.parentDataVersion = parentDataVersion; } + /** + * @return the parentVersionUuid + */ + public String getParentVersionUuid() { + return parentVersionUuid; + } + + /** + * @param parentVersionUuid the parentVersionUuid to set + */ + public void setParentVersionUuid(String parentVersionUuid) { + this.parentVersionUuid = parentVersionUuid; + } + + /** + * @return the parentVersionVersion + */ + public String getParentVersionVersion() { + return parentVersionVersion; + } + + /** + * @param parentVersionVersion the parentVersionVersion to set + */ + public void setParentVersionVersion(String parentVersionVersion) { + this.parentVersionVersion = parentVersionVersion; + } + }

1 0

r67 - trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine
by glandais＠users.labs.libre-entreprise.org Dec. 17, 2007

Dec. 17, 2007

Author: glandais Date: 2007-12-17 10:57:51 +0000 (Mon, 17 Dec 2007) New Revision: 67 Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java Log: Import/export g?\195?\169r?\195?\169s dans le service Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java 2007-12-17 10:57:23 UTC (rev 66) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java 2007-12-17 10:57:51 UTC (rev 67) @@ -117,25 +117,6 @@ database.deleteElement(element); } - // Import/Export - - public MetaDataEntity importElement(InputStream inputStream) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - public void exportElement(MetaDataEntity element, OutputStream os) - throws Exception { - // Retrieve element factory - /* - * Class<? extends LoggableElement> clazz = element.getClass(); - * LoggableElementFactory<LoggableElement> elementFactory = - * (LoggableElementFactory<LoggableElement>) LoggableElementFactory - * .getFactory(clazz); elementFactory.getElementXML(element, os); - */ - } - // Tools public String storeTempData(InputStream stream) throws Exception {

1 0

r66 - in trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database: . lucene
by glandais＠users.labs.libre-entreprise.org Dec. 17, 2007

Dec. 17, 2007

Author: glandais Date: 2007-12-17 10:57:23 +0000 (Mon, 17 Dec 2007) New Revision: 66 Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/SearcherPool.java Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java Log: MAJ interface Database Searcher pool Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java 2007-12-11 17:08:37 UTC (rev 65) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java 2007-12-17 10:57:23 UTC (rev 66) @@ -22,9 +22,20 @@ * * @throws Exception */ - public abstract void open() throws Exception; + public void open() throws Exception { + open(false); + } /** + * Open database connection + * + * @param create + * Create database + * @throws Exception + */ + public abstract void open(boolean create) throws Exception; + + /** * Close database connection * * @throws Exception @@ -47,18 +58,13 @@ * Element to insert * @throws Exception */ - public abstract void insertElement(MetaDataEntity element, List<Reader> readers, boolean forceCommit) throws Exception; + public abstract void insertElement(MetaDataEntity element, + List<Reader> readers) throws Exception; - public void insertElement(MetaDataEntity element, List<Reader> readers) - throws Exception { - insertElement(element, readers, true); - } - // Read /** - * Get an element from its id and its version - * null if not fund + * Get an element from its id and its version null if not fund * * @param uuid * Id @@ -71,8 +77,8 @@ throws Exception; /** - * Get all version of an element thanks to its id - * Empty list if no element with this id + * Get all version of an element thanks to its id Empty list if no element + * with this id * * @param uuid * Id @@ -82,8 +88,7 @@ public abstract List<Version> getVersions(String uuid) throws Exception; /** - * Retrieve an element in its latest version - * null if not fund + * Retrieve an element in its latest version null if not fund * * @param uuid * Id @@ -115,8 +120,7 @@ } /** - * Retrieve all elements by id - * Empty list if no element with this id + * Retrieve all elements by id Empty list if no element with this id * * @param uuid * Id @@ -127,7 +131,7 @@ throws Exception; /** - * Retrieve elements with specific properties + * Retrieve elements with specific properties * Empty list if no element * * @param properties @@ -135,11 +139,41 @@ * @return Element list * @throws Exception */ + public Set<MetaDataEntity> findElementsByProperties( + Map<String, String> properties) throws Exception { + return findElementsByProperties(properties, 0, -1); + } + + /** + * Retrieve elements with specific properties + * Empty list if no element + * + * @param properties + * Matching properties needed + * @param start + * Index of first element returned + * @param count + * Number of elements to return + * @return Element list + * @throws Exception + */ public abstract Set<MetaDataEntity> findElementsByProperties( + Map<String, String> properties, int start, int count) + throws Exception; + + /** + * Retrieve element count specific properties + * + * @param properties + * Matching properties needed + * @return Number of elements fund + * @throws Exception + */ + public abstract int findElementsByPropertiesCount( Map<String, String> properties) throws Exception; /** - * Retrieve elements by content search + * Retrieve elements by content search * Empty list if no element * * @param searchedText @@ -147,9 +181,38 @@ * @return Element list * @throws Exception */ + public Set<MetaDataEntity> findElementsByContentSearch(String queryText) + throws Exception { + return findElementsByContentSearch(queryText, 0, -1); + } + + /** + * Retrieve elements by content search + * Empty list if no element + * + * @param queryText + * Searched text + * @param start + * Index of first element returned + * @param count + * Number of elements to return + * @return Element list + * @throws Exception + */ public abstract Set<MetaDataEntity> findElementsByContentSearch( - String queryText) throws Exception; + String queryText, int start, int count) throws Exception; + /** + * Retrieve element count with specific content + * + * @param queryText + * Searched text + * @return Number of elements fund + * @throws Exception + */ + public abstract int findElementsByContentSearchCount(String queryText) + throws Exception; + // Delete /** Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java 2007-12-11 17:08:37 UTC (rev 65) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java 2007-12-17 10:57:23 UTC (rev 66) @@ -1,7 +1,8 @@ package fr.cemagref.simexplorer.is.storage.database.lucene; +import java.io.File; +import java.io.IOException; import java.io.Reader; -import java.io.StringReader; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -12,12 +13,11 @@ import java.util.Map.Entry; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; -import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queryParser.QueryParser; @@ -30,6 +30,8 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.NoLockFactory; import fr.cemagref.simexplorer.is.storage.database.Database; import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; @@ -44,77 +46,125 @@ public class LuceneDatabase extends Database { /** - * Content analyzer + * Lock factory for Lucene writer */ - private static Analyzer analyser; + private static LockFactory lockFactory = new NoLockFactory(); /** - * Index location + * Location of index */ - private static String indexDirectory; + private Directory directory; /** - * Database writer + * Index writer */ - private static IndexWriter writer; + private IndexWriter writer = null; /** - * Database searcher + * Pool of searcher */ - private static Searcher searcher; + private SearcherPool searcherPool = new SearcherPool(); /** - * Init flag + * Searcher that should be used for new queries */ - private static boolean initok = false; + private Searcher cachedSearcher = null; + /** + * Date of latest optimization + */ + private Date lastOptimize = new Date(); - @Override - public void open() throws Exception { - if (!initok) { - // Create a per field analyze - analyser = new PerFieldAnalyzerWrapper(new SimpleAnalyzer()); + /** + * Retrieve searcher for a new query + * + * @return Searcher + * @throws CorruptIndexException + * @throws IOException + */ + private Searcher getSearcher() throws CorruptIndexException, IOException { + synchronized (this) { + // If cached searcher is null (as after an index write) + if (cachedSearcher == null) { + // Instanciate a new searcher + cachedSearcher = new IndexSearcher(directory); + } + // Mark searcher as being used in a new query + searcherPool.capture(cachedSearcher); + return cachedSearcher; + } + } - // Specific analyzers - // analyser.addAnalyzer(KEY_, new KeywordAnalyzer()); - // analyser.addAnalyzer(KEY_, new KeywordAnalyzer()); + /** + * Release searcher, after the end of a query execution + * + * @param searcher + * Searcher to be released + * @throws IOException + */ + private void releaseSearcher(Searcher searcher) throws IOException { + synchronized (this) { + searcherPool.release(searcher); + } + } - // FIXME index location - indexDirectory = "./index/"; - - // Create index if it does not exist - boolean create = false; - if (!IndexReader.indexExists(indexDirectory)) { - create = true; + /** + * Called to invalidate reader + * + * @throws IOException + */ + private void resetReader() throws IOException { + synchronized (this) { + // Reset searcher + if (cachedSearcher != null) { + // Mark as should be closed asap + searcherPool.mark(cachedSearcher); + cachedSearcher = null; } + } + } - // Create a Store for Lucene DB - Directory dir = FSDirectory.getDirectory(indexDirectory); + @Override + public void open(boolean create) throws Exception { + // Create an analyzer + Analyzer analyzer = new SimpleAnalyzer(); - // Create Lucene indexer - writer = new IndexWriter(dir, true, analyser, create); - - searcher = new IndexSearcher(indexDirectory); - - // FIXME - System.out.println("writer.docCount() : " + writer.docCount()); - - initok = true; - } + // FIXME index location + File indexDir = new File("./index/"); + + // Base directory index + directory = FSDirectory.getDirectory(indexDir, lockFactory); + + // Instanciate unique writer + writer = new IndexWriter(directory, true, analyzer, create); } @Override public void close() throws Exception { - // Close writer writer.close(); - searcher.close(); - initok = false; + resetReader(); } + private synchronized void synchedCommit() throws CorruptIndexException, + IOException { + // Flush in ram data + writer.flush(); + // Check if an optimization is required + Date now = new Date(); + long elapsed = now.getTime() - lastOptimize.getTime(); + // One optimize per hour, if readers are closed + if (elapsed > 3600 * 1000 && searcherPool.allClosed()) { + writer.optimize(); + lastOptimize = new Date(); + } + //System.out.println(writer.docCount()); + // Invalidate reader + resetReader(); + } + @Override public void commit() throws Exception { - // write index - writer.flush(); - searcher = new IndexSearcher(indexDirectory); + synchedCommit(); } - + @Override - public void insertElement(MetaDataEntity element, List<Reader> readers, boolean forceCommit) throws Exception { + public void insertElement(MetaDataEntity element, List<Reader> readers) + throws Exception { // Save element to a Lucene document Document document = saveLuceneElement(element, readers); @@ -129,9 +179,6 @@ // add document to index writer.addDocument(document); - if (forceCommit) { - commit(); - } } @Override @@ -139,10 +186,17 @@ throws Exception { MetaDataEntity result = null; - Hits hits = getHitsByIdVersion(uuid, version); - if (hits.length() != 0) { - // convert first document to element - result = loadLuceneElement(hits.doc(0)); + // Get current searcher instance + Searcher searcher = getSearcher(); + try { + Hits hits = getHitsByIdVersion(uuid, version, searcher); + if (hits != null && hits.length() != 0) { + // convert first document to element + result = loadLuceneElement(hits.doc(0)); + } + } finally { + // Release searcher instance + releaseSearcher(searcher); } return result; @@ -152,17 +206,18 @@ * @param uuid * @param version * @return + * @throws IOException * @throws Exception */ - private Hits getHitsByIdVersion(String uuid, Version version) - throws Exception { + private Hits getHitsByIdVersion(String uuid, Version version, + Searcher searcher) throws IOException { // Retrieve element with id/version Map<String, String> properties = new HashMap<String, String>(); properties.put(KEY_UUID, uuid); properties.put(KEY_VERSION, version.toString()); // Search elements - Hits hits = findHits(properties); + Hits hits = findHits(properties, searcher); return hits; } @@ -171,13 +226,24 @@ // Retrieve all document corresponding to id Map<String, String> properties = new HashMap<String, String>(); properties.put(KEY_UUID, uuid); - Hits hits = findHits(properties); - // Add all versions to a list - List<Version> versions = new ArrayList<Version>(); - for (int i = 0; i < hits.length(); i++) { - Document doc = hits.doc(i); - versions.add(new Version(doc.get(KEY_VERSION))); + List<Version> versions = null; + // Get current searcher instance + Searcher searcher = getSearcher(); + try { + Hits hits = findHits(properties, searcher); + + // Add all versions to a list + versions = new ArrayList<Version>(); + if (hits != null) { + for (int i = 0; i < hits.length(); i++) { + Document doc = hits.doc(i); + versions.add(new Version(doc.get(KEY_VERSION))); + } + } + } finally { + // Release searcher instance + releaseSearcher(searcher); } return versions; @@ -188,40 +254,118 @@ // Create hash map with id Map<String, String> properties = new HashMap<String, String>(); properties.put(KEY_UUID, uuid); + + Set<MetaDataEntity> result = null; + + // Get current searcher instance + Searcher searcher = getSearcher(); + try { + Hits hits = findHits(properties, searcher); + result = convertHitsToElements(hits, 0, -1); + } finally { + // Release searcher instance + releaseSearcher(searcher); + } + // Return all elements parsed from documents fund - return convertHitsToElements(findHits(properties)); + return result; } @Override + public int findElementsByPropertiesCount(Map<String, String> properties) + throws Exception { + int result = 0; + + // Get current searcher instance + Searcher searcher = getSearcher(); + try { + Hits hits = findHits(properties, searcher); + result = hits.length(); + } finally { + // Release searcher instance + releaseSearcher(searcher); + } + + // Return all elements parsed from documents fund + return result; + } + + @Override public Set<MetaDataEntity> findElementsByProperties( - Map<String, String> properties) throws Exception { + Map<String, String> properties, int start, int count) + throws Exception { + Set<MetaDataEntity> result = null; + + // Get current searcher instance + Searcher searcher = getSearcher(); + try { + Hits hits = findHits(properties, searcher); + result = convertHitsToElements(hits, start, count); + } finally { + // Release searcher instance + releaseSearcher(searcher); + } + // Return all elements parsed from documents fund - return convertHitsToElements(findHits(properties)); + return result; } @Override - public Set<MetaDataEntity> findElementsByContentSearch(String queryText) throws Exception { + public int findElementsByContentSearchCount(String queryText) + throws Exception { Analyzer analyzer = new SimpleAnalyzer(); QueryParser parser = new QueryParser(KEY_SEARCHABLE_CONTENT, analyzer); Query luceneQuery = parser.parse(queryText); - // Create a bean with hits and searcher - Hits luceneHits = searcher.search(luceneQuery); + int result = 0; - // Convert hits to elements - Set<MetaDataEntity> result = convertHitsToElements(luceneHits); + Hits hits = null; + // Get current searcher instance + Searcher searcher = getSearcher(); + try { + hits = searcher.search(luceneQuery); + // Return hits length + result = hits.length(); + } finally { + // Release searcher instance + releaseSearcher(searcher); + } return result; } @Override + public Set<MetaDataEntity> findElementsByContentSearch(String queryText, + int start, int count) throws Exception { + Analyzer analyzer = new SimpleAnalyzer(); + QueryParser parser = new QueryParser(KEY_SEARCHABLE_CONTENT, analyzer); + Query luceneQuery = parser.parse(queryText); + + Set<MetaDataEntity> result = null; + + Hits hits = null; + // Get current searcher instance + Searcher searcher = getSearcher(); + try { + hits = searcher.search(luceneQuery); + // Convert hits to elements + result = convertHitsToElements(hits, start, count); + } finally { + // Release searcher instance + releaseSearcher(searcher); + } + + return result; + } + + @Override public void deleteElement(String uuid, Version version) throws Exception { // Delete element in db with term search Term[] terms = new Term[2]; terms[0] = new Term(KEY_UUID, uuid); terms[1] = new Term(KEY_VERSION, version.toString()); + writer.deleteDocuments(terms); - commit(); } /** @@ -230,10 +374,11 @@ * @param properties * criteria * @return Documents and search handle + * @throws IOException * @throws Exception */ - private Hits findHits(Map<String, String> properties) - throws Exception { + private Hits findHits(Map<String, String> properties, Searcher searcher) + throws IOException { // Create a query with all parameters BooleanQuery query = new BooleanQuery(); for (Entry<String, String> kv : properties.entrySet()) { @@ -241,9 +386,8 @@ BooleanClause.Occur.MUST); } - // Create a bean with hits and searcher - Hits luceneHits = searcher.search(query); - return luceneHits; + Hits hits = searcher.search(query); + return hits; } /** @@ -251,17 +395,26 @@ * * @param hits * Document collection + * @param start + * Index of first document to return + * @param count + * Number of documents to return * @return Entities list + * @throws IOException + * @throws CorruptIndexException * @throws Exception */ - private Set<MetaDataEntity> convertHitsToElements(Hits hits) - throws Exception { + private Set<MetaDataEntity> convertHitsToElements(Hits hits, int start, + int count) throws CorruptIndexException, IOException { Set<MetaDataEntity> elements = new HashSet<MetaDataEntity>(); - // For all documents - for (int i = 0; i < hits.length(); i++) { - Document document = hits.doc(i); - // Convert to element - elements.add(loadLuceneElement(document)); + if (hits != null) { + // For documents in range + for (int i = start; i < hits.length() + && (i < start + count || count == -1); i++) { + Document document = hits.doc(i); + // Convert to element + elements.add(loadLuceneElement(document)); + } } return elements; } @@ -276,8 +429,7 @@ * @return Instance of element * @throws Exception */ - private MetaDataEntity loadLuceneElement(Document document) - throws Exception { + private MetaDataEntity loadLuceneElement(Document document) { MetaDataEntity element = new MetaDataEntity(); element.setUuid(document.get(KEY_UUID)); element.setName(document.get(KEY_NAME)); @@ -295,14 +447,14 @@ List<Fieldable> fields = document.getFields(); for (Fieldable fieldable : fields) { - + if (fieldable.name().startsWith(KEY_DESCRIPTOR)) { String field = fieldable.name().replace(KEY_DESCRIPTOR + ".", ""); String value = fieldable.stringValue(); descriptors.put(field, value); } - + if (fieldable.name().startsWith(KEY_ATTACHMENT)) { String field = fieldable.name().replace(KEY_ATTACHMENT + ".", ""); @@ -310,26 +462,17 @@ attachments.put(field, value); } } - + element.setDescriptors(descriptors); element.setAttachments(attachments); - String parentId = document.get(KEY_PARENTDATA_UUID); - String parentVersion = document.get(KEY_PARENTDATA_VERSION); - if (parentId != null && parentVersion != null) { - MetaDataEntity parentData = getElement(parentId, new Version( - parentVersion)); - element.setParentData(parentData); - } + element.setParentDataUuid(document.get(KEY_PARENTDATA_UUID)); + element.setParentDataVersion(document.get(KEY_PARENTDATA_VERSION)); - parentId = document.get(KEY_PARENTVERSION_UUID); - parentVersion = document.get(KEY_PARENTVERSION_VERSION); - if (parentId != null && parentVersion != null) { - MetaDataEntity parentByVersion = getElement(document - .get(KEY_PARENTVERSION_UUID), new Version(document - .get(KEY_PARENTVERSION_VERSION))); - element.setParentVersion(parentByVersion); - } + element.setParentVersionUuid(document.get(KEY_PARENTVERSION_UUID)); + element + .setParentVersionVersion(document + .get(KEY_PARENTVERSION_VERSION)); return element; } @@ -346,7 +489,8 @@ * @return * @throws Exception */ - private Document saveLuceneElement(MetaDataEntity element, List<Reader> readers) throws Exception { + private Document saveLuceneElement(MetaDataEntity element, + List<Reader> readers) { Document document = new Document(); document.add(simpleField(KEY_UUID, element.getUuid())); @@ -372,29 +516,27 @@ document.add(simpleField(key, value)); } - if (element.getParentData() != null) { + if (element.getParentDataUuid() != null) { document.add(simpleField(KEY_PARENTDATA_UUID, element - .getParentData().getUuid())); + .getParentDataUuid())); + } + if (element.getParentDataVersion() != null) { document.add(simpleField(KEY_PARENTDATA_VERSION, element - .getParentData().getVersion().toString())); + .getParentDataVersion())); } - if (element.getParentVersion() != null) { + if (element.getParentVersionUuid() != null) { document.add(simpleField(KEY_PARENTVERSION_UUID, element - .getParentVersion().getUuid())); + .getParentVersionUuid())); + } + if (element.getParentVersionVersion() != null) { document.add(simpleField(KEY_PARENTVERSION_VERSION, element - .getParentVersion().getVersion().toString())); + .getParentVersionVersion())); } - StringBuffer buf = new StringBuffer(); for (Reader reader : readers) { - for(int c = reader.read(); c != -1; c = reader.read()) { - buf.append((char)c); - } + document.add(new Field(KEY_SEARCHABLE_CONTENT, reader)); } - - Reader reader = new StringReader(buf.toString()); - document.add(new Field(KEY_SEARCHABLE_CONTENT, reader)); - + return document; } Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/SearcherPool.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/SearcherPool.java (rev 0) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/SearcherPool.java 2007-12-17 10:57:23 UTC (rev 66) @@ -0,0 +1,131 @@ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.lucene.search.Searcher; + +/** + * Pool of searchers, being randomly accessed + * Manage life cycle of searchers + * + * @author landais + * + */ +public class SearcherPool { + + /** + * Searcher record + * + * @author landais + * + */ + private class SearcherInfo { + /** + * Numbre of queries using this searcher + */ + public int count = 0; + /** + * If marked, searcher is not valid anymore + * It should be closed upon last query ended + */ + public boolean marked = false; + } + + /** + * Searchers used by database + */ + Map<Searcher, SearcherInfo> searchers = new HashMap<Searcher, SearcherInfo>(); + + /** + * Called when a searcher will be used for a query + * + * @param searcher + * Searcher used + */ + public void capture(Searcher searcher) { + SearcherInfo searcherInfo = null; + // Check is searcher is already referenced + if (searchers.containsKey(searcher)) { + // Retrieve searcher info + searcherInfo = searchers.get(searcher); + } else { + // Create searcher info otherwise + searcherInfo = new SearcherInfo(); + // Store it + searchers.put(searcher, searcherInfo); + } + // Increment usage + searcherInfo.count = searcherInfo.count + 1; + } + + /** + * Check if searcher should be closed + * + * @param searcher + * Searcher aiming to be closed + * @param searcherInfo + * Searcher info related + * @throws IOException + */ + private void checkClose(Searcher searcher, SearcherInfo searcherInfo) + throws IOException { + // searcher should be marked as invalid and not used anymore + if (searcherInfo.marked && searcherInfo.count == 0) { + // Remove searcher + searchers.remove(searcher); + // Close it + searcher.close(); + } + } + + /** + * Release a searcher from a query + * + * @param searcher + * Searcher released + * + * @throws IOException + */ + public void release(Searcher searcher) throws IOException { + // Retrieve searcher info + SearcherInfo searcherInfo = searchers.get(searcher); + // Decrement usage + searcherInfo.count = searcherInfo.count - 1; + // Check if searcher should be closed + checkClose(searcher, searcherInfo); + } + + /** + * Mark searcher as being invalid anymore + * + * @param searcher + * @throws IOException + */ + public void mark(Searcher searcher) throws IOException { + // Retrieve searcher + SearcherInfo searcherInfo = searchers.get(searcher); + // Mark as invalid + searcherInfo.marked = true; + // Check if searcher should be closed + checkClose(searcher, searcherInfo); + } + + /** + * Check if any reader is opened + * + * @return true if no reader is open + */ + public boolean allClosed() { + for (Entry<Searcher, SearcherInfo> kv : searchers.entrySet()) { + if (kv.getValue().count > 0) { + return false; + } + } + return true; + } + +}

1 0

r65 - in trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage: . service xml
by glandais＠users.labs.libre-entreprise.org Dec. 11, 2007

Dec. 11, 2007

Author: glandais Date: 2007-12-11 17:08:37 +0000 (Tue, 11 Dec 2007) New Revision: 65 Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java Log: Initialisation du point d'entr?\195?\169e du service de stockage Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java (rev 0) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/service/StorageService.java 2007-12-11 17:08:37 UTC (rev 65) @@ -0,0 +1,44 @@ +package fr.cemagref.simexplorer.is.storage.service; + +import java.io.InputStream; + +import org.w3c.dom.Document; + +import fr.cemagref.simexplorer.is.storage.engine.StorageEngine; +import fr.cemagref.simexplorer.is.storage.entities.data.LoggableElement; +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; +import fr.cemagref.simexplorer.is.storage.xml.MetaDataParser; + +public class StorageService { + + private StorageEngine storageEngine; + + public StorageService() { + super(); + storageEngine = new StorageEngine(); + } + + public void open() throws Exception { + storageEngine.open(); + } + + public void close() throws Exception { + storageEngine.close(); + } + + public LoggableElement saveElement(InputStream xmlFile, + InputStream attachment) throws Exception { + String idxml = storageEngine.storeTempData(xmlFile); + String idattachment = null; + if (attachment != null) { + idattachment = storageEngine.storeTempData(attachment); + } + + MetaDataEntity metaData = MetaDataParser.parse(storageEngine + .retrieveTempData(idxml)); + LoggableElement result = null; + + return result; + } + +} Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java (rev 0) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/MetaDataParser.java 2007-12-11 17:08:37 UTC (rev 65) @@ -0,0 +1,30 @@ +package fr.cemagref.simexplorer.is.storage.xml; + +import java.io.InputStream; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; + +public class MetaDataParser extends Parser { + + private static String KEY_METADATA = "metadata"; + + private static String KEY_UUID = "metadata"; + private static String KEY_VERSION = "version"; + + public static MetaDataEntity parse(InputStream xmlStream) throws Exception { + MetaDataEntity mde = new MetaDataEntity(); + + Document document = getBuilder().parse(xmlStream); + Element rootElement = (Element) document.getFirstChild(); + Element metadataElement = getElementByTagName(rootElement, KEY_METADATA); + + mde.setUuid(getProperty(metadataElement, KEY_UUID)); + mde.setVersion(getProperty(metadataElement, KEY_VERSION)); + + return mde; + } + +} Added: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java (rev 0) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/xml/Parser.java 2007-12-11 17:08:37 UTC (rev 65) @@ -0,0 +1,72 @@ +package fr.cemagref.simexplorer.is.storage.xml; + +import java.util.HashSet; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; + +public class Parser { + + private static DocumentBuilder documentBuilder = null; + + public static synchronized DocumentBuilder getBuilder() throws Exception { + if (documentBuilder == null) { + DocumentBuilderFactory factory = DocumentBuilderFactory + .newInstance(); + documentBuilder = factory.newDocumentBuilder(); + return documentBuilder; + } + return documentBuilder; + } + + protected static String getProperty(Element xmlElement, String tagName) { + Element element = getElementByTagName(xmlElement, tagName); + if (element != null) { + return element.getFirstChild().getNodeValue(); + } else { + return null; + } + } + + protected static void setProperty(Document document, Element xmlElement, + String tagName, Object value) { + if (value != null) { + Element xmlProperty = document.createElement(tagName); + Text xmlDescriptionText = document.createTextNode(value.toString()); + xmlProperty.appendChild(xmlDescriptionText); + xmlElement.appendChild(xmlProperty); + } + } + + protected static Element getElementByTagName(Element xmlElement, String tagName) { + Set<Element> elements = getElementsByTagName(xmlElement, tagName); + if (elements.size() > 0) { + return elements.iterator().next(); + } else { + return null; + } + } + + protected static Set<Element> getElementsByTagName(Element xmlElement, + String tagName) { + Set<Element> elements = new HashSet<Element>(); + + NodeList nodes = xmlElement.getChildNodes(); + for (int i = 0; i < nodes.getLength(); i++) { + Node node = nodes.item(i); + if (node instanceof Element + && tagName.equals(((Element) node).getTagName())) { + elements.add((Element) node); + } + } + return elements; + } + +}

1 0

r64 - trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test
by glandais＠users.labs.libre-entreprise.org Dec. 11, 2007

Dec. 11, 2007

Author: glandais Date: 2007-12-11 17:07:52 +0000 (Tue, 11 Dec 2007) New Revision: 64 Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java Log: Maintenance Modified: trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java =================================================================== --- trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2007-12-11 17:07:26 UTC (rev 63) +++ trunk/simexplorer-si-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2007-12-11 17:07:52 UTC (rev 64) @@ -13,18 +13,20 @@ import java.util.UUID; import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.storage.database.Database; import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; import fr.cemagref.simexplorer.is.storage.entities.metadata.Version; public class LuceneDatabaseTestCase extends TestCase { - private LuceneDatabase database; + private Database database; // private ElementGenerator elementGenerator; private Random r = new Random(); protected void setUp() throws Exception { super.setUp(); + database = new LuceneDatabase(); database.open(); // elementGenerator = new ElementGenerator(); @@ -68,6 +70,7 @@ .toString()); } me.setDescriptors(descriptors); + me.setAttachments(new HashMap<String, String>()); me.setParentData(null); me.setParentVersion(null); @@ -93,14 +96,48 @@ public void testInsertElementGetElement() throws Exception { MetaDataEntity me = generateMetaDataEntity(); - database.insertElement(me); + database.insertElement(me, new ArrayList<Reader>()); MetaDataEntity me2 = database.getElement(me.getUuid(), me.getVersion()); assertEquals(me.getDescription(), me2.getDescription()); + database.insertElement(me, new ArrayList<Reader>()); } + public void testInsertElementGetElementExtra() throws Exception { + int c = 10; + Date begin = new Date(); + List<String> ids = new ArrayList<String>(); + for (int i = 0; i < c; i++) { + MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); + for (int j = 0; j < mes.length; j++) { + database.insertElement(mes[j], new ArrayList<Reader>(), false); + } + ids.add(mes[0].getUuid()); + } + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert : Time taken : " + time + "ms"); + double timePerElement = time / (4 * c); + System.out.println("Insert : Time taken per element : " + + timePerElement + "ms"); + begin = new Date(); + + for (String uuid : ids) { + MetaDataEntity me = database.getElementLatestVersion(uuid); + assertEquals(uuid, me.getUuid()); + } + + end = new Date(); + time = end.getTime() - begin.getTime(); + System.out.println("Search : Time taken : " + time + "ms"); + timePerElement = time / (4 * c); + System.out.println("Search : Time taken per element : " + + timePerElement + "ms"); + } + public void testIndexAddFindElementsByContentSearch() throws Exception { MetaDataEntity me = generateMetaDataEntity(); - database.insertElement(me); String keyphrase = Long.toHexString(r.nextLong()); @@ -109,7 +146,8 @@ List<Reader> readers = new ArrayList<Reader>(); readers.add(reader); - database.indexAdd(me, readers); + + database.insertElement(me, readers); Set<MetaDataEntity> mes = database .findElementsByContentSearch(keyphrase); @@ -120,7 +158,7 @@ public void testGetVersions() throws Exception { MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); for (int i = 0; i < mes.length; i++) { - database.insertElement(mes[i]); + database.insertElement(mes[i], new ArrayList<Reader>()); } List<Version> versions = database.getVersions(mes[0].getUuid()); assertEquals(mes.length, versions.size()); @@ -134,15 +172,17 @@ public void testFindElementsById() throws Exception { MetaDataEntity[] mes = generateVersionnedMetaDataEntity(); for (int i = 0; i < mes.length; i++) { - database.insertElement(mes[i]); + database.insertElement(mes[i], new ArrayList<Reader>()); } Set<MetaDataEntity> elements = database.findElementsById(mes[0] .getUuid()); assertEquals(elements.size(), mes.length); } - public void testFindElementsByProperties() { - fail("Not yet implemented"); + public void testFindElementsByProperties() throws Exception { + MetaDataEntity me = generateMetaDataEntity(); + + database.insertElement(me, new ArrayList<Reader>()); } public void testDeleteElements() {

1 0

r63 - in trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage: attachment/handler attachment/type database database/lucene engine entities/data entities/metadata
by glandais＠users.labs.libre-entreprise.org Dec. 11, 2007

Dec. 11, 2007

Author: glandais Date: 2007-12-11 17:07:26 +0000 (Tue, 11 Dec 2007) New Revision: 63 Removed: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneHits.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/FSStorageEngine.java Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/AttachmentHandler.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/type/ContentTypeFactory.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/DatabaseConstants.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Component.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationApplication.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationData.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Library.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/LoggableElement.java trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java Log: Refactoring et optimisations Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/AttachmentHandler.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/AttachmentHandler.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/AttachmentHandler.java 2007-12-11 17:07:26 UTC (rev 63) @@ -24,7 +24,7 @@ * Content * @throws Exception */ - public abstract void storeData(Database database, MetaDataEntity entity, String field, + public abstract void storeData(MetaDataEntity entity, String field, InputStream is) throws Exception; /** @@ -49,7 +49,7 @@ * Unique field for content * @throws Exception */ - public abstract void deleteData(Database database, MetaDataEntity entity, String field) + public abstract void deleteData(MetaDataEntity entity, String field) throws Exception; } \ No newline at end of file Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/handler/FileSystemAttachmentHandler.java 2007-12-11 17:07:26 UTC (rev 63) @@ -8,7 +8,6 @@ import java.io.InputStream; import fr.cemagref.simexplorer.is.storage.database.Database; -import fr.cemagref.simexplorer.is.storage.entities.data.LoggableElement; import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; /** @@ -60,8 +59,8 @@ } @Override - public void storeData(Database database, MetaDataEntity entity, - String field, InputStream is) throws Exception { + public void storeData(MetaDataEntity entity, String field, InputStream is) + throws Exception { // Simple stream on file FileOutputStream fos = new FileOutputStream(getFile(entity, field)); @@ -82,8 +81,8 @@ } @Override - public void deleteData(Database database, MetaDataEntity entity, - String field) throws Exception { + public void deleteData(MetaDataEntity entity, String field) + throws Exception { // Simple delete on file getFile(entity, field).delete(); } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/type/ContentTypeFactory.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/type/ContentTypeFactory.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/attachment/type/ContentTypeFactory.java 2007-12-11 17:07:26 UTC (rev 63) @@ -16,28 +16,34 @@ */ private static Map<String, ContentType> contentTypes = null; + private static String PACKAGE_TYPES = "fr.cemagref.simexplorer.is.storage.attachment.type"; + /** * Retrieve a content type instance * - * @param contentTypeClass + * @param contentTypeClassSimpleName * Class required * @return Instance * @throws Exception */ public static ContentType getContentTypeInstance( - Class<? extends ContentType> contentTypeClass) throws Exception { + String contentTypeClassSimpleName) throws Exception { + // Create cache if doesn't exist if (contentTypes == null) { contentTypes = new HashMap<String, ContentType>(); } // Check cache - ContentType result = contentTypes.get(contentTypeClass.getSimpleName()); + ContentType result = contentTypes.get(contentTypeClassSimpleName); // Create instance if doesn't exist, and puts it in cache if (result == null) { - result = contentTypeClass.newInstance(); + Class<?> contentTypeClass = Class.forName(PACKAGE_TYPES + "." + + contentTypeClassSimpleName); + result = (ContentType) contentTypeClass.newInstance(); contentTypes.put(contentTypeClass.getSimpleName(), result); } return result; + } } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/Database.java 2007-12-11 17:07:26 UTC (rev 63) @@ -31,6 +31,13 @@ */ public abstract void close() throws Exception; + /** + * Commit pending modifications + * + * @throws Exception + */ + public abstract void commit() throws Exception; + // Create / Update /** @@ -40,25 +47,18 @@ * Element to insert * @throws Exception */ - public abstract void insertElement(MetaDataEntity element) throws Exception; + public abstract void insertElement(MetaDataEntity element, List<Reader> readers, boolean forceCommit) throws Exception; - /** - * Index content for an element - * - * @param entity - * Related entity - * @param field - * Related field - * @param is - * Content - */ - public abstract void indexAdd(MetaDataEntity entity, List<Reader> readers) - throws Exception; + public void insertElement(MetaDataEntity element, List<Reader> readers) + throws Exception { + insertElement(element, readers, true); + } // Read /** * Get an element from its id and its version + * null if not fund * * @param uuid * Id @@ -72,6 +72,7 @@ /** * Get all version of an element thanks to its id + * Empty list if no element with this id * * @param uuid * Id @@ -82,6 +83,7 @@ /** * Retrieve an element in its latest version + * null if not fund * * @param uuid * Id @@ -114,6 +116,7 @@ /** * Retrieve all elements by id + * Empty list if no element with this id * * @param uuid * Id @@ -125,6 +128,7 @@ /** * Retrieve elements with specific properties + * Empty list if no element * * @param properties * Matching properties needed @@ -136,6 +140,7 @@ /** * Retrieve elements by content search + * Empty list if no element * * @param searchedText * Searched text @@ -159,15 +164,6 @@ } /** - * Delete all elements by id - * - * @param uuid - * Id - * @throws Exception - */ - public abstract void deleteElements(String uuid) throws Exception; - - /** * Delete an element by id and version * * @param uuid Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/DatabaseConstants.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/DatabaseConstants.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/DatabaseConstants.java 2007-12-11 17:07:26 UTC (rev 63) @@ -41,6 +41,10 @@ */ public static final String KEY_DESCRIPTOR = "simexplorer.descriptor"; /** + * Field name for attachment + */ + public static final String KEY_ATTACHMENT = "simexplorer.attachment"; + /** * Field name for id */ public static final String KEY_PARENTDATA_UUID = "simexplorer.parentdata.uuid"; Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabase.java 2007-12-11 17:07:26 UTC (rev 63) @@ -23,6 +23,7 @@ import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Hits; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Searcher; @@ -45,19 +46,23 @@ /** * Content analyzer */ - private Analyzer analyser; + private static Analyzer analyser; /** * Index location */ - private String indexDirectory; + private static String indexDirectory; /** * Database writer */ - private IndexWriter writer; + private static IndexWriter writer; /** + * Database searcher + */ + private static Searcher searcher; + /** * Init flag */ - private boolean initok = false; + private static boolean initok = false; @Override public void open() throws Exception { @@ -66,8 +71,8 @@ analyser = new PerFieldAnalyzerWrapper(new SimpleAnalyzer()); // Specific analyzers - // analyser.addAnalyzer("firstname", new KeywordAnalyzer()); - // analyser.addAnalyzer("lastname", new KeywordAnalyzer()); + // analyser.addAnalyzer(KEY_, new KeywordAnalyzer()); + // analyser.addAnalyzer(KEY_, new KeywordAnalyzer()); // FIXME index location indexDirectory = "./index/"; @@ -83,6 +88,12 @@ // Create Lucene indexer writer = new IndexWriter(dir, true, analyser, create); + + searcher = new IndexSearcher(indexDirectory); + + // FIXME + System.out.println("writer.docCount() : " + writer.docCount()); + initok = true; } } @@ -91,12 +102,21 @@ public void close() throws Exception { // Close writer writer.close(); + searcher.close(); + initok = false; } @Override - public void insertElement(MetaDataEntity element) throws Exception { + public void commit() throws Exception { + // write index + writer.flush(); + searcher = new IndexSearcher(indexDirectory); + } + + @Override + public void insertElement(MetaDataEntity element, List<Reader> readers, boolean forceCommit) throws Exception { // Save element to a Lucene document - Document document = saveLuceneElement(element); + Document document = saveLuceneElement(element, readers); // Retrieve existing element with same id/version MetaDataEntity oldElement = getElement(element.getUuid(), element @@ -109,8 +129,9 @@ // add document to index writer.addDocument(document); - // write index - writer.flush(); + if (forceCommit) { + commit(); + } } @Override @@ -118,14 +139,12 @@ throws Exception { MetaDataEntity result = null; - LuceneHits hits = getHitsByIdVersion(uuid, version); - if (hits.getHits().length() != 0) { + Hits hits = getHitsByIdVersion(uuid, version); + if (hits.length() != 0) { // convert first document to element - result = loadLuceneElement(hits.getHits().doc(0)); + result = loadLuceneElement(hits.doc(0)); } - hits.closeSearcher(); - return result; } @@ -135,7 +154,7 @@ * @return * @throws Exception */ - private LuceneHits getHitsByIdVersion(String uuid, Version version) + private Hits getHitsByIdVersion(String uuid, Version version) throws Exception { // Retrieve element with id/version Map<String, String> properties = new HashMap<String, String>(); @@ -143,7 +162,7 @@ properties.put(KEY_VERSION, version.toString()); // Search elements - LuceneHits hits = findHits(properties); + Hits hits = findHits(properties); return hits; } @@ -152,12 +171,12 @@ // Retrieve all document corresponding to id Map<String, String> properties = new HashMap<String, String>(); properties.put(KEY_UUID, uuid); - LuceneHits hits = findHits(properties); + Hits hits = findHits(properties); // Add all versions to a list List<Version> versions = new ArrayList<Version>(); - for (int i = 0; i < hits.getHits().length(); i++) { - Document doc = hits.getHits().doc(i); + for (int i = 0; i < hits.length(); i++) { + Document doc = hits.doc(i); versions.add(new Version(doc.get(KEY_VERSION))); } @@ -182,16 +201,12 @@ @Override public Set<MetaDataEntity> findElementsByContentSearch(String queryText) throws Exception { - // Create a searcher - Searcher searcher = new IndexSearcher(indexDirectory); - Analyzer analyzer = new SimpleAnalyzer(); QueryParser parser = new QueryParser(KEY_SEARCHABLE_CONTENT, analyzer); Query luceneQuery = parser.parse(queryText); // Create a bean with hits and searcher - LuceneHits luceneHits = new LuceneHits(searcher.search(luceneQuery), - searcher); + Hits luceneHits = searcher.search(luceneQuery); // Convert hits to elements Set<MetaDataEntity> result = convertHitsToElements(luceneHits); @@ -200,20 +215,13 @@ } @Override - public void deleteElements(String uuid) throws Exception { - // Delete all documents by term search - writer.deleteDocuments(new Term(KEY_UUID, uuid)); - writer.flush(); - } - - @Override public void deleteElement(String uuid, Version version) throws Exception { // Delete element in db with term search Term[] terms = new Term[2]; terms[0] = new Term(KEY_UUID, uuid); terms[1] = new Term(KEY_VERSION, version.toString()); writer.deleteDocuments(terms); - writer.flush(); + commit(); } /** @@ -224,11 +232,8 @@ * @return Documents and search handle * @throws Exception */ - private LuceneHits findHits(Map<String, String> properties) + private Hits findHits(Map<String, String> properties) throws Exception { - // Create a searcher - Searcher searcher = new IndexSearcher(indexDirectory); - // Create a query with all parameters BooleanQuery query = new BooleanQuery(); for (Entry<String, String> kv : properties.entrySet()) { @@ -237,7 +242,7 @@ } // Create a bean with hits and searcher - LuceneHits luceneHits = new LuceneHits(searcher.search(query), searcher); + Hits luceneHits = searcher.search(query); return luceneHits; } @@ -249,17 +254,15 @@ * @return Entities list * @throws Exception */ - private Set<MetaDataEntity> convertHitsToElements(LuceneHits hits) + private Set<MetaDataEntity> convertHitsToElements(Hits hits) throws Exception { - // For all document Set<MetaDataEntity> elements = new HashSet<MetaDataEntity>(); - for (int i = 0; i < hits.getHits().length(); i++) { - Document document = hits.getHits().doc(i); + // For all documents + for (int i = 0; i < hits.length(); i++) { + Document document = hits.doc(i); // Convert to element elements.add(loadLuceneElement(document)); } - // Close handle - hits.closeSearcher(); return elements; } @@ -288,16 +291,28 @@ element.setHash(document.get(KEY_HASH)); Map<String, String> descriptors = new HashMap<String, String>(); + Map<String, String> attachments = new HashMap<String, String>(); + List<Fieldable> fields = document.getFields(); for (Fieldable fieldable : fields) { + if (fieldable.name().startsWith(KEY_DESCRIPTOR)) { String field = fieldable.name().replace(KEY_DESCRIPTOR + ".", ""); String value = fieldable.stringValue(); descriptors.put(field, value); } + + if (fieldable.name().startsWith(KEY_ATTACHMENT)) { + String field = fieldable.name().replace(KEY_ATTACHMENT + ".", + ""); + String value = fieldable.stringValue(); + attachments.put(field, value); + } } + element.setDescriptors(descriptors); + element.setAttachments(attachments); String parentId = document.get(KEY_PARENTDATA_UUID); String parentVersion = document.get(KEY_PARENTDATA_VERSION); @@ -331,7 +346,7 @@ * @return * @throws Exception */ - private Document saveLuceneElement(MetaDataEntity element) throws Exception { + private Document saveLuceneElement(MetaDataEntity element, List<Reader> readers) throws Exception { Document document = new Document(); document.add(simpleField(KEY_UUID, element.getUuid())); @@ -350,6 +365,13 @@ document.add(simpleField(key, value)); } + Map<String, String> attachments = element.getAttachments(); + for (Map.Entry<String, String> entry : attachments.entrySet()) { + String key = KEY_ATTACHMENT + "." + entry.getKey(); + String value = entry.getValue(); + document.add(simpleField(key, value)); + } + if (element.getParentData() != null) { document.add(simpleField(KEY_PARENTDATA_UUID, element .getParentData().getUuid())); @@ -363,45 +385,17 @@ .getParentVersion().getVersion().toString())); } + StringBuffer buf = new StringBuffer(); + for (Reader reader : readers) { + for(int c = reader.read(); c != -1; c = reader.read()) { + buf.append((char)c); + } + } + + Reader reader = new StringReader(buf.toString()); + document.add(new Field(KEY_SEARCHABLE_CONTENT, reader)); + return document; } - @Override - public void indexAdd(MetaDataEntity entity, List<Reader> readers) - throws Exception { - - // search associated document - LuceneHits hits = getHitsByIdVersion(entity.getUuid(), entity - .getVersion()); - - if (hits.getHits().length() > 0) { - - // retrieve document - Document document = hits.getHits().doc(0); - - StringBuffer buf = new StringBuffer(); - for (Reader reader : readers) { - for(int c = reader.read(); c != -1; c = reader.read()) { - buf.append((char)c); - } - } - - Reader reader = new StringReader(buf.toString()); - document.add(new Field(KEY_SEARCHABLE_CONTENT, reader)); - - hits.closeSearcher(); - - // update document - // delete old document - deleteElement(document.get(KEY_UUID), new Version(document - .get(KEY_VERSION))); - // add document to index - writer.addDocument(document); - // write index - writer.flush(); - - } - - } - } Deleted: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneHits.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneHits.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneHits.java 2007-12-11 17:07:26 UTC (rev 63) @@ -1,74 +0,0 @@ -package fr.cemagref.simexplorer.is.storage.database.lucene; - -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.Searcher; - -/** - * Bean handling hits collection and searcher handle - * - * @author landais - * - */ -public class LuceneHits { - - /** - * Hits collection - */ - private Hits hits; - /** - * Searcher handle - */ - private Searcher searcher; - - /** - * Default constructor - * - * @param hits - * @param searcher - */ - public LuceneHits(Hits hits, Searcher searcher) { - super(); - this.hits = hits; - this.searcher = searcher; - } - - /** - * Close handle - * - * @throws Exception - */ - public void closeSearcher() throws Exception { - searcher.close(); - } - - /** - * @return the hits - */ - public Hits getHits() { - return hits; - } - - /** - * @param hits - * the hits to set - */ - public void setHits(Hits hits) { - this.hits = hits; - } - - /** - * @return the searcher - */ - public Searcher getSearcher() { - return searcher; - } - - /** - * @param searcher - * the searcher to set - */ - public void setSearcher(Searcher searcher) { - this.searcher = searcher; - } - -} Deleted: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/FSStorageEngine.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/FSStorageEngine.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/FSStorageEngine.java 2007-12-11 17:07:26 UTC (rev 63) @@ -1,239 +0,0 @@ -package fr.cemagref.simexplorer.is.storage.engine; - -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Collections; -import java.util.List; - -import fr.cemagref.simexplorer.is.storage.attachment.handler.AttachmentHandler; -import fr.cemagref.simexplorer.is.storage.attachment.handler.FileSystemAttachmentHandler; -import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; -import fr.cemagref.simexplorer.is.storage.entities.data.LoggableElement; -import fr.cemagref.simexplorer.is.storage.entities.metadata.Version; -import fr.cemagref.simexplorer.is.storage.factories.DataEntityFactory; -import fr.cemagref.simexplorer.is.storage.factories.LoggableElementFactory; - -public class FSStorageEngine extends StorageEngine { - - @Override - public void close() throws Exception { - // TODO Auto-generated method stub - - } - - @Override - public void deleteData(LoggableElement entity, String field) - throws Exception { - // TODO Auto-generated method stub - - } - - @Override - public void exportElement(LoggableElement element, OutputStream os) - throws Exception { - // TODO Auto-generated method stub - - } - - @Override - public LoggableElement getElement(String uuid) throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public LoggableElement getElementVersion(String uuid, Version version) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public List<Version> getVersions(String uuid) throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public LoggableElement importElement(InputStream inputStream) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public LoggableElement newElement(LoggableElement element) throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public void open() throws Exception { - // TODO Auto-generated method stub - - } - - @Override - public InputStream retrieveData(LoggableElement entity, String field) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public LoggableElement saveElement(LoggableElement element) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public void storeData(LoggableElement entity, String field, InputStream is) - throws Exception { - // TODO Auto-generated method stub - - } - - @Override - public LoggableElement synchronizeElement(LoggableElement element) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - /* - private static AttachmentHandler handler; - private static boolean handlerInit = false; - - public static AttachmentHandler getHandler() { - if (!handlerInit) { - handler = new FileSystemAttachmentHandler(); - } - return handler; - } - - @Override - public void close() throws Exception { - database.close(); - } - - @Override - public void storeData(LoggableElement entity, String field, InputStream is) - throws Exception { - getHandler().storeData(database, entity, field, is); - } - - - @Override - public void exportElement(LoggableElement element, OutputStream os) - throws Exception { - LoggableElement completeElement = database.getElement(element - .getUuid(), element.getVersion()); - LoggableElementFactory<LoggableElement> factory = (LoggableElementFactory<LoggableElement>) DataEntityFactory - .getFactory(completeElement.getClass()); - factory.getElementXML(element, os); - } - - @Override - public LoggableElement getElement(String uuid) throws Exception { - return database.getElementLatestVersion(uuid); - } - - @Override - public LoggableElement getElementVersion(String uuid, Version version) - throws Exception { - - - // FIXME - String strXMLElement = document.get(KEY_XML); - InputStream is = new ByteArrayInputStream(strXMLElement.getBytes()); - org.w3c.dom.Document xmlDocument = DataEntityFactory.getBuilder().parse(is); - LoggableElement element = factory.loadXMLElement((Element) xmlDocument - .getFirstChild()); - - - return database.getElement(uuid, version); - } - - @Override - public List<Version> getVersions(String uuid) throws Exception { - return database.getVersions(uuid); - } - - @Override - public LoggableElement importElement(InputStream inputStream) - throws Exception { - LoggableElement importedElement = null; - - database.insertElement(importedElement); - return importedElement; - } - - @Override - public LoggableElement newElement(LoggableElement element) throws Exception { - if (getElement(element.getUuid()) != null) { - throw new Exception(element.getUuid() + " already exist"); - } - element.setVersion("1"); - - // FIXME - //document.add(new Field(KEY_XML, factory.getElementXML(element), - // Field.Store.COMPRESS, Field.Index.NO)); - - - database.insertElement(element); - return getElement(element.getUuid()); - } - - @Override - public void open() throws Exception { - database = new LuceneDatabase(); - database.open(); - } - - @Override - public LoggableElement saveElement(LoggableElement element) - throws Exception { - Version version = null; - List<Version> versions = getVersions(element.getUuid()); - if (versions.size() > 0) { - Collections.sort(versions); - Collections.reverse(versions); - - version = versions.get(0); - version.incVersion(0); - } else { - version = new Version("1"); - } - - element.setVersion(version.toString()); - - // FIXME - //document.add(new Field(KEY_XML, factory.getElementXML(element), - // Field.Store.COMPRESS, Field.Index.NO)); - - - database.insertElement(element); - return getElement(element.getUuid()); - } - - @Override - public LoggableElement synchronizeElement(LoggableElement element) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public void deleteData(FileEntity entity, String field) throws Exception { - getHandler().deleteData(database, entity, field); - } - - @Override - public InputStream retrieveData(FileEntity entity, String field) - throws Exception { - return getHandler().retrieveData(entity, field); - } -*/ - -} Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/engine/StorageEngine.java 2007-12-11 17:07:26 UTC (rev 63) @@ -2,79 +2,155 @@ import java.io.InputStream; import java.io.OutputStream; +import java.io.Reader; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import fr.cemagref.simexplorer.is.storage.attachment.handler.AttachmentHandler; +import fr.cemagref.simexplorer.is.storage.attachment.handler.FileSystemAttachmentHandler; +import fr.cemagref.simexplorer.is.storage.attachment.type.ContentType; +import fr.cemagref.simexplorer.is.storage.attachment.type.ContentTypeFactory; import fr.cemagref.simexplorer.is.storage.database.Database; -import fr.cemagref.simexplorer.is.storage.entities.data.LoggableElement; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; +import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; import fr.cemagref.simexplorer.is.storage.entities.metadata.Version; -public abstract class StorageEngine { +public class StorageEngine { protected Database database; - public abstract void open() throws Exception; + protected AttachmentHandler attachmentHandler; - public abstract void close() throws Exception; + private MetaDataEntity mdTmp; - public abstract LoggableElement getElement(String uuid) throws Exception; + public StorageEngine() { + super(); + database = new LuceneDatabase(); + attachmentHandler = new FileSystemAttachmentHandler(); + mdTmp = new MetaDataEntity(); + mdTmp.setUuid(UUID.randomUUID().toString()); + mdTmp.setVersion("0"); + } - public abstract List<Version> getVersions(String uuid) throws Exception; + public void open() throws Exception { + database.open(); + } - public abstract LoggableElement getElementVersion(String uuid, - Version version) throws Exception; + public void close() throws Exception { + database.close(); + } - public abstract LoggableElement newElement(LoggableElement element) - throws Exception; + public void commit() throws Exception { + database.commit(); + } - public abstract LoggableElement saveElement(LoggableElement element) - throws Exception; + // Create / update - public abstract LoggableElement synchronizeElement(LoggableElement element) - throws Exception; + public void saveElement(MetaDataEntity element, + Map<String, InputStream> attachments) throws Exception { - public abstract void exportElement(LoggableElement element, OutputStream os) - throws Exception; + for (Map.Entry<String, InputStream> entry : attachments.entrySet()) { + attachmentHandler.storeData(element, entry.getKey(), entry + .getValue()); + } - public abstract LoggableElement importElement(InputStream inputStream) - throws Exception; + List<Reader> readers = new ArrayList<Reader>(); + for (Map.Entry<String, InputStream> entry : attachments.entrySet()) { + String field = entry.getKey(); + InputStream content = attachmentHandler + .retrieveData(element, field); + String type = element.getAttachments().get(field); + if (type != null) { + ContentType contentType = ContentTypeFactory + .getContentTypeInstance(type); + Reader reader = contentType.renderToText(content); + readers.add(reader); + } + } - /** - * Attatch content to an element FIXME move to storage engine - * - * @param entity - * Related entity - * @param field - * Target field - * @param is - * Content - * @throws Exception - */ - public abstract void storeData(LoggableElement entity, String field, - InputStream is) throws Exception; + database.insertElement(element, readers); + } - /** - * Retrieve content - * - * @param entity - * DataEntity related to content - * @param field - * Unique field for content - * @return Content - * @throws Exception - */ - public abstract InputStream retrieveData(LoggableElement entity, String field) - throws Exception; + // Read - /** - * Delete content - * - * @param entity - * DataEntity related to content - * @param field - * Unique field for content - * @throws Exception - */ - public abstract void deleteData(LoggableElement entity, String field) - throws Exception; + public MetaDataEntity getElement(String uuid) throws Exception { + MetaDataEntity mde = database.getElementLatestVersion(uuid); + return mde; + } + public List<Version> getVersions(String uuid) throws Exception { + return database.getVersions(uuid); + } + + public MetaDataEntity getElementVersion(String uuid, Version version) + throws Exception { + MetaDataEntity mde = database.getElement(uuid, version); + return mde; + } + + public InputStream retrieveData(MetaDataEntity entity, String field) + throws Exception { + InputStream result = attachmentHandler.retrieveData(entity, field); + return result; + } + + // Delete + + public void deleteElements(String uuid) throws Exception { + List<Version> versions = getVersions(uuid); + for (Version version : versions) { + deleteElement(uuid, version); + } + } + + public void deleteElement(String uuid, Version version) throws Exception { + MetaDataEntity element = getElementVersion(uuid, version); + Map<String, String> attachments = element.getAttachments(); + for (Map.Entry<String, String> entry : attachments.entrySet()) { + attachmentHandler.deleteData(element, entry.getKey()); + } + database.deleteElement(element); + } + + // Import/Export + + public MetaDataEntity importElement(InputStream inputStream) + throws Exception { + // TODO Auto-generated method stub + return null; + } + + public void exportElement(MetaDataEntity element, OutputStream os) + throws Exception { + // Retrieve element factory + /* + * Class<? extends LoggableElement> clazz = element.getClass(); + * LoggableElementFactory<LoggableElement> elementFactory = + * (LoggableElementFactory<LoggableElement>) LoggableElementFactory + * .getFactory(clazz); elementFactory.getElementXML(element, os); + */ + } + + // Tools + + public String storeTempData(InputStream stream) throws Exception { + String id = UUID.randomUUID().toString(); + attachmentHandler.storeData(mdTmp, id, stream); + return id; + } + + public InputStream retrieveTempData(String id) throws Exception { + InputStream is = attachmentHandler.retrieveData(mdTmp, id); + return is; + } + + public void deleteTempData(String id) throws Exception { + attachmentHandler.deleteData(mdTmp, id); + } + } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Component.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Component.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Component.java 2007-12-11 17:07:26 UTC (rev 63) @@ -41,10 +41,4 @@ this.libraries = libraries; } - /* - * @Override public Set<LoggableElement> getChildren() { Set<LoggableElement> - * elements = new HashSet<LoggableElement>(); elements.addAll(libraries); - * return elements; } - */ - } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationApplication.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationApplication.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationApplication.java 2007-12-11 17:07:26 UTC (rev 63) @@ -23,11 +23,4 @@ this.components = components; } - /* - * @Override public Set<LoggableElement> getChildren() { Set<LoggableElement> - * elements = new HashSet<LoggableElement>(); - * elements.addAll(explorations); elements.addAll(components); return - * elements; } - */ - } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationData.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationData.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/ExplorationData.java 2007-12-11 17:07:26 UTC (rev 63) @@ -1,6 +1,5 @@ package fr.cemagref.simexplorer.is.storage.entities.data; -import java.util.HashSet; import java.util.Set; public class ExplorationData extends LoggableElement { @@ -52,9 +51,4 @@ this.valuesMap = valuesMap; } - /* - * @Override public Set<LoggableElement> getChildren() { Set<LoggableElement> - * elements = new HashSet<LoggableElement>(); return elements; } - */ - } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Library.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Library.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/Library.java 2007-12-11 17:07:26 UTC (rev 63) @@ -2,14 +2,4 @@ public class Library extends LoggableElement { - /* - * @Override public Set<LoggableElement> getChildren() { Set<LoggableElement> - * elements = new HashSet<LoggableElement>(); return elements; } - * - * @Override public Map<String, ContentType> getContentTypes() throws - * Exception { Map<String, ContentType> contentTypes = - * super.getContentTypes(); contentTypes.put("library", ContentTypeFactory - * .getContentTypeInstance(RawType.class)); return contentTypes; } - */ - } Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/LoggableElement.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/LoggableElement.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/data/LoggableElement.java 2007-12-11 17:07:26 UTC (rev 63) @@ -1,15 +1,11 @@ package fr.cemagref.simexplorer.is.storage.entities.data; -import java.util.Set; - import fr.cemagref.simexplorer.is.storage.entities.metadata.MetaDataEntity; public abstract class LoggableElement extends DataEntity { private MetaDataEntity metaData; - //public abstract Set<LoggableElement> getChildren(); - /** * @return the metaData */ Modified: trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java =================================================================== --- trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java 2007-12-10 15:21:40 UTC (rev 62) +++ trunk/simexplorer-si-storage/src/java/fr/cemagref/simexplorer/is/storage/entities/metadata/MetaDataEntity.java 2007-12-11 17:07:26 UTC (rev 63) @@ -19,6 +19,7 @@ private Date creationDate; private String hash; private Map<String, String> descriptors; + private Map<String, String> attachments; private MetaDataEntity parentData; private MetaDataEntity parentVersion; @@ -172,11 +173,18 @@ this.parentVersion = parentVersion; } - public Map<String, ContentType> getContentTypes() throws Exception { - Map<String, ContentType> contentTypes = new HashMap<String, ContentType>(); - contentTypes.put("xml", ContentTypeFactory - .getContentTypeInstance(RawType.class)); - return contentTypes; + /** + * @return the attachments + */ + public Map<String, String> getAttachments() { + return attachments; } + /** + * @param attachments the attachments to set + */ + public void setAttachments(Map<String, String> attachments) { + this.attachments = attachments; + } + }

1 0