Author: bpoussin Date: 2012-02-04 13:17:17 +0100 (Sat, 04 Feb 2012) New Revision: 1419 Url: http://nuiton.org/repositories/revision/wikitty/1419 Log: amelioration de l'implementation in memory pour faire passer plus de test - sorted result - sorted facet Modified: trunk/pom.xml trunk/wikitty-api/pom.xml trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2012-02-04 01:32:53 UTC (rev 1418) +++ trunk/pom.xml 2012-02-04 12:17:17 UTC (rev 1419) @@ -65,6 +65,7 @@ <nuitonProcessessorVersion>1.2.2</nuitonProcessessorVersion> <nuiton-struts2>1.3</nuiton-struts2> + <cxf.version>2.5.1</cxf.version> <!--Multilanguage maven-site --> <locales>fr,en</locales> @@ -73,8 +74,14 @@ <dependencyManagement> <dependencies> - <!-- pour le parser de requete --> <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>11.0</version> + </dependency> + + <!-- pour le parser de requete --> + <dependency> <groupId>org.parboiled</groupId> <artifactId>parboiled-core</artifactId> <version>1.0.2</version> Modified: trunk/wikitty-api/pom.xml =================================================================== --- trunk/wikitty-api/pom.xml 2012-02-04 01:32:53 UTC (rev 1418) +++ trunk/wikitty-api/pom.xml 2012-02-04 12:17:17 UTC (rev 1419) @@ -17,7 +17,7 @@ <dependencies> - <!-- COMPILE --> + <!-- COMPILE --> <!-- needed to mutualise some util method (tagValueToString) --> <dependency> @@ -26,6 +26,11 @@ <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <!-- pour le parser de requete --> <dependency> <groupId>org.parboiled</groupId> Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2012-02-04 01:32:53 UTC (rev 1418) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2012-02-04 12:17:17 UTC (rev 1419) @@ -24,10 +24,13 @@ */ package org.nuiton.wikitty.storage; +import com.google.common.collect.Multiset; +import com.google.common.collect.TreeMultiset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Deque; import java.util.HashMap; import java.util.Iterator; @@ -38,10 +41,8 @@ import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.collections.Bag; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Factory; -import org.apache.commons.collections.bag.HashBag; import org.apache.commons.collections.map.LazyMap; import org.apache.commons.lang.ClassUtils; import org.apache.commons.lang3.StringUtils; @@ -49,11 +50,14 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.wikitty.WikittyException; import org.nuiton.wikitty.WikittyUtil; +import org.nuiton.wikitty.entities.ElementExtension; +import org.nuiton.wikitty.entities.ElementId; import org.nuiton.wikitty.entities.FieldType; -import org.nuiton.wikitty.entities.WikittyTypes; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyExtension; +import org.nuiton.wikitty.entities.WikittyTypes; import org.nuiton.wikitty.query.FacetQuery; +import org.nuiton.wikitty.query.FacetSortType; import org.nuiton.wikitty.query.FacetTopic; import org.nuiton.wikitty.query.WikittyQuery; import org.nuiton.wikitty.query.WikittyQueryResult; @@ -64,8 +68,6 @@ import org.nuiton.wikitty.query.conditions.ConditionValueString; import org.nuiton.wikitty.query.conditions.ContainsAll; import org.nuiton.wikitty.query.conditions.ContainsOne; -import org.nuiton.wikitty.entities.ElementExtension; -import org.nuiton.wikitty.entities.ElementId; import org.nuiton.wikitty.query.conditions.Equals; import org.nuiton.wikitty.query.conditions.Greater; import org.nuiton.wikitty.query.conditions.GreaterOrEquals; @@ -131,11 +133,93 @@ return result; } + /** comparateur generic qui accept deux objets de meme type en argument */ + static public Comparator<Object> genericComparator = new Comparator<Object>() { + public int compare(Object v1, Object v2) { + if (v1 == null && v2 == null) { + return 0; + } else if (v1 == null) { + return -1; + } else if (v2 == null) { + return 1; + } + + if (v1 == v2 || v1.equals(v2)) { + return 0; + } + + if (v1 instanceof Collection) { + v1 = CollectionUtils.get(v1, 0); + } + if (v2 instanceof Collection) { + v2 = CollectionUtils.get(v2, 0); + } + Comparable c1; + Comparable c2; + if (v1 instanceof Comparable) { + c1 = (Comparable)v1; + } else { + return 0; // non comparable on retourne 0 + } + if (v2 instanceof Comparable) { + c2 = (Comparable)v2; + } else { + return 0; // non comparable on retourne 0 + } + + int result = c1.compareTo(c2); + return result; + } + }; + + static public class WikittyComparator implements Comparator<Wikitty> { + + protected List<org.nuiton.wikitty.entities.Element> asc; + protected List<org.nuiton.wikitty.entities.Element> desc; + + public WikittyComparator(List<org.nuiton.wikitty.entities.Element> asc, List<org.nuiton.wikitty.entities.Element> desc) { + this.asc = asc; + this.desc = desc; + } + + /** compare un champs de deux objets Wikitty */ + protected int compareValue(Wikitty o1, Wikitty o2, org.nuiton.wikitty.entities.Element e, boolean asc) { + Object v1 = o1.getFqField(e.getValue()); + Object v2 = o2.getFqField(e.getValue()); + + int result = genericComparator.compare(v1, v2); + if (!asc) { + result = result * -1; + } + return result; + } + + public int compare(Wikitty o1, Wikitty o2) { + int result = 0; + + for (org.nuiton.wikitty.entities.Element e : asc) { + result = compareValue(o1, o2, e, true); + if (result != 0) { + return result; + } + } + + for (org.nuiton.wikitty.entities.Element e : desc) { + result = compareValue(o1, o2, e, false); + if (result != 0) { + return result; + } + } + return result; + } + + } + static public class FacetPredicate { WikittyTransaction tx; WikittySearchEngine searchEngine; WikittyQuery query; - Map<String, Bag> topic; + Map<String, Multiset> topic; public FacetPredicate(WikittySearchEngine searchEngine, WikittyTransaction tx, WikittyQuery query) { @@ -145,7 +229,7 @@ topic = LazyMap.decorate(new HashMap(), new Factory() { public Object create() { - return new HashBag(); + return TreeMultiset.create(genericComparator); } }); @@ -155,17 +239,21 @@ Map<String, List<FacetTopic>> result = new HashMap<String, List<FacetTopic>>(); for (String facetName : topic.keySet()) { List<FacetTopic> list = new ArrayList<FacetTopic>(); - Bag b = topic.get(facetName); - for (Object topicName : b.uniqueSet()) { - int count = b.getCount(topicName); + Multiset b = topic.get(facetName); + for (Object topicName : b.elementSet()) { + int count = b.count(topicName); // pour ajouter le topic il faut un minimum indique dans la query if (count >= query.getFacetMinCount()) { FacetTopic ft = new FacetTopic(facetName, String.valueOf(topicName), count); list.add(ft); } } - // on tri les facets selon l'ordre demande ... - Collections.sort(list, query.getFacetSort().compartor); + + // on re-tri si besoin, sinon l'ordre par defaut est sur la valeur des topics + if (query.getFacetSort() == FacetSortType.count) { + Collections.sort(list, query.getFacetSort().compartor); + } + // ... et on en prend que le nombre demande if (list.size() > query.getFacetLimit()) { list = list.subList(0, query.getFacetLimit()); @@ -238,10 +326,19 @@ List<String> ids = new LinkedList<String>(); FacetPredicate facets = new FacetPredicate(this, transaction, query); + ArrayList<Wikitty> wikitties = + new ArrayList<Wikitty>(wikittyStorage.getWikitties().values()); + + // on tri les wikitties selon l'ordre demande ... + if (!query.getSortAscending().isEmpty() + || !query.getSortDescending().isEmpty() ) { + Collections.sort(wikitties, new WikittyComparator( + query.getSortAscending(), query.getSortDescending())); + } + int totalResult = 0; - for (Entry<String, Wikitty> entry : wikittyStorage.getWikitties().entrySet()) { - Wikitty w = entry.getValue(); - String id = entry.getKey(); + for (Wikitty w : wikitties) { + String id = w.getId(); Condition c = query.getCondition(); if (!w.isDeleted()) { if (checkRestriction(this, transaction, c, w)) { @@ -255,8 +352,6 @@ } } - // TODO poussin 20120106 faire le tri des resultats en fonction de ce qui est demande - result = new WikittyQueryResult<String>(query.getName(), first, totalResult, query.getCondition().toString(), facets.getFacets(), ids); } Modified: trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java =================================================================== --- trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java 2012-02-04 01:32:53 UTC (rev 1418) +++ trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java 2012-02-04 12:17:17 UTC (rev 1419) @@ -331,7 +331,7 @@ } @Test - public void testSearchExtensionFacet() throws Exception { + public void testSearchExtensionFacetExplicitly() throws Exception { // essai de facettiser sur les extensions WikittyQuery query = new WikittyQueryMaker().keyword("*").end(); query.setFirst(0); @@ -345,6 +345,21 @@ Assert.assertNotNull(result.getFacets().get(Element.EXTENSION.getValue())); } + @Test + public void testSearchExtensionFacet() throws Exception { + // essai de facettiser sur les extensions + WikittyQuery query = new WikittyQueryMaker().keyword("*").end(); + query.setFirst(0); + query.setLimit(0); + query.setFacetField(Element.EXTENSION); + WikittyQueryResult<String> result = wikittyClient.findAllByQuery(query); + Assert.assertEquals(1, result.getFacetNames().size()); + //TODO echatellier 20120112 : revue this code + //Assert.assertNotNull(result.getFacets().get(Element.EXTENSION)); + // is more convenient, but wont work + Assert.assertNotNull(result.getFacets().get(Element.EXTENSION.getValue())); + } + /** * Test qu'une recherche fonctionne et que la même recherche apres * suppression ne trouve plus l'element. @@ -1497,8 +1512,6 @@ */ @Test public void testSearchWithSort2() { - assumeTrueSearchEngineCanRunTest(); // sorting - //Create ext sortable String sortableExtName = "sortable"; String numFieldName = "num"; @@ -1537,8 +1550,6 @@ */ @Test public void testSearchWithSortTestData() { - assumeTrueSearchEngineCanRunTest(); // sorting - WikittyQuery query1 = WikittyQueryParser.parse("Product.price <= 100"); query1.setSortDescending(Product.ELEMENT_FIELD_PRODUCT_PRICE); WikittyQueryResult<Product> results1 = wikittyClient.findAllByQuery(Product.class, query1); @@ -1647,8 +1658,6 @@ @Test public void testQueryMarkerSelectSortAndAggregate() throws Exception { - assumeTrueSearchEngineCanRunTest(); // pas le meme comportement, a verifier (il manque le sort in memory) - WikittyExtension ext = new WikittyExtension("Test", "1.0", // version WikittyUtil.tagValuesToMap(" version=\"1.0\""), // tag/values @@ -1717,35 +1726,35 @@ } { WikittyQuery q = new WikittyQueryMaker().select("Test.number", Aggregate.MAX).end(); - WikittyQueryResult<String> result = wikittyClient.findAllByQuery(q); + WikittyQueryResult<Double> result = wikittyClient.findAllByQuery(Double.class, q); System.out.println("q: " + result); - Assert.assertEquals("10.0", result.peek()); + Assert.assertEquals(Double.valueOf(10), result.peek()); } { WikittyQuery q = new WikittyQueryMaker().select("Test.number", Aggregate.MAX).end() .addSortDescending(new ElementField("Test", "number")); - WikittyQueryResult<String> result = wikittyClient.findAllByQuery(q); + WikittyQueryResult<Double> result = wikittyClient.findAllByQuery(Double.class, q); System.out.println("q: " + result); - Assert.assertEquals("10.0", result.peek()); + Assert.assertEquals(Double.valueOf(10), result.peek()); } { WikittyQuery q = new WikittyQueryMaker().select("Test.number", Aggregate.MIN).end(); - WikittyQueryResult<String> result = wikittyClient.findAllByQuery(q); + WikittyQueryResult<Double> result = wikittyClient.findAllByQuery(Double.class, q); System.out.println("q: " + result); - Assert.assertEquals("-4.0", result.peek()); + Assert.assertEquals(Double.valueOf(-4.0), result.peek()); } { WikittyQuery q = new WikittyQueryMaker().select("Test.number", Aggregate.MIN).end() .addSortDescending(new ElementField("Test", "number")); - WikittyQueryResult<String> result = wikittyClient.findAllByQuery(q); + WikittyQueryResult<Double> result = wikittyClient.findAllByQuery(Double.class, q); System.out.println("q: " + result); - Assert.assertEquals("-4.0", result.peek()); + Assert.assertEquals(Double.valueOf(-4.0), result.peek()); } { WikittyQuery q = new WikittyQueryMaker().select("Test.number", Aggregate.SUM).end(); - WikittyQueryResult<String> result = wikittyClient.findAllByQuery(q); + WikittyQueryResult<Double> result = wikittyClient.findAllByQuery(Double.class, q); System.out.println("q: " + result); - Assert.assertEquals("12", result.peek()); + Assert.assertEquals(Double.valueOf(12), result.peek()); } } Modified: trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2012-02-04 01:32:53 UTC (rev 1418) +++ trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2012-02-04 12:17:17 UTC (rev 1419) @@ -752,7 +752,7 @@ // la facet sur les extensions est directement et convenablement // gere comme un champs - if (CollectionUtils.isNotEmpty(facetField)) { + if (isFacetExtension || CollectionUtils.isNotEmpty(facetField)) { for (FacetField facet : resp.getFacetFields()) { String facetName = elementModifier.convertToField(facet.getName()); List<FacetTopic> topics = new ArrayList<FacetTopic>();