Author: bpoussin Date: 2012-01-02 16:08:12 +0100 (Mon, 02 Jan 2012) New Revision: 1272 Url: http://nuiton.org/repositories/revision/wikitty/1272 Log: Evolution #1868: Add facet topic sort order in query (ajout de l'implantation des facets dans le InMemory) Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetSortType.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineHelper.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicCountComparator.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicNameComparator.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQuery.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java trunk/wikitty-api/src/test/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemoryTest.java trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetSortType.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetSortType.java (rev 0) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetSortType.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -0,0 +1,29 @@ +package org.nuiton.wikitty.query; + +import java.util.Comparator; + +/** + * Use to specify sort order of Facet in result + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public enum FacetSortType { + /** sort on count of topic */ + count("count", FacetTopicCountComparator.instance), + /** sort on name of topic */ + name("index", FacetTopicNameComparator.instance); + + public String solrValue; + public Comparator<FacetTopic> compartor; + + private FacetSortType(String solrValue, Comparator<FacetTopic> compartor) { + this.solrValue = solrValue; + this.compartor = compartor; + } + + +} Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicCountComparator.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicCountComparator.java 2011-12-31 14:06:23 UTC (rev 1271) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicCountComparator.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -20,6 +20,14 @@ /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(FacetTopicCountComparator.class); + final static public Comparator<FacetTopic> instance = new FacetTopicCountComparator(); + + /** + * You must use {@link instance} + */ + public FacetTopicCountComparator() { + } + public int compare(FacetTopic o1, FacetTopic o2) { int thisVal = o1.getCount(); int anotherVal = o2.getCount(); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicNameComparator.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicNameComparator.java 2011-12-31 14:06:23 UTC (rev 1271) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/FacetTopicNameComparator.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -17,14 +17,22 @@ */ public class FacetTopicNameComparator implements Comparator<FacetTopic> { - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(FacetTopicNameComparator.class); + final static public Comparator<FacetTopic> instance = + new FacetTopicNameComparator(); + final static public Comparator<FacetTopic> instanceCaseSensitive = + new FacetTopicNameComparator(false); protected boolean ignoreCase = true; + /** + * You must use {@link instance} or {@link instanceCaseSensitive} + */ public FacetTopicNameComparator() { } + /** + * You must use {@link instance} or {@link instanceCaseSensitive} + */ public FacetTopicNameComparator(boolean ignoreCase) { this.ignoreCase = ignoreCase; } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQuery.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQuery.java 2011-12-31 14:06:23 UTC (rev 1271) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQuery.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -53,10 +53,12 @@ */ protected int facetMinCount = 1; /** - * Nombre maximum de facet a retourner apres la requete. Par default on en + * Nombre maximum de topic par facet a retourner. Par default on en * retourne 100. */ protected int facetLimit = 100; + /** sort topic order, default is sorted on count */ + protected FacetSortType facetSort = FacetSortType.count; /** Facet on condition. */ protected List<FacetQuery> facetQuery; @@ -113,6 +115,7 @@ ObjectUtils.equals(this.getSortAscending(), other.getSortAscending()) && ObjectUtils.equals(this.getSortDescending(), other.getSortDescending()) && ObjectUtils.equals(this.getFacetLimit(), other.getFacetLimit()) && + ObjectUtils.equals(this.getFacetSort(), other.getFacetSort()) && ObjectUtils.equals(this.getFacetMinCount(), other.getFacetMinCount()) && ObjectUtils.equals(this.getFacetQuery(), other.getFacetQuery()) && ObjectUtils.equals(this.getCondition(), other.getCondition()); @@ -200,6 +203,14 @@ return this; } + public void setFacetSort(FacetSortType facetSort) { + this.facetSort = facetSort; + } + + public FacetSortType getFacetSort() { + return facetSort; + } + public List<FacetQuery> getFacetQuery() { if (facetQuery == null) { facetQuery = new LinkedList<FacetQuery>(); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java 2011-12-31 14:06:23 UTC (rev 1271) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -71,6 +71,7 @@ q.setFacetMinCount(o.getFacetMinCount()); q.setFirst(o.getFirst()); q.setLimit(o.getLimit()); + q.setFacetSort(o.getFacetSort()); q.setName(o.getName()); q.setSortAscending(new ArrayList<ElementField>(o.getSortAscending())); q.setSortDescending(new ArrayList<ElementField>(o.getSortDescending())); Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineHelper.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineHelper.java (rev 0) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineHelper.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -0,0 +1,107 @@ +package org.nuiton.wikitty.storage; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.wikitty.WikittyException; +import org.nuiton.wikitty.entities.WikittyExtension; +import org.nuiton.wikitty.query.FacetSortType; +import org.nuiton.wikitty.query.FacetTopic; +import org.nuiton.wikitty.query.WikittyQuery; +import org.nuiton.wikitty.query.WikittyQueryMaker; +import org.nuiton.wikitty.query.WikittyQueryResult; +import org.nuiton.wikitty.query.conditions.Select; +import org.nuiton.wikitty.services.WikittyTransaction; + +/** + * Ensemble de methode reutilisable dans differente implantation de + * {@link WikittySearchEngine} + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class WikittySearchEngineHelper { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(WikittySearchEngineHelper.class); + + /** + * Gere le travail pour les requetes ayant un {@link Select} + * + * @param searchEngine le searchEngine a utiliser pour les sous requetes + * @param transaction la transaction a utiliser + * @param query la requete qui debute par un {@link Select} + * @return + */ + static public WikittyQueryResult<String> findAllByQueryWithSelect( + WikittySearchEngine searchEngine, WikittyTransaction transaction, + WikittyQuery query) { + + if (!(query.getCondition() instanceof Select)) { + throw new WikittyException("Query don't start with Select condition"); + } else { + // gere les conditions qui commence par select + // il faut recreer deux query, une pour le select, une pour les facettes s'il y en a + // car le select est execute via une facette, mais cette facette + // n'a pas les meme limites que les autres facette + + Select select = (Select)query.getCondition(); + String extName = WikittyExtension.extractExtensionName( + select.getElement().getValue()); + WikittyQueryMaker newCond = new WikittyQueryMaker() + .and() + .exteq(extName) + .condition(select.getSubCondition()); + + // copy de la query pour les facettes + WikittyQuery queryFacet = query.copy(); + queryFacet.setCondition(newCond.getCondition()); + queryFacet.setLimit(0); + + // copy de la query pour le select + // on part de facet qui a deja la bonne condition + WikittyQuery querySelect = queryFacet.copy(); + querySelect.setLimit(0); + querySelect.setFacetMinCount(0); + querySelect.setFacetLimit(Integer.MAX_VALUE); + // on force le sort pour toujours utiliser le meme + querySelect.setFacetSort(FacetSortType.count); + querySelect.setFacetQuery(); + querySelect.setFacetField(select.getElement()); + + // execution des requetes + WikittyQueryResult<String> resultFacet = + searchEngine.findAllByQuery(transaction, queryFacet); + WikittyQueryResult<String> resultSelect = + searchEngine.findAllByQuery(transaction, querySelect); + + // creation des resultats via la facette select + List<FacetTopic> topics = resultSelect.getFacets().get(select.getElement().getValue()); + + List<String> selectList = new ArrayList<String>( + Math.min(topics.size(), query.getLimit())); + if (query.getFirst() < topics.size()) { + int first = query.getFirst(); + int last = Math.min(topics.size(), query.getFirst() + query.getLimit()); + for (FacetTopic topic : topics.subList(first, last)) { + selectList.add(topic.getTopicName()); + } + } + + // fusion des resultats + WikittyQueryResult<String> result = new WikittyQueryResult<String>( + query.getName(), + query.getFirst(), + topics.size(), + resultSelect.getQueryString(), + resultFacet.getFacets(), + selectList); + + return result; + } + } +} Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2011-12-31 14:06:23 UTC (rev 1271) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -38,7 +38,16 @@ import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.collections.Bag; +import org.apache.commons.collections.BagUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Factory; +import org.apache.commons.collections.Predicate; +import org.apache.commons.collections.bag.HashBag; +import org.apache.commons.collections.map.LazyMap; +import org.apache.commons.collections.map.LazySortedMap; +import org.apache.commons.collections.map.PredicatedMap; +import org.apache.commons.collections.set.PredicatedSet; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -48,6 +57,8 @@ import org.nuiton.wikitty.entities.FieldType.TYPE; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyExtension; +import org.nuiton.wikitty.query.FacetQuery; +import org.nuiton.wikitty.query.FacetTopic; import org.nuiton.wikitty.query.WikittyQuery; import org.nuiton.wikitty.query.WikittyQueryResult; import org.nuiton.wikitty.query.WikittyQueryResultTreeNode; @@ -56,6 +67,7 @@ import org.nuiton.wikitty.query.conditions.ContainsAll; import org.nuiton.wikitty.query.conditions.ContainsOne; import org.nuiton.wikitty.query.conditions.ElementExtension; +import org.nuiton.wikitty.query.conditions.ElementField; import org.nuiton.wikitty.query.conditions.ElementId; import org.nuiton.wikitty.query.conditions.Equals; import org.nuiton.wikitty.query.conditions.Greater; @@ -113,38 +125,126 @@ // do nothing } - private boolean checkRestriction(WikittyTransaction transaction, - Condition condition, Wikitty w) { + static private boolean checkRestriction(WikittySearchEngine searchEngine, + WikittyTransaction transaction, Condition condition, Wikitty w) { WikittyQueryVisitorCheckCondition v = - new WikittyQueryVisitorCheckCondition(this, transaction, w); + new WikittyQueryVisitorCheckCondition(searchEngine, transaction, w); condition.accept(v); boolean result = v.getResult(); return result; } + + static public class FacetPredicate { + WikittyTransaction tx; + WikittySearchEngine searchEngine; + WikittyQuery query; + Map<String, Bag> topic; + + public FacetPredicate(WikittySearchEngine searchEngine, + WikittyTransaction tx, WikittyQuery query) { + this.searchEngine = searchEngine; + this.tx = tx; + this.query = query; + + topic = LazyMap.decorate(new HashMap(), new Factory() { + public Object create() { + return new HashBag(); + } + }); + + } + + public Map<String, List<FacetTopic>> getFacets() { + Map<String, List<FacetTopic>> result = new HashMap<String, List<FacetTopic>>(); + for (String facetName : topic.keySet()) { + List<FacetTopic> list = new ArrayList<FacetTopic>(); + Bag b = topic.get(facetName); + for (Object topicName : b.uniqueSet()) { + int count = b.getCount(topicName); + // pour ajouter le topic il faut un minimum indique dans la query + if (count >= query.getFacetMinCount()) { + FacetTopic ft = new FacetTopic(facetName, String.valueOf(topicName), count); + list.add(ft); + } + } + // on tri les facets selon l'ordre demande ... + Collections.sort(list, query.getFacetSort().compartor); + // ... et on en prend que le nombre demande + if (list.size() > query.getFacetLimit()) { + list = list.subList(0, query.getFacetLimit()); + } + result.put(facetName, list); + } + return result; + } + + public boolean add(Wikitty w) { + boolean result = false; + + // create facet extension + if (query.isFacetExtension()) { + String facetName = ElementExtension.EXTENSION.getValue(); + for (String extName : w.getExtensionNames()) { + topic.get(facetName).add(extName); + result = true; + } + } + + // create facet field + for (ElementField e : query.getFacetField()) { + String fqf = e.getValue(); + Object value = w.getFqField(fqf); + topic.get(fqf).add(value); + result = true; + } + + // create facet query + for (FacetQuery q : query.getFacetQuery()) { + if (checkRestriction(searchEngine, tx, q.getCondition(), w)) { + String facetName = q.getName(); + if (facetName == null) { + facetName = q.getCondition().toString(); + } + topic.get(facetName).add(facetName); + } + } + + return result; + } + } @Override public WikittyQueryResult<String> findAllByQuery(WikittyTransaction transaction, WikittyQuery query) { - // throw new UnsupportedOperationException("Not supported yet."); - int first = query.getFirst(); - int limit = query.getLimit(); - List<String> ids = new LinkedList<String>(); - int currentIndex = 0; - for (Entry<String, Wikitty> entry : wikittyStorage.getWikitties().entrySet()) { - Wikitty w = entry.getValue(); - String id = entry.getKey(); - Condition c = query.getCondition(); - if (!w.isDeleted() && checkRestriction(transaction, c, w)) { - currentIndex++; - if (currentIndex > first) { - ids.add(id); + WikittyQueryResult<String> result; + + if (query.getCondition() instanceof Select) { + result = WikittySearchEngineHelper.findAllByQueryWithSelect( + this, transaction, query); + } else { + int first = query.getFirst(); + int limit = query.getLimit(); + List<String> ids = new LinkedList<String>(); + FacetPredicate facets = new FacetPredicate(this, transaction, query); + + int totalResult = 0; + for (Entry<String, Wikitty> entry : wikittyStorage.getWikitties().entrySet()) { + Wikitty w = entry.getValue(); + String id = entry.getKey(); + Condition c = query.getCondition(); + if (!w.isDeleted() && checkRestriction(this, transaction, c, w)) { + totalResult++; + if (totalResult > first && ids.size() < limit) { + // ajout en tant que resultat + ids.add(id); + } + facets.add(w); } - if (ids.size() >= limit) { - break; - } } + + result = new WikittyQueryResult<String>(query.getName(), + first, totalResult, query.getCondition().toString(), facets.getFacets(), ids); } - return new WikittyQueryResult<String>(query.getName(), - first, ids.size(), query.getCondition().toString(), null, ids); + return result; } @Override Modified: trunk/wikitty-api/src/test/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemoryTest.java =================================================================== --- trunk/wikitty-api/src/test/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemoryTest.java 2011-12-31 14:06:23 UTC (rev 1271) +++ trunk/wikitty-api/src/test/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemoryTest.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -35,6 +35,9 @@ public void testfindAllByQuery() throws Exception { // initialisation des données + // 2 label: l1("toutou", "titi") et l2() + // 3 group: g1("MonGroup1", [l1]), g2("MonGroup2", [l2]), g3("MonGroup3", [l1,l2]) + WikittyLabelImpl l1 = new WikittyLabelImpl(new WikittyImpl("l1")); l1.addLabels("toutou"); l1.addLabels("titi"); @@ -170,6 +173,19 @@ g3.getWikittyId()))); } + { // select (et donc les facets) + WikittyQuery q = new WikittyQueryMaker() + .select(WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME) + .eq(WikittyGroup.FQ_FIELD_WIKITTYGROUP_MEMBERS, l1) + .end(); + WikittyQueryResult<String> result = se.findAllByQuery(null, q); + System.out.println("dd:" + result.getAll()); + // g3 doit etre retrouve via le join, et g1 est exclue via la not(ideq) + Assert.assertEquals(2, result.size()); + Assert.assertTrue(result.getAll().containsAll(Arrays.asList( + g1.getName(), g3.getName()))); + } + // TODO poussin 20111228 do more test to test all possible condition } Modified: trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-12-31 14:06:23 UTC (rev 1271) +++ trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2012-01-02 15:08:12 UTC (rev 1272) @@ -69,6 +69,7 @@ import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; import org.nuiton.wikitty.query.FacetQuery; +import org.nuiton.wikitty.query.FacetSortType; import org.nuiton.wikitty.query.FacetTopic; import org.nuiton.wikitty.query.WikittyQuery; import org.nuiton.wikitty.query.WikittyQueryMaker; @@ -78,6 +79,7 @@ import org.nuiton.wikitty.query.conditions.Select; import org.nuiton.wikitty.search.Search; import org.nuiton.wikitty.search.TreeNodeResult; +import org.nuiton.wikitty.storage.WikittySearchEngineHelper; /** * @@ -662,58 +664,8 @@ WikittyQueryResult<String> result; if (query.getCondition() instanceof Select) { - // gere les conditions qui commence par select - // il faut recreer deux query, une pour le select, une pour les facettes s'il y en a - // car le select est execute via une facette, mais cette facette - // n'a pas les meme limites que les autres facette - - Select select = (Select)query.getCondition(); - String extName = WikittyExtension.extractExtensionName( - select.getElement().getValue()); - WikittyQueryMaker newCond = new WikittyQueryMaker() - .and() - .exteq(extName) - .condition(select.getSubCondition()); - - // copy de la query pour les facettes - WikittyQuery queryFacet = query.copy(); - queryFacet.setCondition(newCond.getCondition()); - queryFacet.setLimit(0); - - // copy de la query pour le select - // on part de facet qui a deja la bonne condition - WikittyQuery querySelect = queryFacet.copy(); - querySelect.setLimit(0); - querySelect.setFacetMinCount(0); - querySelect.setFacetLimit(Integer.MAX_VALUE); - querySelect.setFacetQuery(); - querySelect.setFacetField(select.getElement()); - - // execution des requetes - WikittyQueryResult<String> resultFacet = findAllByQuery(transaction, queryFacet); - WikittyQueryResult<String> resultSelect = findAllByQuery(transaction, querySelect); - - // creation des resultats via la facette select - List<FacetTopic> topics = resultSelect.getFacets().get(select.getElement().getValue()); - - List<String> selectList = new ArrayList<String>( - Math.min(topics.size(), query.getLimit())); - if (query.getFirst() < topics.size()) { - int first = query.getFirst(); - int last = Math.min(topics.size(), query.getFirst() + query.getLimit()); - for (FacetTopic topic : topics.subList(first, last)) { - selectList.add(topic.getTopicName()); - } - } - - // fusion des resultats - result = new WikittyQueryResult<String>( - query.getName(), - query.getFirst(), - topics.size(), - resultSelect.getQueryString(), - resultFacet.getFacets(), - selectList); + result = WikittySearchEngineHelper.findAllByQueryWithSelect( + this, transaction, query); } else { // Create querySolr @@ -768,6 +720,7 @@ querySolr.setFacet(true); querySolr.setFacetMinCount(query.getFacetMinCount()); querySolr.setFacetLimit(query.getFacetLimit()); + querySolr.setFacetSort(query.getFacetSort().solrValue); if (isFacetExtension) { querySolr.addFacetField(WikittySolrConstant.SOLR_EXTENSIONS);