Author: kmorin Date: 2009-11-04 11:19:47 +0100 (Wed, 04 Nov 2009) New Revision: 308 Added: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java Removed: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEnginTest.java Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/jdbc.properties wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/solrconfig.xml Log: Separation of solr Deleted: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java 2009-11-04 10:19:47 UTC (rev 308) @@ -1,389 +0,0 @@ -package org.sharengo.wikitty.jdbc; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.mortbay.log.Log; -import org.sharengo.wikitty.WikittyException; -import org.sharengo.wikitty.search.*; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * @author "Nicolas Chapurlat" <nicolas.chapurlat@logica.com> - * @author "Guillaume Dufrêne" <dufrene@argia.fr> - * <p/> - * This class is used to parse Restriction to create lucene request on - * content. Every operators describe in RestrictionName is handle. Parsing may - * throw exception when restriction parameters are incorrect. - */ -public class Restriction2Solr { - - private static final int MAX_SUBQUERY_RESULT = 100; - - /** - * use to permit client to modify fieldname during query generation - */ - static public interface FieldModifier { - public String convertToSolr(String fieldname); - - public String convertToField(String solrName); - } - - final static protected FieldModifier dummyFieldModifier = new FieldModifier() { - public String convertToSolr(String fieldname) { - return fieldname; - } - - public String convertToField(String solrName) { - return solrName; - } - }; - - protected FieldModifier fieldModifer; - - public Restriction2Solr() { - this(dummyFieldModifier); - } - - public Restriction2Solr(FieldModifier fieldModifer) { - this.fieldModifer = fieldModifer; - } - - public String toSolr(Restriction restriction) { - return toSolr(restriction, null); - } - - public String toSolr(Restriction restriction, SolrServer solr) - throws WikittyException { - // ParameterValidator.checkNullParameter(restriction, "restriction"); - switch (restriction.getName()) { - case TRUE: - return true2solr(); - case FALSE: - return false2solr(); - case NOT: - Not not = (Not) restriction; - return not2solr(not); - case AND: - And and = (And) restriction; - return and2solr(and); - case OR: - Or or = (Or) restriction; - return or2solr(or); - case EQUALS: - Equals eq = (Equals) restriction; - return eq2solr(eq); - case NOT_EQUALS: - NotEquals neq = (NotEquals) restriction; - return neq2solr(neq); - case LESS: - Less less = (Less) restriction; - return less2solr(less); - case LESS_OR_EQUAL: - LessOrEqual lessEq = (LessOrEqual) restriction; - return lessEq2solr(lessEq); - case GREATER: - Greater great = (Greater) restriction; - return great2solr(great); - case GREATER_OR_EQUAL: - GreaterOrEqual greatEq = (GreaterOrEqual) restriction; - return greatEq2solr(greatEq); - case BETWEEN: - Between between = (Between) restriction; - return between2solr(between); - case CONTAINS: - Contains contains = (Contains) restriction; - return contains2solr(contains); - case IN: - In in = (In) restriction; - return in2solr(in); - case STARTS_WITH: - StartsWith start = (StartsWith) restriction; - return start2solr(start); - case ENDS_WITH: - EndsWith end = (EndsWith) restriction; - return end2solr(end); - case ASSOCIATED: - AssociatedRestriction associated = (AssociatedRestriction) restriction; - return associated2solr(associated, solr); - case KEYWORD: - Keyword keyword = (Keyword) restriction; - return keyword2solr(keyword); - default: - throw new WikittyException("this kind of restriction is not supported : " - + restriction.getName().toString()); - } - } - - private String in2solr(In in) { - boolean first = true; - String result = in.getElement().getName() + ":["; - for (String value : in.getValue()) { - if (!first) { - result += ", "; - first = false; - } - result += value; - } - result += "]"; - return result; - } - - private String associated2solr(AssociatedRestriction associated, SolrServer solr) throws WikittyException { - String subQuery = toSolr(associated.getRestriction()); - SolrQuery query = new SolrQuery(subQuery); - query.setRows(MAX_SUBQUERY_RESULT); - QueryResponse resp = null; - try { - Log.info("Execute sub-query : " + subQuery); - resp = solr.query(query); - } catch (SolrServerException e) { - throw new WikittyException("Unable to execute associative query on " + associated.getElement().getName(), e); - } - SolrDocumentList solrResults = resp.getResults(); - - Restriction generatedRestriction = null; - long size = solrResults.size(); - if (size == 0) { - throw new WikittyException("Associated " + associated.getElement().getName() + " do not retrieved any result"); - } - if (size == 1) { - generatedRestriction = RestrictionHelper.eq(associated.getElement(), (String) solrResults.get(0).getFieldValue("id")); - } else { - List<String> ids = new ArrayList<String>(solrResults.size()); - for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue("id"); // FIXME : set a constant field reference - ids.add(id); - } - generatedRestriction = new In(associated.getElement(), ids); - } - Restriction parent = associated.getParentRestrictionDto(); - And and; - if (parent instanceof And) { - and = (And) parent; - and.getRestrictions().add(generatedRestriction); - } else { - and = RestrictionHelper.and(Arrays.asList(new Restriction[]{associated.getParentRestrictionDto(), generatedRestriction})); - } - return toSolr(and); - } - - private String not2solr(Not not) throws WikittyException { - if (not.getRestriction() == null) { - throw new WikittyException("not.restriction"); - } - return "( *:* - " + toSolr(not.getRestriction()) + " )"; - } - - private String and2solr(And and) throws WikittyException { - if (and.getRestrictions() == null) { - throw new WikittyException("and.restrictions is null"); - } - if (and.getRestrictions().size() < 2) { - throw new WikittyException("AND is an operator that handle 2 operand at least"); - } - boolean first = true; - StringBuffer result = new StringBuffer(); - for (Restriction restriction : and.getRestrictions()) { - if (first) { - result.append("( ").append(toSolr(restriction)); - first = false; - } else { - result.append(" AND ").append(toSolr(restriction)); - } - } - return result.append(" )").toString(); - } - - private String or2solr(Or or) throws WikittyException { - if (or.getRestrictions() == null) { - throw new WikittyException("or.restrictions is null"); - } - if (or.getRestrictions().size() < 2) { - throw new WikittyException("OR is an operator that handle 2 operand at least"); - } - boolean first = true; - StringBuffer result = new StringBuffer(); - for (Restriction restriction : or.getRestrictions()) { - if (first) { - result.append("( "); - first = false; - } else { - result.append(" OR "); - } - result.append(toSolr(restriction)); - } - return result.append(" )").toString(); - } - - private String eq2solr(Equals eq) throws WikittyException { - return element2solr(eq.getElement()) + ":" + value2solr(eq.getValue()); - } - - private String neq2solr(NotEquals neq) - throws WikittyException { - return "( *:* - " + element2solr(neq.getElement()) + ":" - + value2solr(neq.getValue()) + " )"; - } - - private String less2solr(Less less) throws WikittyException { - return element2solr(less.getElement()) + ":{* TO " - + value2solr(less.getValue()) + "}"; - } - - private String lessEq2solr(LessOrEqual lessEq) - throws WikittyException { - return element2solr(lessEq.getElement()) + ":[* TO " - + value2solr(lessEq.getValue()) + "]"; - } - - private String great2solr(Greater great) - throws WikittyException { - return element2solr(great.getElement()) + ":{" - + value2solr(great.getValue()) + " TO *}"; - } - - private String greatEq2solr(GreaterOrEqual greatEq) - throws WikittyException { - return element2solr(greatEq.getElement()) + ":[" - + value2solr(greatEq.getValue()) + " TO *]"; - } - - private String between2solr(Between between) - throws WikittyException { - if (between.getElement() == null) { - throw new WikittyException("contains.element"); - } - if (between.getMin() == null) { - throw new WikittyException("contains.min"); - } - if (between.getMax() == null) { - throw new WikittyException("contains.max"); - } - return element2solr(between.getElement()) + ":[" - + value2solr(between.getMin()) + " TO " - + value2solr(between.getMax()) + "]"; - } - - private String contains2solr(Contains contains) - throws WikittyException { - if (contains.getElement() == null) { - throw new WikittyException("contains.element"); - } - if (contains.getValue() == null) { - throw new WikittyException("contains.values"); - } - if (contains.getValue().size() < 1) { - throw new WikittyException("CONTAINS is an operator that handle 1 operand at least"); - } - boolean first = true; - StringBuffer result = new StringBuffer(); - for (String value : contains.getValue()) { - if (first) { - result.append("( "); - first = false; - } else { - result.append(" AND "); - } - result.append("( "); - result.append(element2solr(contains.getElement())) - .append(":").append(value2solr(value)); - result.append(" OR "); - result.append(element2solr(contains.getElement())) - .append(":").append(value2solr(value)); - result.append(" OR "); - result.append(element2solr(contains.getElement())) - .append(":").append(value2solr(value)); - result.append(" )"); - } - return result.append(" )").toString(); - } - - private String start2solr(StartsWith start) - throws WikittyException { - return element2solr(start.getElement()) + ":" - + value2solr(start.getValue(), true, false); - } - - private String end2solr(EndsWith end) { - return element2solr(end.getElement()) + ":" + value2solr(end.getValue()); - } - - private String true2solr() { - return "( *:* )"; - } - - private String false2solr() { - return "( *:* - *:* )"; - } - - private String keyword2solr(Keyword keyword) { - return value2solr(keyword.getValue()); - } - - private String value2solr(String value) { - return value2solr(value, false, false); - } - - private String value2solr(String value, boolean starOnEnd, - boolean revert) { - String result; - if (value != null) { - if (value.equals("*")) { - return "*"; - } - - if (revert) { - result = Restriction2Solr.escapeValue(reverse(value)); - } else { - result = Restriction2Solr.escapeValue(value); - // result = value; - } - } else { - result = ""; - } - if (starOnEnd) { - result += "*"; - } - if (result.contains(" ")) { - result = "\"" + result + "\""; - } - return result; - } - - private String element2solr(Element element) throws WikittyException { - String result = element.getName(); - if (Element.ELT_EXTENSION.equals(result)) { - result = WikittySearchEnginSolr.SOLR_EXTENSIONS; - } else if (Element.ELT_ID.equals(result)) { - result = WikittySearchEnginSolr.SOLR_ID; - } else { - result = element.getName(); - } - result = fieldModifer.convertToSolr(result); - return result; - } - - private static String escapeValue(String value) { - final String LUCENE_REPLACE_PATTERN = "\\+" + "|-" + "|&&" + "|\\|\\|" - + "|!" + "|\\(|\\)" + "|\\[|\\]" + "|\\{|\\}" + "|\\^" + "|\"" - + "|\\~" + "|\\*" + "|\\?" + "|:" + "|\\\\"; - return value.replaceAll(LUCENE_REPLACE_PATTERN, "\\\\$0"); - } - - private String reverse(String source) { - int i, len = source.length(); - StringBuffer dest = new StringBuffer(len); - - for (i = (len - 1); i >= 0; i--) - dest.append(source.charAt(i)); - return dest.toString(); - } -} - Deleted: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java 2009-11-04 10:19:47 UTC (rev 308) @@ -1,71 +0,0 @@ -package org.sharengo.wikitty.jdbc; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.Query; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.LuceneQParserPlugin; -import org.apache.solr.search.QParser; -import org.apache.solr.search.QueryParsing; -import org.apache.solr.search.SolrQueryParser; - - -/** - * Add allow leading wildcard - * setAllowLeadingWildcard(true); - * setLowercaseExpandedTerms(true); - * <br>Example: <code>{!wikitty q.op=AND df=text sort='price asc'}myfield:foo +bar -baz</code> - * More information @see LuceneQParserPlugin - */ -public class WikittyQueryParser extends LuceneQParserPlugin { - - public static String NAME = "wikitty"; - - @Override - public void init(NamedList args) { - } - - @Override - public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { - return new SolrQParser(qstr, localParams, params, req); - } -} - -class SolrQParser extends QParser { - - static private Log log = LogFactory.getLog(SolrQParser.class); - String sortStr; - SolrQueryParser lparser; - - public SolrQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { - super(qstr, localParams, params, req); - } - - @Override - public Query parse() throws ParseException { - - String defaultField = getParam(CommonParams.DF); - lparser = new SolrQueryParser(this, defaultField); - lparser.setAllowLeadingWildcard(true); - lparser.setLowercaseExpandedTerms(true); - - String opParam = getParam(QueryParsing.OP); - if (opParam != null) { - lparser.setDefaultOperator("AND".equals(opParam) ? QueryParser.Operator.AND : QueryParser.Operator.OR); - } - - String qstr = getString(); - log.debug("Query parse : " + qstr); - return lparser.parse(qstr); - } - - @Override - public String[] getDefaultHighlightFields() { - return new String[]{lparser.getField()}; - } -} Deleted: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java 2009-11-04 10:19:47 UTC (rev 308) @@ -1,659 +0,0 @@ -/* *##% - * Copyright (c) 2009 poussin. All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - *##%*/ - -package org.sharengo.wikitty.jdbc; - - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.core.CoreContainer; -import org.sharengo.wikitty.*; -import org.sharengo.wikitty.FieldType.TYPE; -import org.sharengo.wikitty.search.Search; - -import java.io.IOException; -import java.util.*; - -/** - * @author poussin - * @version $Revision$ - * <p/> - * Last update: $Date$ - * by : $Author$ - */ -public class WikittySearchEnginSolr implements WikittySearchEngin { - - /** - * to use log facility, just put in your code: log.info(\"...\"); - */ - static private Log log = LogFactory.getLog(WikittySearchEnginSolr.class); - - /** - * id field in solr - */ - static final protected String SOLR_ID = "id"; - - /** - * extensions field name in solr - */ - static final public String SOLR_EXTENSIONS = "extensions"; - - /** - * extension use to store field without extension to search on all extesnion - */ - static final public String SOLR_ALL_EXTENSIONS = "all"; - - /** - * Precise the query parser to use, is allow leading wildcard - */ - static final public String SOLR_QUERY_PARSER = "{!wikitty}"; - - // Use for indexation tree node - static final public String TREENODE_PREFIX = TreeNode.EXT_TREENODE + "."; - static final public String TREENODE_CHILD = TREENODE_PREFIX + "wikittyId"; - static final public String TREENODE_EMPTY = TREENODE_PREFIX + "empty"; - static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; - - static protected class CommandSolr implements Command { - /** - * if is store command toStore is true, if is delete toStore is false - */ - boolean toStore; - String id; - Wikitty wikitty; - - /** - * use to store command - */ - public CommandSolr(Wikitty wikitty) { - this.wikitty = wikitty; - this.toStore = true; - } - - /** - * use for deletion command - */ - public CommandSolr(String id) { - this.id = id; - this.toStore = false; - } - } - - static protected class TypeFieldModifer implements Restriction2Solr.FieldModifier { - protected WikittyExtensionStorage extensionStorage; - - public TypeFieldModifer(WikittyExtensionStorage extensionStorage) { - this.extensionStorage = extensionStorage; - } - - public String convertToSolr(String fqfieldname) { - String result = fqfieldname; - String[] searchField = fqfieldname.split("\\."); - - if (searchField.length == 2) { - String extName = searchField[0]; - String fieldName = searchField[1]; - - String version = extensionStorage.getLastVersion(extName); - if (version != null) { // not valid extension is version == null - WikittyExtension ext = extensionStorage.restore( - WikittyExtension.computeId(extName, version)); - FieldType type = ext.getFieldType(fieldName); - if (type != null) { // type can be null if extension version differ - result = WikittySearchEnginSolr.getSolrFieldName( - fqfieldname, type); - } else { - log.debug(String.format( - "Field '%s' could not be found in '%s'", - fqfieldname, extName)); - } - } else { - log.warn("No version for extension " + extName); - } - } - - if (searchField.length == 3) { - String extName = searchField[0]; - String fieldName = searchField[1]; - String fieldType = searchField[2]; - - if (Criteria.ALL_EXTENSIONS.equals(extName)) { - TYPE type = FieldType.TYPE.valueOf(fieldType); - String solrFieldName = SOLR_ALL_EXTENSIONS + "." + fieldName; - result = WikittySearchEnginSolr.getSolrFieldName(solrFieldName, type); - } else { - log.warn("All extension not found " + extName); - } - } - - return result; - } - - public String convertToField(String solrName) { - String fieldname = solrName.replaceAll("(_b$)|(_dt$)|(_t$)|(_l$)", ""); - return fieldname; - } - - } - - /** - * Store parent for each node and search in index if not known parent - */ - protected static class ParentMap extends HashMap<String, String> { - - protected WikittySearchEnginSolr searchEnginSolr; - - public ParentMap(WikittySearchEnginSolr searchEnginSolr) { - this.searchEnginSolr = searchEnginSolr; - } - - @Override - public String get(Object key) { - String id = String.valueOf(key); - String parent = super.get(id); - - // If not found in map, search in index - if (parent == null) { - SolrDocument doc = searchEnginSolr.findById(id); - if (doc == null) { - // is root - return null; - } - parent = (String) doc.getFieldValue(TreeNode.FQ_FIELD_PARENT); - put(id, parent); - } - return parent; - } - - /** - * Add parent map in transaction if is not exists and return it - */ - public static ParentMap getParentMap( - WikittySearchEnginSolr searchEnginSolr, - WikittyTransaction transaction) { - ParentMap value = (ParentMap) transaction.getTagValue(ParentMap.class); - if (value == null) { - value = new ParentMap(searchEnginSolr); - transaction.setTagValue(ParentMap.class, value); - } - return value; - } - } - - /** - * solr server - */ - protected SolrServer solrServer; - /** - * Criteria transformer to solr - */ - protected Restriction2Solr restriction2Solr; - /** - * Field modifier use to transform to solr format - */ - protected TypeFieldModifer fieldModifier; - - public WikittySearchEnginSolr(WikittyExtensionStorage extensionStorage) { - super(); - try { - CoreContainer.Initializer initializer = new CoreContainer.Initializer(); - CoreContainer coreContainer = initializer.initialize(); - solrServer = new EmbeddedSolrServer(coreContainer, ""); - - fieldModifier = new TypeFieldModifer(extensionStorage); - restriction2Solr = new Restriction2Solr(fieldModifier); - - } catch (Exception eee) { - throw new WikittyException(eee); - } - } - - public void clear() { - try { - solrServer.deleteByQuery("*:*"); - solrServer.commit(true, true); - } catch (Exception eee) { - throw new WikittyException(eee); - } - } - - public List<Command> prepare(WikittyTransaction transaction, - Collection<Wikitty> wikitties) { - List<Command> result = new ArrayList<Command>(wikitties.size()); - for (Wikitty w : wikitties) { - - // Store known parent in transaction - if (w.hasExtension(TreeNode.EXT_TREENODE)) { - ParentMap parentMap = ParentMap.getParentMap(this, transaction); - String wikittyId = w.getId(); - String parentId = w.getFieldAsWikitty( - TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); - parentMap.put(wikittyId, parentId); - } - - result.add(new CommandSolr(w)); - } - return result; - } - - public UpdateResponse commit(WikittyTransaction transaction, - List<Command> wikittyIndexationCommandList) { - try { - for (Command c : wikittyIndexationCommandList) { - CommandSolr cs = (CommandSolr) c; - if (cs.toStore) { - SolrInputDocument doc = createIndexDocument(cs.wikitty); - solrServer.add(doc); - - Collection<SolrInputDocument> docs = createTreeNodeIndex(transaction, cs.wikitty); - if (!docs.isEmpty()) { - solrServer.add(docs); - } - - } else { - // Delete all reference on tree - solrServer.deleteByQuery(TREENODE_CHILD + ":" + cs.id); - // FIXME: jru 20091027 delete node in middle in tree - solrServer.deleteByQuery(TREENODE_PREFIX + cs.id + ":" + TREENODE_EMPTY); - - solrServer.deleteById(cs.id); - } - } - solrServer.commit(); - - // no specific result needed - UpdateResponse result = new UpdateResponse(); - return result; - } catch (Exception eee) { - throw new WikittyException(eee); - } - } - - public List<Command> delete(WikittyTransaction transaction, - List<String> idList) throws WikittyException { - List<Command> result = new ArrayList<Command>(idList.size()); - for (String id : idList) { - result.add(new CommandSolr(id)); - } - return result; - } - - public PagedResult<String> findAllByCriteria(Criteria criteria) { - try { - - // Create query with restriction - String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); - - // Add pagged - int firstIndex = criteria.getFirstIndex(); - int endIndex = criteria.getEndIndex(); - - query.setStart(firstIndex); - int nbRows; - if (endIndex == -1) { - // WARNING It is necessary to substract 'start' otherwise, there is a capacity overlow in solR - nbRows = Integer.MAX_VALUE - firstIndex; - } else { - nbRows = endIndex - firstIndex + 1; - } - query.setRows(nbRows); - - // Add sorting - List<String> sortAscending = criteria.getSortAscending(); - if (sortAscending != null) { - for (String sort : sortAscending) { - String tranform = fieldModifier.convertToSolr(sort); - query.setSortField(tranform, SolrQuery.ORDER.asc); - } - } - - List<String> sortDescending = criteria.getSortDescending(); - if (sortDescending != null) { - for (String sort : sortDescending) { - String tranform = fieldModifier.convertToSolr(sort); - query.setSortField(tranform, SolrQuery.ORDER.desc); - } - } - - // Add faceting - List<String> facetField = criteria.getFacetField(); - List<Criteria> facetCriteria = criteria.getFacetCriteria(); - - // use to map query string to criteria facet name - Map<String, String> facetQueryToName = new HashMap<String, String>(); - - if ((facetField != null && !facetField.isEmpty()) - || (facetCriteria != null && !facetCriteria.isEmpty())) { - query.setFacet(true); - query.setFacetMinCount(1); - // query.setFacetLimit(8); // no limit actualy - - // field facetisation - if (facetField != null) { - for (String fqfieldName : facetField) { - String tranform = fieldModifier.convertToSolr(fqfieldName); - query.addFacetField(tranform); - } - } - - // query facetisation - if (facetCriteria != null) { - for (Criteria facet : facetCriteria) { - String queryFacet = - restriction2Solr.toSolr(facet.getRestriction()); - facetQueryToName.put(queryFacet, facet.getName()); - query.addFacetQuery(queryFacet); - } - } - } - - QueryResponse resp = solrServer.query(query); - SolrDocumentList solrResults = resp.getResults(); - - Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); - if (facetField != null && !facetField.isEmpty()) { - for (FacetField facet : resp.getFacetFields()) { - String facetName = fieldModifier.convertToField(facet.getName()); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - if (facet.getValues() != null) { - for (FacetField.Count value : facet.getValues()) { - String topicName = value.getName(); - if (!topicName.endsWith(TREENODE_EMPTY)) { - int topicCount = (int) value.getCount(); - FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); - topics.add(topic); - } - } - } - facets.put(facetName, topics); - } - } - if (facetCriteria != null && !facetCriteria.isEmpty()) { - for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { - String facetName = facet.getKey(); - // don't use contains because, map can have key with null value - if (null != facetQueryToName.get(facetName)) { - facetName = facetQueryToName.get(facetName); - } - Integer count = facet.getValue(); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - FacetTopic topic = new FacetTopic(facetName, facetName, count); - topics.add(topic); - facets.put(facetName, topics); - } - } - - List<String> ids = new ArrayList<String>(solrResults.size()); - for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue(SOLR_ID); - ids.add(id); - } - - int numFound = (int) resp.getResults().getNumFound(); - PagedResult<String> result = new PagedResult<String>( - firstIndex, numFound, queryString, facets, ids); - - return result; - } catch (SolrServerException e) { - throw new WikittyException(e); - } - } - - public Map<String, Integer> findAllChildren(Wikitty w) { - String wikittyId = w.getId(); - - String parent = w.getFieldAsWikitty(TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); - if (parent == null) { - parent = TREENODE_ROOT; - } else { - parent = TREENODE_PREFIX + parent; - } - - // Find count with facet, if the node not contain recurcively content, - // the node not found with facet - Criteria criteria = Search.query().eq(parent, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(0) - .addFacetField(TREENODE_PREFIX + wikittyId); - PagedResult<String> search = findAllByCriteria(criteria); - - Map<String, Integer> counts = new HashMap<String, Integer>(); - List<FacetTopic> topics = search.getTopic(TREENODE_PREFIX + wikittyId); - if (topics != null) { - for (FacetTopic topic : topics) { - String topicName = topic.getTopicName(); - int topicCount = topic.getCount(); - counts.put(topicName, topicCount); - } - } - - log.debug("Facet result " + counts); - - // Find all children, add the other node not found with facet - criteria = Search.query().eq(TreeNode.FQ_FIELD_PARENT, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); - search = findAllByCriteria(criteria); - - List<String> children = search.getAll(); - for (String child : children) { - if (!counts.containsKey(child)) { - counts.put(child, 0); - } - } - - return counts; - } - - /** - * Create all index document to used to modify indexation. - * this method don't modify index - * - * @param wikitties all wikitties object to index - * @return list of SolrInputDocument used to modify index - */ - protected SolrInputDocument createIndexDocument(Wikitty w) { - log.debug("index wikitty " + w.getId()); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField(SOLR_ID, w.getId()); - - for (String name : w.getExtensionNames()) { - doc.addField(SOLR_EXTENSIONS, name); - } - - for (String fqfieldName : w.fieldNames()) { - FieldType type = w.getFieldType(fqfieldName); - // do not index wikitty field reference -// if (type.getType() == FieldType.TYPE.WIKITTY) { -// continue; -// } - String solrFqFieldName = getSolrFieldName(fqfieldName, type); - String[] extAndName = solrFqFieldName.split("\\."); - String solrFieldName = SOLR_ALL_EXTENSIONS + "." + extAndName[1]; - if (type.isCollection()) { - Collection collection = (Collection) w.getFqField(fqfieldName); - for (Object objectValue : collection) { - if (objectValue != null) { - // String value = WikittyUtil.toString(type, objectValue); - Object value = objectValue; - doc.addField(solrFqFieldName, value); - doc.addField(solrFieldName, value); - } - } - } else { - Object objectValue = w.getFqField(fqfieldName); - if (objectValue != null) { - // String value = WikittyUtil.toString(type, objectValue); - Object value = objectValue; - log.debug("index field " + solrFqFieldName + " with value '" + value + "'"); - doc.addField(solrFqFieldName, value); - doc.addField(solrFieldName, value); - } - } - } - - return doc; - } - - /** - * Create a doc for each child for a tree node, his contains the path for - * the child. Update old doc when the node is moved and child is deleted. - */ - protected Collection<SolrInputDocument> createTreeNodeIndex( - WikittyTransaction transaction, Wikitty w) - throws IOException, SolrServerException { - - Collection<SolrInputDocument> result = new ArrayList<SolrInputDocument>(); - if (w.hasExtension(TreeNode.EXT_TREENODE)) { - String treeNodeId = w.getId(); - ParentMap parentMap = (ParentMap) transaction.getTagValue(ParentMap.class); - - // Init new children with all children in node - List<String> children = w.getFieldAsList(TreeNode.EXT_TREENODE, - TreeNode.FIELD_CHILDREN, String.class); - List<String> newChildren = new ArrayList<String>(); - if (children != null) { - newChildren.addAll(children); - } - - // Find old document to do diff - SolrDocument treeNodeDoc = findById(treeNodeId); - if (treeNodeDoc != null) { - Collection oldChildren = treeNodeDoc. - getFieldValues(TreeNode.FQ_FIELD_CHILDREN + "_t"); - - // Remove old children not in current node - List<String> removeChildren = new ArrayList<String>(); - if (oldChildren != null) { - removeChildren.addAll(oldChildren); - removeChildren.removeAll(children); - for (String removeChild : removeChildren) { - solrServer.deleteByQuery(TREENODE_CHILD + ":" + removeChild); - } - - // Remove old child in new children - newChildren.removeAll(oldChildren); - } - - // Update path other node refer to current node - // FIXME jru 20091926 test change path - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PREFIX + treeNodeId + ":*"); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - String child = (String) solrDocument.getFieldValue(TreeNode.EXT_TREENODE + ".wikittyId"); - // Child is not deleted - if (!removeChildren.contains(child)) { - SolrInputDocument doc = createTreeNodeDocument(treeNodeId, child, parentMap); - - // Copy other field after node - String parent = TREENODE_PREFIX + treeNodeId; - String childParent = (String) solrDocument.getFieldValue(parent); - while (childParent != null) { - doc.setField(parent, childParent); - parent = TREENODE_PREFIX + childParent; - childParent = (String) solrDocument.getFieldValue(parent); - } - result.add(doc); - } - } - } - - // New children - for (String newChild : newChildren) { - SolrInputDocument doc = createTreeNodeDocument(treeNodeId, newChild, parentMap); - result.add(doc); - } - - } - return result; - } - - /** - * Create a doc between node and child - */ - public SolrInputDocument createTreeNodeDocument(String treeNodeId, String child, ParentMap parentMap) { - SolrInputDocument doc = new SolrInputDocument(); - doc.setField(SOLR_ID, TreeNode.EXT_TREENODE + "-" + treeNodeId + "-" + child); - doc.setField(TREENODE_CHILD, child); - doc.setField(TREENODE_PREFIX + treeNodeId, TREENODE_EMPTY); - - // Add path - String childParent = treeNodeId; - String parent = parentMap.get(childParent); - while (parent != null) { - doc.setField(TREENODE_PREFIX + parent, childParent); - childParent = parent; - parent = parentMap.get(childParent); - } - doc.setField(TREENODE_ROOT, childParent); - - return doc; - } - - /** - * Find solr document by id - */ - protected SolrDocument findById(String id) { - SolrQuery query = new SolrQuery("id:" + id); - QueryResponse response; - try { - response = solrServer.query(query); - } catch (SolrServerException eee) { - throw new WikittyException(eee); - } - - SolrDocumentList results = response.getResults(); - long numFound = results.getNumFound(); - if (numFound == 1) { - return results.get(0); - } - - return null; - } - - static public String getSolrFieldName(String fqfieldName, FieldType fieldType) { - TYPE type = fieldType.getType(); - String result = getSolrFieldName(fqfieldName, type); - return result; - } - - public static String getSolrFieldName(String fqfieldName, TYPE type) { - switch (type) { - case BOOLEAN: - return fqfieldName + "_b"; - case DATE: - return fqfieldName + "_dt"; - case STRING: - return fqfieldName + "_t"; - case NUMERIC: - return fqfieldName + "_d"; - default: - return fqfieldName; - } - } -} Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java 2009-11-04 10:19:47 UTC (rev 308) @@ -19,10 +19,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.sharengo.wikitty.AbstractWikittyService; -import org.sharengo.wikitty.WikittyExtensionStorage; -import org.sharengo.wikitty.WikittySearchEngin; -import org.sharengo.wikitty.WikittyStorage; +import org.sharengo.wikitty.*; +import org.sharengo.wikitty.solr.WikittySearchEngineSolr; /** * HBase must have 2 definitions, one for extension and one for wikitty object. @@ -65,7 +63,7 @@ public WikittyServiceJDBC() { extensionStorage = new WikittyExtensionStorageJDBC(); wikittyStorage = new WikittyStorageJDBC(extensionStorage); - searchEngin = new WikittySearchEnginSolr(extensionStorage); + searchEngin = new WikittySearchEngineSolr(extensionStorage); } @Override Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml 2009-11-04 10:19:47 UTC (rev 308) @@ -678,7 +678,7 @@ </queryResponseWriter> - <queryParser name="wikitty" class="org.sharengo.wikitty.jdbc.WikittyQueryParser"/> + <queryParser name="wikitty" class="org.sharengo.wikitty.solr.WikittyQueryParser"/> <!-- example of registering a query parser <queryParser name="lucene" class="org.apache.solr.search.LuceneQParserPlugin"/> Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java 2009-11-04 10:19:47 UTC (rev 308) @@ -21,7 +21,6 @@ import org.sharengo.wikitty.Criteria; import org.sharengo.wikitty.Wikitty; import org.sharengo.wikitty.WikittyService; -import org.sharengo.wikitty.search.Element; import org.sharengo.wikitty.search.Search; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; @@ -37,7 +36,7 @@ public void testExport() throws Exception { final List<Wikitty> wikitties = createSampleWikitty(ws); - Criteria criteria = Search.query().eq(Element.ELT_EXTENSION, EXTNAME).criteria(); + Criteria criteria = Search.query().eqExt(EXTNAME).criteria(); String result = ws.syncExportAllByCriteria(criteria); log.debug("[XML] " + result); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java 2009-11-04 10:19:47 UTC (rev 308) @@ -26,18 +26,19 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.sharengo.internal.TreeNodeImpl; import org.sharengo.wikitty.Criteria; import org.sharengo.wikitty.ExtensionFactory; import org.sharengo.wikitty.FacetTopic; import org.sharengo.wikitty.PagedResult; import org.sharengo.wikitty.Tree; import org.sharengo.wikitty.TreeNode; -import org.sharengo.wikitty.TreeNodeImpl; import org.sharengo.wikitty.Wikitty; import org.sharengo.wikitty.WikittyExtension; import org.sharengo.wikitty.WikittyService; import org.sharengo.wikitty.FieldType.TYPE; import org.sharengo.wikitty.search.Search; +import org.sharengo.wikitty.solr.WikittySearchEngineSolr; import org.sharengo.wikitty.test.StorageTest; /** @@ -95,8 +96,8 @@ Wikitty found = findNode(name); if(found == null) { TreeNodeImpl node = new TreeNodeImpl(); - node.setName(name); - node.setParent(parent); + node.setTreeNodeName(name); + node.setTreeNodeParent(parent); found = node.getWikitty(); log.debug("[Storing] " + name + " with id " + found.getId()); @@ -161,7 +162,7 @@ Wikitty root = findNode("root"); String rootId = root.getId(); - Criteria criteria = Search.query().eq(WikittySearchEnginSolr.TREENODE_ROOT, rootId).criteria() + Criteria criteria = Search.query().eq(WikittySearchEngineSolr.TREENODE_ROOT, rootId).criteria() .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS) .addFacetField(rootId); PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); Deleted: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEnginTest.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEnginTest.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEnginTest.java 2009-11-04 10:19:47 UTC (rev 308) @@ -1,292 +0,0 @@ -/* *##% - * Copyright (c) 2009 poussin. All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - *##%*/ - -package org.sharengo.wikitty.jdbc; - - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import junit.framework.TestCase; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.sharengo.wikitty.Criteria; -import org.sharengo.wikitty.FacetTopic; -import org.sharengo.wikitty.PagedResult; -import org.sharengo.wikitty.Wikitty; -import org.sharengo.wikitty.WikittyService; -import org.sharengo.wikitty.WikittyUtil; -import org.sharengo.wikitty.search.Search; -import org.sharengo.wikitty.test.util.GenTestData; -import org.sharengo.wikitty.test.util.ImportFeedback; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class WikittySearchEnginTest extends TestCase { - - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(WikittySearchEnginTest.class); - - /** 10 search by field */ - static final int MAX = 10; - - static String[] allFieldNames = new String[]{ - "Personne.civilite", - "Personne.nom", - "Personne.prenom", - "Personne.date_naissance", - "Personne.poids", - "Employe.matricule", - "Employe.arrivee", - "Employe.salaire" - }; - - /** - * used to count some element during data loading, this information - * is used to assert statement - */ - static public class AssertReference implements ImportFeedback { - - /** total number imported elements */ - public int count = 0; - /** key: fqfieldname value: map(fieldvalue, count this value) */ - public Map<String, Map<Object, Integer>> fieldValueCounters = - new HashMap<String, Map<Object, Integer>>(); - - public void importPerformed(long nbImportedElements, long deltaTime) { - // do nothing - } - - public void parsed(Wikitty w) { - count++; - - for (String fqf : allFieldNames) { - Map<Object, Integer> fvc = fieldValueCounters.get(fqf); - if (fvc == null) { - fvc = new HashMap<Object, Integer>(); - fieldValueCounters.put(fqf, fvc); - } - Object key = w.getFqField(fqf); - Integer n = fvc.get(key); - if (n == null) { - n = 1; - } else { - n = n + 1; - } - fvc.put(key, n); - } - } - - } - - WikittyService ws; - AssertReference assertReference; - - // TODO poussin 20090901 this method is call for each test method, this method can be call only once for all test methods - @Override - public void setUp() throws Exception { - // import data and prepare expected result - ws = new WikittyServiceJDBC(); - ((WikittyServiceJDBC)ws).getSearchEngin().clear(); - assertReference = new AssertReference(); - GenTestData.importGeneratedFile( - GenTestData.getDataFile("selected-data.txt"), ws, assertReference); - - } - - /** - * Do MAX search for each field in object and compare result size with - * number found during import - * - * @throws Exception - */ - public void testFindByExample() throws Exception { - for (String fqf : allFieldNames) { - int cpt = 0; // cpt to stop loop after amount of test - Wikitty w = new Wikitty(); - w.addExtension(GenTestData.ext_personne); - w.addExtension(GenTestData.ext_employe); - - Map<Object, Integer> fvc = assertReference.fieldValueCounters.get(fqf); - for (Object value : fvc.keySet()) { - System.out.println("search on: " + fqf + " with value: " + value); - if(++cpt >= MAX) { - break; - } - int nb = fvc.get(value); - w.setFqField(fqf, value); - Criteria criteria = Search.query(w).criteria() - .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS); - PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); - System.out.println("Count: " + result.size() + " Query: " + result.getQueryString()); - assertEquals(nb, result.size()); - } - } - } - - /** - * Do MAX search for each field in object and compare result size with - * number found during import - * - * @throws Exception - */ - public void testFindByCriteria() throws Exception { - for (String fqf : allFieldNames) { - int cpt = 0; // cpt to stop loop after amount of test - - int sum = 0; // sum number of finding object for this fqf - List values = new LinkedList(); - - Map<Object, Integer> fvc = assertReference.fieldValueCounters.get(fqf); - for (Object value : fvc.keySet()) { - if(++cpt >= MAX) { - break; - } - int nb = fvc.get(value); - - // stock number and value for last test - sum += nb; - values.add(value); - - Criteria criteria = Search.query() - .eq(fqf, WikittyUtil.toString(value)) - .criteria() - .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS); - PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); - - assertEquals(nb, result.size()); - } - - Search query = Search.query().or(); - for (Object value : values) { - query.eq(fqf, WikittyUtil.toString(value)); - } - Criteria criteria = query.criteria() - .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS); - PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); - System.out.println("query: " + result.getQueryString()); - assertEquals(sum, result.size()); - } - } - - /** - * Do search for 'Personne.nom==*' for limited result number, total number - * available in solr must be equals to number found during import - * - * @throws Exception - */ - public void testFindByCriteriaNumFound() throws Exception { - String fqf = "Personne.nom"; - Criteria criteria = Search.query().neq(fqf, "aaaaa").criteria() - .setFirstIndex(0).setEndIndex(2); - - PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); - - assertEquals(assertReference.count, result.getNumFound()); - } - - /** - * Do search on Personne.nom and ask facet for Personne.nom, all Topic - * must be equals (name and count) to name and number found during import - * - * @throws Exception - */ - public void testFindByCriteriaFieldFacet() throws Exception { - String fqf = "Personne.nom"; - Map<Object, Integer> fvc = assertReference.fieldValueCounters.get(fqf); - Criteria criteria = Search.query().neq(fqf, "aaaaa").criteria() - .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS) - .addFacetField(fqf); - - PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); - - assertEquals(new ArrayList(Arrays.asList(fqf)), - new ArrayList(result.getFacetNames())); - List<FacetTopic> topics = result.getTopic(fqf); - - for (FacetTopic t : topics) { - String name = t.getTopicName(); - int num = t.getCount(); - - System.out.println("name: " + name + " keys: " + fvc.keySet()); - assertNotNull(fvc.get(name)); - int realNum = fvc.get(name); - assertEquals(realNum, num); - } - } - - /** - * Do search on Personne.nom and ask facet for Personne.civilite 2 - * named facet criteria: - * - homme (civilite==Mr) - * - femme (civilite==Me OR civilite==Mlle) - * - * @throws Exception - */ - public void testFindByCriteriaCriteriaFacet() throws Exception { - String fqf = "Personne.nom"; - String facetName = "Personne.civilite"; - - Criteria homme = Search.query().eq(facetName, "Mr").criteria("homme"); - Criteria femme = Search.query() - .eq(facetName, "Me").or().eq(facetName, "Mlle") - .criteria("femme"); - - Criteria criteria = Search.query().neq(fqf, "aaaaa").criteria() - .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS) - .addFacetCriteria(homme).addFacetCriteria(femme); - - PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); - - assertEquals(2, result.getFacetNames().size()); - - System.out.println("facet names: " + result.getFacetNames()); - - assertNotNull(assertReference.fieldValueCounters.get(facetName).get("Mr")); - assertNotNull(assertReference.fieldValueCounters.get(facetName).get("Me")); - assertNotNull(assertReference.fieldValueCounters.get(facetName).get("Mlle")); - - int numMr = assertReference.fieldValueCounters.get(facetName).get("Mr"); - int numMe = assertReference.fieldValueCounters.get(facetName).get("Me"); - int numMlle = assertReference.fieldValueCounters.get(facetName).get("Mlle"); - - log.info("numMr: " + numMr); - log.info("numMe: " + numMe); - log.info("numMlle: " + numMlle); - - List<FacetTopic> hommeTopics = result.getTopic("homme"); - List<FacetTopic> femmeTopics = result.getTopic("femme"); - - assertEquals(1, hommeTopics.size()); - assertEquals(1, femmeTopics.size()); - - FacetTopic hommeTopic = hommeTopics.get(0); - FacetTopic femmeTopic = femmeTopics.get(0); - - assertEquals(numMr, hommeTopic.getCount()); - assertEquals(numMe + numMlle, femmeTopic.getCount()); - } -} Added: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java 2009-11-04 10:19:47 UTC (rev 308) @@ -0,0 +1,292 @@ +/* *##% + * Copyright (c) 2009 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.sharengo.wikitty.jdbc; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import junit.framework.TestCase; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.sharengo.wikitty.Criteria; +import org.sharengo.wikitty.FacetTopic; +import org.sharengo.wikitty.PagedResult; +import org.sharengo.wikitty.Wikitty; +import org.sharengo.wikitty.WikittyService; +import org.sharengo.wikitty.WikittyUtil; +import org.sharengo.wikitty.search.Search; +import org.sharengo.wikitty.test.util.GenTestData; +import org.sharengo.wikitty.test.util.ImportFeedback; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class WikittySearchEngineTest extends TestCase { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(WikittySearchEngineTest.class); + + /** 10 search by field */ + static final int MAX = 10; + + static String[] allFieldNames = new String[]{ + "Personne.civilite", + "Personne.nom", + "Personne.prenom", + "Personne.date_naissance", + "Personne.poids", + "Employe.matricule", + "Employe.arrivee", + "Employe.salaire" + }; + + /** + * used to count some element during data loading, this information + * is used to assert statement + */ + static public class AssertReference implements ImportFeedback { + + /** total number imported elements */ + public int count = 0; + /** key: fqfieldname value: map(fieldvalue, count this value) */ + public Map<String, Map<Object, Integer>> fieldValueCounters = + new HashMap<String, Map<Object, Integer>>(); + + public void importPerformed(long nbImportedElements, long deltaTime) { + // do nothing + } + + public void parsed(Wikitty w) { + count++; + + for (String fqf : allFieldNames) { + Map<Object, Integer> fvc = fieldValueCounters.get(fqf); + if (fvc == null) { + fvc = new HashMap<Object, Integer>(); + fieldValueCounters.put(fqf, fvc); + } + Object key = w.getFqField(fqf); + Integer n = fvc.get(key); + if (n == null) { + n = 1; + } else { + n = n + 1; + } + fvc.put(key, n); + } + } + + } + + WikittyService ws; + AssertReference assertReference; + + // TODO poussin 20090901 this method is call for each test method, this method can be call only once for all test methods + @Override + public void setUp() throws Exception { + // import data and prepare expected result + ws = new WikittyServiceJDBC(); + ((WikittyServiceJDBC)ws).getSearchEngin().clear(); + assertReference = new AssertReference(); + GenTestData.importGeneratedFile( + GenTestData.getDataFile("selected-data.txt"), ws, assertReference); + + } + + /** + * Do MAX search for each field in object and compare result size with + * number found during import + * + * @throws Exception + */ + public void testFindByExample() throws Exception { + for (String fqf : allFieldNames) { + int cpt = 0; // cpt to stop loop after amount of test + Wikitty w = new Wikitty(); + w.addExtension(GenTestData.ext_personne); + w.addExtension(GenTestData.ext_employe); + + Map<Object, Integer> fvc = assertReference.fieldValueCounters.get(fqf); + for (Object value : fvc.keySet()) { + System.out.println("search on: " + fqf + " with value: " + value); + if(++cpt >= MAX) { + break; + } + int nb = fvc.get(value); + w.setFqField(fqf, value); + Criteria criteria = Search.query(w).criteria() + .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS); + PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); + System.out.println("Count: " + result.size() + " Query: " + result.getQueryString()); + assertEquals(nb, result.size()); + } + } + } + + /** + * Do MAX search for each field in object and compare result size with + * number found during import + * + * @throws Exception + */ + public void testFindByCriteria() throws Exception { + for (String fqf : allFieldNames) { + int cpt = 0; // cpt to stop loop after amount of test + + int sum = 0; // sum number of finding object for this fqf + List values = new LinkedList(); + + Map<Object, Integer> fvc = assertReference.fieldValueCounters.get(fqf); + for (Object value : fvc.keySet()) { + if(++cpt >= MAX) { + break; + } + int nb = fvc.get(value); + + // stock number and value for last test + sum += nb; + values.add(value); + + Criteria criteria = Search.query() + .eq(fqf, WikittyUtil.toString(value)) + .criteria() + .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS); + PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); + + assertEquals(nb, result.size()); + } + + Search query = Search.query().or(); + for (Object value : values) { + query.eq(fqf, WikittyUtil.toString(value)); + } + Criteria criteria = query.criteria() + .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS); + PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); + System.out.println("query: " + result.getQueryString()); + assertEquals(sum, result.size()); + } + } + + /** + * Do search for 'Personne.nom==*' for limited result number, total number + * available in solr must be equals to number found during import + * + * @throws Exception + */ + public void testFindByCriteriaNumFound() throws Exception { + String fqf = "Personne.nom"; + Criteria criteria = Search.query().neq(fqf, "aaaaa").criteria() + .setFirstIndex(0).setEndIndex(2); + + PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); + + assertEquals(assertReference.count, result.getNumFound()); + } + + /** + * Do search on Personne.nom and ask facet for Personne.nom, all Topic + * must be equals (name and count) to name and number found during import + * + * @throws Exception + */ + public void testFindByCriteriaFieldFacet() throws Exception { + String fqf = "Personne.nom"; + Map<Object, Integer> fvc = assertReference.fieldValueCounters.get(fqf); + Criteria criteria = Search.query().neq(fqf, "aaaaa").criteria() + .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS) + .addFacetField(fqf); + + PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); + + assertEquals(new ArrayList(Arrays.asList(fqf)), + new ArrayList(result.getFacetNames())); + List<FacetTopic> topics = result.getTopic(fqf); + + for (FacetTopic t : topics) { + String name = t.getTopicName(); + int num = t.getCount(); + + System.out.println("name: " + name + " keys: " + fvc.keySet()); + assertNotNull(fvc.get(name)); + int realNum = fvc.get(name); + assertEquals(realNum, num); + } + } + + /** + * Do search on Personne.nom and ask facet for Personne.civilite 2 + * named facet criteria: + * - homme (civilite==Mr) + * - femme (civilite==Me OR civilite==Mlle) + * + * @throws Exception + */ + public void testFindByCriteriaCriteriaFacet() throws Exception { + String fqf = "Personne.nom"; + String facetName = "Personne.civilite"; + + Criteria homme = Search.query().eq(facetName, "Mr").criteria("homme"); + Criteria femme = Search.query() + .eq(facetName, "Me").or().eq(facetName, "Mlle") + .criteria("femme"); + + Criteria criteria = Search.query().neq(fqf, "aaaaa").criteria() + .setFirstIndex(0).setEndIndex(WikittyService.ALL_ELEMENTS) + .addFacetCriteria(homme).addFacetCriteria(femme); + + PagedResult<Wikitty> result = ws.findAllByCriteria(criteria); + + assertEquals(2, result.getFacetNames().size()); + + System.out.println("facet names: " + result.getFacetNames()); + + assertNotNull(assertReference.fieldValueCounters.get(facetName).get("Mr")); + assertNotNull(assertReference.fieldValueCounters.get(facetName).get("Me")); + assertNotNull(assertReference.fieldValueCounters.get(facetName).get("Mlle")); + + int numMr = assertReference.fieldValueCounters.get(facetName).get("Mr"); + int numMe = assertReference.fieldValueCounters.get(facetName).get("Me"); + int numMlle = assertReference.fieldValueCounters.get(facetName).get("Mlle"); + + log.info("numMr: " + numMr); + log.info("numMe: " + numMe); + log.info("numMlle: " + numMlle); + + List<FacetTopic> hommeTopics = result.getTopic("homme"); + List<FacetTopic> femmeTopics = result.getTopic("femme"); + + assertEquals(1, hommeTopics.size()); + assertEquals(1, femmeTopics.size()); + + FacetTopic hommeTopic = hommeTopics.get(0); + FacetTopic femmeTopic = femmeTopics.get(0); + + assertEquals(numMr, hommeTopic.getCount()); + assertEquals(numMe + numMlle, femmeTopic.getCount()); + } +} Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/jdbc.properties =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/jdbc.properties 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/jdbc.properties 2009-11-04 10:19:47 UTC (rev 308) @@ -8,8 +8,7 @@ jdbc.queries.creation.extension.admin=CREATE TABLE IF NOT EXISTS extension_admin (\ id text NOT NULL,\ name text NOT NULL,\ -version varchar(8) NOT NULL,\ -requires text);\ +version varchar(8) NOT NULL);\ ALTER TABLE extension_admin ADD PRIMARY KEY (id); #table extension_data creation query @@ -66,7 +65,7 @@ (id, fieldName, %s) VALUES ('%s', '%s.%s', %s); #extension insertion in the admin table query jdbc.queries.insert.extension.admin=INSERT INTO extension_admin\ -(id, name, version, requires) VALUES ('%s', '%s', '%s', %s); +(id, name, version) VALUES ('%s', '%s', '%s'); #extension insertion in the data table query jdbc.queries.insert.extension.data=INSERT INTO extension_data\ (id, fieldName, fieldType) VALUES ('%s', '%s', '%s'); \ No newline at end of file Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/solrconfig.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/solrconfig.xml 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/solrconfig.xml 2009-11-04 10:19:47 UTC (rev 308) @@ -675,9 +675,11 @@ --> <queryResponseWriter name="xslt" class="org.apache.solr.request.XSLTResponseWriter"> <int name="xsltCacheLifetimeSeconds">5</int> - </queryResponseWriter> + </queryResponseWriter> + <queryParser name="wikitty" class="org.sharengo.wikitty.solr.WikittyQueryParser"/> + <!-- example of registering a query parser <queryParser name="lucene" class="org.apache.solr.search.LuceneQParserPlugin"/> -->
participants (1)
-
kmorin＠users.nuiton.org