Author: kmorin Date: 2009-11-04 11:20:45 +0100 (Wed, 04 Nov 2009) New Revision: 311 Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/protwords.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/spellwords.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/synonyms.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/wikitty/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/wikitty/solr/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/wikitty/solr/test/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/META-INF/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/META-INF/spring/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties Log: Separation of solr Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4"> + <component name="NewModuleRootManager" inherit-compiler-output="false"> + <output url="file://$MODULE_DIR$/target/classes" /> + <output-test url="file://$MODULE_DIR$/target/test-classes" /> + <content url="file://$MODULE_DIR$"> + <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" /> + <sourceFolder url="file://$MODULE_DIR$/src/main/resources" isTestSource="false" /> + <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" /> + <sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" /> + <excludeFolder url="file://$MODULE_DIR$/target" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + <orderEntry type="module" module-name="core-wikitty-api" exported="" /> + <orderEntry type="library" exported="" scope="RUNTIME" name="Maven: log4j:log4j:1.2.14" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-logging:commons-logging:1.1.1" level="project" /> + <orderEntry type="library" exported="" name="Maven: xpp3:xpp3:1.1.4c" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-core:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-common:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-solrj:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-codec:commons-codec:1.3" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.commons:commons-io:1.3.2" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-fileupload:commons-fileupload:1.2" level="project" /> + <orderEntry type="library" exported="" name="Maven: stax:stax:1.2.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: stax:stax-api:1.0.1" level="project" /> + <orderEntry type="library" exported="" name="Maven: net.java.dev.stax-utils:stax-utils:20040917" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-analyzers:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-core:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-highlighter:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-queries:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-snowball:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-spellchecker:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-commons-csv:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: woodstox:wstx-asl:3.2.7" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.geronimo.specs:geronimo-stax-api_1.0_spec:1.0.1" level="project" /> + <orderEntry type="library" exported="" scope="RUNTIME" name="Maven: javax.servlet:servlet-api:2.5" level="project" /> + </component> +</module> + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,92 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" +xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + + <modelVersion>4.0.0</modelVersion> + + <!-- ************************************************************* --> + <!-- *** POM Relationships *************************************** --> + <!-- ************************************************************* --> + + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty-solr-impl</artifactId> + + <parent> + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty</artifactId> + <version>1.0-CL-SNAPSHOT</version> + </parent> + + <dependencies> + <!-- COMPILE --> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>core-wikitty-api</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + + <!-- SOLR --> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-core</artifactId> + <version>1.3.0</version> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.5</version> + <scope>runtime</scope> + </dependency> + + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + </dependencies> + + <!-- ************************************************************* --> + <!-- *** Project Information ************************************* --> + <!-- ************************************************************* --> + + <name>org.sharengo.wikengo.core-wikitty-solr-impl</name> + + <description>hbase impl of wikitty</description> + <inceptionYear>2009</inceptionYear> + + <!-- ************************************************************* --> + <!-- *** Build Settings ****************************************** --> + <!-- ************************************************************* --> + + <packaging>jar</packaging> + + <properties> + <maven.test.skip>true</maven.test.skip> + <maven.test.failure.ignore>true</maven.test.failure.ignore> + </properties> + + <!-- ************************************************************* --> + <!-- *** Build Environment ************************************** --> + <!-- ************************************************************* --> + + <repositories> + <repository> + <id>sharengo-maven2-repository</id> + <name>Sharengo's repository</name> + <url>http://maven2.sharengo.org</url> + <snapshots> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </snapshots> + <releases> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </releases> + </repository> + </repositories> + +</project> + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,92 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" +xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + + <modelVersion>4.0.0</modelVersion> + + <!-- ************************************************************* --> + <!-- *** POM Relationships *************************************** --> + <!-- ************************************************************* --> + + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty-solr-impl</artifactId> + + <parent> + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty</artifactId> + <version>1.0-CL-SNAPSHOT</version> + </parent> + + <dependencies> + <!-- COMPILE --> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>core-wikitty-api</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + + <!-- SOLR --> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-core</artifactId> + <version>1.3.0</version> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.5</version> + <scope>runtime</scope> + </dependency> + + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + </dependencies> + + <!-- ************************************************************* --> + <!-- *** Project Information ************************************* --> + <!-- ************************************************************* --> + + <name>org.sharengo.wikengo.core-wikitty-solr-impl</name> + + <description>hbase impl of wikitty</description> + <inceptionYear>2009</inceptionYear> + + <!-- ************************************************************* --> + <!-- *** Build Settings ****************************************** --> + <!-- ************************************************************* --> + + <packaging>jar</packaging> + + <properties> + <maven.test.skip>true</maven.test.skip> + <maven.test.failure.ignore>true</maven.test.failure.ignore> + </properties> + + <!-- ************************************************************* --> + <!-- *** Build Environment ************************************** --> + <!-- ************************************************************* --> + + <repositories> + <repository> + <id>sharengo-maven2-repository</id> + <name>Sharengo's repository</name> + <url>http://maven2.sharengo.org</url> + <snapshots> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </snapshots> + <releases> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </releases> + </repository> + </repositories> + +</project> + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,397 @@ +package org.sharengo.wikitty.solr; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.sharengo.wikitty.WikittyException; +import org.sharengo.wikitty.search.And; +import org.sharengo.wikitty.search.AssociatedRestriction; +import org.sharengo.wikitty.search.Between; +import org.sharengo.wikitty.search.Contains; +import org.sharengo.wikitty.search.Element; +import org.sharengo.wikitty.search.EndsWith; +import org.sharengo.wikitty.search.Equals; +import org.sharengo.wikitty.search.Greater; +import org.sharengo.wikitty.search.GreaterOrEqual; +import org.sharengo.wikitty.search.In; +import org.sharengo.wikitty.search.Keyword; +import org.sharengo.wikitty.search.Less; +import org.sharengo.wikitty.search.LessOrEqual; +import org.sharengo.wikitty.search.Not; +import org.sharengo.wikitty.search.NotEquals; +import org.sharengo.wikitty.search.Or; +import org.sharengo.wikitty.search.Restriction; +import org.sharengo.wikitty.search.RestrictionHelper; +import org.sharengo.wikitty.search.StartsWith; + +/** + * @author "Nicolas Chapurlat" <nicolas.chapurlat@logica.com> + * @author "Guillaume Dufrêne" <dufrene@argia.fr> + * + * This class is used to parse Restriction to create lucene request on + * content. Every operators describe in RestrictionName is handle. Parsing may + * throw exception when restriction parameters are incorrect. + */ +public class Restriction2Solr { + + private static final int MAX_SUBQUERY_RESULT = 100; + + /** use to permit client to modify fieldname during query generation */ + static public interface FieldModifier { + public String convertToSolr(String fieldname); + public String convertToField(String solrName); + } + final static protected FieldModifier dummyFieldModifier = new FieldModifier() { + public String convertToSolr(String fieldname) { + return fieldname; + } + public String convertToField(String solrName) { + return solrName; + } + }; + + protected FieldModifier fieldModifer; + + public Restriction2Solr() { + this(dummyFieldModifier); + } + + public Restriction2Solr(FieldModifier fieldModifer) { + this.fieldModifer = fieldModifer; + } + + public String toSolr(Restriction restriction) { + return toSolr(restriction, null); + } + + public String toSolr(Restriction restriction, SolrServer solr) + throws WikittyException { + // ParameterValidator.checkNullParameter(restriction, "restriction"); + switch (restriction.getName()) { + case TRUE: + return true2solr(); + case FALSE: + return false2solr(); + case NOT: + Not not = (Not) restriction; + return not2solr(not); + case AND: + And and = (And) restriction; + return and2solr(and); + case OR: + Or or = (Or) restriction; + return or2solr(or); + case EQUALS: + Equals eq = (Equals) restriction; + return eq2solr(eq); + case NOT_EQUALS: + NotEquals neq = (NotEquals) restriction; + return neq2solr(neq); + case LESS: + Less less = (Less) restriction; + return less2solr(less); + case LESS_OR_EQUAL: + LessOrEqual lessEq = (LessOrEqual) restriction; + return lessEq2solr(lessEq); + case GREATER: + Greater great = (Greater) restriction; + return great2solr(great); + case GREATER_OR_EQUAL: + GreaterOrEqual greatEq = (GreaterOrEqual) restriction; + return greatEq2solr(greatEq); + case BETWEEN: + Between between = (Between) restriction; + return between2solr(between); + case CONTAINS: + Contains contains = (Contains) restriction; + return contains2solr(contains); + case IN: + In in = (In) restriction; + return in2solr(in); + case STARTS_WITH: + StartsWith start = (StartsWith) restriction; + return start2solr(start); + case ENDS_WITH: + EndsWith end = (EndsWith) restriction; + return end2solr(end); + case ASSOCIATED: + AssociatedRestriction associated = (AssociatedRestriction) restriction; + return associated2solr(associated, solr); + case KEYWORD: + Keyword keyword = (Keyword) restriction; + return keyword2solr(keyword); + default: + throw new WikittyException("this kind of restriction is not supported : " + + restriction.getName().toString()); + } + } + + private String in2solr(In in) { + boolean first = true; + String result = in.getElement().getName() + ":["; + for( String value : in.getValue() ) { + if ( !first ) { result += ", "; first = false; } + result += value; + } + result +="]"; + return result; + } + + private String associated2solr(AssociatedRestriction associated, SolrServer solr) throws WikittyException { + String subQuery = toSolr( associated.getRestriction() ); + SolrQuery query = new SolrQuery(subQuery); + query.setRows(MAX_SUBQUERY_RESULT); + QueryResponse resp = null; + try { + resp = solr.query(query); + } catch (SolrServerException e) { + throw new WikittyException("Unable to execute associative query on " + associated.getElement().getName(), e); + } + SolrDocumentList solrResults = resp.getResults(); + + Restriction generatedRestriction = null; + long size = solrResults.size(); + if ( size == 0 ) { + throw new WikittyException("Associated " + associated.getElement().getName() + " do not retrieved any result"); + } + if ( size == 1 ) { + generatedRestriction = RestrictionHelper.eq( associated.getElement(), (String) solrResults.get(0).getFieldValue("id") ); + } else { + List<String> ids = new ArrayList<String>(solrResults.size()); + for (SolrDocument doc : solrResults) { + String id = (String) doc.getFieldValue("id"); // FIXME : set a constant field reference + ids.add(id); + } + generatedRestriction = new In(associated.getElement(), ids); + } + Restriction parent = associated.getParentRestrictionDto(); + And and; + if ( parent instanceof And ) { + and = (And) parent; + and.getRestrictions().add( generatedRestriction ); + } else { + and = RestrictionHelper.and( Arrays.asList(new Restriction[]{ associated.getParentRestrictionDto(), generatedRestriction }) ); + } + return toSolr(and); + } + + private String not2solr(Not not) throws WikittyException { + if (not.getRestriction() == null) { + throw new WikittyException( "not.restriction" ); + } + return "( *:* - " + toSolr(not.getRestriction()) + " )"; + } + + private String and2solr(And and) throws WikittyException { + if (and.getRestrictions() == null) { + throw new WikittyException( "and.restrictions is null" ); + } + if (and.getRestrictions().size() < 2) { + throw new WikittyException( "AND is an operator that handle 2 operand at least"); + } + boolean first = true; + StringBuffer result = new StringBuffer(); + for (Restriction restriction : and.getRestrictions()) { + if (first) { + result.append("( ").append(toSolr(restriction)); + first = false; + } else { + result.append(" AND ").append(toSolr(restriction)); + } + } + return result.append(" )").toString(); + } + + private String or2solr(Or or) throws WikittyException { + if (or.getRestrictions() == null) { + throw new WikittyException("or.restrictions is null"); + } + if (or.getRestrictions().size() < 2) { + throw new WikittyException("OR is an operator that handle 2 operand at least"); + } + boolean first = true; + StringBuffer result = new StringBuffer(); + for (Restriction restriction : or.getRestrictions()) { + if (first) { + result.append("( "); + first = false; + } else { + result.append(" OR "); + } + result.append(toSolr(restriction)); + } + return result.append(" )").toString(); + } + + private String eq2solr(Equals eq) throws WikittyException { + return element2solr(eq.getElement()) + ":" + value2solr(eq.getValue()); + } + + private String neq2solr(NotEquals neq) + throws WikittyException { + return "( *:* - " + element2solr(neq.getElement()) + ":" + + value2solr(neq.getValue()) + " )"; + } + + private String less2solr(Less less) throws WikittyException { + return element2solr(less.getElement()) + ":{* TO " + + value2solr(less.getValue()) + "}"; + } + + private String lessEq2solr(LessOrEqual lessEq) + throws WikittyException { + return element2solr(lessEq.getElement()) + ":[* TO " + + value2solr(lessEq.getValue()) + "]"; + } + + private String great2solr(Greater great) + throws WikittyException { + return element2solr(great.getElement()) + ":{" + + value2solr(great.getValue()) + " TO *}"; + } + + private String greatEq2solr(GreaterOrEqual greatEq) + throws WikittyException { + return element2solr(greatEq.getElement()) + ":[" + + value2solr(greatEq.getValue()) + " TO *]"; + } + + private String between2solr(Between between) + throws WikittyException { + if (between.getElement() == null) { + throw new WikittyException("contains.element"); + } + if (between.getMin() == null) { + throw new WikittyException("contains.min"); + } + if (between.getMax() == null) { + throw new WikittyException("contains.max"); + } + return element2solr(between.getElement()) + ":[" + + value2solr(between.getMin()) + " TO " + + value2solr(between.getMax()) + "]"; + } + + private String contains2solr(Contains contains) + throws WikittyException { + if (contains.getElement() == null) { + throw new WikittyException("contains.element"); + } + if (contains.getValue() == null) { + throw new WikittyException("contains.values"); + } + if (contains.getValue().size() < 1) { + throw new WikittyException("CONTAINS is an operator that handle 1 operand at least"); + } + boolean first = true; + StringBuffer result = new StringBuffer(); + for (String value : contains.getValue()) { + if (first) { + result.append("( "); + first = false; + } else { + result.append(" AND "); + } + result.append("( "); + result.append(element2solr(contains.getElement())) + .append(":").append(value2solr(value)); + result.append(" OR "); + result.append(element2solr(contains.getElement())) + .append(":").append(value2solr(value)); + result.append(" OR "); + result.append(element2solr(contains.getElement())) + .append(":").append(value2solr(value)); + result.append(" )"); + } + return result.append(" )").toString(); + } + + private String start2solr(StartsWith start) + throws WikittyException { + return element2solr(start.getElement()) + ":" + + value2solr(start.getValue(), true, false); + } + + private String end2solr(EndsWith end) { + return element2solr(end.getElement()) + ":" + value2solr(end.getValue()); + } + + private String true2solr() { + return "( *:* )"; + } + + private String false2solr() { + return "( *:* - *:* )"; + } + + private String keyword2solr(Keyword keyword) { + return value2solr(keyword.getValue()); + } + + private String value2solr(String value) { + return value2solr(value, false, false); + } + + private String value2solr(String value, boolean starOnEnd, + boolean revert) { + String result; + if (value != null) { + if (value.equals("*")) { + return "*"; + } + + if (revert) { + result = Restriction2Solr.escapeValue(reverse(value)); + } else { + result = Restriction2Solr.escapeValue(value); + // result = value; + } + } else { + result = ""; + } + if (starOnEnd) { + result += "*"; + } + if (result.contains(" ")) { + result = "\"" + result + "\""; + } + return result; + } + + private String element2solr(Element element) throws WikittyException { + String result = element.getName(); + if(Element.ELT_EXTENSION.equals(result)) { + result = WikittySearchEngineSolr.SOLR_EXTENSIONS; + } else if(Element.ELT_ID.equals(result)) { + result = WikittySearchEngineSolr.SOLR_ID; + } else { + result = element.getName(); + } + result = fieldModifer.convertToSolr(result); + return result; + } + + private static String escapeValue(String value) { + final String LUCENE_REPLACE_PATTERN = "\\+" + "|-" + "|&&" + "|\\|\\|" + + "|!" + "|\\(|\\)" + "|\\[|\\]" + "|\\{|\\}" + "|\\^" + "|\"" + + "|\\~" + "|\\*" + "|\\?" + "|:" + "|\\\\"; + return value.replaceAll(LUCENE_REPLACE_PATTERN, "\\\\$0"); + } + + private String reverse(String source) { + int i, len = source.length(); + StringBuffer dest = new StringBuffer(len); + + for (i = (len - 1); i >= 0; i--) + dest.append(source.charAt(i)); + return dest.toString(); + } +} + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,70 @@ +package org.sharengo.wikitty.solr; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Query; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.LuceneQParserPlugin; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SolrQueryParser; + +/** + * Add allow leading wildcard + * setAllowLeadingWildcard(true); + * setLowercaseExpandedTerms(true); + * <br>Example: <code>{!wikitty q.op=AND df=text sort='price asc'}myfield:foo +bar -baz</code> + * More information @see LuceneQParserPlugin + */ +public class WikittyQueryParser extends LuceneQParserPlugin { + + public static String NAME = "wikitty"; + + @Override + public void init(NamedList args) { + } + + @Override + public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new SolrQParser(qstr, localParams, params, req); + } +} + +class SolrQParser extends QParser { + + static private Log log = LogFactory.getLog(SolrQParser.class); + String sortStr; + SolrQueryParser lparser; + + public SolrQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + @Override + public Query parse() throws ParseException { + + String defaultField = getParam(CommonParams.DF); + lparser = new SolrQueryParser(this, defaultField); + lparser.setAllowLeadingWildcard(true); + lparser.setLowercaseExpandedTerms(true); + + String opParam = getParam(QueryParsing.OP); + if (opParam != null) { + lparser.setDefaultOperator("AND".equals(opParam) ? QueryParser.Operator.AND : QueryParser.Operator.OR); + } + + String qstr = getString(); + log.debug("Query parse : " + qstr); + return lparser.parse(qstr); + } + + @Override + public String[] getDefaultHighlightFields() { + return new String[]{lparser.getField()}; + } +} Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,652 @@ +/* *##% + * Copyright (c) 2009 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.sharengo.wikitty.solr; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.core.CoreContainer; +import org.sharengo.wikitty.Criteria; +import org.sharengo.wikitty.FacetTopic; +import org.sharengo.wikitty.FieldType; +import org.sharengo.wikitty.FieldType.TYPE; +import org.sharengo.wikitty.PagedResult; +import org.sharengo.wikitty.TreeNode; +import org.sharengo.wikitty.UpdateResponse; +import org.sharengo.wikitty.Wikitty; +import org.sharengo.wikitty.WikittyException; +import org.sharengo.wikitty.WikittyExtension; +import org.sharengo.wikitty.WikittyExtensionStorage; +import org.sharengo.wikitty.WikittySearchEngin; +import org.sharengo.wikitty.WikittyTransaction; +import org.sharengo.wikitty.search.Search; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class WikittySearchEngineSolr implements WikittySearchEngin { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(WikittySearchEngineSolr.class); + + /** id field in solr */ + static final protected String SOLR_ID = "id"; + + /** extensions field name in solr */ + static final public String SOLR_EXTENSIONS = "extensions"; + + /** extension use to store field without extension to search on all extesnion */ + static final public String SOLR_ALL_EXTENSIONS = "all"; + + /** Precise the query parser to use, is allow leading wildcard */ + static final public String SOLR_QUERY_PARSER = "{!wikitty}"; + + // Use for indexation tree node + static final public String TREENODE_PREFIX = TreeNode.EXT_TREENODE + "."; + static final public String TREENODE_CHILD = TREENODE_PREFIX + "wikittyId"; + static final public String TREENODE_EMPTY = TREENODE_PREFIX + "empty"; + static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; + + static protected class CommandSolr implements Command { + /** if is store command toStore is true, if is delete toStore is false */ + boolean toStore; + String id; + Wikitty wikitty; + /** use to store command */ + public CommandSolr(Wikitty wikitty) { + this.wikitty = wikitty; + this.toStore = true; + } + /** use for deletion command */ + public CommandSolr(String id) { + this.id = id; + this.toStore = false; + } + } + + static protected class TypeFieldModifer implements Restriction2Solr.FieldModifier { + protected WikittyExtensionStorage extensionStorage; + public TypeFieldModifer(WikittyExtensionStorage extensionStorage) { + this.extensionStorage = extensionStorage; + } + + public String convertToSolr(String fqfieldname) { + String result = fqfieldname; + String[] searchField = fqfieldname.split("\\."); + + if (searchField.length == 2) { + String extName = searchField[0]; + String fieldName = searchField[1]; + + String version = extensionStorage.getLastVersion(extName); + if (version != null) { // not valid extension is version == null + WikittyExtension ext = extensionStorage.restore( + WikittyExtension.computeId(extName, version)); + FieldType type = ext.getFieldType(fieldName); + if (type != null) { // type can be null if extension version differ + result = WikittySearchEngineSolr.getSolrFieldName( + fqfieldname, type); + } else { + log.debug(String.format( + "Field '%s' could not be found in '%s'", + fqfieldname, extName)); + } + } else { + log.warn("No version for extension " + extName); + } + } + + if (searchField.length == 3) { + String extName = searchField[0]; + String fieldName = searchField[1]; + String fieldType = searchField[2]; + + if(Criteria.ALL_EXTENSIONS.equals(extName)) { + TYPE type = FieldType.TYPE.valueOf(fieldType); + String solrFieldName = SOLR_ALL_EXTENSIONS + "." + fieldName; + result = WikittySearchEngineSolr.getSolrFieldName(solrFieldName, type); + } else { + log.warn("All extension not found " + extName); + } + } + + return result; + } + + public String convertToField(String solrName) { + String fieldname = solrName.replaceAll("(_b$)|(_dt$)|(_t$)|(_l$)", ""); + return fieldname; + } + + } + + /** + * Store parent for each node and search in index if not known parent + */ + protected static class ParentMap extends HashMap<String, String> { + + protected WikittySearchEngineSolr searchEngineSolr; + + public ParentMap(WikittySearchEngineSolr searchEngineSolr) { + this.searchEngineSolr = searchEngineSolr; + } + + @Override + public String get(Object key) { + String id = String.valueOf(key); + String parent = super.get(id); + + // If not found in map, search in index + if(parent == null) { + SolrDocument doc = searchEngineSolr.findById(id); + if(doc == null) { + // is root + return null; + } + parent = (String) doc.getFieldValue(TreeNode.FQ_FIELD_PARENT); + put(id, parent); + } + return parent; + } + + /** + * Add parent map in transaction if is not exists and return it + */ + public static ParentMap getParentMap( + WikittySearchEngineSolr searchEngineSolr, + WikittyTransaction transaction) { + ParentMap value = (ParentMap) transaction.getTagValue(ParentMap.class); + if(value == null) { + value = new ParentMap(searchEngineSolr); + transaction.setTagValue(ParentMap.class, value); + } + return value; + } + } + + /** solr server */ + protected SolrServer solrServer; + /** Criteria transformer to solr */ + protected Restriction2Solr restriction2Solr; + /** Field modifier use to transform to solr format */ + protected TypeFieldModifer fieldModifier; + + public WikittySearchEngineSolr(WikittyExtensionStorage extensionStorage) { + super(); + try { + CoreContainer.Initializer initializer = new CoreContainer.Initializer(); + CoreContainer coreContainer = initializer.initialize(); + solrServer = new EmbeddedSolrServer(coreContainer, ""); + + fieldModifier = new TypeFieldModifer(extensionStorage); + restriction2Solr = new Restriction2Solr(fieldModifier); + + } catch (Exception eee) { + throw new WikittyException(eee); + } + } + + public void clear() { + try { + solrServer.deleteByQuery("*:*"); + solrServer.commit(true, true); + } catch (Exception eee) { + throw new WikittyException(eee); + } + } + + public List<Command> prepare(WikittyTransaction transaction, + Collection<Wikitty> wikitties) { + List<Command> result = new ArrayList<Command>(wikitties.size()); + for (Wikitty w : wikitties) { + + // Store known parent in transaction + if(w.hasExtension(TreeNode.EXT_TREENODE)) { + ParentMap parentMap = ParentMap.getParentMap(this, transaction); + String wikittyId = w.getId(); + String parentId = w.getFieldAsWikitty( + TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); + parentMap.put(wikittyId, parentId); + } + + result.add(new CommandSolr(w)); + } + return result; + } + + public UpdateResponse commit(WikittyTransaction transaction, + List<Command> wikittyIndexationCommandList) { + try { + for (Command c : wikittyIndexationCommandList) { + CommandSolr cs = (CommandSolr) c; + if (cs.toStore) { + SolrInputDocument doc = createIndexDocument(cs.wikitty); + solrServer.add(doc); + + Collection<SolrInputDocument> docs = createTreeNodeIndex(transaction, cs.wikitty); + if(!docs.isEmpty()) { + solrServer.add(docs); + } + + } else { + // Delete all reference on tree + solrServer.deleteByQuery(TREENODE_CHILD + ":" + cs.id); + // FIXME: jru 20091027 delete node in middle in tree + solrServer.deleteByQuery(TREENODE_PREFIX + cs.id + ":" + TREENODE_EMPTY); + + solrServer.deleteById(cs.id); + } + } + solrServer.commit(); + + // no specific result needed + UpdateResponse result = new UpdateResponse(); + return result; + } catch (Exception eee) { + throw new WikittyException(eee); + } + } + + public List<Command> delete(WikittyTransaction transaction, + List<String> idList) throws WikittyException { + List<Command> result = new ArrayList<Command>(idList.size()); + for (String id : idList) { + result.add(new CommandSolr(id)); + } + return result; + } + + public PagedResult<String> findAllByCriteria(Criteria criteria) { + try { + + // Create query with restriction + String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); + + // Add pagged + int firstIndex = criteria.getFirstIndex(); + int endIndex = criteria.getEndIndex(); + + query.setStart(firstIndex); + int nbRows; + if (endIndex == -1) { + // WARNING It is necessary to substract 'start' otherwise, there is a capacity overlow in solR + nbRows = Integer.MAX_VALUE - firstIndex; + } else { + nbRows = endIndex - firstIndex + 1; + } + query.setRows(nbRows); + + // Add sorting + List<String> sortAscending = criteria.getSortAscending(); + if(sortAscending != null) { + for (String sort : sortAscending) { + String tranform = fieldModifier.convertToSolr(sort); + query.setSortField(tranform, SolrQuery.ORDER.asc); + } + } + + List<String> sortDescending = criteria.getSortDescending(); + if(sortDescending != null) { + for (String sort : sortDescending) { + String tranform = fieldModifier.convertToSolr(sort); + query.setSortField(tranform, SolrQuery.ORDER.desc); + } + } + + // Add faceting + List<String> facetField = criteria.getFacetField(); + List<Criteria> facetCriteria = criteria.getFacetCriteria(); + + // use to map query string to criteria facet name + Map<String, String> facetQueryToName = new HashMap<String, String>(); + + if ((facetField != null && !facetField.isEmpty()) + || (facetCriteria != null && !facetCriteria.isEmpty())) { + query.setFacet(true); + query.setFacetMinCount(1); + // query.setFacetLimit(8); // no limit actualy + + // field facetisation + if (facetField != null) { + for (String fqfieldName : facetField) { + String tranform = fieldModifier.convertToSolr(fqfieldName); + query.addFacetField(tranform); + } + } + + // query facetisation + if (facetCriteria != null) { + for (Criteria facet : facetCriteria) { + String queryFacet = + restriction2Solr.toSolr(facet.getRestriction()); + facetQueryToName.put(queryFacet, facet.getName()); + query.addFacetQuery(queryFacet); + } + } + } + + QueryResponse resp = solrServer.query(query); + SolrDocumentList solrResults = resp.getResults(); + + Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); + if (facetField != null && !facetField.isEmpty()) { + for (FacetField facet : resp.getFacetFields()) { + String facetName = fieldModifier.convertToField(facet.getName()); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + if (facet.getValues() != null) { + for (FacetField.Count value : facet.getValues()) { + String topicName = value.getName(); + if(!topicName.endsWith(TREENODE_EMPTY)) { + int topicCount = (int) value.getCount(); + FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); + topics.add(topic); + } + } + } + facets.put(facetName, topics); + } + } + if (facetCriteria != null && !facetCriteria.isEmpty()) { + for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { + String facetName = facet.getKey(); + // don't use contains because, map can have key with null value + if (null != facetQueryToName.get(facetName)) { + facetName = facetQueryToName.get(facetName); + } + Integer count = facet.getValue(); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + FacetTopic topic = new FacetTopic(facetName, facetName, count); + topics.add(topic); + facets.put(facetName, topics); + } + } + + List<String> ids = new ArrayList<String>(solrResults.size()); + for (SolrDocument doc : solrResults) { + String id = (String) doc.getFieldValue(SOLR_ID); + ids.add(id); + } + + int numFound = (int)resp.getResults().getNumFound(); + PagedResult<String> result = new PagedResult<String>( + firstIndex, numFound, queryString, facets, ids); + + return result; + } catch (SolrServerException e) { + throw new WikittyException(e); + } + } + + public Map<String, Integer> findAllChildren(Wikitty w) { + String wikittyId = w.getId(); + + String parent = w.getFieldAsWikitty(TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); + if(parent == null) { + parent = TREENODE_ROOT; + } else { + parent = TREENODE_PREFIX + parent; + } + + // Find count with facet, if the node not contain recurcively content, + // the node not found with facet + Criteria criteria = Search.query().eq(parent, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(0) + .addFacetField(TREENODE_PREFIX + wikittyId); + PagedResult<String> search = findAllByCriteria(criteria); + + Map<String, Integer> counts = new HashMap<String, Integer>(); + List<FacetTopic> topics = search.getTopic(TREENODE_PREFIX + wikittyId); + if(topics != null) { + for (FacetTopic topic : topics) { + String topicName = topic.getTopicName(); + int topicCount = topic.getCount(); + counts.put(topicName, topicCount); + } + } + + log.debug("Facet result " + counts); + + // Find all children, add the other node not found with facet + criteria = Search.query().eq(TreeNode.FQ_FIELD_PARENT, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); + search = findAllByCriteria(criteria); + + List<String> children = search.getAll(); + for (String child : children) { + if(!counts.containsKey(child)) { + counts.put(child, 0); + } + } + + return counts; + } + + /** + * Create all index document to used to modify indexation. + * this method don't modify index + * @param wikitties all wikitties object to index + * @return list of SolrInputDocument used to modify index + */ + protected SolrInputDocument createIndexDocument(Wikitty w) { + log.debug("index wikitty " + w.getId()); + + SolrInputDocument doc = new SolrInputDocument(); + doc.addField(SOLR_ID, w.getId()); + + for (String name : w.getExtensionNames()) { + doc.addField(SOLR_EXTENSIONS, name); + } + + for (String fqfieldName : w.fieldNames()) { + FieldType type = w.getFieldType(fqfieldName); + // do not index wikitty field reference +// if (type.getType() == FieldType.TYPE.WIKITTY) { +// continue; +// } + String solrFqFieldName = getSolrFieldName(fqfieldName, type ); + String[] extAndName = solrFqFieldName.split("\\."); + String solrFieldName = SOLR_ALL_EXTENSIONS + "." + extAndName[1]; + if (type.isCollection()) { + Collection collection = (Collection) w.getFqField(fqfieldName); + if (collection != null) { + for (Object objectValue : collection) { + if (objectValue != null) { + // String value = WikittyUtil.toString(type, objectValue); + Object value = objectValue; + doc.addField(solrFqFieldName, value); + doc.addField(solrFieldName, value); + } + } + } + } else { + Object objectValue = w.getFqField(fqfieldName); + if (objectValue != null) { + // String value = WikittyUtil.toString(type, objectValue); + Object value = objectValue; + log.debug("index field " + solrFqFieldName + " with value '" + value + "'"); + doc.addField(solrFqFieldName, value); + doc.addField(solrFieldName, value); + } + } + } + + return doc; + } + + /** + * Create a doc for each child for a tree node, his contains the path for + * the child. Update old doc when the node is moved and child is deleted. + */ + protected Collection<SolrInputDocument> createTreeNodeIndex( + WikittyTransaction transaction, Wikitty w) + throws IOException, SolrServerException { + + Collection<SolrInputDocument> result = new ArrayList<SolrInputDocument>(); + if(w.hasExtension(TreeNode.EXT_TREENODE)) { + String treeNodeId = w.getId(); + ParentMap parentMap = (ParentMap) transaction.getTagValue(ParentMap.class); + + // Init new children with all children in node + List<String> children = w.getFieldAsList(TreeNode.EXT_TREENODE, + TreeNode.FIELD_CHILDREN, String.class); + List<String> newChildren = new ArrayList<String>(); + if(children != null) { + newChildren.addAll(children); + } + + // Find old document to do diff + SolrDocument treeNodeDoc = findById(treeNodeId); + if(treeNodeDoc != null) { + Collection oldChildren = treeNodeDoc. + getFieldValues(TreeNode.FQ_FIELD_CHILDREN + "_t"); + + // Remove old children not in current node + List<String> removeChildren = new ArrayList<String>(); + if(oldChildren != null) { + removeChildren.addAll(oldChildren); + removeChildren.removeAll(children); + for (String removeChild : removeChildren) { + solrServer.deleteByQuery(TREENODE_CHILD + ":" + removeChild); + } + + // Remove old child in new children + newChildren.removeAll(oldChildren); + } + + // Update path other node refer to current node + // FIXME jru 20091926 test change path + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PREFIX + treeNodeId + ":*"); + QueryResponse response = solrServer.query(query); + SolrDocumentList updateDocs = response.getResults(); + for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { + SolrDocument solrDocument = iterator.next(); + String child = (String) solrDocument.getFieldValue(TreeNode.EXT_TREENODE + ".wikittyId"); + // Child is not deleted + if(!removeChildren.contains(child)) { + SolrInputDocument doc = createTreeNodeDocument(treeNodeId, child, parentMap); + + // Copy other field after node + String parent = TREENODE_PREFIX + treeNodeId; + String childParent = (String) solrDocument.getFieldValue(parent); + while (childParent != null) { + doc.setField(parent, childParent); + parent = TREENODE_PREFIX + childParent; + childParent = (String) solrDocument.getFieldValue(parent); + } + result.add(doc); + } + } + } + + // New children + for (String newChild : newChildren) { + SolrInputDocument doc = createTreeNodeDocument(treeNodeId, newChild, parentMap); + result.add(doc); + } + + } + return result; + } + + /** + * Create a doc between node and child + */ + public SolrInputDocument createTreeNodeDocument(String treeNodeId, String child, ParentMap parentMap) { + SolrInputDocument doc = new SolrInputDocument(); + doc.setField(SOLR_ID, TreeNode.EXT_TREENODE + "-" + treeNodeId + "-" + child); + doc.setField(TREENODE_CHILD, child); + doc.setField(TREENODE_PREFIX + treeNodeId, TREENODE_EMPTY); + + // Add path + String childParent = treeNodeId; + String parent = parentMap.get(childParent); + while (parent != null) { + doc.setField(TREENODE_PREFIX + parent, childParent); + childParent = parent; + parent = parentMap.get(childParent); + } + doc.setField(TREENODE_ROOT, childParent); + + return doc; + } + + /** + * Find solr document by id + */ + protected SolrDocument findById(String id) { + SolrQuery query = new SolrQuery("id:" + id); + QueryResponse response; + try { + response = solrServer.query(query); + } catch (SolrServerException eee) { + throw new WikittyException(eee); + } + + SolrDocumentList results = response.getResults(); + long numFound = results.getNumFound(); + if(numFound == 1) { + return results.get(0); + } + + return null; + } + + static public String getSolrFieldName(String fqfieldName, FieldType fieldType) { + TYPE type = fieldType.getType(); + String result = getSolrFieldName(fqfieldName, type); + return result; + } + + public static String getSolrFieldName(String fqfieldName, TYPE type) { + switch (type) { + case BOOLEAN: + return fqfieldName + "_b"; + case DATE: + return fqfieldName + "_dt"; + case STRING: + return fqfieldName + "_t"; + case NUMERIC: + return fqfieldName + "_d"; + default: + return fqfieldName; + } + } +} Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,308 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + This is the Solr schema file. This file should be named "schema.xml" and + should be in the conf directory under the solr home + (i.e. ./solr/conf/schema.xml by default) + or located where the classloader for the Solr webapp can find it. + + This example schema is the recommended starting point for users. + It should be kept correct and concise, usable out-of-the-box. + + For more information, on how to customize this file, please see + http://wiki.apache.org/solr/SchemaXml +--> + +<schema name="wikitty" version="1.1"> + <!-- attribute "name" is the name of this schema and is only used for display purposes. + Applications should change this to reflect the nature of the search collection. + version="1.1" is Solr's version number for the schema syntax and semantics. It should + not normally be changed by applications. + 1.0: multiValued attribute did not exist, all fields are multiValued by nature + 1.1: multiValued attribute introduced, false by default --> + + <types> + <!-- field type definitions. The "name" attribute is + just a label to be used by field definitions. The "class" + attribute and any other attributes determine the real + behavior of the fieldType. + Class names starting with "solr" refer to java classes in the + org.apache.solr.analysis package. + --> + + <!-- The StrField type is not analyzed, but indexed/stored verbatim. + - StrField and TextField support an optional compressThreshold which + limits compression (if enabled in the derived fields) to values which + exceed a certain size (in characters). + --> + <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/> + + <!-- boolean type: "true" or "false" --> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/> + + <!-- The optional sortMissingLast and sortMissingFirst attributes are + currently supported on types that are sorted internally as strings. + - If sortMissingLast="true", then a sort on this field will cause documents + without the field to come after documents with the field, + regardless of the requested sort order (asc or desc). + - If sortMissingFirst="true", then a sort on this field will cause documents + without the field to come before documents with the field, + regardless of the requested sort order. + - If sortMissingLast="false" and sortMissingFirst="false" (the default), + then default lucene sorting will be used which places docs without the + field first in an ascending sort and last in a descending sort. + --> + + + <!-- numeric field types that store and index the text + value verbatim (and hence don't support range queries, since the + lexicographic ordering isn't equal to the numeric ordering) --> + <fieldType name="integer" class="solr.IntField" omitNorms="true"/> + <fieldType name="long" class="solr.LongField" omitNorms="true"/> + <fieldType name="float" class="solr.FloatField" omitNorms="true"/> + <fieldType name="double" class="solr.DoubleField" omitNorms="true"/> + + + <!-- Numeric field types that manipulate the value into + a string value that isn't human-readable in its internal form, + but with a lexicographic ordering the same as the numeric ordering, + so that range queries work correctly. --> + <fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/> + + + <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and + is a more restricted form of the canonical representation of dateTime + http://www.w3.org/TR/xmlschema-2/#dateTime + The trailing "Z" designates UTC time and is mandatory. + Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z + All other components are mandatory. + + Expressions can also be used to denote calculations that should be + performed relative to "NOW" to determine the value, ie... + + NOW/HOUR + ... Round to the start of the current hour + NOW-1DAY + ... Exactly 1 day prior to now + NOW/DAY+6MONTHS+3DAYS + ... 6 months and 3 days in the future from the start of + the current day + + Consult the DateField javadocs for more information. + --> + <fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/> + + + <!-- The "RandomSortField" is not used to store or search any + data. You can declare fields of this type it in your schema + to generate psuedo-random orderings of your docs for sorting + purposes. The ordering is generated based on the field name + and the version of the index, As long as the index version + remains unchanged, and the same field name is reused, + the ordering of the docs will be consistent. + If you want differend psuedo-random orderings of documents, + for the same version of the index, use a dynamicField and + change the name + --> + <fieldType name="random" class="solr.RandomSortField" indexed="true" /> + + <!-- solr.TextField allows the specification of custom text analyzers + specified as a tokenizer and a list of token filters. Different + analyzers may be specified for indexing and querying. + + The optional positionIncrementGap puts space between multiple fields of + this type on the same document, with the purpose of preventing false phrase + matching across fields. + + For more info on customizing your analyzer chain, please see + http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters + --> + + <!-- One can also specify an existing Analyzer class that has a + default constructor via the class attribute on the analyzer element + <fieldType name="text_greek" class="solr.TextField"> + <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/> + </fieldType> + --> + + <!-- A text field that only splits on whitespace for exact matching of words --> + <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + </analyzer> + </fieldType> + + <!-- A text field that uses WordDelimiterFilter to enable splitting and matching of + words on case-change, alpha numeric boundaries, and non-alphanumeric chars, + so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi". + Synonyms and stopwords are customized by external files, and stemming is enabled. + Duplicate tokens at the same position (which may result from Stemmed Synonyms or + WordDelim parts) are removed. + --> + <fieldType name="text" class="solr.TextField" positionIncrementGap="100"> + <analyzer type="index"> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <!-- in this example, we will only use synonyms at query time + <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/> + --> + <!-- Case insensitive stop word removal. + enablePositionIncrements=true ensures that a 'gap' is left to + allow for accurate phrase queries. + --> + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="stopwords.txt" + enablePositionIncrements="true" + /> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + </fieldType> + + <fieldType name="text_core" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory" /> + <filter class="solr.LowerCaseFilterFactory" /> + <!-- filter class="solr.WordDelimiterFilterFactory" + generateWordParts="1" generateNumberParts="1" + catenateWords="1" catenateNumbers="1" + catenateAll="0" splitOnCaseChange="1" / --> + </analyzer> + </fieldType> + + + <!-- Less flexible matching, but less false matches. Probably not ideal for product names, + but may be good for SKUs. Can insert dashes in the wrong place and still match. --> + <fieldType name="textTight" class="solr.TextField" positionIncrementGap="100" > + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + </fieldType> + + <!-- + Setup simple analysis for spell checking + --> + <fieldType name="textSpell" class="solr.TextField" positionIncrementGap="100" > + <analyzer> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + </fieldType> + + <!-- This is an example of using the KeywordTokenizer along + With various TokenFilterFactories to produce a sortable field + that does not include some properties of the source text + --> + <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true"> + <analyzer> + <!-- KeywordTokenizer does no actual tokenizing, so the entire + input string is preserved as a single token + --> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <!-- The LowerCase TokenFilter does what you expect, which can be + when you want your sorting to be case insensitive + --> + <filter class="solr.LowerCaseFilterFactory" /> + <!-- The TrimFilter removes any leading or trailing whitespace --> + <filter class="solr.TrimFilterFactory" /> + <!-- The PatternReplaceFilter gives you the flexibility to use + Java Regular expression to replace any sequence of characters + matching a pattern with an arbitrary replacement string, + which may include back refrences to portions of the orriginal + string matched by the pattern. + + See the Java Regular Expression documentation for more + infomation on pattern and replacement string syntax. + + http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/package-summary.html + --> + <filter class="solr.PatternReplaceFilterFactory" + pattern="([^a-z])" replacement="" replace="all" + /> + </analyzer> + </fieldType> + + <!-- since fields of this type are by default not stored or indexed, any data added to + them will be ignored outright + --> + <fieldtype name="ignored" stored="false" indexed="false" class="solr.StrField" /> + + </types> + + + <fields> + <field name="id" type="string" indexed="true" stored="true" required="true" /> + <field name="extensions" type="string" indexed="true" stored="false" multiValued="true"/> + + <!-- catchall field, containing all other searchable text fields (implemented + via copyField further on in this schema --> + <field name="text" type="text" indexed="true" stored="false" multiValued="true"/> + + <dynamicField name="Label.label*" type="text" indexed="true" stored="true" multiValued="true"/> + + <dynamicField name="*_i" type="sint" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_s" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_l" type="slong" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_t" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_b" type="boolean" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_f" type="sfloat" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_d" type="sdouble" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_dt" type="date" indexed="true" stored="true" multiValued="true"/> + + <!-- all wikitty field --> + <dynamicField name="*" type="string" indexed="true" stored="true" multiValued="true"/> + + <!-- copy fields for optimisation --> + <dynamicField name="_*" type="string" indexed="true" stored="true" multiValued="true"/> + + </fields> + + <!-- Field to use to determine and enforce document uniqueness. + Unless this field is marked with required="false", it will be a required field + --> + <uniqueKey>id</uniqueKey> + + <!-- field for the QueryParser to use when an explicit fieldname is absent --> + <defaultSearchField>id</defaultSearchField> + + <!-- SolrQueryParser configuration: defaultOperator="AND|OR" --> + <solrQueryParser defaultOperator="OR"/> + +</schema> Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,700 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<config> + <!-- Set this to 'false' if you want solr to continue working after it has + encountered an severe configuration error. In a production environment, + you may want solr to keep working even if one handler is mis-configured. + + You may also set this to false using by setting the system property: + -Dsolr.abortOnConfigurationError=false + --> + <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError> + + <!-- Used to specify an alternate directory to hold all index data + other than the default ./data under the Solr home. + If replication is in use, this should match the replication configuration. --> + <dataDir>${solr.data.dir:./solr/data}</dataDir> + + + <indexDefaults> + <!-- Values here affect all index writers and act as a default unless overridden. --> + <useCompoundFile>false</useCompoundFile> + + <mergeFactor>10</mergeFactor> + <!-- + If both ramBufferSizeMB and maxBufferedDocs is set, then Lucene will flush based on whichever limit is hit first. + + --> + <!--<maxBufferedDocs>1000</maxBufferedDocs>--> + <!-- Tell Lucene when to flush documents to disk. + Giving Lucene more memory for indexing means faster indexing at the cost of more RAM + + If both ramBufferSizeMB and maxBufferedDocs is set, then Lucene will flush based on whichever limit is hit first. + + --> + <ramBufferSizeMB>32</ramBufferSizeMB> + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + <writeLockTimeout>1000</writeLockTimeout> + <commitLockTimeout>10000</commitLockTimeout> + + <!-- + Expert: Turn on Lucene's auto commit capability. + This causes intermediate segment flushes to write a new lucene + index descriptor, enabling it to be opened by an external + IndexReader. + NOTE: Despite the name, this value does not have any relation to Solr's autoCommit functionality + --> + <!--<luceneAutoCommit>false</luceneAutoCommit>--> + <!-- + Expert: + The Merge Policy in Lucene controls how merging is handled by Lucene. The default in 2.3 is the LogByteSizeMergePolicy, previous + versions used LogDocMergePolicy. + + LogByteSizeMergePolicy chooses segments to merge based on their size. The Lucene 2.2 default, LogDocMergePolicy chose when + to merge based on number of documents + + Other implementations of MergePolicy must have a no-argument constructor + --> + <!--<mergePolicy>org.apache.lucene.index.LogByteSizeMergePolicy</mergePolicy>--> + + <!-- + Expert: + The Merge Scheduler in Lucene controls how merges are performed. The ConcurrentMergeScheduler (Lucene 2.3 default) + can perform merges in the background using separate threads. The SerialMergeScheduler (Lucene 2.2 default) does not. + --> + <!--<mergeScheduler>org.apache.lucene.index.ConcurrentMergeScheduler</mergeScheduler>--> + + <!-- + This option specifies which Lucene LockFactory implementation to use. + + single = SingleInstanceLockFactory - suggested for a read-only index + or when there is no possibility of another process trying + to modify the index. + native = NativeFSLockFactory + simple = SimpleFSLockFactory + + (For backwards compatibility with Solr 1.2, 'simple' is the default + if not specified.) + --> + <lockType>single</lockType> + </indexDefaults> + + <mainIndex> + <!-- options specific to the main on-disk lucene index --> + <useCompoundFile>false</useCompoundFile> + <ramBufferSizeMB>32</ramBufferSizeMB> + <mergeFactor>10</mergeFactor> + <!-- Deprecated --> + <!--<maxBufferedDocs>1000</maxBufferedDocs>--> + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + + <!-- If true, unlock any held write or commit locks on startup. + This defeats the locking mechanism that allows multiple + processes to safely access a lucene index, and should be + used with care. + This is not needed if lock type is 'none' or 'single' + --> + <unlockOnStartup>false</unlockOnStartup> + </mainIndex> + + <!-- Enables JMX if and only if an existing MBeanServer is found, use + this if you want to configure JMX through JVM parameters. Remove + this to disable exposing Solr configuration and statistics to JMX. + + If you want to connect to a particular server, specify the agentId + e.g. <jmx agentId="myAgent" /> + + If you want to start a new MBeanServer, specify the serviceUrl + e.g <jmx serviceurl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr" /> + + For more details see http://wiki.apache.org/solr/SolrJmx + --> + <jmx /> + + <!-- the default high-performance update handler --> + <updateHandler class="solr.DirectUpdateHandler2"> + + <!-- A prefix of "solr." for class names is an alias that + causes solr to search appropriate packages, including + org.apache.solr.(search|update|request|core|analysis) + --> + + <!-- Perform a <commit/> automatically under certain conditions: + maxDocs - number of updates since last commit is greater than this + maxTime - oldest uncommited update (in ms) is this long ago + <autoCommit> + <maxDocs>10000</maxDocs> + <maxTime>1000</maxTime> + </autoCommit> + --> + + <!-- The RunExecutableListener executes an external command. + exe - the name of the executable to run + dir - dir to use as the current working directory. default="." + wait - the calling thread waits until the executable returns. default="true" + args - the arguments to pass to the program. default=nothing + env - environment variables to set. default=nothing + --> + <!-- A postCommit event is fired after every commit or optimize command + <listener event="postCommit" class="solr.RunExecutableListener"> + <str name="exe">solr/bin/snapshooter</str> + <str name="dir">.</str> + <bool name="wait">true</bool> + <arr name="args"> <str>arg1</str> <str>arg2</str> </arr> + <arr name="env"> <str>MYVAR=val1</str> </arr> + </listener> + --> + <!-- A postOptimize event is fired only after every optimize command, useful + in conjunction with index distribution to only distribute optimized indicies + <listener event="postOptimize" class="solr.RunExecutableListener"> + <str name="exe">snapshooter</str> + <str name="dir">solr/bin</str> + <bool name="wait">true</bool> + </listener> + --> + + </updateHandler> + + + <query> + <!-- Maximum number of clauses in a boolean query... can affect + range or prefix queries that expand to big boolean + queries. An exception is thrown if exceeded. --> + <maxBooleanClauses>1024</maxBooleanClauses> + + + <!-- Cache used by SolrIndexSearcher for filters (DocSets), + unordered sets of *all* documents that match a query. + When a new searcher is opened, its caches may be prepopulated + or "autowarmed" using data from caches in the old searcher. + autowarmCount is the number of items to prepopulate. For LRUCache, + the autowarmed items will be the most recently accessed items. + Parameters: + class - the SolrCache implementation (currently only LRUCache) + size - the maximum number of entries in the cache + initialSize - the initial capacity (number of entries) of + the cache. (seel java.util.HashMap) + autowarmCount - the number of entries to prepopulate from + and old cache. + --> + <filterCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="128"/> + + <!-- queryResultCache caches results of searches - ordered lists of + document ids (DocList) based on a query, a sort, and the range + of documents requested. --> + <queryResultCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="32"/> + + <!-- documentCache caches Lucene Document objects (the stored fields for each document). + Since Lucene internal document ids are transient, this cache will not be autowarmed. --> + <documentCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + + <!-- If true, stored fields that are not requested will be loaded lazily. + + This can result in a significant speed improvement if the usual case is to + not load all stored fields, especially if the skipped fields are large compressed + text fields. + --> + <enableLazyFieldLoading>true</enableLazyFieldLoading> + + <!-- Example of a generic cache. These caches may be accessed by name + through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert(). + The purpose is to enable easy caching of user/application level data. + The regenerator argument should be specified as an implementation + of solr.search.CacheRegenerator if autowarming is desired. --> + <!-- + <cache name="myUserCache" + class="solr.LRUCache" + size="4096" + initialSize="1024" + autowarmCount="1024" + regenerator="org.mycompany.mypackage.MyRegenerator" + /> + --> + + <!-- An optimization that attempts to use a filter to satisfy a search. + If the requested sort does not include score, then the filterCache + will be checked for a filter matching the query. If found, the filter + will be used as the source of document ids, and then the sort will be + applied to that. + <useFilterForSortedQuery>true</useFilterForSortedQuery> + --> + + <!-- An optimization for use with the queryResultCache. When a search + is requested, a superset of the requested number of document ids + are collected. For example, if a search for a particular query + requests matching documents 10 through 19, and queryWindowSize is 50, + then documents 0 through 49 will be collected and cached. Any further + requests in that range can be satisfied via the cache. --> + <queryResultWindowSize>50</queryResultWindowSize> + + <!-- Maximum number of documents to cache for any entry in the + queryResultCache. --> + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + + <!-- This entry enables an int hash representation for filters (DocSets) + when the number of items in the set is less than maxSize. For smaller + sets, this representation is more memory efficient, more efficient to + iterate over, and faster to take intersections. --> + <HashDocSet maxSize="3000" loadFactor="0.75"/> + + <!-- a newSearcher event is fired whenever a new searcher is being prepared + and there is a current searcher handling requests (aka registered). --> + <!-- QuerySenderListener takes an array of NamedList and executes a + local query request for each NamedList in sequence. --> + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + + <!-- a firstSearcher event is fired whenever a new searcher is being + prepared but there is no current registered searcher to handle + requests or to gain autowarming data from. --> + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static firstSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + + <!-- If a search request comes in and there is no current registered searcher, + then immediately register the still warming searcher and use it. If + "false" then all requests will block until the first searcher is done + warming. --> + <useColdSearcher>false</useColdSearcher> + + <!-- Maximum number of searchers that may be warming in the background + concurrently. An error is returned if this limit is exceeded. Recommend + 1-2 for read-only slaves, higher for masters w/o cache warming. --> + <maxWarmingSearchers>2</maxWarmingSearchers> + + </query> + + <!-- + Let the dispatch filter handler /select?qt=XXX + handleSelect=true will use consistent error handling for /select and /update + handleSelect=false will use solr1.1 style error formatting + --> + <requestDispatcher handleSelect="true" > + <!--Make sure your system has some authentication before enabling remote streaming! --> + <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" /> + + <!-- Set HTTP caching related parameters (for proxy caches and clients). + + To get the behaviour of Solr 1.2 (ie: no caching related headers) + use the never304="true" option and do not specify a value for + <cacheControl> + --> + <!-- <httpCaching never304="true"> --> + <httpCaching lastModifiedFrom="openTime" + etagSeed="Solr"> + <!-- lastModFrom="openTime" is the default, the Last-Modified value + (and validation against If-Modified-Since requests) will all be + relative to when the current Searcher was opened. + You can change it to lastModFrom="dirLastMod" if you want the + value to exactly corrispond to when the physical index was last + modified. + + etagSeed="..." is an option you can change to force the ETag + header (and validation against If-None-Match requests) to be + differnet even if the index has not changed (ie: when making + significant changes to your config file) + + lastModifiedFrom and etagSeed are both ignored if you use the + never304="true" option. + --> + <!-- If you include a <cacheControl> directive, it will be used to + generate a Cache-Control header, as well as an Expires header + if the value contains "max-age=" + + By default, no Cache-Control header is generated. + + You can use the <cacheControl> option even if you have set + never304="true" + --> + <!-- <cacheControl>max-age=30, public</cacheControl> --> + </httpCaching> + </requestDispatcher> + + + <!-- requestHandler plugins... incoming queries will be dispatched to the + correct handler based on the path or the qt (query type) param. + Names starting with a '/' are accessed with the a path equal to the + registered name. Names without a leading '/' are accessed with: + http://host/app/select?qt=name + If no qt is defined, the requestHandler that declares default="true" + will be used. + --> + <requestHandler name="standard" class="solr.SearchHandler" default="true"> + <!-- default values for query parameters --> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <!-- + <int name="rows">10</int> + <str name="fl">*</str> + <str name="version">2.1</str> + --> + </lst> + </requestHandler> + + + <!-- DisMaxRequestHandler allows easy searching across multiple fields + for simple user-entered phrases. It's implementation is now + just the standard SearchHandler with a default query type + of "dismax". + see http://wiki.apache.org/solr/DisMaxRequestHandler + --> + <requestHandler name="dismax" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <float name="tie">0.01</float> + <str name="qf"> + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + </str> + <str name="pf"> + text^0.2 features^1.1 name^1.5 manu^1.4 manu_exact^1.9 + </str> + <str name="bf"> + ord(popularity)^0.5 recip(rord(price),1,1000,1000)^0.3 + </str> + <str name="fl"> + id,name,price,score + </str> + <str name="mm"> + 2<-1 5<-2 6<90% + </str> + <int name="ps">100</int> + <str name="q.alt">*:*</str> + <!-- example highlighter config, enable per-query with hl=true --> + <str name="hl.fl">text features name</str> + <!-- for this field, we want no fragmenting, just highlighting --> + <str name="f.name.hl.fragsize">0</str> + <!-- instructs Solr to return the field itself if no query terms are + found --> + <str name="f.name.hl.alternateField">name</str> + <str name="f.text.hl.fragmenter">regex</str> <!-- defined below --> + </lst> + </requestHandler> + + <!-- Note how you can register the same handler multiple times with + different names (and different init parameters) + --> + <requestHandler name="partitioned" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <str name="qf">text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0</str> + <str name="mm">2<-1 5<-2 6<90%</str> + <!-- This is an example of using Date Math to specify a constantly + moving date range in a config... + --> + <str name="bq">incubationdate_dt:[* TO NOW/DAY-1MONTH]^2.2</str> + </lst> + <!-- In addition to defaults, "appends" params can be specified + to identify values which should be appended to the list of + multi-val params from the query (or the existing "defaults"). + + In this example, the param "fq=instock:true" will be appended to + any query time fq params the user may specify, as a mechanism for + partitioning the index, independent of any user selected filtering + that may also be desired (perhaps as a result of faceted searching). + + NOTE: there is *absolutely* nothing a client can do to prevent these + "appends" values from being used, so don't use this mechanism + unless you are sure you always want it. + --> + <lst name="appends"> + <str name="fq">inStock:true</str> + </lst> + <!-- "invariants" are a way of letting the Solr maintainer lock down + the options available to Solr clients. Any params values + specified here are used regardless of what values may be specified + in either the query, the "defaults", or the "appends" params. + + In this example, the facet.field and facet.query params are fixed, + limiting the facets clients can use. Faceting is not turned on by + default - but if the client does specify facet=true in the request, + these are the only facets they will be able to see counts for; + regardless of what other facet.field or facet.query params they + may specify. + + NOTE: there is *absolutely* nothing a client can do to prevent these + "invariants" values from being used, so don't use this mechanism + unless you are sure you always want it. + --> + <lst name="invariants"> + <str name="facet.field">cat</str> + <str name="facet.field">manu_exact</str> + <str name="facet.query">price:[* TO 500]</str> + <str name="facet.query">price:[500 TO *]</str> + </lst> + </requestHandler> + + + <!-- + Search components are registered to SolrCore and used by Search Handlers + + By default, the following components are avaliable: + + <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" /> + <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" /> + <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" /> + <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" /> + <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" /> + + Default configuration in a requestHandler would look like: + <arr name="components"> + <str>query</str> + <str>facet</str> + <str>mlt</str> + <str>highlight</str> + <str>debug</str> + </arr> + + If you register a searchComponent to one of the standard names, that will be used instead. + To insert handlers before or after the 'standard' components, use: + + <arr name="first-components"> + <str>myFirstComponentName</str> + </arr> + + <arr name="last-components"> + <str>myLastComponentName</str> + </arr> + --> + + <!-- The spell check component can return a list of alternative spelling + suggestions. --> + <searchComponent name="spellcheck" class="solr.SpellCheckComponent"> + + <str name="queryAnalyzerFieldType">textSpell</str> + + <lst name="spellchecker"> + <str name="name">default</str> + <str name="field">spell</str> + <str name="spellcheckIndexDir">./spellchecker1</str> + + </lst> + <lst name="spellchecker"> + <str name="name">jarowinkler</str> + <str name="field">spell</str> + <!-- Use a different Distance Measure --> + <str name="distanceMeasure">org.apache.lucene.search.spell.JaroWinklerDistance</str> + <str name="spellcheckIndexDir">./spellchecker2</str> + + </lst> + + <lst name="spellchecker"> + <str name="classname">solr.FileBasedSpellChecker</str> + <str name="name">file</str> + <str name="sourceLocation">spellings.txt</str> + <str name="characterEncoding">UTF-8</str> + <str name="spellcheckIndexDir">./spellcheckerFile</str> + </lst> + </searchComponent> + + <!-- a request handler utilizing the spellcheck component --> + <requestHandler name="/spellCheckCompRH" class="solr.SearchHandler"> + <lst name="defaults"> + <!-- omp = Only More Popular --> + <str name="spellcheck.onlyMorePopular">false</str> + <!-- exr = Extended Results --> + <str name="spellcheck.extendedResults">false</str> + <!-- The number of suggestions to return --> + <str name="spellcheck.count">1</str> + </lst> + <arr name="last-components"> + <str>spellcheck</str> + </arr> + </requestHandler> + + <!-- a search component that enables you to configure the top results for + a given query regardless of the normal lucene scoring.--> + +<!-- poussin 20090902 remove elevate this file is empty, what need ? + <searchComponent name="elevator" class="solr.QueryElevationComponent" > + <str name="queryFieldType">string</str> + <str name="config-file">elevate.xml</str> + </searchComponent> + --> + <!-- a request handler utilizing the elevator component --> +<!-- + <requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + </lst> + <arr name="last-components"> + <str>elevator</str> + </arr> + </requestHandler> + --> + + <!-- Update request handler. + + Note: Since solr1.1 requestHandlers requires a valid content type header if posted in + the body. For example, curl now requires: -H 'Content-type:text/xml; charset=utf-8' + The response format differs from solr1.1 formatting and returns a standard error code. + + To enable solr1.1 behavior, remove the /update handler or change its path + --> + <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" /> + + <!-- + Analysis request handler. Since Solr 1.3. Use to returnhow a document is analyzed. Useful + for debugging and as a token server for other types of applications + --> + <requestHandler name="/analysis" class="solr.AnalysisRequestHandler" /> + + + <!-- CSV update handler, loaded on demand --> + <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" /> + + + <!-- + Admin Handlers - This will register all the standard admin RequestHandlers. Adding + this single handler is equivolent to registering: + + <requestHandler name="/admin/luke" class="org.apache.solr.handler.admin.LukeRequestHandler" /> + <requestHandler name="/admin/system" class="org.apache.solr.handler.admin.SystemInfoHandler" /> + <requestHandler name="/admin/plugins" class="org.apache.solr.handler.admin.PluginInfoHandler" /> + <requestHandler name="/admin/threads" class="org.apache.solr.handler.admin.ThreadDumpHandler" /> + <requestHandler name="/admin/properties" class="org.apache.solr.handler.admin.PropertiesRequestHandler" /> + <requestHandler name="/admin/file" class="org.apache.solr.handler.admin.ShowFileRequestHandler" > + + If you wish to hide files under ${solr.home}/conf, explicitly register the ShowFileRequestHandler using: + <requestHandler name="/admin/file" class="org.apache.solr.handler.admin.ShowFileRequestHandler" > + <lst name="invariants"> + <str name="hidden">synonyms.txt</str> + <str name="hidden">anotherfile.txt</str> + </lst> + </requestHandler> + --> + <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" /> + + <!-- ping/healthcheck --> + <requestHandler name="/admin/ping" class="PingRequestHandler"> + <lst name="defaults"> + <str name="qt">standard</str> + <str name="q">solrpingquery</str> + <str name="echoParams">all</str> + </lst> + </requestHandler> + + <!-- Echo the request contents back to the client --> + <requestHandler name="/debug/dump" class="solr.DumpRequestHandler" > + <lst name="defaults"> + <str name="echoParams">explicit</str> <!-- for all params (including the default etc) use: 'all' --> + <str name="echoHandler">true</str> + </lst> + </requestHandler> + + <highlighting> + <!-- Configure the standard fragmenter --> + <!-- This could most likely be commented out in the "default" case --> + <fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true"> + <lst name="defaults"> + <int name="hl.fragsize">100</int> + </lst> + </fragmenter> + + <!-- A regular-expression-based fragmenter (f.i., for sentence extraction) --> + <fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter"> + <lst name="defaults"> + <!-- slightly smaller fragsizes work better because of slop --> + <int name="hl.fragsize">70</int> + <!-- allow 50% slop on fragment sizes --> + <float name="hl.regex.slop">0.5</float> + <!-- a basic sentence pattern --> + <str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str> + </lst> + </fragmenter> + + <!-- Configure the standard formatter --> + <formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true"> + <lst name="defaults"> + <str name="hl.simple.pre"><![CDATA[<em>]]></str> + <str name="hl.simple.post"><![CDATA[</em>]]></str> + </lst> + </formatter> + </highlighting> + + + <!-- queryResponseWriter plugins... query responses will be written using the + writer specified by the 'wt' request parameter matching the name of a registered + writer. + The "default" writer is the default and will be used if 'wt' is not specified + in the request. XMLResponseWriter will be used if nothing is specified here. + The json, python, and ruby writers are also available by default. + + <queryResponseWriter name="xml" class="org.apache.solr.request.XMLResponseWriter" default="true"/> + <queryResponseWriter name="json" class="org.apache.solr.request.JSONResponseWriter"/> + <queryResponseWriter name="python" class="org.apache.solr.request.PythonResponseWriter"/> + <queryResponseWriter name="ruby" class="org.apache.solr.request.RubyResponseWriter"/> + <queryResponseWriter name="php" class="org.apache.solr.request.PHPResponseWriter"/> + <queryResponseWriter name="phps" class="org.apache.solr.request.PHPSerializedResponseWriter"/> + + <queryResponseWriter name="custom" class="com.example.MyResponseWriter"/> + --> + + <!-- XSLT response writer transforms the XML output by any xslt file found + in Solr's conf/xslt directory. Changes to xslt files are checked for + every xsltCacheLifetimeSeconds. + --> + <queryResponseWriter name="xslt" class="org.apache.solr.request.XSLTResponseWriter"> + <int name="xsltCacheLifetimeSeconds">5</int> + </queryResponseWriter> + + + <queryParser name="wikitty" class="org.sharengo.wikitty.solr.WikittyQueryParser"/> + + <!-- example of registering a query parser + <queryParser name="lucene" class="org.apache.solr.search.LuceneQParserPlugin"/> + --> + + <!-- example of registering a custom function parser + <valueSourceParser name="myfunc" class="com.mycompany.MyValueSourceParser" /> + --> + + <!-- config for the admin interface --> + <admin> + <defaultQuery>solr</defaultQuery> + + <!-- configure a healthcheck file for servers behind a loadbalancer + <healthcheck type="file">server-enabled</healthcheck> + --> + </admin> + +</config> Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,9 @@ +# Appender and Layout +log4j.appender.logConsole=org.apache.log4j.ConsoleAppender +log4j.appender.logConsole.layout=org.apache.log4j.PatternLayout +log4j.appender.logConsole.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# Configuration by components +log4j.rootLogger=ERROR, logConsole +log4j.category.org.sharengo.wikitty=DEBUG +#log4j.category.org.apache.solr=DEBUG