[Wikitty-commits] r330 - in branches/wikitty-eugene-migration/wikitty-solr-impl: . src/main/java/org/nuiton/wikitty/solr src/test/java/org/nuiton/wikitty/solr/test src/test/resources

Author: bleny Date: 2010-09-22 10:14:49 +0200 (Wed, 22 Sep 2010) New Revision: 330 Url: http://nuiton.org/repositories/revision/wikitty/330 Log: removed Spring dependency (code+pom), fix solrconfig for tests, added a non-regression test (child not removed in index) Added: branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/resources/solrconfig.xml Removed: branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/resources/META-INF/ Modified: branches/wikitty-eugene-migration/wikitty-solr-impl/pom.xml branches/wikitty-eugene-migration/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/AbstractTestSolr.java branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrServerTest.java branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/TreeTest.java Modified: branches/wikitty-eugene-migration/wikitty-solr-impl/pom.xml =================================================================== --- branches/wikitty-eugene-migration/wikitty-solr-impl/pom.xml 2010-09-21 15:40:14 UTC (rev 329) +++ branches/wikitty-eugene-migration/wikitty-solr-impl/pom.xml 2010-09-22 08:14:49 UTC (rev 330) @@ -60,21 +60,6 @@ <groupId>junit</groupId> <artifactId>junit</artifactId> </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-test</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-beans</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-context</artifactId> - <scope>test</scope> - </dependency> </dependencies> Modified: branches/wikitty-eugene-migration/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java =================================================================== --- branches/wikitty-eugene-migration/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java 2010-09-21 15:40:14 UTC (rev 329) +++ branches/wikitty-eugene-migration/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java 2010-09-22 08:14:49 UTC (rev 330) @@ -130,7 +130,7 @@ if (searchField.length >= 3) { String fieldNameType = searchField[2]; - TYPE type = FieldType.TYPE.valueOf(fieldNameType); + TYPE type = FieldType.TYPE.valueOf(fieldNameType); result = WikittySearchEnginSolr.getSolrFieldName(fqfieldname, type); return result; } @@ -165,13 +165,18 @@ /** * Helper to get information nodes and elements for reindexation. */ - protected class ReindexChildTreeNode { + static protected class ReindexChildTreeNode { + protected SolrResource solrResource; + protected SolrServer solrServer; + protected Map<String, Collection<String>> includedNodeIds; protected Map<String, Collection<String>> excludedNodeIds; protected Map<String, String> parents; - public ReindexChildTreeNode() { + public ReindexChildTreeNode(SolrServer solrServer, SolrResource solrResource) { + this.solrServer = solrServer; + this.solrResource = solrResource; includedNodeIds = new HashMap<String, Collection<String>>(); excludedNodeIds = new HashMap<String, Collection<String>>(); parents = new HashMap<String, String>(); @@ -235,7 +240,7 @@ // If not found in map, search in index if(parentId == null) { - SolrDocument doc = WikittySearchEnginSolr.this.findById(nodeId); + SolrDocument doc = findById(solrServer, nodeId); if(doc == null) { // is root return null; @@ -278,19 +283,25 @@ doc = new SolrInputDocument(); // Copy old field value - SolrDocument found = findById(id); - Collection<String> fieldNames = found.getFieldNames(); - for (String fieldName : fieldNames) { - Collection<Object> fieldValues = found.getFieldValues(fieldName); + SolrDocument found = findById(solrServer, id); + if (found != null) { + Collection<String> fieldNames = found.getFieldNames(); + for (String fieldName : fieldNames) { + Collection<Object> fieldValues = found.getFieldValues(fieldName); + + if(!fieldName.startsWith(TREENODE_PREFIX)) { + for (Object fieldValue : fieldValues) { + doc.addField(fieldName, fieldValue); + } + } + } - if(!fieldName.startsWith(TREENODE_PREFIX)) { - for (Object fieldValue : fieldValues) { - doc.addField(fieldName, fieldValue); - } + solrResource.addDoc(id, doc); + } else { + if (log.isWarnEnabled()) { + log.warn("Can't find wikitty id '" + id + "' in index. Skip this wikitty."); } } - - solrResource.addDoc(id, doc); } // Add tree node fields @@ -298,7 +309,8 @@ Collection<String> excludedChildNodeIds = getExcludedNodeIds(id); // Find all node contain child - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TreeNode.FQ_FIELD_TREENODE_CHILDREN + "_s:" + id); + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + + TreeNode.FQ_FIELD_TREENODE_CHILDREN + "_s:" + id); QueryResponse response = solrServer.query(query); SolrDocumentList updateDocs = response.getResults(); @@ -346,12 +358,14 @@ /** * Use to plug solr indexation in JTA transaction. */ - protected class SolrResource implements OnePhaseResource { + static protected class SolrResource implements OnePhaseResource { + protected SolrServer solrServer; protected ThreadLocal<Map<String, SolrInputDocument>> addedDocs; protected ThreadLocal<List<String>> deletedDocs; - public SolrResource() { + public SolrResource(SolrServer solrServer) { + this.solrServer = solrServer; addedDocs = new ThreadLocal<Map<String, SolrInputDocument>>(); deletedDocs = new ThreadLocal<List<String>>(); @@ -482,7 +496,7 @@ solrServer = new EmbeddedSolrServer(coreContainer, ""); fieldModifier = new TypeFieldModifer(extensionStorage); - solrResource = new SolrResource(); + solrResource = new SolrResource(solrServer); } catch (Exception eee) { throw new WikittyException(eee); @@ -503,7 +517,8 @@ public UpdateResponse store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { try { solrResource.init(); - ReindexChildTreeNode reindexChildTreeNode = new ReindexChildTreeNode(); + ReindexChildTreeNode reindexChildTreeNode = + new ReindexChildTreeNode(solrServer, solrResource); for (Wikitty w : wikitties) { String id = w.getId(); @@ -513,7 +528,7 @@ reindexChildTreeNode.putIncludedChildren(id, children); // Search deleted children - SolrDocument treeNodeDoc = findById(id); + SolrDocument treeNodeDoc = findById(solrServer, id); if(treeNodeDoc != null) { Collection oldChildren = treeNodeDoc.getFieldValues(TreeNode.FQ_FIELD_TREENODE_CHILDREN + "_s"); if(oldChildren != null) { @@ -557,7 +572,8 @@ public UpdateResponse delete(WikittyTransaction transaction, Collection<String> ids) throws WikittyException { try { solrResource.init(); - ReindexChildTreeNode reindexChildTreeNode = new ReindexChildTreeNode(); + ReindexChildTreeNode reindexChildTreeNode = + new ReindexChildTreeNode(solrServer, solrResource); for (String id : ids) { // Find child in node id @@ -753,7 +769,7 @@ parent = TREENODE_PREFIX + parent; } - // Find count with facet, if the node not contain recurcively content, + // Find count with facet, if the node not contain recursively content, // the node not found with facet Criteria criteria = Search.query(filter).eq(parent, wikittyId).criteria() .setFirstIndex(0).setEndIndex(0) @@ -862,7 +878,7 @@ /** * Find solr document by id */ - protected SolrDocument findById(String id) { + static protected SolrDocument findById(SolrServer solrServer, String id) { SolrQuery query = new SolrQuery("id:" + id); QueryResponse response; try { Modified: branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/AbstractTestSolr.java =================================================================== --- branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/AbstractTestSolr.java 2010-09-21 15:40:14 UTC (rev 329) +++ branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/AbstractTestSolr.java 2010-09-22 08:14:49 UTC (rev 330) @@ -1,31 +1,12 @@ package org.nuiton.wikitty.solr.test; -import org.junit.runner.RunWith; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - import org.junit.Before; -import org.nuiton.wikitty.WikittySearchEngin; import org.nuiton.wikitty.WikittyService; -import org.springframework.beans.factory.annotation.Autowired; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations="classpath:META-INF/spring/wikitty-test.xml") public abstract class AbstractTestSolr { - @Autowired - protected WikittyService ws; + protected WikittyService ws = new WikittyServiceSolr(); - public void setWikittyService(WikittyService service) { - ws = service; - } - - public WikittyService getWikittyService() { - return ws; - } - - WikittySearchEngin engine; - @Before public void deleteAll() throws Exception { WikittyServiceSolr hbaseService = ((WikittyServiceSolr) ws); Modified: branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java =================================================================== --- branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java 2010-09-21 15:40:14 UTC (rev 329) +++ branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java 2010-09-22 08:14:49 UTC (rev 330) @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Calendar; -import java.util.Date; import java.util.List; import org.apache.commons.logging.Log; @@ -29,7 +28,7 @@ public class SolrSearchTest extends AbstractTestSolr { - static private Log log = LogFactory.getLog(SolrSearchTest.class); + private static final Log log = LogFactory.getLog(SolrSearchTest.class); protected WikittyExtension extCategory; protected WikittyExtension extProduct; Modified: branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrServerTest.java =================================================================== --- branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrServerTest.java 2010-09-21 15:40:14 UTC (rev 329) +++ branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrServerTest.java 2010-09-22 08:14:49 UTC (rev 330) @@ -34,7 +34,6 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import org.springframework.test.annotation.ExpectedException; /** * Test behaviour SolrServer embedded. @@ -45,12 +44,12 @@ * Last update: $Date$ * by : $Author$ */ -public class SolrServerTest extends AbstractTestSolr { +public class SolrServerTest { /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(SolrServerTest.class); + private static final Log log = LogFactory.getLog(SolrServerTest.class); - public SolrServer getSolrServer() throws Exception { + public SolrServer newSolrServer() throws Exception { CoreContainer.Initializer initializer = new CoreContainer.Initializer(); CoreContainer coreContainer = initializer.initialize(); SolrServer solrServer = new EmbeddedSolrServer(coreContainer, ""); @@ -112,16 +111,15 @@ @Before public void clearIndex() throws Exception { - SolrServer solrServer = getSolrServer(); + SolrServer solrServer = newSolrServer(); solrServer.deleteByQuery("*:*"); solrServer.commit(); } - @Test - @ExpectedException(value=SolrServerException.class) + @Test(expected=SolrServerException.class) public void testLockObtainFailed() throws Exception { - SolrServer solrServer1 = getSolrServer(); - SolrServer solrServer2 = getSolrServer(); + SolrServer solrServer1 = newSolrServer(); + SolrServer solrServer2 = newSolrServer(); addDocument(solrServer1, "1", "1"); addDocument(solrServer2, "2", "2"); @@ -136,9 +134,10 @@ * De plus ce test est la seule cause des erreurs de build de ces derniers temps. * Et il y a un commentaire laissant planer le doute un peu plus bas. */ - //@Test + @Ignore + @Test public void test2Threads1SolrServer() throws Exception { - SolrServer solrServer = getSolrServer(); + SolrServer solrServer = newSolrServer(); ThreadAddDocument thread1 = new ThreadAddDocument(solrServer, "1", true); ThreadAddDocument thread2 = new ThreadAddDocument(solrServer, "2", false); @@ -158,8 +157,8 @@ @Test public void test2Threads1Writer2SolrServers() throws Exception { - SolrServer solrServer1 = getSolrServer(); - SolrServer solrServer2 = getSolrServer(); + SolrServer solrServer1 = newSolrServer(); + SolrServer solrServer2 = newSolrServer(); ThreadAddDocument thread1 = new ThreadAddDocument(solrServer1, "1", true); ThreadAddDocument thread2 = new ThreadAddDocument(solrServer2, "2", false); @@ -175,7 +174,7 @@ result = findById(solrServer2, "2"); assertNull(result); - SolrServer solrServer3 = getSolrServer(); + SolrServer solrServer3 = newSolrServer(); result = findById(solrServer3, "1"); assertNotNull(result); result = findById(solrServer3, "2"); @@ -186,8 +185,8 @@ @Ignore @Test public void test2Threads2Writers2SolrServers() throws Exception { - SolrServer solrServer1 = getSolrServer(); - SolrServer solrServer2 = getSolrServer(); + SolrServer solrServer1 = newSolrServer(); + SolrServer solrServer2 = newSolrServer(); ThreadAddDocument thread1 = new ThreadAddDocument(solrServer1, "1", true); ThreadAddDocument thread2 = new ThreadAddDocument(solrServer2, "2", true); @@ -205,7 +204,7 @@ result = findById(solrServer2, "2"); assertNotNull(result); - SolrServer solrServer3 = getSolrServer(); + SolrServer solrServer3 = newSolrServer(); result = findById(solrServer3, "1"); assertNotNull(result); result = findById(solrServer3, "2"); @@ -215,8 +214,8 @@ @Ignore // FIXME 20100806 bleny randomly fail @Test public void testReader() throws Exception { - SolrServer solrServer1 = getSolrServer(); - SolrServer solrServer2 = getSolrServer(); + SolrServer solrServer1 = newSolrServer(); + SolrServer solrServer2 = newSolrServer(); addDocument(solrServer1, "1", "1"); solrServer1.commit(); Modified: branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/TreeTest.java =================================================================== --- branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/TreeTest.java 2010-09-21 15:40:14 UTC (rev 329) +++ branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/TreeTest.java 2010-09-22 08:14:49 UTC (rev 330) @@ -35,6 +35,7 @@ import org.nuiton.wikitty.FieldType.TYPE; import org.nuiton.wikitty.Tree; import org.nuiton.wikitty.TreeNode; +import org.nuiton.wikitty.TreeNodeHelper; import org.nuiton.wikitty.TreeNodeImpl; import org.nuiton.wikitty.Wikitty; import org.nuiton.wikitty.WikittyExtension; @@ -79,11 +80,12 @@ // Create tree as following : // root // |_ node 1 - // | |_ node 11 + // | |_ node 11 (2) + // | | |_ node 111 (1) // | |_ node 12 - // | | |_ node 121 + // | | |_ node 121 (2) // | |_ node 13 - // |_ node 2 + // |_ node 2 (1) createBranch("root/node1/node11/node111"); createBranch("root/node1/node12/node121"); @@ -172,9 +174,29 @@ String nodeId = node.getId(); Map<TreeNode, Integer> children = ws.restoreChildren(null, nodeId, null); + /* for (Integer count : children.values()) { sum += count; } + */ + + for (Map.Entry<TreeNode, Integer> e : children.entrySet()) { + log.debug("*treeNode = " + e.getKey().getName() + " " + e.getValue() + + " -> " + e.getKey().getChildren()); + } + + for (TreeNode treeNode : children.keySet()) { + Set<String> treeNodeChildren = treeNode.getChildren(); + log.debug("+treeNode = " + treeNode.getName() + " " + (treeNodeChildren==null?0:treeNodeChildren.size()) + + " -> " + treeNodeChildren); +// if (treeNodeChildren == null) { +// sum += 0; +// } else { +// sum += treeNodeChildren.size(); +// } + sum += sum(((TreeNodeImpl)treeNode).getWikitty()); + } + return sum; } @@ -341,18 +363,54 @@ Wikitty root = findNode("root"); int childInit = sum(root); - // Remove a value on node2 + // Remove a value on node11 Wikitty node = findNode("node11"); List<String> leafs = node.getFieldAsList(TreeNode.EXT_TREENODE, TreeNode.FIELD_TREENODE_CHILDREN, String.class); + + log.info("leafs = " + leafs); + node.removeFromField(TreeNode.EXT_TREENODE, TreeNode.FIELD_TREENODE_CHILDREN, leafs.get(0)); + + leafs = node.getFieldAsList(TreeNode.EXT_TREENODE, TreeNode.FIELD_TREENODE_CHILDREN, String.class); + log.info("leafs after remove = " + leafs); + ws.store(null, node); + + node = ws.restore(null, node.getId()); + leafs = node.getFieldAsList(TreeNode.EXT_TREENODE, TreeNode.FIELD_TREENODE_CHILDREN, String.class); + log.info("leafs after restore = " + leafs); // now, there is one more value for the root node int newSum = sum(root); assertEquals(childInit - 1, newSum); } + /** regression test, for unknown reason the child may not be removed in the index */ @Test + public void testSimpleDeleteChild() throws Exception { + + TreeNodeImpl parent = new TreeNodeImpl(); + ws.store(null, parent.getWikitty()); + + TreeNodeImpl child = new TreeNodeImpl(); + child.setParent(parent.getWikittyId()); + ws.store(null, child.getWikitty()); + + Map<TreeNode, Integer> children = ws.restoreChildren(null, parent.getWikittyId(), null); + + assertEquals(1, children.size()); + assertEquals(0, children.get(child).intValue()); + + child.setParent(null); + + ws.store(null, child.getWikitty()); + + children = ws.restoreChildren(null, parent.getWikittyId(), null); + + assertEquals(0, children.size()); + } + + @Test public void testDeleteValue() throws Exception { // Get the initial number of values for Root node Wikitty root = findNode("root"); Added: branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/resources/solrconfig.xml =================================================================== --- branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/resources/solrconfig.xml (rev 0) +++ branches/wikitty-eugene-migration/wikitty-solr-impl/src/test/resources/solrconfig.xml 2010-09-22 08:14:49 UTC (rev 330) @@ -0,0 +1,700 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<config> + <!-- Set this to 'false' if you want solr to continue working after it has + encountered an severe configuration error. In a production environment, + you may want solr to keep working even if one handler is mis-configured. + + You may also set this to false using by setting the system property: + -Dsolr.abortOnConfigurationError=false + --> + <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError> + + <!-- Used to specify an alternate directory to hold all index data + other than the default ./data under the Solr home. + If replication is in use, this should match the replication configuration. --> + <dataDir>${solr.data.dir:./target/tests-temp-dir/solr/data}</dataDir> + + + <indexDefaults> + <!-- Values here affect all index writers and act as a default unless overridden. --> + <useCompoundFile>false</useCompoundFile> + + <mergeFactor>10</mergeFactor> + <!-- + If both ramBufferSizeMB and maxBufferedDocs is set, then Lucene will flush based on whichever limit is hit first. + + --> + <!--<maxBufferedDocs>1000</maxBufferedDocs>--> + <!-- Tell Lucene when to flush documents to disk. + Giving Lucene more memory for indexing means faster indexing at the cost of more RAM + + If both ramBufferSizeMB and maxBufferedDocs is set, then Lucene will flush based on whichever limit is hit first. + + --> + <ramBufferSizeMB>32</ramBufferSizeMB> + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + <writeLockTimeout>1000</writeLockTimeout> + <commitLockTimeout>10000</commitLockTimeout> + + <!-- + Expert: Turn on Lucene's auto commit capability. + This causes intermediate segment flushes to write a new lucene + index descriptor, enabling it to be opened by an external + IndexReader. + NOTE: Despite the name, this value does not have any relation to Solr's autoCommit functionality + --> + <!--<luceneAutoCommit>false</luceneAutoCommit>--> + <!-- + Expert: + The Merge Policy in Lucene controls how merging is handled by Lucene. The default in 2.3 is the LogByteSizeMergePolicy, previous + versions used LogDocMergePolicy. + + LogByteSizeMergePolicy chooses segments to merge based on their size. The Lucene 2.2 default, LogDocMergePolicy chose when + to merge based on number of documents + + Other implementations of MergePolicy must have a no-argument constructor + --> + <!--<mergePolicy>org.apache.lucene.index.LogByteSizeMergePolicy</mergePolicy>--> + + <!-- + Expert: + The Merge Scheduler in Lucene controls how merges are performed. The ConcurrentMergeScheduler (Lucene 2.3 default) + can perform merges in the background using separate threads. The SerialMergeScheduler (Lucene 2.2 default) does not. + --> + <!--<mergeScheduler>org.apache.lucene.index.ConcurrentMergeScheduler</mergeScheduler>--> + + <!-- + This option specifies which Lucene LockFactory implementation to use. + + single = SingleInstanceLockFactory - suggested for a read-only index + or when there is no possibility of another process trying + to modify the index. + native = NativeFSLockFactory + simple = SimpleFSLockFactory + + (For backwards compatibility with Solr 1.2, 'simple' is the default + if not specified.) + --> + <lockType>simple</lockType> + </indexDefaults> + + <mainIndex> + <!-- options specific to the main on-disk lucene index --> + <useCompoundFile>false</useCompoundFile> + <ramBufferSizeMB>32</ramBufferSizeMB> + <mergeFactor>10</mergeFactor> + <!-- Deprecated --> + <!--<maxBufferedDocs>1000</maxBufferedDocs>--> + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + + <!-- If true, unlock any held write or commit locks on startup. + This defeats the locking mechanism that allows multiple + processes to safely access a lucene index, and should be + used with care. + This is not needed if lock type is 'none' or 'single' + --> + <unlockOnStartup>true</unlockOnStartup> + </mainIndex> + + <!-- Enables JMX if and only if an existing MBeanServer is found, use + this if you want to configure JMX through JVM parameters. Remove + this to disable exposing Solr configuration and statistics to JMX. + + If you want to connect to a particular server, specify the agentId + e.g. <jmx agentId="myAgent" /> + + If you want to start a new MBeanServer, specify the serviceUrl + e.g <jmx serviceurl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr" /> + + For more details see http://wiki.apache.org/solr/SolrJmx + --> + <jmx /> + + <!-- the default high-performance update handler --> + <updateHandler class="solr.DirectUpdateHandler2"> + + <!-- A prefix of "solr." for class names is an alias that + causes solr to search appropriate packages, including + org.apache.solr.(search|update|request|core|analysis) + --> + + <!-- Perform a <commit/> automatically under certain conditions: + maxDocs - number of updates since last commit is greater than this + maxTime - oldest uncommited update (in ms) is this long ago + <autoCommit> + <maxDocs>10000</maxDocs> + <maxTime>1000</maxTime> + </autoCommit> + --> + + <!-- The RunExecutableListener executes an external command. + exe - the name of the executable to run + dir - dir to use as the current working directory. default="." + wait - the calling thread waits until the executable returns. default="true" + args - the arguments to pass to the program. default=nothing + env - environment variables to set. default=nothing + --> + <!-- A postCommit event is fired after every commit or optimize command + <listener event="postCommit" class="solr.RunExecutableListener"> + <str name="exe">solr/bin/snapshooter</str> + <str name="dir">.</str> + <bool name="wait">true</bool> + <arr name="args"> <str>arg1</str> <str>arg2</str> </arr> + <arr name="env"> <str>MYVAR=val1</str> </arr> + </listener> + --> + <!-- A postOptimize event is fired only after every optimize command, useful + in conjunction with index distribution to only distribute optimized indicies + <listener event="postOptimize" class="solr.RunExecutableListener"> + <str name="exe">snapshooter</str> + <str name="dir">solr/bin</str> + <bool name="wait">true</bool> + </listener> + --> + + </updateHandler> + + + <query> + <!-- Maximum number of clauses in a boolean query... can affect + range or prefix queries that expand to big boolean + queries. An exception is thrown if exceeded. --> + <maxBooleanClauses>1024</maxBooleanClauses> + + + <!-- Cache used by SolrIndexSearcher for filters (DocSets), + unordered sets of *all* documents that match a query. + When a new searcher is opened, its caches may be prepopulated + or "autowarmed" using data from caches in the old searcher. + autowarmCount is the number of items to prepopulate. For LRUCache, + the autowarmed items will be the most recently accessed items. + Parameters: + class - the SolrCache implementation (currently only LRUCache) + size - the maximum number of entries in the cache + initialSize - the initial capacity (number of entries) of + the cache. (seel java.util.HashMap) + autowarmCount - the number of entries to prepopulate from + and old cache. + --> + <filterCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="128"/> + + <!-- queryResultCache caches results of searches - ordered lists of + document ids (DocList) based on a query, a sort, and the range + of documents requested. --> + <queryResultCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="32"/> + + <!-- documentCache caches Lucene Document objects (the stored fields for each document). + Since Lucene internal document ids are transient, this cache will not be autowarmed. --> + <documentCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + + <!-- If true, stored fields that are not requested will be loaded lazily. + + This can result in a significant speed improvement if the usual case is to + not load all stored fields, especially if the skipped fields are large compressed + text fields. + --> + <enableLazyFieldLoading>true</enableLazyFieldLoading> + + <!-- Example of a generic cache. These caches may be accessed by name + through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert(). + The purpose is to enable easy caching of user/application level data. + The regenerator argument should be specified as an implementation + of solr.search.CacheRegenerator if autowarming is desired. --> + <!-- + <cache name="myUserCache" + class="solr.LRUCache" + size="4096" + initialSize="1024" + autowarmCount="1024" + regenerator="org.mycompany.mypackage.MyRegenerator" + /> + --> + + <!-- An optimization that attempts to use a filter to satisfy a search. + If the requested sort does not include score, then the filterCache + will be checked for a filter matching the query. If found, the filter + will be used as the source of document ids, and then the sort will be + applied to that. + <useFilterForSortedQuery>true</useFilterForSortedQuery> + --> + + <!-- An optimization for use with the queryResultCache. When a search + is requested, a superset of the requested number of document ids + are collected. For example, if a search for a particular query + requests matching documents 10 through 19, and queryWindowSize is 50, + then documents 0 through 49 will be collected and cached. Any further + requests in that range can be satisfied via the cache. --> + <queryResultWindowSize>50</queryResultWindowSize> + + <!-- Maximum number of documents to cache for any entry in the + queryResultCache. --> + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + + <!-- This entry enables an int hash representation for filters (DocSets) + when the number of items in the set is less than maxSize. For smaller + sets, this representation is more memory efficient, more efficient to + iterate over, and faster to take intersections. --> + <HashDocSet maxSize="3000" loadFactor="0.75"/> + + <!-- a newSearcher event is fired whenever a new searcher is being prepared + and there is a current searcher handling requests (aka registered). --> + <!-- QuerySenderListener takes an array of NamedList and executes a + local query request for each NamedList in sequence. --> + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + + <!-- a firstSearcher event is fired whenever a new searcher is being + prepared but there is no current registered searcher to handle + requests or to gain autowarming data from. --> + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static firstSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + + <!-- If a search request comes in and there is no current registered searcher, + then immediately register the still warming searcher and use it. If + "false" then all requests will block until the first searcher is done + warming. --> + <useColdSearcher>false</useColdSearcher> + + <!-- Maximum number of searchers that may be warming in the background + concurrently. An error is returned if this limit is exceeded. Recommend + 1-2 for read-only slaves, higher for masters w/o cache warming. --> + <maxWarmingSearchers>2</maxWarmingSearchers> + + </query> + + <!-- + Let the dispatch filter handler /select?qt=XXX + handleSelect=true will use consistent error handling for /select and /update + handleSelect=false will use solr1.1 style error formatting + --> + <requestDispatcher handleSelect="true" > + <!--Make sure your system has some authentication before enabling remote streaming! --> + <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" /> + + <!-- Set HTTP caching related parameters (for proxy caches and clients). + + To get the behaviour of Solr 1.2 (ie: no caching related headers) + use the never304="true" option and do not specify a value for + <cacheControl> + --> + <!-- <httpCaching never304="true"> --> + <httpCaching lastModifiedFrom="openTime" + etagSeed="Solr"> + <!-- lastModFrom="openTime" is the default, the Last-Modified value + (and validation against If-Modified-Since requests) will all be + relative to when the current Searcher was opened. + You can change it to lastModFrom="dirLastMod" if you want the + value to exactly corrispond to when the physical index was last + modified. + + etagSeed="..." is an option you can change to force the ETag + header (and validation against If-None-Match requests) to be + differnet even if the index has not changed (ie: when making + significant changes to your config file) + + lastModifiedFrom and etagSeed are both ignored if you use the + never304="true" option. + --> + <!-- If you include a <cacheControl> directive, it will be used to + generate a Cache-Control header, as well as an Expires header + if the value contains "max-age=" + + By default, no Cache-Control header is generated. + + You can use the <cacheControl> option even if you have set + never304="true" + --> + <!-- <cacheControl>max-age=30, public</cacheControl> --> + </httpCaching> + </requestDispatcher> + + + <!-- requestHandler plugins... incoming queries will be dispatched to the + correct handler based on the path or the qt (query type) param. + Names starting with a '/' are accessed with the a path equal to the + registered name. Names without a leading '/' are accessed with: + http://host/app/select?qt=name + If no qt is defined, the requestHandler that declares default="true" + will be used. + --> + <requestHandler name="standard" class="solr.SearchHandler" default="true"> + <!-- default values for query parameters --> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <!-- + <int name="rows">10</int> + <str name="fl">*</str> + <str name="version">2.1</str> + --> + </lst> + </requestHandler> + + + <!-- DisMaxRequestHandler allows easy searching across multiple fields + for simple user-entered phrases. It's implementation is now + just the standard SearchHandler with a default query type + of "dismax". + see http://wiki.apache.org/solr/DisMaxRequestHandler + --> + <requestHandler name="dismax" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <float name="tie">0.01</float> + <str name="qf"> + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + </str> + <str name="pf"> + text^0.2 features^1.1 name^1.5 manu^1.4 manu_exact^1.9 + </str> + <str name="bf"> + ord(popularity)^0.5 recip(rord(price),1,1000,1000)^0.3 + </str> + <str name="fl"> + id,name,price,score + </str> + <str name="mm"> + 2<-1 5<-2 6<90% + </str> + <int name="ps">100</int> + <str name="q.alt">*:*</str> + <!-- example highlighter config, enable per-query with hl=true --> + <str name="hl.fl">text features name</str> + <!-- for this field, we want no fragmenting, just highlighting --> + <str name="f.name.hl.fragsize">0</str> + <!-- instructs Solr to return the field itself if no query terms are + found --> + <str name="f.name.hl.alternateField">name</str> + <str name="f.text.hl.fragmenter">regex</str> <!-- defined below --> + </lst> + </requestHandler> + + <!-- Note how you can register the same handler multiple times with + different names (and different init parameters) + --> + <requestHandler name="partitioned" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <str name="qf">text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0</str> + <str name="mm">2<-1 5<-2 6<90%</str> + <!-- This is an example of using Date Math to specify a constantly + moving date range in a config... + --> + <str name="bq">incubationdate_dt:[* TO NOW/DAY-1MONTH]^2.2</str> + </lst> + <!-- In addition to defaults, "appends" params can be specified + to identify values which should be appended to the list of + multi-val params from the query (or the existing "defaults"). + + In this example, the param "fq=instock:true" will be appended to + any query time fq params the user may specify, as a mechanism for + partitioning the index, independent of any user selected filtering + that may also be desired (perhaps as a result of faceted searching). + + NOTE: there is *absolutely* nothing a client can do to prevent these + "appends" values from being used, so don't use this mechanism + unless you are sure you always want it. + --> + <lst name="appends"> + <str name="fq">inStock:true</str> + </lst> + <!-- "invariants" are a way of letting the Solr maintainer lock down + the options available to Solr clients. Any params values + specified here are used regardless of what values may be specified + in either the query, the "defaults", or the "appends" params. + + In this example, the facet.field and facet.query params are fixed, + limiting the facets clients can use. Faceting is not turned on by + default - but if the client does specify facet=true in the request, + these are the only facets they will be able to see counts for; + regardless of what other facet.field or facet.query params they + may specify. + + NOTE: there is *absolutely* nothing a client can do to prevent these + "invariants" values from being used, so don't use this mechanism + unless you are sure you always want it. + --> + <lst name="invariants"> + <str name="facet.field">cat</str> + <str name="facet.field">manu_exact</str> + <str name="facet.query">price:[* TO 500]</str> + <str name="facet.query">price:[500 TO *]</str> + </lst> + </requestHandler> + + + <!-- + Search components are registered to SolrCore and used by Search Handlers + + By default, the following components are avaliable: + + <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" /> + <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" /> + <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" /> + <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" /> + <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" /> + + Default configuration in a requestHandler would look like: + <arr name="components"> + <str>query</str> + <str>facet</str> + <str>mlt</str> + <str>highlight</str> + <str>debug</str> + </arr> + + If you register a searchComponent to one of the standard names, that will be used instead. + To insert handlers before or after the 'standard' components, use: + + <arr name="first-components"> + <str>myFirstComponentName</str> + </arr> + + <arr name="last-components"> + <str>myLastComponentName</str> + </arr> + --> + + <!-- The spell check component can return a list of alternative spelling + suggestions. --> + <searchComponent name="spellcheck" class="solr.SpellCheckComponent"> + + <str name="queryAnalyzerFieldType">textSpell</str> + + <lst name="spellchecker"> + <str name="name">default</str> + <str name="field">spell</str> + <str name="spellcheckIndexDir">./spellchecker1</str> + + </lst> + <lst name="spellchecker"> + <str name="name">jarowinkler</str> + <str name="field">spell</str> + <!-- Use a different Distance Measure --> + <str name="distanceMeasure">org.apache.lucene.search.spell.JaroWinklerDistance</str> + <str name="spellcheckIndexDir">./spellchecker2</str> + + </lst> + + <lst name="spellchecker"> + <str name="classname">solr.FileBasedSpellChecker</str> + <str name="name">file</str> + <str name="sourceLocation">spellings.txt</str> + <str name="characterEncoding">UTF-8</str> + <str name="spellcheckIndexDir">./spellcheckerFile</str> + </lst> + </searchComponent> + + <!-- a request handler utilizing the spellcheck component --> + <requestHandler name="/spellCheckCompRH" class="solr.SearchHandler"> + <lst name="defaults"> + <!-- omp = Only More Popular --> + <str name="spellcheck.onlyMorePopular">false</str> + <!-- exr = Extended Results --> + <str name="spellcheck.extendedResults">false</str> + <!-- The number of suggestions to return --> + <str name="spellcheck.count">1</str> + </lst> + <arr name="last-components"> + <str>spellcheck</str> + </arr> + </requestHandler> + + <!-- a search component that enables you to configure the top results for + a given query regardless of the normal lucene scoring.--> + +<!-- poussin 20090902 remove elevate this file is empty, what need ? + <searchComponent name="elevator" class="solr.QueryElevationComponent" > + <str name="queryFieldType">string</str> + <str name="config-file">elevate.xml</str> + </searchComponent> + --> + <!-- a request handler utilizing the elevator component --> +<!-- + <requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + </lst> + <arr name="last-components"> + <str>elevator</str> + </arr> + </requestHandler> + --> + + <!-- Update request handler. + + Note: Since solr1.1 requestHandlers requires a valid content type header if posted in + the body. For example, curl now requires: -H 'Content-type:text/xml; charset=utf-8' + The response format differs from solr1.1 formatting and returns a standard error code. + + To enable solr1.1 behavior, remove the /update handler or change its path + --> + <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" /> + + <!-- + Analysis request handler. Since Solr 1.3. Use to returnhow a document is analyzed. Useful + for debugging and as a token server for other types of applications + --> + <requestHandler name="/analysis" class="solr.AnalysisRequestHandler" /> + + + <!-- CSV update handler, loaded on demand --> + <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" /> + + + <!-- + Admin Handlers - This will register all the standard admin RequestHandlers. Adding + this single handler is equivolent to registering: + + <requestHandler name="/admin/luke" class="org.apache.solr.handler.admin.LukeRequestHandler" /> + <requestHandler name="/admin/system" class="org.apache.solr.handler.admin.SystemInfoHandler" /> + <requestHandler name="/admin/plugins" class="org.apache.solr.handler.admin.PluginInfoHandler" /> + <requestHandler name="/admin/threads" class="org.apache.solr.handler.admin.ThreadDumpHandler" /> + <requestHandler name="/admin/properties" class="org.apache.solr.handler.admin.PropertiesRequestHandler" /> + <requestHandler name="/admin/file" class="org.apache.solr.handler.admin.ShowFileRequestHandler" > + + If you wish to hide files under ${solr.home}/conf, explicitly register the ShowFileRequestHandler using: + <requestHandler name="/admin/file" class="org.apache.solr.handler.admin.ShowFileRequestHandler" > + <lst name="invariants"> + <str name="hidden">synonyms.txt</str> + <str name="hidden">anotherfile.txt</str> + </lst> + </requestHandler> + --> + <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" /> + + <!-- ping/healthcheck --> + <requestHandler name="/admin/ping" class="PingRequestHandler"> + <lst name="defaults"> + <str name="qt">standard</str> + <str name="q">solrpingquery</str> + <str name="echoParams">all</str> + </lst> + </requestHandler> + + <!-- Echo the request contents back to the client --> + <requestHandler name="/debug/dump" class="solr.DumpRequestHandler" > + <lst name="defaults"> + <str name="echoParams">explicit</str> <!-- for all params (including the default etc) use: 'all' --> + <str name="echoHandler">true</str> + </lst> + </requestHandler> + + <highlighting> + <!-- Configure the standard fragmenter --> + <!-- This could most likely be commented out in the "default" case --> + <fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true"> + <lst name="defaults"> + <int name="hl.fragsize">100</int> + </lst> + </fragmenter> + + <!-- A regular-expression-based fragmenter (f.i., for sentence extraction) --> + <fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter"> + <lst name="defaults"> + <!-- slightly smaller fragsizes work better because of slop --> + <int name="hl.fragsize">70</int> + <!-- allow 50% slop on fragment sizes --> + <float name="hl.regex.slop">0.5</float> + <!-- a basic sentence pattern --> + <str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str> + </lst> + </fragmenter> + + <!-- Configure the standard formatter --> + <formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true"> + <lst name="defaults"> + <str name="hl.simple.pre"><![CDATA[<em>]]></str> + <str name="hl.simple.post"><![CDATA[</em>]]></str> + </lst> + </formatter> + </highlighting> + + + <!-- queryResponseWriter plugins... query responses will be written using the + writer specified by the 'wt' request parameter matching the name of a registered + writer. + The "default" writer is the default and will be used if 'wt' is not specified + in the request. XMLResponseWriter will be used if nothing is specified here. + The json, python, and ruby writers are also available by default. + + <queryResponseWriter name="xml" class="org.apache.solr.request.XMLResponseWriter" default="true"/> + <queryResponseWriter name="json" class="org.apache.solr.request.JSONResponseWriter"/> + <queryResponseWriter name="python" class="org.apache.solr.request.PythonResponseWriter"/> + <queryResponseWriter name="ruby" class="org.apache.solr.request.RubyResponseWriter"/> + <queryResponseWriter name="php" class="org.apache.solr.request.PHPResponseWriter"/> + <queryResponseWriter name="phps" class="org.apache.solr.request.PHPSerializedResponseWriter"/> + + <queryResponseWriter name="custom" class="com.example.MyResponseWriter"/> + --> + + <!-- XSLT response writer transforms the XML output by any xslt file found + in Solr's conf/xslt directory. Changes to xslt files are checked for + every xsltCacheLifetimeSeconds. + --> + <queryResponseWriter name="xslt" class="org.apache.solr.request.XSLTResponseWriter"> + <int name="xsltCacheLifetimeSeconds">5</int> + </queryResponseWriter> + + + <queryParser name="wikitty" class="org.nuiton.wikitty.solr.WikittyQueryParser"/> + + <!-- example of registering a query parser + <queryParser name="lucene" class="org.apache.solr.search.LuceneQParserPlugin"/> + --> + + <!-- example of registering a custom function parser + <valueSourceParser name="myfunc" class="com.mycompany.MyValueSourceParser" /> + --> + + <!-- config for the admin interface --> + <admin> + <defaultQuery>solr</defaultQuery> + + <!-- configure a healthcheck file for servers behind a loadbalancer + <healthcheck type="file">server-enabled</healthcheck> + --> + </admin> + +</config>