[Wikitty-commits] r642 - in trunk/wikitty-solr-impl/src: main/java/org/nuiton/wikitty/storage/solr main/resources test/java/org/nuiton/wikitty/storage/solr

22 Dec 2010

Author: bpoussin
Date: 2010-12-22 04:40:56 +0100 (Wed, 22 Dec 2010)
New Revision: 642

Url: http://nuiton.org/repositories/revision/wikitty/642

Log:
- search case unsensitive and accent unsensitive work for like and unlike


Modified:
   trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java
   trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java
   trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java
   trunk/wikitty-solr-impl/src/main/resources/schema.xml
   trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java
   trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java

Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java
===================================================================

--- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java	2010-12-21 15:34:09 UTC (rev 641)
+++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java	2010-12-22 03:40:56 UTC (rev 642)
@@ -285,7 +285,7 @@
                     element2solr += WikittySolrConstant.SUFFIX_STRING_FULLTEXT;
                     break;
                 case ToLowerCase:
-                    element2solr += WikittySolrConstant.SUFFIX_STRING_FULLTEXT;
+                    element2solr += WikittySolrConstant.SUFFIX_STRING_LOWERCASE;
                     break;
             }
         }

Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java
===================================================================
--- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java	2010-12-21 15:34:09 UTC (rev 641)
+++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java	2010-12-22 03:40:56 UTC (rev 642)
@@ -62,6 +62,7 @@
 import org.nuiton.wikitty.search.Search;
 
 import java.io.File;
+import java.util.Collections;
 import org.nuiton.util.ApplicationConfig;
 import org.nuiton.wikitty.WikittyConfig;
 import org.nuiton.wikitty.WikittyUtil;
@@ -244,7 +245,6 @@
                     // Copy old field value
                     SolrDocument found = SolrUtil.findById(solrServer, id);
                     if (found != null) {
-                        // FIXME poussin 20101209 a finir en ne copiant que le necessaire voir javadoc du package
                         doc = SolrUtil.copySolrDocument(found, true, fieldToCopyPattern);
     	                solrResource.addDoc(id, doc);
     	            } else {
@@ -263,7 +263,7 @@
                 Collection<String> includedChildNodeIds = getIncludedNodeIds(id);
                 Collection<String> excludedChildNodeIds = getExcludedNodeIds(id);
 
-                // Find all node contain child
+                // Find all node contain is as attachment
                 SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER
                         + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT
                         + SUFFIX_WIKITTY + ":" + id);
@@ -389,31 +389,56 @@
                 if (w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) {
 
                     Set<String> attachments = WikittyTreeNodeHelper.getAttachment(w);
-                    reindexChildTreeNode.putIncludedAttachments(id, attachments);
+                    if (attachments == null) {
+                        attachments = Collections.emptySet();
+                    }
 
-                    // Search deleted children
+                    //
+                    // recherche des anciennes attachments
+                    //
+                    Collection oldAttachments = null;
+                    Set<String> oldAttachmentsCopy;
+
+                    // Search deleted attachments
                     SolrDocument treeNodeDoc = SolrUtil.findById(solrServer, id);
                     if (treeNodeDoc != null) {
-                        Collection oldAttachments = treeNodeDoc.getFieldValues(
+                        oldAttachments = treeNodeDoc.getFieldValues(
                                 WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT + SUFFIX_WIKITTY);
-                        if (oldAttachments != null) {
-                            // if no more children, remove all old children
-                            if(attachments == null) {
-                                reindexChildTreeNode.putExcludedAttachments(id, oldAttachments);
-                            } else {
-                                // exclude only the removed children
-                                for (Object oldAttachment : oldAttachments) {
-                                    if(!attachments.contains(oldAttachment)) {
-                                        reindexChildTreeNode.putExcludedAttachment(id,(String) oldAttachment);
-                                    }
-                                }
-                            }
-                        }
                     }
+                    // make copy to modify it
+                    if (oldAttachments == null) {
+                        oldAttachments = Collections.emptySet();
+                    }
+                    oldAttachmentsCopy = new HashSet<String>(oldAttachments);
 
-                    // Get new parent id (may be the same old parent)
-                    String parentId = WikittyTreeNodeHelper.getParent(w);
-                    reindexChildTreeNode.putParent(id, parentId);
+
+
+                    // les attachments seulement dans old doivent etre exclus
+                    oldAttachmentsCopy.removeAll(attachments);
+                    reindexChildTreeNode.putExcludedAttachments(id, oldAttachmentsCopy);
+
+                    if (w.getDirty().contains(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT)
+                            || null == WikittyTreeNodeHelper.getParent(w)
+                            || WikittyUtil.versionGreaterThan("1", w.getVersion())) {
+                        // si le pere a change
+                        // ou qu'il est null (creation d'un nouvel arbre)
+                        // ou que l'objet n'a jamais ete sauve 1 > version
+                        // il faut indexer les attachments
+
+                        // il faut reindexer tous les attachments
+                        reindexChildTreeNode.putIncludedAttachments(id, attachments);
+
+                        // Get new parent id (may be the same old parent)
+                        String parentId = WikittyTreeNodeHelper.getParent(w);
+                        reindexChildTreeNode.putParent(id, parentId);
+                    } else {
+                        // les attachments seulement dans les nouveaux doivent etre inclus
+                        Set<String> attachmentsCopy = new HashSet<String>(attachments);
+                        attachmentsCopy.removeAll(oldAttachments);
+                        reindexChildTreeNode.putIncludedAttachments(id, attachmentsCopy);
+                    }
+
+
                 }
 
                 // Index

Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java
===================================================================
--- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java	2010-12-21 15:34:09 UTC (rev 641)
+++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java	2010-12-22 03:40:56 UTC (rev 642)
@@ -65,8 +65,6 @@
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import org.nuiton.util.ApplicationConfig;
-import org.nuiton.util.CollectionUtil;
-import org.nuiton.util.StringUtil;
 import org.nuiton.wikitty.WikittyConfig;
 import org.nuiton.wikitty.WikittyUtil;
 import org.nuiton.wikitty.entities.WikittyTreeNodeHelper;
@@ -257,6 +255,14 @@
         return result;
     }
 
+    /**
+     * Reindexe les noeuds passe en parametre et reindexe automatiquement leurs
+     * sous noeuds
+     *
+     * @param solrResource
+     * @param treeNodeToIndex
+     * @throws Exception
+     */
     protected void reindexTreeNode(SolrResource solrResource,
             LinkedHashSet<String> treeNodeToIndex) throws Exception {
         LinkedList<String> todo = new LinkedList<String>(treeNodeToIndex);
@@ -341,6 +347,7 @@
             for (String id : ids) {
                 SolrDocument doc = SolrUtil.findById(solrServer, id);
                 if (doc.containsKey(TREENODE_ROOT)) {
+                    // child are automaticaly done in reindexTreeNode method
                     treeNodeToIndex.add(id);
                 }
                 solrResource.deleteDoc(id);

Modified: trunk/wikitty-solr-impl/src/main/resources/schema.xml
===================================================================
--- trunk/wikitty-solr-impl/src/main/resources/schema.xml	2010-12-21 15:34:09 UTC (rev 641)
+++ trunk/wikitty-solr-impl/src/main/resources/schema.xml	2010-12-22 03:40:56 UTC (rev 642)
@@ -58,6 +58,7 @@
                sortMissingLast="true">
       <analyzer> <!-- no type to indicated that used it for both type: index and query -->
         <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
       </analyzer>
     </fieldType>
@@ -75,6 +76,7 @@
                positionIncrementGap="100">
       <analyzer type="index">
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.StopFilterFactory"
                 ignoreCase="true"
                 words="stopwords_fr.txt"
@@ -82,7 +84,8 @@
                 />
         <filter class="solr.WordDelimiterFilterFactory"
                 generateWordParts="1" generateNumberParts="1" catenateWords="1"
-                catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+                catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"
+                preserveOriginal="1"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.EnglishPorterFilterFactory"
                 protected="protwords.txt"/>
@@ -90,13 +93,15 @@
       </analyzer>
       <analyzer type="query">
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.SynonymFilterFactory"
                 synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
         <filter class="solr.StopFilterFactory"
                 ignoreCase="true" words="stopwords_fr.txt"/>
         <filter class="solr.WordDelimiterFilterFactory"
                 generateWordParts="1" generateNumberParts="1" catenateWords="0"
-                catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+                catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"
+                preserveOriginal="1"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.EnglishPorterFilterFactory"
                 protected="protwords.txt"/>

Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java
===================================================================
--- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java	2010-12-21 15:34:09 UTC (rev 641)
+++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java	2010-12-22 03:40:56 UTC (rev 642)
@@ -540,9 +540,32 @@
         {
             // avec accent specifiquement sur le champs name mais sans accent pour la recherche
             Criteria criteria = Search.query().eq(
-                    WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME,"ceca").criteria();
+                    WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME
+                    + WikittySolrConstant.SUFFIX_STRING
+                    + WikittySolrConstant.SUFFIX_STRING_FULLTEXT, "ceca").criteria();
             PagedResult<String> result = ws.findAllByCriteria(null, criteria);
             Assert.assertEquals(1, result.size());
         }
+        {
+            // avec accent specifiquement sur le champs name
+            // mais sans accent pour la recherche
+            // et et majuscule
+            Criteria criteria = Search.query().eq(
+                    WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME
+                    + WikittySolrConstant.SUFFIX_STRING
+                    + WikittySolrConstant.SUFFIX_STRING_FULLTEXT, "CECA").criteria();
+            PagedResult<String> result = ws.findAllByCriteria(null, criteria);
+            Assert.assertEquals(1, result.size());
+        }
+        {
+            System.out.println("#######");
+            // avec accent specifiquement sur le champs name
+            // mais sans accent pour la recherche
+            // et et majuscule
+            Criteria criteria = Search.query().like(
+                    WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME, "CECA").criteria();
+            PagedResult<String> result = ws.findAllByCriteria(null, criteria);
+            Assert.assertEquals(1, result.size());
+        }
     }
 }

Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java
===================================================================
--- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java	2010-12-21 15:34:09 UTC (rev 641)
+++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java	2010-12-22 03:40:56 UTC (rev 642)
@@ -506,4 +506,19 @@
         sum = sum(node32);
         assertEquals(1, sum);
     }
+
+    // verifier que l'indexation de l'arbre fonctionne bien et la reindexation aussi
+    @Test
+    public void testReindexation() throws Exception {
+//        FIXME poussin 20101222 a faire
+        // creer deux arbres (3 noeud)
+        // mettre 3 attachments par noeud (les memes) dans les 2 arbres
+
+        // enlever un attachment par noeud et supprime le noeud intermediaire
+        // en meme temps et store les arbres
+
+        // verifier que tout est bien reindexe et que le dernier noeud est bien
+        // devenu un noeud root
+    }
+
 }

    

[Wikitty-commits] r642 - in trunk/wikitty-solr-impl/src: main/java/org/nuiton/wikitty/storage/solr main/resources test/java/org/nuiton/wikitty/storage/solr

bpoussin＠users.nuiton.org