Author: bpoussin Date: 2010-12-02 02:29:28 +0100 (Thu, 02 Dec 2010) New Revision: 551 Url: http://nuiton.org/repositories/revision/wikitty/551 Log: - rename all engin to engine - rename syncEngin to syncSearchEngine - sample configuration update for that Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java Removed: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngin.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEnginInMemory.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemory.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java trunk/wikitty-api/src/main/resources/wikitty-config-sample-inmemory.properties trunk/wikitty-api/src/main/resources/wikitty-config-sample-server-slave.properties trunk/wikitty-api/src/main/resources/wikitty-config-sample-server.properties trunk/wikitty-api/src/test/java/org/nuiton/wikitty/services/WikittyServiceHelperTest.java trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyServiceJDBC.java trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemoryJdbcSolr.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/WikittyServiceSolr.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-02 01:29:28 UTC (rev 551) @@ -200,7 +200,7 @@ _("WikittyServiceStorage component to use (ExtensionStorage, WikittyStorage, SearchEngine)"), "org.nuiton.wikitty.jdbc.WikittyExtensionStorageJDBC," + "org.nuiton.wikitty.jdbc.WikittyStorageJDBC," - + "org.nuiton.wikitty.solr.WikittySearchEnginSolr", String.class, false, false), + + "org.nuiton.wikitty.solr.WikittySearchEngineSolr", String.class, false, false), WIKITTY_WIKITTYSERVICENOTIFIER_COMPONENTS( "wikitty.WikittyServiceNotifier.components", Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java 2010-12-02 01:29:28 UTC (rev 551) @@ -664,8 +664,8 @@ * * @return update response */ - public void syncEngin() { - wikittyService.syncEngin(securityToken); + public void syncSearchEngine() { + wikittyService.syncSearchEngine(securityToken); } /** Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java 2010-12-02 01:29:28 UTC (rev 551) @@ -382,6 +382,6 @@ * * @param securityToken security token */ - public void syncEngin(String securityToken); + public void syncSearchEngine(String securityToken); } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java 2010-12-02 01:29:28 UTC (rev 551) @@ -396,8 +396,8 @@ } @Override - public void syncEngin(String securityToken) { - ws.syncEngin(securityToken); + public void syncSearchEngine(String securityToken) { + ws.syncSearchEngine(securityToken); } @Override Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java 2010-12-02 01:29:28 UTC (rev 551) @@ -198,8 +198,8 @@ } @Override - public void syncEngin(String securityToken) { - delegate.syncEngin(securityToken); + public void syncSearchEngine(String securityToken) { + delegate.syncSearchEngine(securityToken); } @Override Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemory.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemory.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemory.java 2010-12-02 01:29:28 UTC (rev 551) @@ -25,7 +25,7 @@ package org.nuiton.wikitty.services; -import org.nuiton.wikitty.storage.WikittySearchEnginInMemory; +import org.nuiton.wikitty.storage.WikittySearchEngineInMemory; import org.nuiton.wikitty.storage.WikittyExtensionStorageInMemory; import org.nuiton.wikitty.storage.WikittyStorageInMemory; import java.io.File; @@ -71,7 +71,7 @@ super(new WikittyExtensionStorageInMemory(), new WikittyStorageInMemory(), null); - searchEngin = new WikittySearchEnginInMemory( + searchEngine = new WikittySearchEngineInMemory( (WikittyStorageInMemory) wikittyStorage); if (config != null) { Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java 2010-12-02 01:29:28 UTC (rev 551) @@ -330,9 +330,9 @@ } @Override - public void syncEngin(String securityToken) { + public void syncSearchEngine(String securityToken) { // no notification - ws.syncEngin(securityToken); + ws.syncSearchEngine(securityToken); } /** Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java 2010-12-02 01:29:28 UTC (rev 551) @@ -596,11 +596,11 @@ } @Override - public void syncEngin(String securityToken) { + public void syncSearchEngine(String securityToken) { String userId = getUserId(securityToken); if (isAppAdmin(securityToken, userId)) { // seul les AppAdmin on le droit a cette method - ws.syncEngin(securityToken); + ws.syncSearchEngine(securityToken); } else { throw new SecurityException(_("user %s can't sync search engine", getUserId(securityToken))); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2010-12-02 01:29:28 UTC (rev 551) @@ -26,7 +26,7 @@ package org.nuiton.wikitty.services; import org.nuiton.wikitty.storage.WikittyExtensionStorage; -import org.nuiton.wikitty.storage.WikittySearchEngin; +import org.nuiton.wikitty.storage.WikittySearchEngine; import org.nuiton.wikitty.storage.WikittyStorage; import java.util.ArrayList; import java.util.Collection; @@ -76,7 +76,7 @@ protected WikittyExtensionMigration defaultExtensionMigration = new WikittyExtensionMigrationRename(); - protected WikittySearchEngin searchEngin; + protected WikittySearchEngine searchEngine; protected WikittyExtensionStorage extensionStorage; protected WikittyStorage wikittyStorage; @@ -93,14 +93,14 @@ public WikittyServiceStorage(WikittyExtensionStorage extensionStorage, WikittyStorage wikittyStorage, - WikittySearchEngin searchEngin) { + WikittySearchEngine searchEngine) { this.extensionStorage = extensionStorage; this.wikittyStorage = wikittyStorage; - this.searchEngin = searchEngin; + this.searchEngine = searchEngine; } - protected WikittySearchEngin getSearchEngin() { - return searchEngin; + protected WikittySearchEngine getSearchEngine() { + return searchEngine; } protected WikittyExtensionStorage getExtensionStorage() { @@ -198,7 +198,7 @@ getExtensionStorage().store(tx, allExtensions); WikittyEvent wikUpdate = getWikittyStorage().store(tx, wikitties, force); - getSearchEngin().store(tx, wikitties); + getSearchEngine().store(tx, wikitties); WikittyEvent result = new WikittyEvent(this); // prepare update client response @@ -650,7 +650,7 @@ WikittyEvent eventDelete = getWikittyStorage().delete(tx, idSet); - getSearchEngin().delete(tx, idSet); + getSearchEngine().delete(tx, idSet); WikittyEvent eventStore = store(securityToken, storedWikitties, false); @@ -689,7 +689,7 @@ txBeginHere = true; } - getSearchEngin().clear(tx); + getSearchEngine().clear(tx); WikittyEvent eventWik = getWikittyStorage().clear(tx); WikittyEvent eventExt = getExtensionStorage().clear(tx); @@ -730,7 +730,7 @@ } PagedResult<String> result = - getSearchEngin().findAllByCriteria(tx, criteria); + getSearchEngine().findAllByCriteria(tx, criteria); if (txBeginHere) { tx.commit(); } @@ -920,7 +920,7 @@ } WikittyTreeNode node = new WikittyTreeNodeImpl(w); - Integer count = getSearchEngin().findNodeCount(tx, w, filter); + Integer count = getSearchEngine().findNodeCount(tx, w, filter); HashMap.SimpleEntry<WikittyTreeNode, Integer> result = new SimpleEntry<WikittyTreeNode, Integer>(node, count); @@ -969,7 +969,7 @@ Map<WikittyTreeNode, Integer> result = new LinkedHashMap<WikittyTreeNode, Integer>(); - Map<String, Integer> search = getSearchEngin().findAllChildrenCount( + Map<String, Integer> search = getSearchEngine().findAllChildrenCount( tx, w, filter); Set<Entry<String, Integer>> children = search.entrySet(); for (Entry<String, Integer> child : children) { @@ -1011,23 +1011,23 @@ * @param securityToken */ @Override - public void syncEngin(final String securityToken) { + public void syncSearchEngine(final String securityToken) { final WikittyTransaction tx = WikittyTransaction.get(); if (tx.isStarted()) { - throw new WikittyException("Transaction must be not started for syncEngin method"); + throw new WikittyException("Transaction must be not started for syncSearchEngine method"); } boolean txBeginHere = false; try { final int numberForCommit = 1000; - final WikittySearchEngin searchEngin = getSearchEngin(); + final WikittySearchEngine searchEngine = getSearchEngine(); final List<Wikitty> wikitties = new ArrayList<Wikitty>(numberForCommit); if (!tx.isStarted()) { tx.begin(); txBeginHere = true; } - searchEngin.clear(tx); + searchEngine.clear(tx); if (txBeginHere) { tx.commit(); } @@ -1048,7 +1048,7 @@ if(count == numberForCommit) { // Reindex - searchEngin.store(tx, wikitties); + searchEngine.store(tx, wikitties); tx.commit(); // Reinit count = 0; @@ -1060,7 +1060,7 @@ }); // Last wikitties - searchEngin.store(tx, wikitties); + searchEngine.store(tx, wikitties); if (txBeginHere) { tx.commit(); } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java 2010-12-02 01:29:28 UTC (rev 551) @@ -349,7 +349,7 @@ } @Override - public void syncEngin(String securityToken) { + public void syncSearchEngine(String securityToken) { throw new UnsupportedOperationException("Not supported yet."); } Deleted: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngin.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngin.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngin.java 2010-12-02 01:29:28 UTC (rev 551) @@ -1,96 +0,0 @@ -/* - * #%L - * Wikitty :: api - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2009 - 2010 CodeLutin, Benjamin Poussin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ - -package org.nuiton.wikitty.storage; - -import java.util.Collection; -import java.util.Map; -import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.PagedResult; -import org.nuiton.wikitty.entities.Wikitty; -import org.nuiton.wikitty.WikittyException; -import org.nuiton.wikitty.services.WikittyTransaction; - -/** - * WikittySearchEngin is used to abstract search engine used in WikittyService. - * - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public interface WikittySearchEngin { - - /** - * Remove all data in index - */ - public void clear(WikittyTransaction transaction); - - /** - * Store wikitty in storage - * @return information usefull for client side update data - */ - public void store(WikittyTransaction transaction, - Collection<Wikitty> wikitties); - - /** - * Delete all object with idList argument. If id is not valid or don't exist. - * - * @param idList list of ids to delete - * @return <code>UpdateResponse</code>> - * @throws WikittyException - */ - public void delete(WikittyTransaction transaction, - Collection<String> idList) throws WikittyException; - - /** - * Delete all object with idList argument. idList is directly passed to search engine - * and is processed without any other kind of treatment. - * - * @param idList list of ids to delete - * @return <code>UpdateResponse</code> - * @throws WikittyException - */ - public void delete(Collection<String> idList) throws WikittyException; - - public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria); - - /** - * Find count of child for a node - * @param w - * @return - */ - public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter); - - /** - * Find all children ids with count for a node wikitty. - * @param w - * @return - */ - public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter); - -} Deleted: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEnginInMemory.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEnginInMemory.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEnginInMemory.java 2010-12-02 01:29:28 UTC (rev 551) @@ -1,170 +0,0 @@ -/* - * #%L - * Wikitty :: api - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2009 - 2010 CodeLutin, Benjamin Poussin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ -package org.nuiton.wikitty.storage; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import org.nuiton.wikitty.WikittyException; -import org.nuiton.wikitty.entities.FieldType; -import org.nuiton.wikitty.entities.FieldType.TYPE; -import org.nuiton.wikitty.entities.Wikitty; -import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.PagedResult; -import org.nuiton.wikitty.search.operators.And; -import org.nuiton.wikitty.search.operators.BinaryOperator; -import org.nuiton.wikitty.search.operators.Element; -import org.nuiton.wikitty.search.operators.Restriction; -import org.nuiton.wikitty.services.WikittyTransaction; - -public class WikittySearchEnginInMemory implements WikittySearchEngin { - - WikittyStorageInMemory wikittyStorage; - - public WikittySearchEnginInMemory(WikittyStorageInMemory wikittyStorage) { - this.wikittyStorage = wikittyStorage; - } - - @Override - public void clear(WikittyTransaction transaction) { - // do nothing - } - - @Override - public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { - } - - @Override - public void delete(WikittyTransaction transaction, Collection<String> idList) throws WikittyException { - } - - @Override - public void delete(Collection<String> idList) throws WikittyException { - } - - public boolean checkRestriction(Restriction restriction, Wikitty w) { - if (restriction instanceof BinaryOperator) { - BinaryOperator binOp = (BinaryOperator) restriction; - if (binOp.getElement().getName().equals(Element.ELT_EXTENSION)) { - return w.hasExtension(binOp.getValue()); - } - String fqfieldName = binOp.getElement().getName(); - if (Element.ELT_EXTENSION.equals(fqfieldName)) { - return true; - } else if (Element.ELT_ID.equals(fqfieldName)) { - return w.getId().equals(binOp.getValue()); - } // si les wikitty n'ont meme pas l'extension concerné - // Le check restriction, ne doit pas tester les champs - // si les wikitty n'ont meme pas l'extension concerné - String[] extName = fqfieldName.split("\\."); - if (!w.hasField(extName[0], extName[1])) { - return false; - } - Object o = w.getFqField(fqfieldName); - FieldType t = w.getFieldType(fqfieldName); - Object value = t.getValidValue(binOp.getValue()); - boolean checked = false; - switch (restriction.getName()) { - case EQUALS: - checked = value.equals(o); - break; - case LESS: - checked = ((Comparable) o).compareTo(value) < 0; - break; - case LESS_OR_EQUAL: - checked = ((Comparable) o).compareTo(value) <= 0; - break; - case GREATER: - checked = ((Comparable) o).compareTo(value) > 0; - break; - case GREATER_OR_EQUAL: - checked = ((Comparable) o).compareTo(value) >= 0; - break; - case NOT_EQUALS: - checked = !value.equals(o); - break; - case ENDS_WITH: - if (t.getType() != TYPE.STRING) { - throw new WikittyException("Can't search for contents that 'ends with' on attribute type different of String. " + "Attribute " + fqfieldName + " is " + t.getType().name()); - } - checked = ((String) o).endsWith((String) value); - break; - case STARTS_WITH: - if (t.getType() != TYPE.STRING) { - throw new WikittyException("Can't search for contents that 'starts with' on attribute type different of String. " + "Attribute " + fqfieldName + " is " + t.getType().name()); - } - checked = ((String) o).startsWith((String) value); - break; - } - return checked; - } else if (restriction instanceof And) { - And and = (And) restriction; - for (Restriction sub : and.getRestrictions()) { - if (!checkRestriction(sub, w)) { - return false; - } - } - return true; - } else { - throw new UnsupportedOperationException(restriction.getName() + " Search Not yet implemented"); - } - } - - @Override - public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria) { - // throw new UnsupportedOperationException("Not supported yet."); - int firstIndex = criteria.getFirstIndex(); - int endIndex = criteria.getEndIndex(); - List<String> ids = new LinkedList<String>(); - int currentIndex = 0; - for (Entry<String, Wikitty> entry : wikittyStorage.getWikitties().entrySet()) { - Wikitty w = entry.getValue(); - String id = entry.getKey(); - Restriction dto = criteria.getRestriction(); - if (!w.isDeleted() && checkRestriction(dto, w)) { - currentIndex++; - if (currentIndex > firstIndex) { - ids.add(id); - } - if (endIndex >= 0 && currentIndex >= endIndex) { - break; - } - } - } - return new PagedResult<String>(firstIndex, ids.size(), criteria.getRestriction().toString(), null, ids); - } - - @Override - public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - throw new UnsupportedOperationException("Not supported yet."); - } -} Copied: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java (from rev 549, trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngin.java) =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java (rev 0) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java 2010-12-02 01:29:28 UTC (rev 551) @@ -0,0 +1,96 @@ +/* + * #%L + * Wikitty :: api + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2009 - 2010 CodeLutin, Benjamin Poussin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ + +package org.nuiton.wikitty.storage; + +import java.util.Collection; +import java.util.Map; +import org.nuiton.wikitty.search.Criteria; +import org.nuiton.wikitty.search.PagedResult; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.WikittyException; +import org.nuiton.wikitty.services.WikittyTransaction; + +/** + * WikittySearchEngine is used to abstract search engine used in WikittyService. + * + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public interface WikittySearchEngine { + + /** + * Remove all data in index + */ + public void clear(WikittyTransaction transaction); + + /** + * Store wikitty in storage + * @return information usefull for client side update data + */ + public void store(WikittyTransaction transaction, + Collection<Wikitty> wikitties); + + /** + * Delete all object with idList argument. If id is not valid or don't exist. + * + * @param idList list of ids to delete + * @return <code>UpdateResponse</code>> + * @throws WikittyException + */ + public void delete(WikittyTransaction transaction, + Collection<String> idList) throws WikittyException; + + /** + * Delete all object with idList argument. idList is directly passed to search engine + * and is processed without any other kind of treatment. + * + * @param idList list of ids to delete + * @return <code>UpdateResponse</code> + * @throws WikittyException + */ + public void delete(Collection<String> idList) throws WikittyException; + + public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria); + + /** + * Find count of child for a node + * @param w + * @return + */ + public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter); + + /** + * Find all children ids with count for a node wikitty. + * @param w + * @return + */ + public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter); + +} Copied: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java (from rev 549, trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEnginInMemory.java) =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java (rev 0) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2010-12-02 01:29:28 UTC (rev 551) @@ -0,0 +1,170 @@ +/* + * #%L + * Wikitty :: api + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2009 - 2010 CodeLutin, Benjamin Poussin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.wikitty.storage; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.nuiton.wikitty.WikittyException; +import org.nuiton.wikitty.entities.FieldType; +import org.nuiton.wikitty.entities.FieldType.TYPE; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.search.Criteria; +import org.nuiton.wikitty.search.PagedResult; +import org.nuiton.wikitty.search.operators.And; +import org.nuiton.wikitty.search.operators.BinaryOperator; +import org.nuiton.wikitty.search.operators.Element; +import org.nuiton.wikitty.search.operators.Restriction; +import org.nuiton.wikitty.services.WikittyTransaction; + +public class WikittySearchEngineInMemory implements WikittySearchEngine { + + WikittyStorageInMemory wikittyStorage; + + public WikittySearchEngineInMemory(WikittyStorageInMemory wikittyStorage) { + this.wikittyStorage = wikittyStorage; + } + + @Override + public void clear(WikittyTransaction transaction) { + // do nothing + } + + @Override + public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { + } + + @Override + public void delete(WikittyTransaction transaction, Collection<String> idList) throws WikittyException { + } + + @Override + public void delete(Collection<String> idList) throws WikittyException { + } + + public boolean checkRestriction(Restriction restriction, Wikitty w) { + if (restriction instanceof BinaryOperator) { + BinaryOperator binOp = (BinaryOperator) restriction; + if (binOp.getElement().getName().equals(Element.ELT_EXTENSION)) { + return w.hasExtension(binOp.getValue()); + } + String fqfieldName = binOp.getElement().getName(); + if (Element.ELT_EXTENSION.equals(fqfieldName)) { + return true; + } else if (Element.ELT_ID.equals(fqfieldName)) { + return w.getId().equals(binOp.getValue()); + } // si les wikitty n'ont meme pas l'extension concerné + // Le check restriction, ne doit pas tester les champs + // si les wikitty n'ont meme pas l'extension concerné + String[] extName = fqfieldName.split("\\."); + if (!w.hasField(extName[0], extName[1])) { + return false; + } + Object o = w.getFqField(fqfieldName); + FieldType t = w.getFieldType(fqfieldName); + Object value = t.getValidValue(binOp.getValue()); + boolean checked = false; + switch (restriction.getName()) { + case EQUALS: + checked = value.equals(o); + break; + case LESS: + checked = ((Comparable) o).compareTo(value) < 0; + break; + case LESS_OR_EQUAL: + checked = ((Comparable) o).compareTo(value) <= 0; + break; + case GREATER: + checked = ((Comparable) o).compareTo(value) > 0; + break; + case GREATER_OR_EQUAL: + checked = ((Comparable) o).compareTo(value) >= 0; + break; + case NOT_EQUALS: + checked = !value.equals(o); + break; + case ENDS_WITH: + if (t.getType() != TYPE.STRING) { + throw new WikittyException("Can't search for contents that 'ends with' on attribute type different of String. " + "Attribute " + fqfieldName + " is " + t.getType().name()); + } + checked = ((String) o).endsWith((String) value); + break; + case STARTS_WITH: + if (t.getType() != TYPE.STRING) { + throw new WikittyException("Can't search for contents that 'starts with' on attribute type different of String. " + "Attribute " + fqfieldName + " is " + t.getType().name()); + } + checked = ((String) o).startsWith((String) value); + break; + } + return checked; + } else if (restriction instanceof And) { + And and = (And) restriction; + for (Restriction sub : and.getRestrictions()) { + if (!checkRestriction(sub, w)) { + return false; + } + } + return true; + } else { + throw new UnsupportedOperationException(restriction.getName() + " Search Not yet implemented"); + } + } + + @Override + public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria) { + // throw new UnsupportedOperationException("Not supported yet."); + int firstIndex = criteria.getFirstIndex(); + int endIndex = criteria.getEndIndex(); + List<String> ids = new LinkedList<String>(); + int currentIndex = 0; + for (Entry<String, Wikitty> entry : wikittyStorage.getWikitties().entrySet()) { + Wikitty w = entry.getValue(); + String id = entry.getKey(); + Restriction dto = criteria.getRestriction(); + if (!w.isDeleted() && checkRestriction(dto, w)) { + currentIndex++; + if (currentIndex > firstIndex) { + ids.add(id); + } + if (endIndex >= 0 && currentIndex >= endIndex) { + break; + } + } + } + return new PagedResult<String>(firstIndex, ids.size(), criteria.getRestriction().toString(), null, ids); + } + + @Override + public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { + throw new UnsupportedOperationException("Not supported yet."); + } +} Modified: trunk/wikitty-api/src/main/resources/wikitty-config-sample-inmemory.properties =================================================================== --- trunk/wikitty-api/src/main/resources/wikitty-config-sample-inmemory.properties 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/resources/wikitty-config-sample-inmemory.properties 2010-12-02 01:29:28 UTC (rev 551) @@ -37,7 +37,7 @@ org.nuiton.wikitty.services.WikittyServiceSecurity wikitty.WikittyServiceStorage.components=org.nuiton.wikitty.jdbc.WikittyExtensionStorageJDBC,\ org.nuiton.wikitty.jdbc.WikittyStorageJDBC,\ -org.nuiton.wikitty.solr.WikittySearchEnginSolr +org.nuiton.wikitty.solr.WikittySearchEngineSolr wikitty.searchengine.solr.directory.factory=org.nuiton.wikitty.solr.RAMDirectoryFactory wikitty.service.cache.allwaysRestoreCopies=false wikitty.service.event.propagate=false Modified: trunk/wikitty-api/src/main/resources/wikitty-config-sample-server-slave.properties =================================================================== --- trunk/wikitty-api/src/main/resources/wikitty-config-sample-server-slave.properties 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/resources/wikitty-config-sample-server-slave.properties 2010-12-02 01:29:28 UTC (rev 551) @@ -41,7 +41,7 @@ org.nuiton.wikitty.services.WikittyServiceHessianServer wikitty.WikittyServiceStorage.components=org.nuiton.wikitty.jdbc.WikittyExtensionStorageJDBC,\ org.nuiton.wikitty.jdbc.WikittyStorageJDBC,\ -org.nuiton.wikitty.solr.WikittySearchEnginSolr +org.nuiton.wikitty.solr.WikittySearchEngineSolr wikitty.WikittyServiceNotifier.components=org.nuiton.wikitty.services.XMPPNotifierTransporter wikitty.service.server.url=http://services-slave.codelutin.com/wikitty wikitty.service.cache.allwaysRestoreCopies=false Modified: trunk/wikitty-api/src/main/resources/wikitty-config-sample-server.properties =================================================================== --- trunk/wikitty-api/src/main/resources/wikitty-config-sample-server.properties 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/main/resources/wikitty-config-sample-server.properties 2010-12-02 01:29:28 UTC (rev 551) @@ -40,7 +40,7 @@ org.nuiton.wikitty.services.WikittyServiceHessianServer wikitty.WikittyServiceStorage.components=org.nuiton.wikitty.jdbc.WikittyExtensionStorageJDBC,\ org.nuiton.wikitty.jdbc.WikittyStorageJDBC,\ -org.nuiton.wikitty.solr.WikittySearchEnginSolr +org.nuiton.wikitty.solr.WikittySearchEngineSolr wikitty.WikittyServiceNotifier.components=org.nuiton.wikitty.services.XMPPNotifierTransporter wikitty.service.server.url=http://services.codelutin.com/wikitty wikitty.service.cache.listenevents=false Modified: trunk/wikitty-api/src/test/java/org/nuiton/wikitty/services/WikittyServiceHelperTest.java =================================================================== --- trunk/wikitty-api/src/test/java/org/nuiton/wikitty/services/WikittyServiceHelperTest.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-api/src/test/java/org/nuiton/wikitty/services/WikittyServiceHelperTest.java 2010-12-02 01:29:28 UTC (rev 551) @@ -33,7 +33,7 @@ import org.nuiton.wikitty.WikittyService; import org.nuiton.wikitty.WikittyServiceFactory; import org.nuiton.wikitty.storage.WikittyExtensionStorageInMemory; -import org.nuiton.wikitty.storage.WikittySearchEnginInMemory; +import org.nuiton.wikitty.storage.WikittySearchEngineInMemory; import org.nuiton.wikitty.storage.WikittyStorageInMemory; /** @@ -74,7 +74,7 @@ config.setOption(WikittyConfig.WikittyOption.WIKITTY_WIKITTYSERVICESTORAGE_COMPONENTS.getKey(), WikittyStorageInMemory.class.getName() + "," + WikittyExtensionStorageInMemory.class.getName() - + "," + WikittySearchEnginInMemory.class.getName()); + + "," + WikittySearchEngineInMemory.class.getName()); WikittyService ws = WikittyServiceFactory.buildWikittyService(config); Assert.assertEquals(WikittyServiceSecurity.class, ws.getClass()); Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyServiceJDBC.java =================================================================== --- trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyServiceJDBC.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyServiceJDBC.java 2010-12-02 01:29:28 UTC (rev 551) @@ -29,7 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; import org.nuiton.wikitty.services.WikittyServiceStorage; -import org.nuiton.wikitty.solr.WikittySearchEnginSolr; +import org.nuiton.wikitty.solr.WikittySearchEngineSolr; /** * @author poussin @@ -46,7 +46,7 @@ public WikittyServiceJDBC(ApplicationConfig config) { extensionStorage = new WikittyExtensionStorageJDBC(config); wikittyStorage = new WikittyStorageJDBC(config, extensionStorage); - searchEngin = new WikittySearchEnginSolr(config, extensionStorage); + searchEngine = new WikittySearchEngineSolr(config, extensionStorage); } } Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemoryJdbcSolr.java =================================================================== --- trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemoryJdbcSolr.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/services/WikittyServiceInMemoryJdbcSolr.java 2010-12-02 01:29:28 UTC (rev 551) @@ -32,7 +32,7 @@ import org.nuiton.wikitty.WikittyConfig; import org.nuiton.wikitty.jdbc.WikittyExtensionStorageJDBC; import org.nuiton.wikitty.jdbc.WikittyStorageJDBC; -import org.nuiton.wikitty.solr.WikittySearchEnginSolr; +import org.nuiton.wikitty.solr.WikittySearchEngineSolr; /** * In memory implementation that use in memory h2 and in memory solr @@ -67,7 +67,7 @@ extensionStorage = new WikittyExtensionStorageJDBC(config); wikittyStorage = new WikittyStorageJDBC(config, extensionStorage); - searchEngin = new WikittySearchEnginSolr(config, extensionStorage); + searchEngine = new WikittySearchEngineSolr(config, extensionStorage); } } Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java 2010-12-02 01:29:28 UTC (rev 551) @@ -72,7 +72,7 @@ private static final int MAX_SUBQUERY_RESULT = 100; - final static protected WikittySearchEnginSolr.FieldModifier dummyFieldModifier = new WikittySearchEnginSolr.FieldModifier() { + final static protected WikittySearchEngineSolr.FieldModifier dummyFieldModifier = new WikittySearchEngineSolr.FieldModifier() { public String convertToSolr(WikittyTransaction transaction, String fieldname) { return fieldname; } @@ -81,7 +81,7 @@ } }; - protected WikittySearchEnginSolr.FieldModifier fieldModifer; + protected WikittySearchEngineSolr.FieldModifier fieldModifer; protected WikittyTransaction transaction; public Restriction2Solr() { @@ -89,7 +89,7 @@ } // TODO 20101201 jru improve manage transaction and fieldModifeir in helper - public Restriction2Solr(WikittyTransaction transaction, WikittySearchEnginSolr.FieldModifier fieldModifer) { + public Restriction2Solr(WikittyTransaction transaction, WikittySearchEngineSolr.FieldModifier fieldModifer) { this.transaction = transaction; this.fieldModifer = fieldModifer; } @@ -185,7 +185,7 @@ private String associated2solr(AssociatedRestriction associated, SolrServer solr) throws WikittyException { String subQuery = toSolr( associated.getRestriction() ); - SolrQuery query = new SolrQuery(WikittySearchEnginSolr.SOLR_QUERY_PARSER + subQuery); + SolrQuery query = new SolrQuery(WikittySearchEngineSolr.SOLR_QUERY_PARSER + subQuery); query.setRows(MAX_SUBQUERY_RESULT); QueryResponse resp = null; try { @@ -201,11 +201,11 @@ throw new WikittyException("Associated " + associated.getElement().getName() + " do not retrieved any result"); } if ( size == 1 ) { - generatedRestriction = RestrictionHelper.eq( associated.getElement(), (String) solrResults.get(0).getFieldValue(WikittySearchEnginSolr.SOLR_ID) ); + generatedRestriction = RestrictionHelper.eq( associated.getElement(), (String) solrResults.get(0).getFieldValue(WikittySearchEngineSolr.SOLR_ID) ); } else { List<String> ids = new ArrayList<String>(solrResults.size()); for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue(WikittySearchEnginSolr.SOLR_ID); + String id = (String) doc.getFieldValue(WikittySearchEngineSolr.SOLR_ID); ids.add(id); } generatedRestriction = new In(associated.getElement(), ids); @@ -415,11 +415,11 @@ } private String isNull2solr(Null isNull) { - return "( *:* - " + WikittySearchEnginSolr.SOLR_NOT_NULL_FIELDS + ":" + isNull.getFieldName() + ")"; + return "( *:* - " + WikittySearchEngineSolr.SOLR_NOT_NULL_FIELDS + ":" + isNull.getFieldName() + ")"; } private String isNotNull2solr(Null isNotNull) { - return WikittySearchEnginSolr.SOLR_NOT_NULL_FIELDS + ":" + isNotNull.getFieldName(); + return WikittySearchEngineSolr.SOLR_NOT_NULL_FIELDS + ":" + isNotNull.getFieldName(); } private String element2solr(Element element) throws WikittyException { Deleted: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java 2010-12-02 01:29:28 UTC (rev 551) @@ -1,925 +0,0 @@ -/* - * #%L - * Wikitty :: wikitty-solr-impl - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2009 - 2010 CodeLutin, Benjamin POUSSIN - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ - -package org.nuiton.wikitty.solr; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.core.CoreContainer; -import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.FacetTopic; -import org.nuiton.wikitty.entities.FieldType; -import org.nuiton.wikitty.entities.FieldType.TYPE; -import org.nuiton.wikitty.search.PagedResult; -import org.nuiton.wikitty.entities.WikittyTreeNode; -import org.nuiton.wikitty.entities.Wikitty; -import org.nuiton.wikitty.WikittyException; -import org.nuiton.wikitty.entities.WikittyExtension; -import org.nuiton.wikitty.storage.WikittyExtensionStorage; -import org.nuiton.wikitty.storage.WikittySearchEngin; -import org.nuiton.wikitty.services.WikittyTransaction; -import org.nuiton.wikitty.search.operators.Element; -import org.nuiton.wikitty.search.Search; - -import com.arjuna.ats.arjuna.coordinator.BasicAction; -import com.arjuna.ats.arjuna.coordinator.OnePhaseResource; -import com.arjuna.ats.arjuna.coordinator.TwoPhaseOutcome; -import com.arjuna.ats.arjuna.state.InputObjectState; -import com.arjuna.ats.arjuna.state.OutputObjectState; -import com.arjuna.ats.internal.arjuna.abstractrecords.LastResourceRecord; -import java.io.File; -import org.nuiton.util.ApplicationConfig; -import org.nuiton.wikitty.WikittyConfig; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class WikittySearchEnginSolr implements WikittySearchEngin { - - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(WikittySearchEnginSolr.class); - - /** id field in solr */ - static final protected String SOLR_ID = "id"; - - /** extensions field name in solr */ - static final public String SOLR_EXTENSIONS = "extensions"; - - /** group all fields is not null */ - static final public String SOLR_NOT_NULL_FIELDS = "not_null_fields"; - - /** extension use to store field without extension to search on all extesnion */ - static final public String SOLR_ALL_EXTENSIONS = "all"; - - /** Precise the query parser to use, is allow leading wildcard */ - static final public String SOLR_QUERY_PARSER = "{!wikitty}"; - - // Use for indexation tree node - static final public String TREENODE_PREFIX = WikittyTreeNode.EXT_WIKITTYTREENODE + "."; - static final public String TREENODE_EMPTY = TREENODE_PREFIX + "empty"; - static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; - static final public String TREENODE_PATH = TREENODE_PREFIX + "path"; - - /** use to permit client to modify fieldname during query generation */ - static public interface FieldModifier { - public String convertToSolr(WikittyTransaction transaction, String fieldname); - public String convertToField(WikittyTransaction transaction, String solrName); - } - - static protected class TypeFieldModifer implements FieldModifier { - protected WikittyExtensionStorage extensionStorage; - public TypeFieldModifer(WikittyExtensionStorage extensionStorage) { - this.extensionStorage = extensionStorage; - } - - @Override - public String convertToSolr(WikittyTransaction transaction, String fqfieldname) { - String result = fqfieldname; - String[] searchField = fqfieldname.split("\\."); - - if (Element.ELT_EXTENSION.equals(fqfieldname)) { - result = SOLR_EXTENSIONS; - - } else if (Element.ELT_ID.equals(fqfieldname)) { - result = SOLR_ID; - - } else if (searchField.length >= 2) { - String extName = searchField[0]; - String fieldName = searchField[1]; - - if (Criteria.ALL_EXTENSIONS.equals(extName)) { - fqfieldname = SOLR_ALL_EXTENSIONS + "." + fieldName; - } - - if (searchField.length >= 3) { - String fieldNameType = searchField[2]; - TYPE type = FieldType.TYPE.valueOf(fieldNameType); - result = WikittySearchEnginSolr.getSolrFieldName(fqfieldname, type); - return result; - } - - // Search type of field in extension - String version = extensionStorage.getLastVersion(transaction, extName); - if (version != null) { // not valid extension is version == null - WikittyExtension ext = extensionStorage.restore(transaction, extName, version); - FieldType fieldType = ext.getFieldType(fieldName); - log.debug(ext.toDefinition() + " for " + fieldName); - if (fieldType != null) { // type can be null if extension version differ - TYPE type = fieldType.getType(); - result = WikittySearchEnginSolr.getSolrFieldName(fqfieldname, type); - return result; - } - } - } - - return result; - } - - @Override - public String convertToField(WikittyTransaction transaction, String solrName) { - String fieldName = solrName.replaceAll("(_b$)|(_dt$)|(_s$)|(_d$)", ""); - if(SOLR_EXTENSIONS.equals(fieldName)) { - fieldName = Element.ELT_EXTENSION; - } - return fieldName; - } - } - - /** - * Helper to get information nodes and elements for reindexation. - */ - static protected class ReindexChildTreeNode { - - protected SolrResource solrResource; - protected SolrServer solrServer; - - protected Map<String, Collection<String>> includedNodeIds; - protected Map<String, Collection<String>> excludedNodeIds; - protected Map<String, String> parents; - - public ReindexChildTreeNode(SolrServer solrServer, SolrResource solrResource) { - this.solrServer = solrServer; - this.solrResource = solrResource; - includedNodeIds = new HashMap<String, Collection<String>>(); - excludedNodeIds = new HashMap<String, Collection<String>>(); - parents = new HashMap<String, String>(); - } - - public void putIncludedAttachments(String nodeId, Collection<String> attchmentIds) { - putAttachements(includedNodeIds, nodeId, attchmentIds); - } - - public void putExcludedAttachments(String nodeId, Collection<String> attachmentIds) { - putAttachements(excludedNodeIds, nodeId, attachmentIds); - } - - public void putIncludedAttachment(String nodeId, String attachmentId) { - putAttachment(includedNodeIds, nodeId, attachmentId); - } - - public void putExcludedAttachment(String nodeId, String attachmentId) { - putAttachment(excludedNodeIds, nodeId, attachmentId); - } - - public Collection<String> getExcludedNodeIds(String attachmentId) { - Collection<String> result = excludedNodeIds.get(attachmentId); - if (result == null) { - result = new HashSet<String>(); - } - return result; - } - - public Collection<String> getIncludedNodeIds(String attachmentId) { - Collection<String> result = includedNodeIds.get(attachmentId); - if (result == null) { - result = new HashSet<String>(); - } - return result; - } - - protected void putAttachements(Map<String, Collection<String>> map, String nodeId, Collection<String> attachmentIds) { - if (attachmentIds != null) { - for (String attachmentId : attachmentIds) { - putAttachment(map, nodeId, attachmentId); - } - } - } - - protected void putAttachment(Map<String, Collection<String>> map, String nodeId, String attachmentId) { - Collection<String> values = map.get(attachmentId); - if(values == null) { - values = new HashSet<String>(); - map.put(attachmentId, values); - } - values.add(nodeId); - } - - public void putParent(String nodeId, String parentId) { - parents.put(nodeId, parentId); - } - - public String getParent(String nodeId) { - String parentId = parents.get(nodeId); - - // If not found in map, search in index - if(parentId == null) { - SolrDocument doc = findById(solrServer, nodeId); - if(doc == null) { - // is root - return null; - } - parentId = (String) doc.getFieldValue(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT); - parents.put(nodeId, parentId); - } - - Collection<String> deletedDocIds = solrResource.getDeletedDocs(); - if(deletedDocIds.contains(parentId)) { - return null; - } - return parentId; - } - - public Collection<String> getReindexIds() { - Collection<String> result = new HashSet<String>(); - result.addAll(includedNodeIds.keySet()); - result.addAll(excludedNodeIds.keySet()); - result.addAll(solrResource.getAddedDocIds()); - return result; - } - - /** - * Add in doc fields on association between nodes. - * - * For example if you have a element in node with parent, like this - * A -> B -> C => element, the method add field in document solr : - * TreeNode.root : A - * TreeNode.A : B - * TreeNode.B : C - * TreeNode.C : TreeNode.empty - * - * @throws SolrServerException - */ - public void reindex() throws SolrServerException { - for (String id : getReindexIds()) { - - // Get documents - SolrInputDocument doc = solrResource.getAddedDoc(id); - if(doc == null) { - doc = new SolrInputDocument(); - - // Copy old field value - SolrDocument found = findById(solrServer, id); - if (found != null) { - Collection<String> fieldNames = found.getFieldNames(); - for (String fieldName : fieldNames) { - Collection<Object> fieldValues = found.getFieldValues(fieldName); - - if(!fieldName.startsWith(TREENODE_PREFIX)) { - for (Object fieldValue : fieldValues) { - doc.addField(fieldName, fieldValue); - } - } - } - - solrResource.addDoc(id, doc); - } else { - if (log.isWarnEnabled()) { - log.warn("Can't find wikitty id '" + id + "' in index. Skip this wikitty."); - } - } - } - - // Add tree node fields - Collection<String> includedChildNodeIds = getIncludedNodeIds(id); - Collection<String> excludedChildNodeIds = getExcludedNodeIds(id); - - // Find all node contain child - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT + ":" + id); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); - iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - - String nodeId = (String) solrDocument.getFieldValue(SOLR_ID); - includedChildNodeIds.add(nodeId); - } - - // Excluded nodes - includedChildNodeIds.removeAll(excludedChildNodeIds); - includedChildNodeIds.removeAll(solrResource.getDeletedDocs()); - - // Add paths in doc - Map<String, String> paths = new HashMap<String, String>(); - for (String nodeId : includedChildNodeIds) { - doc.addField(TREENODE_PREFIX + nodeId, TREENODE_EMPTY); - - // Add path - String childParent = nodeId; - String parent = getParent(childParent); - while (parent != null) { - String parentPath = paths.get(childParent); - if(parentPath == null) { - doc.addField(TREENODE_PREFIX + parent, childParent); - paths.put(childParent, parent); - } - - childParent = parent; - parent = getParent(childParent); - } - - String parentPath = paths.get(childParent); - if(parentPath == null) { - doc.addField(TREENODE_ROOT, childParent); - paths.put(childParent, TREENODE_ROOT); - } - } - } - } - } - - /** - * Use to plug solr indexation in JTA transaction. - */ - static protected class SolrResource implements OnePhaseResource { - - protected SolrServer solrServer; - protected ThreadLocal<Map<String, SolrInputDocument>> addedDocs; - protected ThreadLocal<List<String>> deletedDocs; - - public SolrResource(SolrServer solrServer) { - this.solrServer = solrServer; - addedDocs = new ThreadLocal<Map<String, SolrInputDocument>>(); - deletedDocs = new ThreadLocal<List<String>>(); - - clear(); - } - - protected void init() { - // Add resource on phase in current transaction - LastResourceRecord lastResourceRecord = new LastResourceRecord(this); - BasicAction.Current().add(lastResourceRecord); - } - - public Map<String, SolrInputDocument> getAddedDocs() { - Map<String, SolrInputDocument> result = addedDocs.get(); - if(result == null) { - result = new HashMap<String, SolrInputDocument>(); - addedDocs.set(result); - } - return result; - } - - public List<String> getDeletedDocs() { - List<String> result = deletedDocs.get(); - if(result == null) { - result = new ArrayList<String>(); - deletedDocs.set(result); - } - return result; - } - - public void clear() { - addedDocs.set(new HashMap<String, SolrInputDocument>()); - deletedDocs.set(new ArrayList<String>()); - } - - public void addDoc(String id, SolrInputDocument doc) { - getAddedDocs().put(id, doc); - } - - public SolrInputDocument getAddedDoc(String id) { - SolrInputDocument result = getAddedDocs().get(id); - return result; - } - - public Collection<String> getAddedDocIds() { - Collection<String> result = getAddedDocs().keySet(); - return result; - } - - public void deleteDoc(String docId) { - getDeletedDocs().add(docId); - } - - @Override - public int commit() { - try { - synchronized(this) { - Collection<SolrInputDocument> docs = getAddedDocs().values(); - if(!docs.isEmpty()) { - solrServer.add(docs); - } - List<String> ids = getDeletedDocs(); - if(!ids.isEmpty()) { - solrServer.deleteById(ids); - } - solrServer.commit(); - } - clear(); - return TwoPhaseOutcome.FINISH_OK; - } catch (Exception eee) { - log.error("Error commit solr", eee); - return TwoPhaseOutcome.FINISH_ERROR; - } - } - - @Override - public int rollback() { - clear(); - return TwoPhaseOutcome.FINISH_OK; - } - - @Override - public void pack(OutputObjectState arg0) throws IOException { - } - - @Override - public void unpack(InputObjectState arg0) throws IOException { - } - } - - /** solr server */ - protected SolrServer solrServer; - - /** Field modifier use to transform to solr format */ - protected TypeFieldModifer fieldModifier; - - /** JTA resource */ - protected SolrResource solrResource; - - /** - * Init wikitty search engin on solr embedded server. - * - * @param extensionStorage extension storage - * @param properties properties (can be null) - */ - public WikittySearchEnginSolr( - ApplicationConfig config, WikittyExtensionStorage extensionStorage) { - - // init system env solr.data.dir - if (config != null) { - // choix du storage (file or Ram) - String solrDirFactoryKey = - WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_FACTORY.getKey(); - String solrDirFactory = config.getOption(solrDirFactoryKey); - if (solrDirFactory != null) { - System.setProperty(solrDirFactoryKey, solrDirFactory); - } - - // on utilise le directory que si on est pas en Ram - if (solrDirFactory != null && !solrDirFactory.contains("RAMDirectoryFactory")) { - String solrDataDirKey = - WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_DATA.getKey(); - String solrDataDir = config.getOption(solrDataDirKey); - // make sure that dir exists - if (solrDataDir != null) { - File file = new File(solrDataDir); - file.mkdirs(); - System.setProperty(solrDataDirKey, solrDataDir); - } - } - } - - try { - CoreContainer.Initializer initializer = new CoreContainer.Initializer(); - CoreContainer coreContainer = initializer.initialize(); - solrServer = new EmbeddedSolrServer(coreContainer, ""); - - fieldModifier = new TypeFieldModifer(extensionStorage); - solrResource = new SolrResource(solrServer); - - } catch (Exception eee) { - throw new WikittyException("SolR initialization error", eee); - } - } - - @Override - public void clear(WikittyTransaction transaction) { - try { - // FIXME poussin 20100618 pourquoi n'est pas fait dans la transaction ? - solrResource.init(); - solrServer.deleteByQuery("*:*"); - } catch (Exception eee) { - throw new WikittyException("Error during clearing SolR data", eee); - } - } - - @Override - public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { - try { - solrResource.init(); - ReindexChildTreeNode reindexChildTreeNode = - new ReindexChildTreeNode(solrServer, solrResource); - for (Wikitty w : wikitties) { - String id = w.getId(); - - if (w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) { - - Set<String> attachments = w.getFieldAsSet(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_ATTACHMENT, String.class); - reindexChildTreeNode.putIncludedAttachments(id, attachments); - - // Search deleted children - SolrDocument treeNodeDoc = findById(solrServer, id); - if (treeNodeDoc != null) { - Collection oldAttachments = treeNodeDoc.getFieldValues(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT); - if (oldAttachments != null) { - // if no more children, remove all old children - if(attachments == null) { - reindexChildTreeNode.putExcludedAttachments(id, oldAttachments); - } else { - // exclude only the removed children - for (Object oldAttachment : oldAttachments) { - if(!attachments.contains(oldAttachment)) { - reindexChildTreeNode.putExcludedAttachment(id,(String) oldAttachment); - } - } - } - } - } - - // Get new parent id (may be the same old parent) - String parentId = w.getFieldAsString(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); - reindexChildTreeNode.putParent(id, parentId); - } - - // Index - SolrInputDocument doc = createIndexDocument(w); - solrResource.addDoc(id, doc); - } - - // Reindex child in tree node - reindexChildTreeNode.reindex(); - } catch (Exception eee) { - throw new WikittyException("Can't store wikitty", eee); - } - } - - @Override - public void delete(WikittyTransaction transaction, Collection<String> ids) throws WikittyException { - try { - solrResource.init(); - ReindexChildTreeNode reindexChildTreeNode = - new ReindexChildTreeNode(solrServer, solrResource); - for (String id : ids) { - - // Find child in node id - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PREFIX + id + ":*"); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - String childId = (String) solrDocument.getFieldValue(SOLR_ID); - reindexChildTreeNode.putExcludedAttachment(id, childId); - } - - solrResource.deleteDoc(id); - } - - // Reindex child in tree node - reindexChildTreeNode.reindex(); - } catch (Exception eee) { - throw new WikittyException("Can't delete wikitty in index", eee); - } - } - - @Override - public void delete(Collection<String> idList) throws WikittyException { - try { - for (String id : idList) { - solrServer.deleteById(id); - } - solrServer.commit(); - } catch (Exception eee) { - throw new WikittyException("Can't delete wikitty in index", eee); - } - } - - @Override - public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria) { - try { - // Create query with restriction - Restriction2Solr restriction2Solr = new Restriction2Solr(transaction, fieldModifier); - String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); - - // Add paged - int firstIndex = criteria.getFirstIndex(); - int endIndex = criteria.getEndIndex(); - - query.setStart(firstIndex); - int nbRows; - if (endIndex == -1) { - // WARNING It is necessary to substract 'start' otherwise, there is a capacity overlow in solR - nbRows = Integer.MAX_VALUE - firstIndex; - } else { - nbRows = endIndex - firstIndex + 1; - } - query.setRows(nbRows); - - // Add sorting - List<String> sortAscending = criteria.getSortAscending(); - if(sortAscending != null) { - for (String sort : sortAscending) { - String tranform = fieldModifier.convertToSolr(transaction, sort); - query.addSortField(tranform, SolrQuery.ORDER.asc); - } - } - - List<String> sortDescending = criteria.getSortDescending(); - if(sortDescending != null) { - for (String sort : sortDescending) { - String tranform = fieldModifier.convertToSolr(transaction, sort); - query.addSortField(tranform, SolrQuery.ORDER.desc); - } - } - - // Add faceting - List<String> facetField = criteria.getFacetField(); - log.debug("facetField : " + facetField); - List<Criteria> facetCriteria = criteria.getFacetCriteria(); - - // use to map query string to criteria facet name - Map<String, String> facetQueryToName = new HashMap<String, String>(); - - if ((facetField != null && !facetField.isEmpty()) - || (facetCriteria != null && !facetCriteria.isEmpty())) { - query.setFacet(true); - query.setFacetMinCount(1); - // query.setFacetLimit(8); // no limit actualy - - // field facetisation - if (facetField != null) { - for (String fqfieldName : facetField) { - String tranform = fieldModifier.convertToSolr(transaction, fqfieldName); - query.addFacetField(tranform); - } - } - - // query facetisation - if (facetCriteria != null) { - for (Criteria facet : facetCriteria) { - String queryFacet = - restriction2Solr.toSolr(facet.getRestriction()); - facetQueryToName.put(queryFacet, facet.getName()); - query.addFacetQuery(queryFacet); - } - } - } - - QueryResponse resp = solrServer.query(query); - SolrDocumentList solrResults = resp.getResults(); - - Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); - if (facetField != null && !facetField.isEmpty()) { - for (FacetField facet : resp.getFacetFields()) { - String facetName = fieldModifier.convertToField(transaction, facet.getName()); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - if (facet.getValues() != null) { - for (FacetField.Count value : facet.getValues()) { - String topicName = value.getName(); - if(!topicName.endsWith(TREENODE_EMPTY)) { - int topicCount = (int) value.getCount(); - FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); - topics.add(topic); - } - } - } - facets.put(facetName, topics); - } - } - if (facetCriteria != null && !facetCriteria.isEmpty()) { - for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { - String facetName = facet.getKey(); - // don't use contains because, map can have key with null value - if (null != facetQueryToName.get(facetName)) { - facetName = facetQueryToName.get(facetName); - } - Integer count = facet.getValue(); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - FacetTopic topic = new FacetTopic(facetName, facetName, count); - topics.add(topic); - facets.put(facetName, topics); - } - } - - List<String> ids = new ArrayList<String>(solrResults.size()); - for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue(SOLR_ID); - ids.add(id); - } - - int numFound = (int)resp.getResults().getNumFound(); - PagedResult<String> result = new PagedResult<String>( - firstIndex, numFound, queryString, facets, ids); - - return result; - } catch (SolrServerException eee) { - throw new WikittyException("Error during find", eee); - } - } - - @Override - public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - String wikittyId = w.getId(); - - String parent = w.getFieldAsWikitty(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); - if(parent == null) { - parent = TREENODE_ROOT; - } else { - parent = TREENODE_PREFIX + parent; - } - - Criteria criteria = Search.query(filter) - .eq(parent, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(0); - PagedResult<String> search = findAllByCriteria(transaction, criteria); - - int numFound = search.getNumFound(); - return numFound; - } - - - @Override - public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - String wikittyId = w.getId(); - - String parent = w.getFieldAsWikitty(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); - if(parent == null) { - parent = TREENODE_ROOT; - } else { - parent = TREENODE_PREFIX + parent; - } - - // Find count with facet, if the node not contain recurcively content, - // the node not found with facet - Criteria criteria = Search.query(filter).eq(parent, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(0) - .addFacetField(TREENODE_PREFIX + wikittyId); - PagedResult<String> search = findAllByCriteria(transaction, criteria); - - Map<String, Integer> counts = new HashMap<String, Integer>(); - List<FacetTopic> topics = search.getTopic(TREENODE_PREFIX + wikittyId); - if(topics != null) { - for (FacetTopic topic : topics) { - String topicName = topic.getTopicName(); - int topicCount = topic.getCount(); - counts.put(topicName, topicCount); - } - } - - log.debug("Facet result " + counts); - - // Find all children, add the other node not found with facet - criteria = Search.query().eq(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); - search = findAllByCriteria(transaction, criteria); - - List<String> children = search.getAll(); - for (String child : children) { - if(!counts.containsKey(child)) { - counts.put(child, 0); - } - } - - return counts; - } - - /** - * Create all index document to used to modify indexation. - * this method don't modify index. - * - * The document looks like : - * SolrId : wikittyId - * extensions : extensionNames - * fieldName : fieldValue - * - * @param w all wikitties object to index - * @return solrInputDocument used to modify index - */ - protected SolrInputDocument createIndexDocument(Wikitty w) { - log.debug("index wikitty " + w.getId()); - - SolrInputDocument doc = new SolrInputDocument(); - String id = w.getId(); - doc.addField(SOLR_ID, id); - - for (String name : w.getExtensionNames()) { - doc.addField(SOLR_EXTENSIONS, name); - } - - for (String fqfieldName : w.fieldNames()) { - FieldType fieldType = w.getFieldType(fqfieldName); - TYPE type = fieldType.getType(); - String solrFqFieldName = getSolrFieldName(fqfieldName, type); - - String[] solrFieldName = solrFqFieldName.split("\\."); - String solrAllFieldName = SOLR_ALL_EXTENSIONS + "." + solrFieldName[1]; - - Object objectValue = w.getFqField(fqfieldName); - if(objectValue != null) { - if (fieldType.isCollection()) { - Collection collectionValue = (Collection) objectValue; - for (Object itemValue : collectionValue) { - if (itemValue != null) { - doc.addField(solrFqFieldName, itemValue); - doc.addField(solrAllFieldName, itemValue); - - // Store string field in differents styles - if(type == TYPE.STRING) { - doc.addField(solrFqFieldName + "_t", itemValue); - doc.addField(solrAllFieldName + "_t", itemValue); - String itemValueLowerCase = itemValue.toString().toLowerCase(); - doc.addField(solrFqFieldName + "_c", itemValueLowerCase); - doc.addField(solrAllFieldName + "_c", itemValueLowerCase); - } - - doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); - log.debug("index field " + solrFqFieldName + " with value '" + itemValue + "'"); - } - } - } else { - doc.addField(solrFqFieldName, objectValue); - doc.addField(solrAllFieldName, objectValue); - - // Store string field in differents styles - if(type == TYPE.STRING) { - doc.addField(solrFqFieldName + "_t", objectValue); - doc.addField(solrAllFieldName + "_t", objectValue); - String objectValueLowerCase = objectValue.toString().toLowerCase(); - doc.addField(solrFqFieldName + "_c", objectValueLowerCase); - doc.addField(solrAllFieldName + "_c", objectValueLowerCase); - } - - doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); - log.debug("index field " + solrFqFieldName + " with value '" + objectValue + "'"); - } - } - } - return doc; - } - - /** - * Find solr document by id - */ - protected static SolrDocument findById(SolrServer solrServer, String id) { - SolrQuery query = new SolrQuery(SOLR_ID + ":" + id); - QueryResponse response; - try { - response = solrServer.query(query); - } catch (SolrServerException eee) { - throw new WikittyException("Error during find", eee); - } - - SolrDocumentList results = response.getResults(); - long numFound = results.getNumFound(); - if(numFound == 1) { - return results.get(0); - } - - return null; - } - - public static String getSolrFieldName(String fqfieldName, TYPE type) { - switch (type) { - case BOOLEAN: - return fqfieldName + "_b"; - case DATE: - return fqfieldName + "_dt"; - case STRING: - return fqfieldName + "_s"; - case NUMERIC: - return fqfieldName + "_d"; - default: - return fqfieldName; - } - } -} Copied: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java (from rev 549, trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEnginSolr.java) =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java (rev 0) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java 2010-12-02 01:29:28 UTC (rev 551) @@ -0,0 +1,925 @@ +/* + * #%L + * Wikitty :: wikitty-solr-impl + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2009 - 2010 CodeLutin, Benjamin POUSSIN + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ + +package org.nuiton.wikitty.solr; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.core.CoreContainer; +import org.nuiton.wikitty.search.Criteria; +import org.nuiton.wikitty.search.FacetTopic; +import org.nuiton.wikitty.entities.FieldType; +import org.nuiton.wikitty.entities.FieldType.TYPE; +import org.nuiton.wikitty.search.PagedResult; +import org.nuiton.wikitty.entities.WikittyTreeNode; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.WikittyException; +import org.nuiton.wikitty.entities.WikittyExtension; +import org.nuiton.wikitty.storage.WikittyExtensionStorage; +import org.nuiton.wikitty.storage.WikittySearchEngine; +import org.nuiton.wikitty.services.WikittyTransaction; +import org.nuiton.wikitty.search.operators.Element; +import org.nuiton.wikitty.search.Search; + +import com.arjuna.ats.arjuna.coordinator.BasicAction; +import com.arjuna.ats.arjuna.coordinator.OnePhaseResource; +import com.arjuna.ats.arjuna.coordinator.TwoPhaseOutcome; +import com.arjuna.ats.arjuna.state.InputObjectState; +import com.arjuna.ats.arjuna.state.OutputObjectState; +import com.arjuna.ats.internal.arjuna.abstractrecords.LastResourceRecord; +import java.io.File; +import org.nuiton.util.ApplicationConfig; +import org.nuiton.wikitty.WikittyConfig; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class WikittySearchEngineSolr implements WikittySearchEngine { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(WikittySearchEngineSolr.class); + + /** id field in solr */ + static final protected String SOLR_ID = "id"; + + /** extensions field name in solr */ + static final public String SOLR_EXTENSIONS = "extensions"; + + /** group all fields is not null */ + static final public String SOLR_NOT_NULL_FIELDS = "not_null_fields"; + + /** extension use to store field without extension to search on all extesnion */ + static final public String SOLR_ALL_EXTENSIONS = "all"; + + /** Precise the query parser to use, is allow leading wildcard */ + static final public String SOLR_QUERY_PARSER = "{!wikitty}"; + + // Use for indexation tree node + static final public String TREENODE_PREFIX = WikittyTreeNode.EXT_WIKITTYTREENODE + "."; + static final public String TREENODE_EMPTY = TREENODE_PREFIX + "empty"; + static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; + static final public String TREENODE_PATH = TREENODE_PREFIX + "path"; + + /** use to permit client to modify fieldname during query generation */ + static public interface FieldModifier { + public String convertToSolr(WikittyTransaction transaction, String fieldname); + public String convertToField(WikittyTransaction transaction, String solrName); + } + + static protected class TypeFieldModifer implements FieldModifier { + protected WikittyExtensionStorage extensionStorage; + public TypeFieldModifer(WikittyExtensionStorage extensionStorage) { + this.extensionStorage = extensionStorage; + } + + @Override + public String convertToSolr(WikittyTransaction transaction, String fqfieldname) { + String result = fqfieldname; + String[] searchField = fqfieldname.split("\\."); + + if (Element.ELT_EXTENSION.equals(fqfieldname)) { + result = SOLR_EXTENSIONS; + + } else if (Element.ELT_ID.equals(fqfieldname)) { + result = SOLR_ID; + + } else if (searchField.length >= 2) { + String extName = searchField[0]; + String fieldName = searchField[1]; + + if (Criteria.ALL_EXTENSIONS.equals(extName)) { + fqfieldname = SOLR_ALL_EXTENSIONS + "." + fieldName; + } + + if (searchField.length >= 3) { + String fieldNameType = searchField[2]; + TYPE type = FieldType.TYPE.valueOf(fieldNameType); + result = WikittySearchEngineSolr.getSolrFieldName(fqfieldname, type); + return result; + } + + // Search type of field in extension + String version = extensionStorage.getLastVersion(transaction, extName); + if (version != null) { // not valid extension is version == null + WikittyExtension ext = extensionStorage.restore(transaction, extName, version); + FieldType fieldType = ext.getFieldType(fieldName); + log.debug(ext.toDefinition() + " for " + fieldName); + if (fieldType != null) { // type can be null if extension version differ + TYPE type = fieldType.getType(); + result = WikittySearchEngineSolr.getSolrFieldName(fqfieldname, type); + return result; + } + } + } + + return result; + } + + @Override + public String convertToField(WikittyTransaction transaction, String solrName) { + String fieldName = solrName.replaceAll("(_b$)|(_dt$)|(_s$)|(_d$)", ""); + if(SOLR_EXTENSIONS.equals(fieldName)) { + fieldName = Element.ELT_EXTENSION; + } + return fieldName; + } + } + + /** + * Helper to get information nodes and elements for reindexation. + */ + static protected class ReindexChildTreeNode { + + protected SolrResource solrResource; + protected SolrServer solrServer; + + protected Map<String, Collection<String>> includedNodeIds; + protected Map<String, Collection<String>> excludedNodeIds; + protected Map<String, String> parents; + + public ReindexChildTreeNode(SolrServer solrServer, SolrResource solrResource) { + this.solrServer = solrServer; + this.solrResource = solrResource; + includedNodeIds = new HashMap<String, Collection<String>>(); + excludedNodeIds = new HashMap<String, Collection<String>>(); + parents = new HashMap<String, String>(); + } + + public void putIncludedAttachments(String nodeId, Collection<String> attchmentIds) { + putAttachements(includedNodeIds, nodeId, attchmentIds); + } + + public void putExcludedAttachments(String nodeId, Collection<String> attachmentIds) { + putAttachements(excludedNodeIds, nodeId, attachmentIds); + } + + public void putIncludedAttachment(String nodeId, String attachmentId) { + putAttachment(includedNodeIds, nodeId, attachmentId); + } + + public void putExcludedAttachment(String nodeId, String attachmentId) { + putAttachment(excludedNodeIds, nodeId, attachmentId); + } + + public Collection<String> getExcludedNodeIds(String attachmentId) { + Collection<String> result = excludedNodeIds.get(attachmentId); + if (result == null) { + result = new HashSet<String>(); + } + return result; + } + + public Collection<String> getIncludedNodeIds(String attachmentId) { + Collection<String> result = includedNodeIds.get(attachmentId); + if (result == null) { + result = new HashSet<String>(); + } + return result; + } + + protected void putAttachements(Map<String, Collection<String>> map, String nodeId, Collection<String> attachmentIds) { + if (attachmentIds != null) { + for (String attachmentId : attachmentIds) { + putAttachment(map, nodeId, attachmentId); + } + } + } + + protected void putAttachment(Map<String, Collection<String>> map, String nodeId, String attachmentId) { + Collection<String> values = map.get(attachmentId); + if(values == null) { + values = new HashSet<String>(); + map.put(attachmentId, values); + } + values.add(nodeId); + } + + public void putParent(String nodeId, String parentId) { + parents.put(nodeId, parentId); + } + + public String getParent(String nodeId) { + String parentId = parents.get(nodeId); + + // If not found in map, search in index + if(parentId == null) { + SolrDocument doc = findById(solrServer, nodeId); + if(doc == null) { + // is root + return null; + } + parentId = (String) doc.getFieldValue(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT); + parents.put(nodeId, parentId); + } + + Collection<String> deletedDocIds = solrResource.getDeletedDocs(); + if(deletedDocIds.contains(parentId)) { + return null; + } + return parentId; + } + + public Collection<String> getReindexIds() { + Collection<String> result = new HashSet<String>(); + result.addAll(includedNodeIds.keySet()); + result.addAll(excludedNodeIds.keySet()); + result.addAll(solrResource.getAddedDocIds()); + return result; + } + + /** + * Add in doc fields on association between nodes. + * + * For example if you have a element in node with parent, like this + * A -> B -> C => element, the method add field in document solr : + * TreeNode.root : A + * TreeNode.A : B + * TreeNode.B : C + * TreeNode.C : TreeNode.empty + * + * @throws SolrServerException + */ + public void reindex() throws SolrServerException { + for (String id : getReindexIds()) { + + // Get documents + SolrInputDocument doc = solrResource.getAddedDoc(id); + if(doc == null) { + doc = new SolrInputDocument(); + + // Copy old field value + SolrDocument found = findById(solrServer, id); + if (found != null) { + Collection<String> fieldNames = found.getFieldNames(); + for (String fieldName : fieldNames) { + Collection<Object> fieldValues = found.getFieldValues(fieldName); + + if(!fieldName.startsWith(TREENODE_PREFIX)) { + for (Object fieldValue : fieldValues) { + doc.addField(fieldName, fieldValue); + } + } + } + + solrResource.addDoc(id, doc); + } else { + if (log.isWarnEnabled()) { + log.warn("Can't find wikitty id '" + id + "' in index. Skip this wikitty."); + } + } + } + + // Add tree node fields + Collection<String> includedChildNodeIds = getIncludedNodeIds(id); + Collection<String> excludedChildNodeIds = getExcludedNodeIds(id); + + // Find all node contain child + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT + ":" + id); + QueryResponse response = solrServer.query(query); + SolrDocumentList updateDocs = response.getResults(); + + for (Iterator<SolrDocument> iterator = updateDocs.iterator(); + iterator.hasNext();) { + SolrDocument solrDocument = iterator.next(); + + String nodeId = (String) solrDocument.getFieldValue(SOLR_ID); + includedChildNodeIds.add(nodeId); + } + + // Excluded nodes + includedChildNodeIds.removeAll(excludedChildNodeIds); + includedChildNodeIds.removeAll(solrResource.getDeletedDocs()); + + // Add paths in doc + Map<String, String> paths = new HashMap<String, String>(); + for (String nodeId : includedChildNodeIds) { + doc.addField(TREENODE_PREFIX + nodeId, TREENODE_EMPTY); + + // Add path + String childParent = nodeId; + String parent = getParent(childParent); + while (parent != null) { + String parentPath = paths.get(childParent); + if(parentPath == null) { + doc.addField(TREENODE_PREFIX + parent, childParent); + paths.put(childParent, parent); + } + + childParent = parent; + parent = getParent(childParent); + } + + String parentPath = paths.get(childParent); + if(parentPath == null) { + doc.addField(TREENODE_ROOT, childParent); + paths.put(childParent, TREENODE_ROOT); + } + } + } + } + } + + /** + * Use to plug solr indexation in JTA transaction. + */ + static protected class SolrResource implements OnePhaseResource { + + protected SolrServer solrServer; + protected ThreadLocal<Map<String, SolrInputDocument>> addedDocs; + protected ThreadLocal<List<String>> deletedDocs; + + public SolrResource(SolrServer solrServer) { + this.solrServer = solrServer; + addedDocs = new ThreadLocal<Map<String, SolrInputDocument>>(); + deletedDocs = new ThreadLocal<List<String>>(); + + clear(); + } + + protected void init() { + // Add resource on phase in current transaction + LastResourceRecord lastResourceRecord = new LastResourceRecord(this); + BasicAction.Current().add(lastResourceRecord); + } + + public Map<String, SolrInputDocument> getAddedDocs() { + Map<String, SolrInputDocument> result = addedDocs.get(); + if(result == null) { + result = new HashMap<String, SolrInputDocument>(); + addedDocs.set(result); + } + return result; + } + + public List<String> getDeletedDocs() { + List<String> result = deletedDocs.get(); + if(result == null) { + result = new ArrayList<String>(); + deletedDocs.set(result); + } + return result; + } + + public void clear() { + addedDocs.set(new HashMap<String, SolrInputDocument>()); + deletedDocs.set(new ArrayList<String>()); + } + + public void addDoc(String id, SolrInputDocument doc) { + getAddedDocs().put(id, doc); + } + + public SolrInputDocument getAddedDoc(String id) { + SolrInputDocument result = getAddedDocs().get(id); + return result; + } + + public Collection<String> getAddedDocIds() { + Collection<String> result = getAddedDocs().keySet(); + return result; + } + + public void deleteDoc(String docId) { + getDeletedDocs().add(docId); + } + + @Override + public int commit() { + try { + synchronized(this) { + Collection<SolrInputDocument> docs = getAddedDocs().values(); + if(!docs.isEmpty()) { + solrServer.add(docs); + } + List<String> ids = getDeletedDocs(); + if(!ids.isEmpty()) { + solrServer.deleteById(ids); + } + solrServer.commit(); + } + clear(); + return TwoPhaseOutcome.FINISH_OK; + } catch (Exception eee) { + log.error("Error commit solr", eee); + return TwoPhaseOutcome.FINISH_ERROR; + } + } + + @Override + public int rollback() { + clear(); + return TwoPhaseOutcome.FINISH_OK; + } + + @Override + public void pack(OutputObjectState arg0) throws IOException { + } + + @Override + public void unpack(InputObjectState arg0) throws IOException { + } + } + + /** solr server */ + protected SolrServer solrServer; + + /** Field modifier use to transform to solr format */ + protected TypeFieldModifer fieldModifier; + + /** JTA resource */ + protected SolrResource solrResource; + + /** + * Init wikitty search engine on solr embedded server. + * + * @param extensionStorage extension storage + * @param properties properties (can be null) + */ + public WikittySearchEngineSolr( + ApplicationConfig config, WikittyExtensionStorage extensionStorage) { + + // init system env solr.data.dir + if (config != null) { + // choix du storage (file or Ram) + String solrDirFactoryKey = + WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_FACTORY.getKey(); + String solrDirFactory = config.getOption(solrDirFactoryKey); + if (solrDirFactory != null) { + System.setProperty(solrDirFactoryKey, solrDirFactory); + } + + // on utilise le directory que si on est pas en Ram + if (solrDirFactory != null && !solrDirFactory.contains("RAMDirectoryFactory")) { + String solrDataDirKey = + WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_DATA.getKey(); + String solrDataDir = config.getOption(solrDataDirKey); + // make sure that dir exists + if (solrDataDir != null) { + File file = new File(solrDataDir); + file.mkdirs(); + System.setProperty(solrDataDirKey, solrDataDir); + } + } + } + + try { + CoreContainer.Initializer initializer = new CoreContainer.Initializer(); + CoreContainer coreContainer = initializer.initialize(); + solrServer = new EmbeddedSolrServer(coreContainer, ""); + + fieldModifier = new TypeFieldModifer(extensionStorage); + solrResource = new SolrResource(solrServer); + + } catch (Exception eee) { + throw new WikittyException("SolR initialization error", eee); + } + } + + @Override + public void clear(WikittyTransaction transaction) { + try { + // FIXME poussin 20100618 pourquoi n'est pas fait dans la transaction ? + solrResource.init(); + solrServer.deleteByQuery("*:*"); + } catch (Exception eee) { + throw new WikittyException("Error during clearing SolR data", eee); + } + } + + @Override + public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { + try { + solrResource.init(); + ReindexChildTreeNode reindexChildTreeNode = + new ReindexChildTreeNode(solrServer, solrResource); + for (Wikitty w : wikitties) { + String id = w.getId(); + + if (w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) { + + Set<String> attachments = w.getFieldAsSet(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_ATTACHMENT, String.class); + reindexChildTreeNode.putIncludedAttachments(id, attachments); + + // Search deleted children + SolrDocument treeNodeDoc = findById(solrServer, id); + if (treeNodeDoc != null) { + Collection oldAttachments = treeNodeDoc.getFieldValues(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT); + if (oldAttachments != null) { + // if no more children, remove all old children + if(attachments == null) { + reindexChildTreeNode.putExcludedAttachments(id, oldAttachments); + } else { + // exclude only the removed children + for (Object oldAttachment : oldAttachments) { + if(!attachments.contains(oldAttachment)) { + reindexChildTreeNode.putExcludedAttachment(id,(String) oldAttachment); + } + } + } + } + } + + // Get new parent id (may be the same old parent) + String parentId = w.getFieldAsString(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); + reindexChildTreeNode.putParent(id, parentId); + } + + // Index + SolrInputDocument doc = createIndexDocument(w); + solrResource.addDoc(id, doc); + } + + // Reindex child in tree node + reindexChildTreeNode.reindex(); + } catch (Exception eee) { + throw new WikittyException("Can't store wikitty", eee); + } + } + + @Override + public void delete(WikittyTransaction transaction, Collection<String> ids) throws WikittyException { + try { + solrResource.init(); + ReindexChildTreeNode reindexChildTreeNode = + new ReindexChildTreeNode(solrServer, solrResource); + for (String id : ids) { + + // Find child in node id + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PREFIX + id + ":*"); + QueryResponse response = solrServer.query(query); + SolrDocumentList updateDocs = response.getResults(); + + for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { + SolrDocument solrDocument = iterator.next(); + String childId = (String) solrDocument.getFieldValue(SOLR_ID); + reindexChildTreeNode.putExcludedAttachment(id, childId); + } + + solrResource.deleteDoc(id); + } + + // Reindex child in tree node + reindexChildTreeNode.reindex(); + } catch (Exception eee) { + throw new WikittyException("Can't delete wikitty in index", eee); + } + } + + @Override + public void delete(Collection<String> idList) throws WikittyException { + try { + for (String id : idList) { + solrServer.deleteById(id); + } + solrServer.commit(); + } catch (Exception eee) { + throw new WikittyException("Can't delete wikitty in index", eee); + } + } + + @Override + public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria) { + try { + // Create query with restriction + Restriction2Solr restriction2Solr = new Restriction2Solr(transaction, fieldModifier); + String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); + + // Add paged + int firstIndex = criteria.getFirstIndex(); + int endIndex = criteria.getEndIndex(); + + query.setStart(firstIndex); + int nbRows; + if (endIndex == -1) { + // WARNING It is necessary to substract 'start' otherwise, there is a capacity overlow in solR + nbRows = Integer.MAX_VALUE - firstIndex; + } else { + nbRows = endIndex - firstIndex + 1; + } + query.setRows(nbRows); + + // Add sorting + List<String> sortAscending = criteria.getSortAscending(); + if(sortAscending != null) { + for (String sort : sortAscending) { + String tranform = fieldModifier.convertToSolr(transaction, sort); + query.addSortField(tranform, SolrQuery.ORDER.asc); + } + } + + List<String> sortDescending = criteria.getSortDescending(); + if(sortDescending != null) { + for (String sort : sortDescending) { + String tranform = fieldModifier.convertToSolr(transaction, sort); + query.addSortField(tranform, SolrQuery.ORDER.desc); + } + } + + // Add faceting + List<String> facetField = criteria.getFacetField(); + log.debug("facetField : " + facetField); + List<Criteria> facetCriteria = criteria.getFacetCriteria(); + + // use to map query string to criteria facet name + Map<String, String> facetQueryToName = new HashMap<String, String>(); + + if ((facetField != null && !facetField.isEmpty()) + || (facetCriteria != null && !facetCriteria.isEmpty())) { + query.setFacet(true); + query.setFacetMinCount(1); + // query.setFacetLimit(8); // no limit actualy + + // field facetisation + if (facetField != null) { + for (String fqfieldName : facetField) { + String tranform = fieldModifier.convertToSolr(transaction, fqfieldName); + query.addFacetField(tranform); + } + } + + // query facetisation + if (facetCriteria != null) { + for (Criteria facet : facetCriteria) { + String queryFacet = + restriction2Solr.toSolr(facet.getRestriction()); + facetQueryToName.put(queryFacet, facet.getName()); + query.addFacetQuery(queryFacet); + } + } + } + + QueryResponse resp = solrServer.query(query); + SolrDocumentList solrResults = resp.getResults(); + + Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); + if (facetField != null && !facetField.isEmpty()) { + for (FacetField facet : resp.getFacetFields()) { + String facetName = fieldModifier.convertToField(transaction, facet.getName()); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + if (facet.getValues() != null) { + for (FacetField.Count value : facet.getValues()) { + String topicName = value.getName(); + if(!topicName.endsWith(TREENODE_EMPTY)) { + int topicCount = (int) value.getCount(); + FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); + topics.add(topic); + } + } + } + facets.put(facetName, topics); + } + } + if (facetCriteria != null && !facetCriteria.isEmpty()) { + for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { + String facetName = facet.getKey(); + // don't use contains because, map can have key with null value + if (null != facetQueryToName.get(facetName)) { + facetName = facetQueryToName.get(facetName); + } + Integer count = facet.getValue(); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + FacetTopic topic = new FacetTopic(facetName, facetName, count); + topics.add(topic); + facets.put(facetName, topics); + } + } + + List<String> ids = new ArrayList<String>(solrResults.size()); + for (SolrDocument doc : solrResults) { + String id = (String) doc.getFieldValue(SOLR_ID); + ids.add(id); + } + + int numFound = (int)resp.getResults().getNumFound(); + PagedResult<String> result = new PagedResult<String>( + firstIndex, numFound, queryString, facets, ids); + + return result; + } catch (SolrServerException eee) { + throw new WikittyException("Error during find", eee); + } + } + + @Override + public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { + String wikittyId = w.getId(); + + String parent = w.getFieldAsWikitty(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); + if(parent == null) { + parent = TREENODE_ROOT; + } else { + parent = TREENODE_PREFIX + parent; + } + + Criteria criteria = Search.query(filter) + .eq(parent, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(0); + PagedResult<String> search = findAllByCriteria(transaction, criteria); + + int numFound = search.getNumFound(); + return numFound; + } + + + @Override + public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { + String wikittyId = w.getId(); + + String parent = w.getFieldAsWikitty(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); + if(parent == null) { + parent = TREENODE_ROOT; + } else { + parent = TREENODE_PREFIX + parent; + } + + // Find count with facet, if the node not contain recurcively content, + // the node not found with facet + Criteria criteria = Search.query(filter).eq(parent, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(0) + .addFacetField(TREENODE_PREFIX + wikittyId); + PagedResult<String> search = findAllByCriteria(transaction, criteria); + + Map<String, Integer> counts = new HashMap<String, Integer>(); + List<FacetTopic> topics = search.getTopic(TREENODE_PREFIX + wikittyId); + if(topics != null) { + for (FacetTopic topic : topics) { + String topicName = topic.getTopicName(); + int topicCount = topic.getCount(); + counts.put(topicName, topicCount); + } + } + + log.debug("Facet result " + counts); + + // Find all children, add the other node not found with facet + criteria = Search.query().eq(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); + search = findAllByCriteria(transaction, criteria); + + List<String> children = search.getAll(); + for (String child : children) { + if(!counts.containsKey(child)) { + counts.put(child, 0); + } + } + + return counts; + } + + /** + * Create all index document to used to modify indexation. + * this method don't modify index. + * + * The document looks like : + * SolrId : wikittyId + * extensions : extensionNames + * fieldName : fieldValue + * + * @param w all wikitties object to index + * @return solrInputDocument used to modify index + */ + protected SolrInputDocument createIndexDocument(Wikitty w) { + log.debug("index wikitty " + w.getId()); + + SolrInputDocument doc = new SolrInputDocument(); + String id = w.getId(); + doc.addField(SOLR_ID, id); + + for (String name : w.getExtensionNames()) { + doc.addField(SOLR_EXTENSIONS, name); + } + + for (String fqfieldName : w.fieldNames()) { + FieldType fieldType = w.getFieldType(fqfieldName); + TYPE type = fieldType.getType(); + String solrFqFieldName = getSolrFieldName(fqfieldName, type); + + String[] solrFieldName = solrFqFieldName.split("\\."); + String solrAllFieldName = SOLR_ALL_EXTENSIONS + "." + solrFieldName[1]; + + Object objectValue = w.getFqField(fqfieldName); + if(objectValue != null) { + if (fieldType.isCollection()) { + Collection collectionValue = (Collection) objectValue; + for (Object itemValue : collectionValue) { + if (itemValue != null) { + doc.addField(solrFqFieldName, itemValue); + doc.addField(solrAllFieldName, itemValue); + + // Store string field in differents styles + if(type == TYPE.STRING) { + doc.addField(solrFqFieldName + "_t", itemValue); + doc.addField(solrAllFieldName + "_t", itemValue); + String itemValueLowerCase = itemValue.toString().toLowerCase(); + doc.addField(solrFqFieldName + "_c", itemValueLowerCase); + doc.addField(solrAllFieldName + "_c", itemValueLowerCase); + } + + doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); + log.debug("index field " + solrFqFieldName + " with value '" + itemValue + "'"); + } + } + } else { + doc.addField(solrFqFieldName, objectValue); + doc.addField(solrAllFieldName, objectValue); + + // Store string field in differents styles + if(type == TYPE.STRING) { + doc.addField(solrFqFieldName + "_t", objectValue); + doc.addField(solrAllFieldName + "_t", objectValue); + String objectValueLowerCase = objectValue.toString().toLowerCase(); + doc.addField(solrFqFieldName + "_c", objectValueLowerCase); + doc.addField(solrAllFieldName + "_c", objectValueLowerCase); + } + + doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); + log.debug("index field " + solrFqFieldName + " with value '" + objectValue + "'"); + } + } + } + return doc; + } + + /** + * Find solr document by id + */ + protected static SolrDocument findById(SolrServer solrServer, String id) { + SolrQuery query = new SolrQuery(SOLR_ID + ":" + id); + QueryResponse response; + try { + response = solrServer.query(query); + } catch (SolrServerException eee) { + throw new WikittyException("Error during find", eee); + } + + SolrDocumentList results = response.getResults(); + long numFound = results.getNumFound(); + if(numFound == 1) { + return results.get(0); + } + + return null; + } + + public static String getSolrFieldName(String fqfieldName, TYPE type) { + switch (type) { + case BOOLEAN: + return fqfieldName + "_b"; + case DATE: + return fqfieldName + "_dt"; + case STRING: + return fqfieldName + "_s"; + case NUMERIC: + return fqfieldName + "_d"; + default: + return fqfieldName; + } + } +} Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/SolrSearchTest.java 2010-12-02 01:29:28 UTC (rev 551) @@ -373,7 +373,7 @@ * WARNING: as the only solr index is clear, all data in all test is reindexed. */ @Test - public void testSyncEngin() { + public void testSyncSearchEngine() { Criteria criteria = new Criteria(); criteria.setRestriction(RestrictionHelper.rTrue()); criteria.setFirstIndex(0); @@ -382,7 +382,7 @@ PagedResult<String> result = ws.findAllByCriteria(null, criteria); assertEquals(13, result.getNumFound()); - ws.syncEngin(null); + ws.syncSearchEngine(null); result = ws.findAllByCriteria(null, criteria); assertTrue(13 <= result.getNumFound()); Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/WikittyServiceSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/WikittyServiceSolr.java 2010-12-02 01:13:56 UTC (rev 550) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/solr/test/WikittyServiceSolr.java 2010-12-02 01:29:28 UTC (rev 551) @@ -27,7 +27,7 @@ import org.nuiton.util.ApplicationConfig; import org.nuiton.wikitty.services.WikittyServiceStorage; -import org.nuiton.wikitty.solr.WikittySearchEnginSolr; +import org.nuiton.wikitty.solr.WikittySearchEngineSolr; import org.nuiton.wikitty.storage.WikittyExtensionStorageInMemory; import org.nuiton.wikitty.storage.WikittyStorageInMemory; @@ -44,6 +44,6 @@ public WikittyServiceSolr(ApplicationConfig config) { extensionStorage = new WikittyExtensionStorageInMemory(); wikittyStorage = new WikittyStorageInMemory(); - searchEngin = new WikittySearchEnginSolr(config, extensionStorage); + searchEngine = new WikittySearchEngineSolr(config, extensionStorage); } }