Author: echatellier Date: 2010-05-31 11:44:30 +0200 (Mon, 31 May 2010) New Revision: 63 Url: http://nuiton.org/repositories/revision/wikitty/63 Log: Add CSV import/export implementation. Modified: trunk/wikitty-api/pom.xml trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ExportTask.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportCSV.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportMethod.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportXML.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportTask.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/WikittyBatchUpdate.java Modified: trunk/wikitty-api/pom.xml =================================================================== --- trunk/wikitty-api/pom.xml 2010-05-28 16:47:31 UTC (rev 62) +++ trunk/wikitty-api/pom.xml 2010-05-31 09:44:30 UTC (rev 63) @@ -71,6 +71,20 @@ <version>1.1.4c</version> </dependency> + <dependency> + <groupId>net.sf.opencsv</groupId> + <artifactId>opencsv</artifactId> + <version>2.1</version> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + <version>1.3</version> + <scope>compile</scope> + </dependency> + <!-- JTA --> <dependency> <groupId>jboss.jbossts</groupId> Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ExportTask.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ExportTask.java 2010-05-28 16:47:31 UTC (rev 62) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ExportTask.java 2010-05-31 09:44:30 UTC (rev 63) @@ -10,6 +10,7 @@ import org.nuiton.wikitty.WikittyService; import org.nuiton.wikitty.WikittyTransaction; import org.nuiton.wikitty.WikittyImportExportService.FORMAT; +import org.nuiton.wikitty.search.Element; public class ExportTask implements Runnable { @@ -35,6 +36,11 @@ public void run() { try { transaction.begin(); + + // use a facet to get only extension used in export + // used for CSV export + criteria.addFacetField(Element.ELT_EXTENSION); + PagedResult<Wikitty> pageResult = ws.findAllByCriteria(transaction, criteria); long time = 0; if (log.isInfoEnabled()) { @@ -42,7 +48,7 @@ log.info("Export started"); } ImportExportMethod exporter = format.ieporter(); - exporter.exportWriter(writer, pageResult.getAll()); + exporter.exportWriter(writer, ws, pageResult); if (log.isInfoEnabled()) { time = System.currentTimeMillis() - time; log.info("Export in (ms)" + time); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportCSV.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportCSV.java 2010-05-28 16:47:31 UTC (rev 62) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportCSV.java 2010-05-31 09:44:30 UTC (rev 63) @@ -17,14 +17,32 @@ package org.nuiton.wikitty.importexport; - import java.io.Reader; import java.io.Writer; +import java.util.Collection; +import java.util.LinkedList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.util.StringUtil; +import org.nuiton.wikitty.Criteria; +import org.nuiton.wikitty.FacetTopic; +import org.nuiton.wikitty.FieldType; +import org.nuiton.wikitty.PagedResult; import org.nuiton.wikitty.Wikitty; +import org.nuiton.wikitty.WikittyExtension; +import org.nuiton.wikitty.WikittyService; +import org.nuiton.wikitty.WikittyUtil; +import org.nuiton.wikitty.search.Element; +import org.nuiton.wikitty.search.Search; +import au.com.bytecode.opencsv.CSVReader; +import au.com.bytecode.opencsv.CSVWriter; + /** * * @author poussin @@ -38,14 +56,228 @@ /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(ImportExportCSV.class); + /** Wikitty id header name. */ + protected final static String FIELD_WIKITTY_ID = "Wikitty.Id"; + + /** Pattern form link queries. */ + protected Pattern queryPattern = Pattern.compile("^((\\w+)\\.(\\w+))=(\"(.+)\"|([^\"]+))$"); + @Override - public void importReader(Reader reader, WikittyBatchUpdate batchUpdate) throws Exception { - throw new UnsupportedOperationException("Not supported yet."); + public void importReader(Reader reader, WikittyService ws, WikittyBatchUpdate batchUpdate) throws Exception { + + // get index of wikitty.id field + int wikittyIdIndex = -1; + + // read header to get extension names + // and build two array for ext and fieldName + CSVReader csvReader = new CSVReader(reader); + String[] header = csvReader.readNext(); + String[] ext = new String[header.length]; + String[] fieldsName = new String[header.length]; + for (int i = 0; i < header.length ; ++i) { + // wikitty id is technical (special management) + if (FIELD_WIKITTY_ID.equals(header[i])) { + wikittyIdIndex = i; + } + else { + ext[i] = header[i].substring(0, header[i].indexOf(".")); + fieldsName[i] = header[i].substring(header[i].indexOf(".") + 1); + } + } + + // create a wikitty for each next line + String[] currentLine = null; + while ((currentLine = csvReader.readNext()) != null) { + + String wikittyId = currentLine[wikittyIdIndex]; + Wikitty currentWikitty = null; + if (StringUtils.isNotEmpty(wikittyId)) { + currentWikitty = new Wikitty(wikittyId); + } + else { + currentWikitty = new Wikitty(); + } + + for (int i = 0; i < header.length ; ++i) { + + // wikitty id column, already managed + if (i == wikittyIdIndex) { + continue; + } + + String extName = ext[i]; + String fieldName = fieldsName[i]; + String value = currentLine[i]; + + // case null or empty + if (StringUtils.isNotEmpty(value)) { + + // extension must exists on wikitty to set a field value + addMissingExtension(ws, currentWikitty, extName); + + // convert link values (if necessary) + value = convertLinkValues(ws, value); + + // add value to correct field + FieldType fieldType = currentWikitty.getFieldType(extName + "." + fieldName); + if(fieldType.isCollection()) { + String[] multiplesValues = StringUtil.split(value); + for (String multiplesValue : multiplesValues) { + // begin and ends with () only if fields + // has multiples values during import + if (multiplesValue.startsWith("(") && multiplesValue.endsWith(")")) { + multiplesValue = multiplesValue.substring(1, multiplesValue.length() - 1); + } + currentWikitty.addToField(extName, fieldName, multiplesValue); + } + } else { + currentWikitty.setField(extName, fieldName, value); + } + } + } + + // add it into datas + batchUpdate.addWikitty(currentWikitty); + } } + + /** + * Recusively add missing extension of not exist and required extension too. + */ + protected void addMissingExtension(WikittyService ws, Wikitty currentWikitty, String extName) { + // extension must exists on wikitty to set a field value + if (!currentWikitty.hasExtension(extName)) { + WikittyExtension extension = ws.restoreExtensionLastVersion(extName); + String requires = extension.getRequires(); + if (StringUtils.isNotEmpty(requires)) { + // add required extensions BEFORE current + for (String require : requires.split(",")) { + String localRequire = require.trim(); + addMissingExtension(ws, currentWikitty, localRequire); + } + } + + currentWikitty.addExtension(extension); + } + } + + /** + * Convert internal queries to search for other real wikitty ids. + * + * @param value + * @return + */ + protected String convertLinkValues(WikittyService ws, String value) { + + String originalValue = value; + String resultValue = ""; + String separator = ""; + boolean correctQueries = true; + + // manage multiples query comma separated + try { + String[] queries = StringUtil.split(value, ","); + + for (String query : queries) { + Matcher m = queryPattern.matcher(query.trim()); + if (m.find()) { + String fqField = m.group(1); + String fValue = m.group(5); + if (fValue == null) { + // quoted value + fValue = m.group(6); + } + Criteria criteria = Search.query().eq(fqField, fValue).criteria(); + Wikitty wikitty = ws.findByCriteria(criteria); + if (wikitty == null) { + correctQueries = false; + } + else { + resultValue += separator + wikitty.getId(); + separator = ","; + } + } + else { + // global parsing fail + // return original value + correctQueries = false; + } + } + } + catch (StringIndexOutOfBoundsException eee) { + if (log.isTraceEnabled()) { + log.trace("Can't split field on , skipping"); + } + } + + // if conversion has not been done, return original value + if (!correctQueries || StringUtils.isEmpty(resultValue)) { + resultValue = originalValue; + } + return resultValue; + } + @Override - public void exportWriter(Writer writer, List<Wikitty> wikitties) throws Exception { - throw new UnsupportedOperationException("Not supported yet."); + public void exportWriter(Writer writer, WikittyService ws, PagedResult<Wikitty> pagedResult) throws Exception { + + CSVWriter csvWriter = new CSVWriter(writer); + + // write all data into writer + List<String> extensionHeader = new LinkedList<String>(); + extensionHeader.add(FIELD_WIKITTY_ID); + for (FacetTopic topic : pagedResult.getTopic(Element.ELT_EXTENSION)) { + String extName = topic.getTopicName(); + + WikittyExtension extension = ws.restoreExtensionLastVersion(extName); + String ext = WikittyExtension.computeName(extName); + for (String fieldName : extension.getFieldNames()) { + extensionHeader.add(ext + "." + fieldName); + } + } + csvWriter.writeNext(extensionHeader.toArray(new String[extensionHeader.size()])); + + if (log.isDebugEnabled()) { + log.debug("Exporting wikitty : " + pagedResult.getAll().size() + " results"); + } + + // Export wikitty data + List<Wikitty> wikitties = pagedResult.getAll(); + for (Wikitty w : wikitties) { + + String[] wikittyField = new String[extensionHeader.size()]; + + // first, add technical id + wikittyField[extensionHeader.indexOf(FIELD_WIKITTY_ID)] = w.getId(); + + // wikitty export must be composed of all field + // corresponding to header extensions names + for (String fieldName : w.fieldNames()) { + String currentField = ""; + + FieldType type = w.getFieldType(fieldName); + if (type.isCollection()) { + Object fqField = w.getFqField(fieldName); + if (fqField != null) { + String separator = ""; + for (Object o : (Collection) fqField) { + String fqFieldValue = WikittyUtil.toString(type, o); + currentField += separator + "(" + fqFieldValue + ")"; + separator = ","; + } + } + } else { + String fqFieldValue = WikittyUtil.toString(type, w.getFqField(fieldName)); + currentField = fqFieldValue; + } + + wikittyField[extensionHeader.indexOf(fieldName)] = currentField; + } + + csvWriter.writeNext(wikittyField); + } + + csvWriter.close(); } } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportMethod.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportMethod.java 2010-05-28 16:47:31 UTC (rev 62) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportMethod.java 2010-05-31 09:44:30 UTC (rev 63) @@ -19,11 +19,14 @@ import java.io.Reader; import java.io.Writer; -import java.util.List; + +import org.nuiton.wikitty.PagedResult; import org.nuiton.wikitty.Wikitty; +import org.nuiton.wikitty.WikittyService; /** - * + * Import / export interface. + * * @author poussin * @version $Revision$ * @@ -31,6 +34,26 @@ * by : $Author$ */ public interface ImportExportMethod { - public void importReader(Reader reader, WikittyBatchUpdate batchUpdate) throws Exception; - public void exportWriter(Writer writer, List<Wikitty> wikitties) throws Exception; + + /** + * Import data from reader. + * + * @param reader reader + * @param ws wikitty service + * @param batchUpdate wikitty batch update helper + * + * @throws Exception + */ + public void importReader(Reader reader, WikittyService ws, WikittyBatchUpdate batchUpdate) throws Exception; + + /** + * Export data to writer. + * + * @param writer writer + * @param ws wikitty service + * @param pagedResult paged result + * + * @throws Exception + */ + public void exportWriter(Writer writer, WikittyService ws, PagedResult<Wikitty> pagedResult) throws Exception; } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportXML.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportXML.java 2010-05-28 16:47:31 UTC (rev 62) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportExportXML.java 2010-05-31 09:44:30 UTC (rev 63) @@ -17,7 +17,6 @@ package org.nuiton.wikitty.importexport; - import java.io.Reader; import java.io.Writer; import java.util.Collection; @@ -27,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; + import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,6 +35,7 @@ import org.nuiton.wikitty.Wikitty; import org.nuiton.wikitty.WikittyException; import org.nuiton.wikitty.WikittyExtension; +import org.nuiton.wikitty.WikittyService; import org.nuiton.wikitty.WikittyUtil; import org.xmlpull.v1.XmlPullParser; import org.xmlpull.v1.XmlPullParserFactory; @@ -53,7 +54,7 @@ static private Log log = LogFactory.getLog(ImportExportXML.class); @Override - public void importReader(Reader reader, WikittyBatchUpdate batchUpdate) throws Exception { + public void importReader(Reader reader, WikittyService ws, WikittyBatchUpdate batchUpdate) throws Exception { XmlPullParserFactory factory = XmlPullParserFactory.newInstance( System.getProperty(XmlPullParserFactory.PROPERTY_NAME), null); factory.setNamespaceAware(true); @@ -68,9 +69,9 @@ do { eventType = xpp.next(); String objectVersion = null; - if (eventType == xpp.START_DOCUMENT) { + if (eventType == XmlPullParser.START_DOCUMENT) { log.info("start XML import at " + new Date()); - } else if (eventType == xpp.START_TAG) { + } else if (eventType == XmlPullParser.START_TAG) { String name = xpp.getName(); if ("extension".equals(name)) { String extName = xpp.getAttributeValue(null, "name"); @@ -95,7 +96,7 @@ w.addExtension(e); } } - } else if (eventType == xpp.END_TAG) { + } else if (eventType == XmlPullParser.END_TAG) { String name = xpp.getName(); if ("extension".equals(name)) { batchUpdate.addExtension(ext); @@ -122,17 +123,19 @@ w.setField(extensionName, fieldName, CDATA); } } - } else if (eventType == xpp.TEXT) { + } else if (eventType == XmlPullParser.TEXT) { CDATA = xpp.getText(); } - } while (eventType != xpp.END_DOCUMENT); + } while (eventType != XmlPullParser.END_DOCUMENT); } @Override - public void exportWriter(Writer result, List<Wikitty> wikitties) throws Exception { + public void exportWriter(Writer result, WikittyService ws, PagedResult<Wikitty> pagedResult) throws Exception { // keep extension already done Set<String> extDone = new HashSet<String>(); result.write("<wikengo>\n"); + + List<Wikitty> wikitties = pagedResult.getAll(); for (Wikitty w : wikitties) { String extensionList = ""; for (WikittyExtension ext : w.getExtensions()) { Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportTask.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportTask.java 2010-05-28 16:47:31 UTC (rev 62) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/ImportTask.java 2010-05-31 09:44:30 UTC (rev 63) @@ -39,7 +39,7 @@ log.info("Import in (ms)" + time); } ImportExportMethod importer = format.ieporter(); - importer.importReader(reader, batchUpdate); + importer.importReader(reader, ws, batchUpdate); if (log.isInfoEnabled()) { time = System.currentTimeMillis() - time; Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/WikittyBatchUpdate.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/WikittyBatchUpdate.java 2010-05-28 16:47:31 UTC (rev 62) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/importexport/WikittyBatchUpdate.java 2010-05-31 09:44:30 UTC (rev 63) @@ -43,7 +43,7 @@ * search extension in local extension list and if missed restore * extension from internal WikittyService * @param id - * @return + * @return extension */ public WikittyExtension getExtension(String id) { WikittyExtension result = exts.get(id);