Author: bleny Date: 2010-05-25 17:24:32 +0200 (Tue, 25 May 2010) New Revision: 51 Url: http://nuiton.org/repositories/revision/diswork/51 Log: menage, documentation, tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-25 15:24:32 UTC (rev 51) @@ -1,28 +0,0 @@ -package org.nuiton.disworkfs; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - -public interface DistributedFileSystem { - - public boolean exists(String path) throws IOException; - - public void write(String path, InputStream source) throws IOException; - - public void write(String parent, String fileName, InputStream source) - throws IOException; - - public InputStream read(String path) throws FileNotFoundException, - IOException; - - public void mkdir(String path) throws IOException; - - public void ln(String path, String target) throws IOException; - - public void remove(String path) throws IOException; - - public List<String> ls(String path) throws IOException; - -} Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-25 15:24:32 UTC (rev 51) @@ -1,14 +0,0 @@ -package org.nuiton.disworkfs; - -import org.nuiton.util.ApplicationConfig; - -public class DisworkConfig extends ApplicationConfig { - - public DisworkConfig() { - setDefaultOption("blocks_size", "10485760"); // 10 MiB - } - - public int getBlockSize() { - return Integer.parseInt(getOption("blocks_size")); - } -} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-25 15:24:32 UTC (rev 51) @@ -12,122 +12,67 @@ import org.nuiton.disworkfs.storage.InMemoryMap; import org.nuiton.disworkfs.storage.Storage; -public class DisworkFileSystem implements DistributedFileSystem { +/** Main class of Diswork File System, provide methods for all operations + * You can use: + * <ul> + * <li>{@link #createDirectory(String)} and {@link DisworkFileSystem#readDirectory(String)} + * to create and browse directories</li> + * <li>{@link #write(String, InputStream)} and {@link #read(String)} + * to write a file and read it</li> + * <li>{@link #createSymbolicLink(String, String)} to create symbolic links</li> + * <li>{@link #exists(String)} and {@link #delete(String)} can be used + * on directories, files and symlinks</li> + * </ul> + */ +public class DisworkFileSystem { + /** + * storage will permit to save and read directories, files and links + */ protected Storage storage; private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException { + /** default constructor (uses default configuration parameters) + * @throws IOException + */ + public DisworkFileSystem() throws IOException { + this(new DisworkFileSystemConfig()); + } + + /** constructor allowing to provide another configuration to use + * + * @param disworkConfig the configuration + * @throws IOException + */ + public DisworkFileSystem(DisworkFileSystemConfig disworkConfig) + throws IOException { storage = new Storage(disworkConfig, new InMemoryMap()); storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); } - @Override + /** tests the existence of a file/dir/link at a given path + * return true if something exists at path p, it will be true if a call + * to <code>mkdir(p)</code>, <code>write(p, ?)</code> or + * <code>ln(p, ?)</code> has been done before. + * @param path a path in the virtual FS + * @return true is something (a link, a file, or a directory) exists at path + * @throws IOException + */ public boolean exists(String path) throws IOException { String entry = walk(path); boolean result = entry != null; return result; } - + /** - * return the entry of the element at the end of <code>path</code> - * - * @param path - * @return null if path is not valid + * use this method to read the content of a file. A call of read may be + * done after a call to {@link #write} + * @param path the path to the file to read + * @return an InputStream on the file + * @throws FileNotFoundException if no file exists at this path + * @throws IOException if path exists but is a directory */ - protected String walk(String path) throws IOException { - String result = walk(path, null, null); - log.info("walking to " + path + " returns " + result); - return result; - } - - // FIXME 20105021 bleny works fine but is not understandable - protected String walk(String path, String current, String content) - throws IOException { - String result = null; - - if (path.equals("/")) { - return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "/" + EntryUtil.ENTRY_SEPARATOR - + "/"; - } - - - String parentPath = EntryUtil.getParentFromPath(path); - - if (content == null) { - // start of walk - content = storage.getRootDirectory(); - result = walk(path, "/", content); - } else if (parentPath.equals(current)) { - // in the last directory - // recuperation de l'element a traiter dans p - - String tail = path.substring(current.length()); - - /* - log.info("tail " + tail); - String[] elementsNames = tail.split("/"); - String p = elementsNames[0]; - */ - String p = EntryUtil.getNameFromPath(tail); - - log.info("in final dir " + current + ", looking for " + p); - - String entry = EntryUtil.findEntryInDirectory(content, p); - result = entry; - } else { - // in middle of path - - String tail; // the path still remaining when in current - - if (current.equals("/")) { - tail = path.substring(current.length()); - } else { - tail = path.substring(current.length() + 1); - } - - log.debug("current = " + current); - log.debug("tail = " + tail); - String[] elementsNames = tail.split("/"); - String p = elementsNames[0]; - - log.info("in intermediate dir " + current + ", looking for " + p); - - // mise a jour de current - if (current.equals("/")) current = ""; - current += "/" + p; - - String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry == null) { - result = null; - } else { - if (EntryUtil.isDirectory(entry)) { - String id = EntryUtil.getId(entry); - content = storage.getDirectory(id); - result = walk(path, current, content); - } else if (EntryUtil.isLink(entry)) { - String id = EntryUtil.getId(entry); - String linkContent = storage.getLink(id); - String newTarget = EntryUtil.resolveLink(current, linkContent); - newTarget += path.substring(current.length()); - - // restart walk from / - result = walk(newTarget, null, null); - } else if (EntryUtil.isFile(entry)) { - // erreur, find file in path like '/dir1/dir2/filename/dir3' - result = null; - } else { - log.warn("strange case : " + entry); - result = null; - } - } - } - return result; - } - - @Override public InputStream read(String path) throws FileNotFoundException, IOException { @@ -140,7 +85,7 @@ if (EntryUtil.isLink(entry)) { log.info("reading link " + path); - String id = EntryUtil.getId(entry); + String id = EntryUtil.getIdFromEntry(entry); String link = storage.getLink(id); String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); @@ -148,21 +93,26 @@ throw new IOException("target is not a file: " + path); } else if (EntryUtil.isFile(entry)) { log.info("reading file " + path); - String id = EntryUtil.getId(entry); + String id = EntryUtil.getIdFromEntry(entry); result = storage.getFile(id); - } + } return result; } - @Override + /** + * write a file on the file system at a given place + * @param path + * @param source + * @throws IOException if file already exists + */ public void write(String path, InputStream source) throws IOException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); write(parent, name, source); } - - @Override - public void write(String parent, String fileName, InputStream source) throws IOException { + + protected void write(String parent, String fileName, InputStream source) + throws IOException { String entryParent = walk(parent); if (entryParent == null) { @@ -170,11 +120,20 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); + + // checking that file do not already exists in this directory + String findResult = EntryUtil.findEntryInDirectory + (content, fileName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + fileName); + } + String newFileId = EntryUtil.generateId(); String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.F, fileName, newFileId); + content, EntryUtil.TYPE.F, fileName, newFileId); // store file before meta info storage.putFile(newFileId, source); @@ -185,14 +144,18 @@ } } - @Override - public void mkdir(String path) throws IOException { + /** + * creates a new empty directory + * @param path the absolute path (ending by the name) of the directory + * @throws IOException if parent path is not correct + */ + public void createDirectory(String path) throws IOException { String parent = EntryUtil.getParentFromPath(path); String dirName = EntryUtil.getNameFromPath(path); - mkdir(parent, dirName); + createDirectory(parent, dirName); } - public void mkdir(String parent, String dirName) throws IOException { + protected void createDirectory(String parent, String dirName) throws IOException { log.info("trying to create directory " + dirName + " in " + parent); @@ -203,7 +166,7 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); String newDirectoryId = EntryUtil.generateId(); String newContent = EntryUtil.addEntryToDirectoryContent( @@ -219,14 +182,25 @@ } } - @Override - public void ln(String path, String target) throws IOException { + /** + * create a new symbolic link + * @param path the path (ending by the name) where the link will be created + * @param target the path where the link point to + * (may be relative or absolute) + * @throws IOException if parent path is not correct + */ + public void createSymbolicLink(String path, String target) + throws IOException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); - ln(parent, name, target); + createSymbolicLink(parent, name, target); } - public void ln(String parent, String name, String target) throws IOException { + /** + * @see #createSymbolicLink(String, String) + */ + protected void createSymbolicLink(String parent, String name, String target) + throws IOException { if (target.startsWith("/")) { // target is absolute @@ -244,11 +218,11 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); String newLinkId = EntryUtil.generateId(); String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.L, name, newLinkId); + content, EntryUtil.TYPE.L, name, newLinkId); // store link before meta info storage.putLink(newLinkId, target); @@ -264,15 +238,23 @@ } } - @Override - public void remove(String path) throws IOException { + /** + * remove a file, directory, or link. Non-empty directories can't be + * removed + * @param path the complete path to the entity to remove + * @throws IOException if path is incorrect or directory not empty + */ + public void delete(String path) throws IOException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); log.info("trying to remove " + path); - remove(parent, name); + delete(parent, name); } - - protected void remove(String parent, String name) throws IOException { + + /** + * @see #delete(String) + */ + protected void delete(String parent, String name) throws IOException { String entryParent = walk(parent); if (entryParent == null) { @@ -280,25 +262,28 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); String entry = EntryUtil.findEntryInDirectory(content, name); - String idToRemove = EntryUtil.getId(entry); + String idToRemove = EntryUtil.getIdFromEntry(entry); String newContent = EntryUtil.removeEntryFromEntries( content, name); // check if not removing a non-empty directory if (EntryUtil.isDirectory(entry)) { - String innerDirectoryId = EntryUtil.getId(entry); - String innerDirectoryContent = storage.getDirectory(innerDirectoryId); - if (! innerDirectoryContent.equals(EntryUtil.newEmptyDirectoryContent())) { + String innerDirectoryId = EntryUtil.getIdFromEntry(entry); + String innerDirectoryContent = + storage.getDirectory(innerDirectoryId); + // checking the emptiness of the directory + if (!innerDirectoryContent.equals( + EntryUtil.newEmptyDirectoryContent())) { // directory is not empty - throw new IOException("trying to remove a non-empty directory"); + throw new IOException + ("trying to remove a non-empty directory"); } } - // update meta info directory storage.putDirectory(parentId, newContent); @@ -309,27 +294,34 @@ throw new IOException(parent + " is not a directory"); } } - - @Override - public List<String> ls(String path) throws IOException { + + /** + * list the content of a directory + * @param path the complete path to the directory + * @return a list of the names of the elements in <code>path</code> + * @throws IOException if path doesn't point to a directory + */ + public List<String> readDirectory(String path) throws IOException { String entry = walk(path); List<String> result = null; - if (entry == null) { throw new IOException(path + " doesn't exists"); } else { + // path may be a link, if it's the case, + // entry become the actual directory if (EntryUtil.isLink(entry)) { - String target = storage.getLink(EntryUtil.getId(entry)); + String target = storage.getLink( + EntryUtil.getIdFromEntry(entry)); entry = walk(target); - } - + } if (EntryUtil.isDirectory(entry)) { - String content = storage.getDirectory(EntryUtil.getId(entry)); + String content = storage.getDirectory( + EntryUtil.getIdFromEntry(entry)); String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR); result = new ArrayList<String>(); for (String elementEntry : entries) { - result.add(EntryUtil.getName(elementEntry)); + result.add(EntryUtil.getNameFromEntry(elementEntry)); } } else { throw new IOException(path + " is not a directory"); @@ -339,4 +331,111 @@ return result; } + /** + * return the entry of the element at the end of <code>path</code> + * @param path + * @return null if path is not valid + */ + protected String walk(String path) throws IOException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + /** + * This method is a recursive function to walk through the tree structure + * of the directories and starting for root directory, following the + * symbolic links to reach the given path + * @param path + * @param current + * @param content + * @return null if path is not valid or the entry corresponding to path + * @throws IOException + */ + + // FIXME 20105021 bleny works fine but is not understandable + protected String walk(String path, String current, String content) + throws IOException { + String result = null; + + if (path.equals("/")) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "/" + EntryUtil.ENTRY_SEPARATOR + + "/"; + } + + String parentPath = EntryUtil.getParentFromPath(path); + + if (content == null) { + // start of walk + content = storage.getRootDirectory(); + result = walk(path, "/", content); + } else if (parentPath.equals(current)) { + // in the last directory + // recuperation de l'element a traiter dans p + + String tail = path.substring(current.length()); + + /* + log.info("tail " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; + */ + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + + String entry = EntryUtil.findEntryInDirectory(content, p); + result = entry; + } else { + // in middle of path + + String tail; // the path still remaining when in current + + if (current.equals("/")) { + tail = path.substring(current.length()); + } else { + tail = path.substring(current.length() + 1); + } + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; + + log.info("in intermediate dir " + current + ", looking for " + p); + + // mise a jour de current + if (current.equals("/")) current = ""; + current += "/" + p; + + String entry = EntryUtil.findEntryInDirectory(content, p); + if (entry == null) { + result = null; + } else { + if (EntryUtil.isDirectory(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + content = storage.getDirectory(id); + result = walk(path, current, content); + } else if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + String linkContent = storage.getLink(id); + String newTarget = + EntryUtil.resolveLink(current, linkContent); + newTarget += path.substring(current.length()); + + // restart walk from / + result = walk(newTarget, null, null); + } else if (EntryUtil.isFile(entry)) { + // error, found file in path like '/dir1/dir2/filename/dir3' + result = null; + } else { + log.warn("strange case: " + entry); + result = null; + } + } + } + return result; + } + } Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java (from rev 50, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,26 @@ +package org.nuiton.disworkfs; + +import org.nuiton.util.ApplicationConfig; + +/** + * This object contains parameters used in {@link DisworkFileSystem}. + * + * Available parameters are: + * <dl> + * <dt>blocks_size</dt> + * <dd> + * The size of blocks to create when splitting big data in bytes + * (by default, 10 MiB) + * </dd> + * </dl> + */ +public class DisworkFileSystemConfig extends ApplicationConfig { + + public DisworkFileSystemConfig() { + setDefaultOption("blocks_size", "10485760"); // 10 MiB + } + + public int getBlockSize() { + return getOptionAsInt("blocks_size"); + } +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,10 @@ +/** + * DisworkFS is a distributed file system. You can use it by instantiating + * {@link DisworkFileSystem} and use it to store directories and their content. + * + * You can change the default DisworkFileSystem behavior by provide a + * {@link DisworkFileSystemConfig} instance at construction. If you don't + * provide one, a instance will be created + */ + +package org.nuiton.disworkfs; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-25 15:24:32 UTC (rev 51) @@ -7,28 +7,51 @@ import org.apache.commons.io.FilenameUtils; /** + * This class contains String utilities to ease operations on + * <dl> + * <dt>a single entry;</dt> + * <dd>a record in the list of content in a directory</dd> + * <dt>multiple entries;</dt> + * <dd>a string containing multiple entry (as above) separated by + * {@link #ENTRIES_SEPARATOR} + * </dd> + * <dt>a metablock;</dt> + * <dd>a metablock is a String that begins with the total size of the data + * it describes and the ids where some blocks containing the data + * are stored into the map</dd> + * <dt>a path.</dt> + * <dd>a string like /a/b/c/d</dd> + * </dl> + * * @author poussin * @version $Revision$ * * Last update: $Date$ - * by : $Author$ + * by: $Author$ */ public class EntryUtil { + static public enum TYPE { /** Directory */ D, + /** Link */ L, + /** File */ F + }; + static final int TYPE_LENGTH = 1; + static final public String ENTRY_SEPARATOR = ":"; static final public String ENTRIES_SEPARATOR = "\n"; static final public String BLOCKIDS_SEPARATOR = ";"; - - static final int TYPE_LENGTH = 1; - /* - // this line may not be accurate for "/" - static final int UUID_LENGTH = UUID.randomUUID().toString().length(); - */ + static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); - static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; + /** + * This charset is forced in string-to-bytes and bytes-to-string conversions + */ + static protected Charset CHARSET = Charset.forName("UTF-8"); - static protected Charset CHARSET = Charset.forName("UTF-8"); + /** + * This is a utility class thus should not be instantiated + */ + protected EntryUtil() {}; /** * generate a new Id usage as a new key for the map @@ -45,7 +68,7 @@ * a link */ static public boolean isDirectory(String entry) { - boolean result = entry.startsWith(TYPE.D.name()); + boolean result = getTypeFromEntry(entry) == TYPE.D; return result; } @@ -55,7 +78,7 @@ * a directory */ static public boolean isLink(String entry) { - boolean result = entry.startsWith(TYPE.L.name()); + boolean result = getTypeFromEntry(entry) == TYPE.L; return result; } @@ -65,47 +88,31 @@ * a directory */ static public boolean isFile(String entry) { - boolean result = entry.startsWith(TYPE.F.name()); + boolean result = getTypeFromEntry(entry) == TYPE.F; return result; } - static public TYPE getType(String entry) { + static public TYPE getTypeFromEntry(String entry) { int index = entry.indexOf(ENTRY_SEPARATOR); String type = entry.substring(0, index); TYPE result = TYPE.valueOf(type); return result; } - static public String getName(String entry) { + static public String getNameFromEntry(String entry) { int start = entry.indexOf(ENTRY_SEPARATOR); int last = entry.lastIndexOf(ENTRY_SEPARATOR); String result = entry.substring(start + ENTRY_SEPARATOR_LENGTH, last); return result; } - static public String getId(String entry) { + static public String getIdFromEntry(String entry) { int index = entry.lastIndexOf(ENTRY_SEPARATOR); String result = entry.substring(index + ENTRY_SEPARATOR_LENGTH); return result; } + - static public int getTotalSizeFromMetaBlock(String metaBlock) { - // dealing with empty meta-block (for empty directory or file) - if (metaBlock.equals("0")) return 0; - String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); - return Integer.valueOf(result); - } - - static public String[] getBlockIdsFromMetaBlock(String metaBlock) { - // dealing with empty meta-block (for empty directory or file) - if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { - return new String[0]; - } - String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length()); - String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); - return blockIds; - } - /** * find file name in directory content description * @@ -126,6 +133,12 @@ return result; } + public static String removeEntryFromEntries(String content, String name) { + String entry = findEntryInDirectory(content, name); + String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, ""); + return newContent; + } + /** * */ @@ -135,51 +148,38 @@ /** * - * @param content the string content of the directory (all entries before add) + * @param content the string content of the directory + * (all entries before add) * @param type the type of the new entry * @param name the name (not a full path) of the new entry * @param id the id where the content of the entry can be found * @return a String with the new content */ static public String addEntryToDirectoryContent - (String content, TYPE type, String name, String id) { - content += type.name() + ENTRY_SEPARATOR + name + - ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR; - return content; + (String content, TYPE type, String name, String id) { + String newEntry = type.name() + ENTRY_SEPARATOR + name + + ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR; + return content + newEntry; } - /** - * Resolve a path from a parent directory, - * - * ie resolve /dir/subdir, ../file returns /dir/file - * @param parent the path to the dir where start from compute the path - * @param link the path to follow (may be absolute or relative to - * <code>parent</code>) - * @return the path to the destination - */ - static public String resolveLink(String parent, String link) { - /* - String result = null; - if (link.startsWith("/")) { - // destination is absolute, ignore parent - result = link; - } else { - // destination is relative, so we have to consider parent - // parent may have a trailing file name, let's remove it - String parentWithNoFile = parent; - if (!parent.endsWith("/")) { - // removing the file at the end of the path, it's not a - // directory and thus should not be part of the path - parentWithNoFile = parent.substring(0, parent.lastIndexOf("/") + 1); - } - - // concatenate and making the complete path consistent - // ie /dir/subdir/../file to /dir/file - result = FilenameUtils.normalize(parentWithNoFile + link); + static public int getTotalSizeFromMetaBlock(String metaBlock) { + // dealing with empty meta-block (for empty directory or file) + if (metaBlock.equals("0")) return 0; + String result = metaBlock.substring(0, + metaBlock.indexOf(BLOCKIDS_SEPARATOR)); + return Integer.valueOf(result); + } + + static public String[] getBlockIdsFromMetaBlock(String metaBlock) { + // dealing with empty meta-block (for empty directory or file) + if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { + return new String[0]; } - return result; - */ - return FilenameUtils.concat(parent, link); + String blockIdsAsString = metaBlock.substring( + metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, + metaBlock.length()); + String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); + return blockIds; } /** @@ -189,20 +189,31 @@ * @return a byte array containing all the bytes in <code>string</code> */ public static byte[] stringToBytes(String string) { - // FIXME 20100519 bleny this code may cause bad conversion - // due to charsets byte[] bytes = string.getBytes(CHARSET); return bytes; } /** - * @see {@link #stringToBytes(String)} - * @param string the string + * @see #stringToBytes(String) + * @param bytes the bytes to convert * @return a byte array containing all the bytes in <code>string</code> */ public static String bytesToString(byte[] bytes) { return new String(bytes, CHARSET); } + + /** + * Resolve a path from a parent directory and a link + * + * ie resolve /dir/subdir, ../file returns /dir/file + * @param parent the path to the dir where start from compute the path + * @param link the path to follow (may be absolute or relative to + * <code>parent</code>) + * @return the path to the destination + */ + static public String resolveLink(String parent, String link) { + return FilenameUtils.concat(parent, link); + } public static String getParentFromPath(String path) { String parent = FilenameUtils.getFullPathNoEndSeparator(path); @@ -215,11 +226,5 @@ public static String getNameFromPath(String path) { return FilenameUtils.getName(path); } - - public static String removeEntryFromEntries(String content, String name) { - String entry = findEntryInDirectory(content, name); - String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, ""); - return newContent; - } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-25 15:24:32 UTC (rev 51) @@ -6,6 +6,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +/** An in-memory implementation of a Map + * This class is a map where put make a copy of the value. It is used for + * stubbing purpose. It emulates the behavior of a DHT and thus can be used + * by {@link Storage} to store data and blocks. + */ public class InMemoryMap extends HashMap<String, byte[]> { private final Log log = LogFactory.getLog(InMemoryMap.class); @@ -17,4 +22,4 @@ log.info("putting " + valueCopy.length + " bytes"); return super.put(key, valueCopy); } -} +} \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 15:24:32 UTC (rev 51) @@ -8,12 +8,20 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkConfig; +import org.nuiton.disworkfs.DisworkFileSystemConfig; +/** + * + */ public class Storage { private static final Log log = LogFactory.getLog(Storage.class); + /** + * This class is a InputStream used to read data from the map. When + * someone read from the filesytem, he will bre returned a + * SplitBlocksInputStream. This InputStream loads blocks only when needed + */ protected class SplitBlocksInputStream extends InputStream { protected Map<String, byte[]> map; @@ -45,6 +53,8 @@ if (currentBlock == null || currentBlock.available() == 0) { blockIdsIndex += 1; if (blockIdsIndex < blockIds.length) { + // closing previously opened stream + IOUtils.closeQuietly(currentBlock); currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex])); log.debug("new current block (size = " + currentBlock.available() + ")"); } else { @@ -53,16 +63,18 @@ } int result = currentBlock.read(); - /* - log.debug("reading block number " + blockIdsIndex + - " (available = " + available + ")" + - " returns " + result); - */ available -= 1; - + + if(log.isDebugEnabled()) { + log.debug("reading block number " + blockIdsIndex + + " (available = " + available + ")" + + " returns " + result); + } return result; } + // This method is called at the first read or the first available + // it initialize all the necessary field protected void checkInitialization() { if (blockIds == null) { byte[] bytes = map.get(id); @@ -86,17 +98,13 @@ protected Map<String, byte[]> map; - protected DisworkConfig disworkConfig; + protected DisworkFileSystemConfig disworkConfig; - public Storage(DisworkConfig disworkConfig, Map<String, byte[]> map) { + public Storage(DisworkFileSystemConfig disworkConfig, Map<String, byte[]> map) { this.map = map; this.disworkConfig = disworkConfig; } - public boolean contains(Object id) { - return map.containsKey(id); - } - /** * @return the content (entries) of the root directory * @throws IOException @@ -108,8 +116,11 @@ - - + /** + * @param id an id of a directory + * @return a the entries of the directory + * @throws IOException + */ public String getDirectory(String id) throws IOException { InputStream in = get(id); String content = IOUtils.toString(in); @@ -162,36 +173,37 @@ protected void put(String key, InputStream value) throws IOException { // TODO 20100519 bleny deal with copy-on-write - // TODO 20100519 bleny deal with null value properly + // TODO 20100519 bleny deal with null value properly if (value == null) { value = new ByteArrayInputStream(new byte[0]); } String blocksIds = ""; - int read = 0; + int readResult = 0; int totalSize = 0; + // creating a buffer of the size of a block int bufferSize = disworkConfig.getBlockSize(); byte[] buffer = new byte[bufferSize]; - while ((read = value.read(buffer)) != -1) { - totalSize += read; + while ((readResult = value.read(buffer)) != -1) { + totalSize += readResult; byte[] newBlock = buffer; - // if the block is shorter, truncate - if (read < buffer.length) { - newBlock = new byte[read]; - System.arraycopy(buffer, 0, newBlock, 0, read); + // if the buffer is not full, truncate the block + if (readResult < buffer.length) { + newBlock = new byte[readResult]; + System.arraycopy(buffer, 0, newBlock, 0, readResult); } String id = EntryUtil.generateId(); blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; - log.debug("saving new block (size = " + newBlock.length - + ") at key " + id); + log.debug("saving new block (size = " + newBlock.length + ")" + + " at key " + id); - // copy buffer in map + // copy block in map map.put(id, newBlock); } @@ -208,5 +220,4 @@ map.remove(key); } - -} +} \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,203 @@ +/** + * <p> + * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a + * way to persistently store data. This is done by the {@link Storage} class + * which permit to store different type of data. + * </p> + * <p> + * We illustrate the use of a file system by setting the content of "/", + * the root directory as follow: + * </p> + * <ul> + * <li> + * a directory named "my_folder" that contains: + * <ul> + * <li> + * a directory named "my_sub_folder" that contains + * <ul> + * <li>a file name "my_deep_file.ext"</li> + * <li>an empty folder named "my_empty_folder"</li> + * </ul> + * </li> + * <li>a file named "my_other_file.ext"</li> + * </ul> + * </li> + * <li> + * a file named "my_file.ext" (containing binary data: 010101) + * </li> + * <li> + * a symbolic link named "my_link" which target the path + * <code>"/my_folder/my_sub_folder/my_deep_file.ext"</code> + * </li> + * </ul> + * + * Storage uses a Map to store data. + * + * <table> + * <caption>Use of map, abstract point of view</caption> + * <tr> + * <th>Key (String)</th> + * <th>Value (byte[])</th> + * </tr> + * <tr> + * <td>"/"</td> + * <td>"D:my_folder:ID1\n<br /> + * F:my_file.ext:ID2\n<br /> + * L:my_link:ID3"</td> + * </tr> + * <tr> + * <td>"ID1"</td> + * <td>"D:my_sub_folder:ID4\n<br /> + * F:my_other_file.ext:ID5"</td> + * </tr> + * <tr> + * <td>"ID2"</td> + * <td>010101</td> + * </tr> + * <tr> + * <td>"ID3"</td> + * <td>"/my_folder/my_sub_folder/my_deep_file.ext"</td> + * </tr> + * <tr> + * <td>"ID4"</td> + * <td>"F:my_deep_file:ID6\n<br /> + * D:my_empty_folder:ID7"</td> + * </tr> + * <tr> + * <td>"ID5"</td> + * <td>011010100111010...</td> + * </tr> + * <tr> + * <td>"ID6"</td> + * <td>1100010101010110110010...</td> + * </tr> + * <tr> + * <td>"ID7"</td> + * <td>""</td> + * </tr> + * </table> + * + * <p> + * An entry is, for example, "D:my_folder:ID1". It contains the type of the + * entry (<strong>D</strong>irectory, <strong>F</strong>ile or + * <strong>L</strong>ink), the name of the element, and an ID to be used as + * a key on the map to get the actual content. Those three informations are + * separated by ":" which is {@link EntryUtil#ENTRY_SEPARATOR}. + * </p> + * + * <p> + * A directory way have multiple <em>entries</em>. Entries of a directory are + * separated by "\n" which is {@link EntryUtil#ENTRIES_SEPARATOR}. + * </p> + * <p> + * The above description shows the main principle used to store a tree + * structure in a map. + * </p> + * <p> + * In fact, the storage of actual data is a bit more complex, due to + * the need of splitting the data that may be too large (the value of "ID6" + * may be a series of 100 * 2<sup>20</sup> bytes for a 200 MiB file). + * Actually, files content (like ID2, ID5, ID6) and directory content + * values ("/", ID1, ID4) will be split and the map will look like that ( + * <strong>note that only the files have been split for readability</strong>): + * </p> + * <table> + * <caption>Use of map, concrete point of view</caption> + * <tr> + * <th>Key (String)</th> + * <th>Value (byte[])</th> + * </tr> + * <tr> + * <td>"/"</td> + * <td>"D:my_folder:ID1\n<br /> + * F:my_file.ext:ID2\n<br /> + * L:my_link:ID3"</td> + * </tr> + * <tr> + * <td>"ID1"</td> + * <td>"D:my_sub_folder:ID4\n<br /> + * F:my_other_file.ext:ID5"</td> + * </tr> + * <tr> + * <td>"ID2"</td> + * <td>"6;ID8"</td> + * </tr> + * <tr> + * <td>"ID3"</td> + * <td>"/my_folder/my_sub_folder/my_deep_file.ext"</td> + * </tr> + * <tr> + * <td>"ID4"</td> + * <td>"F:my_deep_file:ID6\n<br /> + * D:my_empty_folder:ID7"</td> + * </tr> + * <tr> + * <td>"ID5"</td> + * <td>"14689;ID9"</td> + * </tr> + * <tr> + * <td>"ID6"</td> + * <td>"79874567;ID10;ID11;ID12"</td> + * </tr> + * <tr> + * <td>"ID7"</td> + * <td>""</td> + * </tr> + * <tr> + * <td>"ID8"</td> + * <td>010101</td> + * </tr> + * <tr> + * <td>"ID9"</td> + * <td>011010100111010...</td> + * </tr> + * <tr> + * <td>"ID10"</td> + * <td>110001010101011...</td> + * </tr> + * <tr> + * <td>"ID11"</td> + * <td>111010110101101...</td> + * </tr> + * <tr> + * <td>"ID12"</td> + * <td>011101100111010...</td> + * </tr> + * </table> + * + * <p> + * Values of ID2, ID5, ID6 are no longer the file content. It's now a set + * of meta-information information data called <code>metablock</code>. + * A metablock is composed of the total size of the file followed by + * an ordered lists of IDs of the different blocks composing the file. + * Those informations are separated by ";" (see + * {@link EntryUtil#BLOCKIDS_SEPARATOR}). + * </p> + * <p> + * In the above example: + * <ul> + * <li> + * "/my_file.ext" is a file which size is 6 bytes, it is + * composed of one block available at ID8; + * </li> + * <li> + * "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes) + * which content is available at ID9; + * </li> + * <li> + * "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its + * size is 79874567. It has been split in three blocks: ID10, ID11 and + * ID12. + * </li> + * </ul> + * </p> + * <p> + * When reading and writing in storage, split is done transparently. When + * reading, a Stream is returned: it loads data blocks after blocks + * when needed (see {@link Storage.SplitBlocksInputStream}). When writing, + * data are split in blocks of a maximum configurable size + * (see {@link org.nuiton.disworkfs.DisworkConfig}). + * </p> + */ + +package org.nuiton.disworkfs.storage; \ No newline at end of file Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-25 15:24:32 UTC (rev 51) @@ -5,4 +5,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=TRACE \ No newline at end of file +log4j.logger.org.nuiton.disworkfs=WARN \ No newline at end of file Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-25 15:24:32 UTC (rev 51) @@ -1,253 +0,0 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nuiton.util.FileUtil; - - -public class DistributedFileSystemTest { - - /** - * a place to store files for the test it's a subdirectory of the OS temp - * dir e.g. under linux /tmp/disworkfs/tests/ - */ - static protected String tempDirectoryPath = System.getProperty( - "java.io.tmpdir", ".") - + "/disworkfs/tests"; - - /** - * We will create a file at this path for test purpose - */ - static protected String randomFilePath = tempDirectoryPath + "/randomfile"; - - /** - * The file will have this fixed size - */ - static protected int randomFileSize = 10000; - - static protected DistributedFileSystem fileSystem; - - @Before - public void setUp() throws Exception { - File tempDirectory = new File(tempDirectoryPath); - tempDirectory.mkdir(); - - Random random = new Random(); - // creating random data for the file - byte[] randomBytes = new byte[randomFileSize]; - random.nextBytes(randomBytes); - - // dumping random data into the file - File randomFile = new File(randomFilePath); - FileUtils.writeByteArrayToFile(randomFile, randomBytes); - - DisworkConfig disworkConfig = new DisworkConfig(); - - fileSystem = new DisworkFileSystem(disworkConfig); - } - - @After - public void tearDown() throws Exception { - // cleaning - FileUtil.deleteRecursively(tempDirectoryPath); - } - - @Test - public void testWrite() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testExists() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_file")); - assertFalse(fileSystem.exists("/my_other_file")); - } - - /** - * try to read a file that as never been created nor written - */ - @Test(expected = FileNotFoundException.class) - public void testFailAtRead() throws Exception { - fileSystem.read("/not_existing_file"); - } - - /** - * testing {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} - * by storing a "-1" byte, can be buggy due to the use of read() - * @throws IOException - */ - @Test - public void testSplit() throws IOException { - - byte[] bytes = new byte[1]; - bytes[0] = -0x1; - - InputStream source; - - source = new ByteArrayInputStream(bytes); - fileSystem.write("/", "my_file", source); - - source.close(); - - - source = new ByteArrayInputStream(bytes); - InputStream readResult = fileSystem.read("/my_file"); - - int read = 0; - byte[] b = new byte[1]; - - read = readResult.read(b); - - assertEquals(1, read); - assertArrayEquals(bytes, b); - - } - - /** - * writing a file at the root directory and reading it. - * finally, compare original source and read result - * byte-to-byte : contents should be equals - * @throws Exception - */ - @Test - public void testWriteRead() throws Exception { - - InputStream source = new FileInputStream(randomFilePath); - - fileSystem.write("/", "my_file", source); - - source.close(); - - InputStream readResult; - - // to be used for debugging purpose - - source = new FileInputStream(randomFilePath); - readResult = fileSystem.read("/my_file"); - - System.out.println("source.available() = " + source.available()); - System.out.println("readResult.available() = " + readResult.available()); - - byte[] sourceAsBytes = IOUtils.toByteArray(source); - byte[] readResultAsBytes = IOUtils.toByteArray(readResult); - /* - System.out.println("source.available() = " + source.available()); - System.out.println("readResult.available() = " + readResult.available()); - - System.out.println("source (" + sourceAsBytes.length + ") = " + Arrays.toString(sourceAsBytes)); - System.out.println("result (" + readResultAsBytes.length + ") = " + Arrays.toString(readResultAsBytes)); - */ - assertArrayEquals(sourceAsBytes, readResultAsBytes); - - - source = new FileInputStream(randomFilePath); - readResult = fileSystem.read("/my_file"); - - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - source.close(); - readResult.close(); - - assertTrue(actualContentEquality); - - } - - /** - * this use case should raise an exception because my_folder - * doesn't exists - */ - @Test(expected = IOException.class) - public void testFailAtWrite() throws Exception { - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testMkdir() throws Exception { - fileSystem.mkdir("/my_folder"); - assertTrue(fileSystem.exists("/my_folder")); - fileSystem.mkdir("/my_folder/my_sub_folder"); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); - } - - @Test - public void testWriteInFolder() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.mkdir("/my_folder/my_sub_folder"); - fileSystem.write("/my_folder/my_sub_folder", "my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); - } - - @Test - public void testln() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.ln("/my_link", "/my_folder/my_file"); - - InputStream source = new FileInputStream(randomFilePath); - InputStream readResult = fileSystem.read("/my_link"); - - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - source.close(); - readResult.close(); - - assertTrue(actualContentEquality); - } - - @Test(expected = IOException.class) - public void testFailAtLn() throws Exception { - fileSystem.ln("/my_link", "/wrong_target_path"); - } - - @Test - public void testRemove() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.remove("/my_folder/my_file"); - assertTrue(fileSystem.exists("/my_folder")); - assertFalse(fileSystem.exists("/my_folder/my_file")); - fileSystem.remove("/my_folder"); - assertFalse(fileSystem.exists("/my_folder")); - } - - @Test(expected = IOException.class) - public void testFailAtRemove() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - - // trying to remove a non-empty directory should raise an exception - fileSystem.remove("/my_folder"); - } - - @Test - public void testLs() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.mkdir("/my_folder/my_sub_dir"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.ln("/my_folder/my_link", "my_file"); - - // trying to remove a non-empty directory should raise an exception - List<String> lsResult = fileSystem.ls("/my_folder"); - assertEquals(3, lsResult.size()); - } - -} Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java (from rev 50, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,316 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.util.FileUtil; + + +public class DisworkFileSystemTest { + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = + System.getProperty("java.io.tmpdir", ".") // temp directory + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 9999; + + static protected DisworkFileSystem fileSystem; + + /** + * This setUp creates in a temp directory a file of size + * {@link randomFileSize} (fulfilling it with random bytes) + * This file can be found at {@link #randomFilePath} + * At the end of the method {@link #fileSystem} is ready to be used + * @throws Exception + */ + @Before + public void setUp() throws Exception { + // create a temp directory for our test + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + // creating random data for the file + Random random = new Random(); + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); + fileSystem = new DisworkFileSystem(disworkConfig); + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + /** + * writing a file at root directory should not raise any exception + * @throws Exception + */ + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + /** + * First, write a file then test if exists return true. + * @throws Exception + */ + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * tests {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte: contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // now, the checks. We read the original file and the result of + // a read() and then compare it byte-to-byte + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + source.close(); + readResult.close(); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testFailAtWrite() throws Exception { + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + } + + /** + * this use case should raise an exception because writing to a file + * that already exists + */ + @Test(expected = IOException.class) + public void testFailAtDoubleWrite() throws Exception { + InputStream source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + source.close(); + + source = new FileInputStream(randomFilePath); + try { + fileSystem.write("/my_file", source); + } finally { + source.close(); + } + } + + /** + * This test uses mkdir to create dirs and sub-dirs + * @throws Exception + */ + @Test + public void testCreateDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + /** + * Create some folders with mkdir and write files in those directories + * @throws Exception + */ + @Test + public void testWriteInFolder() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", + new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + /** + * create a symbolic link to a file. This test show that we can read the + * file using the link + * @throws Exception + */ + @Test + public void testLinking() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); + + InputStream source = new FileInputStream(randomFilePath); + InputStream readResult = fileSystem.read("/my_link"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + } + + /** + * Trying to create a link to a wrong target, this sould raise an exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtLinking() throws Exception { + fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + } + + /** + * Trying to remove files and directories + * @throws Exception + */ + @Test + public void testRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.delete("/my_folder/my_file"); + assertTrue(fileSystem.exists("/my_folder")); + assertFalse(fileSystem.exists("/my_folder/my_file")); + fileSystem.delete("/my_folder"); + assertFalse(fileSystem.exists("/my_folder")); + } + + /** + * By trying to remove a non-empty directory, this test should raise an + * exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + + // trying to remove a non-empty directory should raise an exception + fileSystem.delete("/my_folder"); + } + + /** + * This tests uses ls + */ + @Test + public void testListDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.createDirectory("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); + + List<String> lsResult = fileSystem.readDirectory("/my_folder"); + + // checking that result contains all the required data + assertTrue(lsResult.contains("my_sub_dir")); + assertTrue(lsResult.contains("my_file")); + assertTrue(lsResult.contains("my_link")); + + // ... and only those + assertEquals(3, lsResult.size()); + } + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-25 15:24:32 UTC (rev 51) @@ -18,6 +18,7 @@ package org.nuiton.disworkfs; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -87,24 +88,24 @@ @Test public void testGetId() { - assertEquals("myid1", EntryUtil.getId(directoryEntry)); - assertEquals("myid2", EntryUtil.getId(fileEntry)); - assertEquals("myid3", EntryUtil.getId(linkEntry)); + assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry)); + assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry)); + assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry)); } @Test public void testGetName() { - assertEquals("mydir", EntryUtil.getName(directoryEntry)); - assertEquals("myfile", EntryUtil.getName(fileEntry)); - assertEquals("mylink", EntryUtil.getName(linkEntry)); + assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry)); + assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry)); + assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry)); } @Test public void testGetType() { - assertEquals(EntryUtil.TYPE.D, EntryUtil.getType(directoryEntry)); - assertEquals(EntryUtil.TYPE.F, EntryUtil.getType(fileEntry)); - assertEquals(EntryUtil.TYPE.L, EntryUtil.getType(linkEntry)); + assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry)); + assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry)); + assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry)); } @Test @@ -123,7 +124,7 @@ @Test public void testBytesToArray() { - String s = "abcdefg@^:éèvwxyz,!;*$"; + String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$"; String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); assertEquals(s, copy); } @@ -137,10 +138,7 @@ public void testGetBlockIdsFromMetaBlock() { String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); - assertEquals(expectedBlocksIds.length, actualBlocksIds.length); - assertEquals(expectedBlocksIds[0], actualBlocksIds[0]); - assertEquals(expectedBlocksIds[1], actualBlocksIds[1]); - assertEquals(expectedBlocksIds[2], actualBlocksIds[2]); + assertArrayEquals(expectedBlocksIds, actualBlocksIds); } @Test