Author: bleny Date: 2010-05-21 15:01:45 +0200 (Fri, 21 May 2010) New Revision: 47 Url: http://nuiton.org/repositories/revision/diswork/47 Log: write, read in deep hierarchy ; mkdir ; less bugs ; tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,51 +1,18 @@ package org.nuiton.disworkfs; -import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; public interface DistributedFileSystem { - - /** - * checks the existence of a file on the virtual FS - * - * @param path - * a path to a file on the virtual FS - * @return true if the file exists, false if not - */ + public boolean exists(String path) throws IOException; -; - /** - * write a file - * - * @param path - * path of the virtual FS where the file should be written - * @param source - * the file to copy on the VFS - * @throws IOException - * if problems occurs while reading the source or writing on VFS - */ public void write(String parent, String file, InputStream source) throws IOException; - /** - * read a file on the VFS, the file is downloaded if needed (may take some - * times) - * - * @param path - * the path in the virtual FS to the file you want to read - * @return ?? - * @throws FileNotFoundException - * if no file have been written to this path (you may use - * {@link #exists(String) to check before read} - * @throws IOException - * if a problem occur while reading the local file - */ public InputStream read(String path) throws FileNotFoundException, IOException; public void mkdir(String path) throws IOException; - } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,12 +1,9 @@ package org.nuiton.disworkfs; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.storage.EntryUtil; @@ -18,57 +15,89 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - public DisworkFileSystem(DisworkConfig disworkConfig) { - storage = new Storage(new HashMap<String, byte[]>()); - // TODO 20100519 bleny create root dir + public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException { + storage = new Storage(new InMemoryMap()); + storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); } @Override public boolean exists(String path) throws IOException { - String entry = walk(path, null, null); + String entry = walk(path); boolean result = entry != null; return result; } - + /** - * Return id of the last element in path. + * return the entry of the element at the end of <code>path</code> * * @param path * @return null if path is not valid */ + protected String walk(String path) throws IOException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + // FIXME 20105021 bleny works fine but is not understandable protected String walk(String path, String current, String content) throws IOException { String result = null; + + if (path.equals("/")) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "/" + EntryUtil.ENTRY_SEPARATOR + + "/"; + } + + String parentPath = EntryUtil.getParentFromPath(path); + if (content == null) { // start of walk content = storage.getRootDirectory(); result = walk(path, "/", content); - } else if (FilenameUtils.getFullPathNoEndSeparator(path).equals(current)) { + } else if (parentPath.equals(current)) { // in the last directory - - // recuperation du directory a traiter dans p + // recuperation de l'element a traiter dans p + String tail = path.substring(current.length()); - String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); - String p = dirNames[0]; - + + /* + log.info("tail " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; + */ + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry != null) { - result = entry; + result = entry; + } else { + // in middle of path + + String tail; // the path still remaining when in current + + if (current.equals("/")) { + tail = path.substring(current.length()); } else { - result = null; + tail = path.substring(current.length() + 1); } - } else { - // in middle of path + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; - // recuperation du directory a traiter dans p - String tail = path.substring(current.length()); - String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); - String p = dirNames[0]; - + log.info("in intermediate dir " + current + ", looking for " + p); + // mise a jour de current + if (current.equals("/")) current = ""; current += "/" + p; String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry != null) { + if (entry == null) { + result = null; + } else { if (EntryUtil.isDirectory(entry)) { String id = EntryUtil.getId(entry); content = storage.getDirectory(id); @@ -81,12 +110,13 @@ // restart walk from / result = walk(newTarget, null, null); - } else { + } else if (EntryUtil.isFile(entry)) { // erreur, find file in path like '/dir1/dir2/filename/dir3' result = null; + } else { + log.warn("strange case : " + entry); + result = null; } - } else { - result = null; } } return result; @@ -94,44 +124,50 @@ @Override public InputStream read(String path) throws FileNotFoundException, - IOException { + IOException { - String entry = walk(path, null, null); + String entry = walk(path); if (entry == null) { throw new FileNotFoundException(path); } - InputStream result; + InputStream result = null; + if (EntryUtil.isLink(entry)) { + log.info("reading link " + path); String id = EntryUtil.getId(entry); String link = storage.getLink(id); String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); - } else if(EntryUtil.isDirectory(entry)) { + } else if (EntryUtil.isDirectory(entry)) { throw new IOException("target is not a file: " + path); - } else { + } else if (EntryUtil.isFile(entry)) { + log.info("reading file " + path); String id = EntryUtil.getId(entry); result = storage.getFile(id); - } + } return result; } @Override - public void write(String parent, String file, InputStream source) throws IOException { - String entryParent = walk(parent, null, null); + public void write(String parent, String fileName, InputStream source) throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + if (EntryUtil.isDirectory(entryParent)) { - String idParent = EntryUtil.getId(entryParent); - String content = storage.getDirectory(idParent); - String id = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent(content, EntryUtil.TYPE.D, file, id); + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + String newFileId = EntryUtil.generateId(); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.F, fileName, newFileId); // store file before meta info - // FIXME 20100519 bleny this should use putFile - storage.put(id, source); + storage.putFile(newFileId, source); // update meta info directory - InputStream parentContent = new ByteArrayInputStream(newContent.getBytes()); - // FIXME 20100519 bleny this should use putDirectory - storage.put(idParent, parentContent); + storage.putDirectory(parentId, newContent); } else { throw new IOException(parent + " is not a directory"); } @@ -139,7 +175,36 @@ @Override public void mkdir(String path) throws IOException { - throw new UnsupportedOperationException("not yet implemented"); + String parent = EntryUtil.getParentFromPath(path); + String dirName = EntryUtil.getNameFromPath(path); + mkdir(parent, dirName); } + public void mkdir(String parent, String dirName) throws IOException { + + log.info("trying to create directory " + dirName + " in " + parent); + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + String newDirectoryId = EntryUtil.generateId(); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.D, dirName, newDirectoryId); + + // store file before meta info + storage.putDirectory(newDirectoryId, + EntryUtil.newEmptyDirectoryContent()); + // update meta info directory + storage.putDirectory(parentId, newContent); + } else { + throw new IOException(parent + " is not a directory"); + } + } + } Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 13:01:45 UTC (rev 47) @@ -0,0 +1,20 @@ +package org.nuiton.disworkfs; + +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class InMemoryMap extends HashMap<String, byte[]> { + + private final Log log = LogFactory.getLog(InMemoryMap.class); + + @Override + public byte[] put(String key, byte[] value) { + log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); + byte[] valueCopy = value.clone(); + log.info("putting " + valueCopy.length + " bytes"); + return super.put(key, valueCopy); + } +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 13:01:45 UTC (rev 47) @@ -87,11 +87,17 @@ } static public int getTotalSizeFromMetaBlock(String metaBlock) { + // FIXME 20100520 bleny this workaround should not be + if (metaBlock.equals("0")) return 0; String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); return Integer.valueOf(result); } static public String[] getBlockIdsFromMetaBlock(String metaBlock) { + // FIXME 20100520 bleny this workaround should not be + if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { + return new String[0]; + } String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length()); String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); return blockIds; @@ -119,16 +125,23 @@ /** * + */ + static public String newEmptyDirectoryContent() { + return ""; + } + + /** + * * @param content the string content of the directory (all entries before add) * @param type the type of the new entry * @param name the name (not a full path) of the new entry * @param id the id where the content of the entry can be found * @return a String with the new content - */ + */ static public String addEntryToDirectoryContent (String content, TYPE type, String name, String id) { - content += ENTRIES_SEPARATOR + type.name() + ENTRY_SEPARATOR + name + - ENTRY_SEPARATOR + id; + content += type.name() + ENTRY_SEPARATOR + name + + ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR; return content; } @@ -142,6 +155,7 @@ * @return the path to the destination */ static public String resolveLink(String parent, String link) { + /* String result = null; if (link.startsWith("/")) { // destination is absolute, ignore parent @@ -161,6 +175,8 @@ result = FilenameUtils.normalize(parentWithNoFile + link); } return result; + */ + return FilenameUtils.concat(parent, link); } /** @@ -170,7 +186,8 @@ * @return a byte array containing all the bytes in <code>string</code> */ public static byte[] stringToBytes(String string) { - // FIXME 20100519 bleny this code may cause inconsistences with charsets + // FIXME 20100519 bleny this code may cause bad conversion + // due to charsets byte[] bytes = string.getBytes(); return bytes; } @@ -184,4 +201,16 @@ return new String(bytes); } + public static String getParentFromPath(String path) { + String parent = FilenameUtils.getFullPathNoEndSeparator(path); + if (parent.isEmpty()) { + parent = "/"; + } + return parent; + } + + public static String getNameFromPath(String path) { + return FilenameUtils.getName(path); + } + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 13:01:45 UTC (rev 47) @@ -6,9 +6,13 @@ import java.util.Map; import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; public class Storage { + private static final Log log = LogFactory.getLog(Storage.class); + protected class SplitBlocksInputStream extends InputStream { protected Map<String, byte[]> map; @@ -17,8 +21,7 @@ protected String[] blockIds; protected int blockIdsIndex; - protected byte[] currentBlock; - protected int currentBlockIndex; + protected ByteArrayInputStream currentBlock; protected int available; @@ -32,31 +35,43 @@ @Override public int read() throws IOException { checkInitialization(); + + if (blockIdsIndex >= blockIds.length) { + return -1; + } // update current block if needed - if (currentBlock == null || currentBlockIndex >= currentBlock.length) { + if (currentBlock == null || currentBlock.available() == 0) { blockIdsIndex += 1; if (blockIdsIndex < blockIds.length) { - currentBlock = map.get(blockIds[blockIdsIndex]); - currentBlockIndex = 0; + currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex])); + log.debug("new current block (size = " + currentBlock.available() + ")"); } else { return -1; } } - byte result = currentBlock[currentBlockIndex]; - currentBlockIndex += 1; + int result = currentBlock.read(); + /* + log.debug("reading block number " + blockIdsIndex + + " (available = " + available + ")" + + " returns " + result); + */ + available -= 1; return result; } protected void checkInitialization() { - if (blockIds == null) { + if (blockIds == null) { byte[] bytes = map.get(id); String metaBlock = EntryUtil.bytesToString(bytes); blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); blockIdsIndex = -1; available = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + log.debug("initializing stream with meta block \"" + metaBlock + + "\" (" + blockIds.length + " blocks, " + available + + " bytes)"); } } @@ -74,10 +89,14 @@ this.map = map; } - public boolean exists(Object key) { - return map.containsKey(key); + public boolean contains(Object id) { + return map.containsKey(id); } + /** + * @return the content (entries) of the root directory + * @throws IOException + */ public String getRootDirectory() throws IOException { String result = getDirectory("/"); return result; @@ -89,8 +108,9 @@ public String getDirectory(String id) throws IOException { InputStream in = get(id); - String result = IOUtils.toString(in); - return result; + String content = IOUtils.toString(in); + log.debug("getDirectory(\"" + id + "\") returns \n" + content); + return content; } public InputStream getFile(String id) { @@ -99,12 +119,28 @@ } public String getLink(String id) throws IOException { - return EntryUtil.bytesToString(map.get(id)); + String content = EntryUtil.bytesToString(map.get(id)); + log.debug("getDirectory(\"" + id + "\") returns \"" + content + "\""); + return content; } - - // TODO 20100519 bleny write getDirectory, getFile, getLink + public void putDirectory(String id, String content) throws IOException { + log.debug("putDirectory(\"" + id + "\", \"" + content + "\")"); + InputStream value = IOUtils.toInputStream(content); + put(id, value); + } + public void putFile(String id, InputStream content) throws IOException { + put(id, content); + } + + public void putLink(String id, String content) { + log.debug("putLink(\"" + id + "\", \"" + content + "\")"); + byte[] contentAsBytes = EntryUtil.stringToBytes(content); + map.put(id, contentAsBytes); + } + + /** * see {@link #get(String)} */ @@ -119,43 +155,47 @@ * @param value * @throws IOException */ - public void put(String key, InputStream value) throws IOException { + protected void put(String key, InputStream value) throws IOException { // TODO 20100519 bleny deal with copy-on-write - // TODO 20100519 bleny deal with null value properly - + // TODO 20100519 bleny deal with null value properly if (value == null) { value = new ByteArrayInputStream(new byte[0]); } String blocksIds = ""; - + int read = 0; + int totalSize = 0; + // TODO 20100519 bleny the size of blocks should be a config directive byte[] buffer = new byte[10 * 1024 * 1024]; - int read = value.read(buffer); - int totalSize = read; - - while (read == buffer.length) { + while ((read = value.read(buffer)) != -1) { + totalSize += read; + + byte[] newBlock = buffer; + + // if the block is shorter, truncate + if (read < buffer.length) { + newBlock = new byte[read]; + System.arraycopy(buffer, 0, newBlock, 0, read); + } + String id = EntryUtil.generateId(); - map.put(id, buffer); blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + log.debug("saving new block (size = " + newBlock.length + + ") at key " + id); - // for next iteration - read = value.read(buffer); - totalSize += read; + // copy buffer in map + map.put(id, newBlock); + } - byte[] lastBuffer = new byte[read]; - System.arraycopy(buffer, 0, lastBuffer, 0, read); + String metaBlock = totalSize + blocksIds; - String id = EntryUtil.generateId(); - map.put(id, buffer); + log.debug("putting metablock " + metaBlock + " at key " + key); - blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; - - String metaBlock = totalSize + blocksIds; - map.put(key, EntryUtil.stringToBytes(metaBlock)); } Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-21 13:01:45 UTC (rev 47) @@ -5,5 +5,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=TRACE -log4j.logger.org.nuiton.disworkfs.storage=WARN \ No newline at end of file +log4j.logger.org.nuiton.disworkfs=TRACE \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,137 +1,207 @@ -//package org.nuiton.disworkfs; -// -//import static org.junit.Assert.assertFalse; -//import static org.junit.Assert.assertTrue; -// -//import java.io.File; -//import java.io.FileInputStream; -//import java.io.FileNotFoundException; -//import java.io.IOException; -//import java.io.InputStream; -//import java.util.Random; -// -//import org.apache.commons.io.FileUtils; -//import org.apache.commons.io.IOUtils; -//import org.junit.After; -//import org.junit.Before; -//import org.junit.Test; -//import org.nuiton.util.FileUtil; -// -//public class DistributedFileSystemTest { -// -// /** -// * a place to store files for the test it's a subdirectory of the OS temp -// * dir e.g. under linux /tmp/disworkfs/tests/ -// */ -// static protected String tempDirectoryPath = System.getProperty( -// "java.io.tmpdir", ".") -// + "/disworkfs/tests"; -// -// /** -// * We will create a file at this path for test purpose -// */ -// static protected String randomFilePath = tempDirectoryPath + "/randomfile"; -// -// /** -// * The file will have this fixed size -// */ -// static protected int randomFileSize = 3000; -// -// static protected DistributedFileSystem fileSystem; -// -// @Before -// public void setUp() throws Exception { -// File tempDirectory = new File(tempDirectoryPath); -// tempDirectory.mkdir(); -// -// Random random = new Random(); -// // creating random data for the file -// byte[] randomBytes = new byte[randomFileSize]; -// random.nextBytes(randomBytes); -// -// // dumping random data into the file -// File randomFile = new File(randomFilePath); -// FileUtils.writeByteArrayToFile(randomFile, randomBytes); -// -// DisworkConfig disworkConfig = new DisworkConfig(); -// -// fileSystem = new DisworkFileSystem(disworkConfig); -// } -// -// @After -// public void tearDown() throws Exception { -// // cleaning -// fileSystem.close(); -// -// FileUtil.deleteRecursively(tempDirectoryPath); -// } -// -// @Test -// public void testWrite() throws Exception { -// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); -// } -// -// @Test -// public void testExists() throws Exception { -// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); -// assertTrue(fileSystem.exists("/my_file")); -// assertFalse(fileSystem.exists("/my_other_file")); -// } -// -// /** -// * try to read a file that as never been created nor written -// */ -// @Test(expected = FileNotFoundException.class) -// public void testFailAtRead() throws Exception { -// fileSystem.read("/not_existing_file"); -// } -// -// /** -// * writing a file at the root directory and reading it. -// * finally, compare original source and read result -// * byte-to-byte : contents should be equals -// * @throws Exception -// */ -// @Test -// public void testWriteRead() throws Exception { -// -// InputStream source = new FileInputStream(randomFilePath); -// -// fileSystem.write("/my_file", source); -// -// InputStream readResult = fileSystem.read("/my_file"); -// -// source = new FileInputStream(randomFilePath); -// boolean actualContentEquality = -// IOUtils.contentEquals(source, readResult); -// -// assertTrue(actualContentEquality); -// } -// -// @Test -// public void testMkdir() throws Exception { -// fileSystem.mkdir("/my_folder"); -// assertTrue(fileSystem.exists("/my_folder")); -// } -// -// /** -// * this use case should raise an exception because my_folder -// * doesn't exists -// */ -// @Test(expected = IOException.class) -// public void testWriteFail() throws Exception { -// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); -// } -// -// @Test -// public void testWriteInFolder() throws Exception { -// fileSystem.mkdir("/my_folder"); -// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); -// fileSystem.mkdir("/my_folder/my_sub_folder"); -// fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); -// -// ((DisworkFileSystem) fileSystem).dump(); -// } -// -// -//} +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertArrayEquals; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.util.FileUtil; + +import com.sun.org.apache.xml.internal.security.keys.storage.StorageResolverException; +import com.sun.xml.internal.ws.util.ByteArrayBuffer; + + +public class DistributedFileSystemTest { + + private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class); + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = System.getProperty( + "java.io.tmpdir", ".") + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 10000; + + static protected DistributedFileSystem fileSystem; + + @Before + public void setUp() throws Exception { + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + Random random = new Random(); + // creating random data for the file + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + DisworkConfig disworkConfig = new DisworkConfig(); + + fileSystem = new DisworkFileSystem(disworkConfig); + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * testing {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte : contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // to be used for debugging purpose + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + System.out.println("source.available() = " + source.available()); + System.out.println("readResult.available() = " + readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + /* + System.out.println("source.available() = " + source.available()); + System.out.println("readResult.available() = " + readResult.available()); + + System.out.println("source (" + sourceAsBytes.length + ") = " + Arrays.toString(sourceAsBytes)); + System.out.println("result (" + readResultAsBytes.length + ") = " + Arrays.toString(readResultAsBytes)); + */ + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testWriteFail() throws Exception { + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + } + + @Test + public void testMkdir() throws Exception { + fileSystem.mkdir("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.mkdir("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + @Test + public void testWriteInFolder() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + fileSystem.mkdir("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -51,7 +51,7 @@ + "myid3"; static protected String metaBlock = - "12345" + EntryUtil.BLOCKIDS_SEPARATOR + "12345" + EntryUtil.BLOCKIDS_SEPARATOR + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR + "myid3"; @@ -65,20 +65,24 @@ @Test public void testFind() { - String content = directoryEntry + "\n" - + fileEntry + "\n" - + linkEntry; + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - { - // recherche et trouve le nom - String entry = EntryUtil.findEntryInDirectory(content, "myfile"); - assertEquals(fileEntry, entry); - } - { - // recherche et ne trouve pas le nom - String entry = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); - assertEquals(null, entry); - } + // seek and return entry + String findResult; + + findResult = EntryUtil.findEntryInDirectory(content, "mydir"); + assertEquals(directoryEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "myfile"); + assertEquals(fileEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "mylink"); + assertEquals(linkEntry, findResult); + + + // return null if file not found + findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); + assertEquals(null, findResult); } @Test @@ -109,8 +113,6 @@ path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); assertEquals("/anotherdir", path); - path = EntryUtil.resolveLink("/dir/afile", "anotherdir"); - assertEquals("/dir/anotherdir", path); path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); assertEquals("/dir/subdir/anotherdir", path); path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -0,0 +1,37 @@ +package org.nuiton.disworkfs; + +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; + +public class InMemoryMapTest { + private final Log log = LogFactory.getLog(InMemoryMapTest.class); + + + @Before + public void setUp() throws Exception {} + + @After + public void tearDown() throws Exception {} + + @Test + public void testPut() { + InMemoryMap map = new InMemoryMap(); + byte[] expected = {0x1, 0x2}; + + map.put("key", expected); + + expected[0] = 0xf; + + byte[] actual = map.get("key"); + + assertFalse(Arrays.equals(expected, actual)); + } + +}