- Diswork-commits - mailman.cloud.codelutin.com

r47 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage main/resources test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 21 May '10

21 May '10

Author: bleny Date: 2010-05-21 15:01:45 +0200 (Fri, 21 May 2010) New Revision: 47 Url: http://nuiton.org/repositories/revision/diswork/47 Log: write, read in deep hierarchy ; mkdir ; less bugs ; tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,51 +1,18 @@ package org.nuiton.disworkfs; -import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; public interface DistributedFileSystem { - - /** - * checks the existence of a file on the virtual FS - * - * @param path - * a path to a file on the virtual FS - * @return true if the file exists, false if not - */ + public boolean exists(String path) throws IOException; -; - /** - * write a file - * - * @param path - * path of the virtual FS where the file should be written - * @param source - * the file to copy on the VFS - * @throws IOException - * if problems occurs while reading the source or writing on VFS - */ public void write(String parent, String file, InputStream source) throws IOException; - /** - * read a file on the VFS, the file is downloaded if needed (may take some - * times) - * - * @param path - * the path in the virtual FS to the file you want to read - * @return ?? - * @throws FileNotFoundException - * if no file have been written to this path (you may use - * {@link #exists(String) to check before read} - * @throws IOException - * if a problem occur while reading the local file - */ public InputStream read(String path) throws FileNotFoundException, IOException; public void mkdir(String path) throws IOException; - } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,12 +1,9 @@ package org.nuiton.disworkfs; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.storage.EntryUtil; @@ -18,57 +15,89 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - public DisworkFileSystem(DisworkConfig disworkConfig) { - storage = new Storage(new HashMap<String, byte[]>()); - // TODO 20100519 bleny create root dir + public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException { + storage = new Storage(new InMemoryMap()); + storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); } @Override public boolean exists(String path) throws IOException { - String entry = walk(path, null, null); + String entry = walk(path); boolean result = entry != null; return result; } - + /** - * Return id of the last element in path. + * return the entry of the element at the end of <code>path</code> * * @param path * @return null if path is not valid */ + protected String walk(String path) throws IOException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + // FIXME 20105021 bleny works fine but is not understandable protected String walk(String path, String current, String content) throws IOException { String result = null; + + if (path.equals("/")) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "/" + EntryUtil.ENTRY_SEPARATOR + + "/"; + } + + String parentPath = EntryUtil.getParentFromPath(path); + if (content == null) { // start of walk content = storage.getRootDirectory(); result = walk(path, "/", content); - } else if (FilenameUtils.getFullPathNoEndSeparator(path).equals(current)) { + } else if (parentPath.equals(current)) { // in the last directory - - // recuperation du directory a traiter dans p + // recuperation de l'element a traiter dans p + String tail = path.substring(current.length()); - String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); - String p = dirNames[0]; - + + /* + log.info("tail " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; + */ + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry != null) { - result = entry; + result = entry; + } else { + // in middle of path + + String tail; // the path still remaining when in current + + if (current.equals("/")) { + tail = path.substring(current.length()); } else { - result = null; + tail = path.substring(current.length() + 1); } - } else { - // in middle of path + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; - // recuperation du directory a traiter dans p - String tail = path.substring(current.length()); - String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); - String p = dirNames[0]; - + log.info("in intermediate dir " + current + ", looking for " + p); + // mise a jour de current + if (current.equals("/")) current = ""; current += "/" + p; String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry != null) { + if (entry == null) { + result = null; + } else { if (EntryUtil.isDirectory(entry)) { String id = EntryUtil.getId(entry); content = storage.getDirectory(id); @@ -81,12 +110,13 @@ // restart walk from / result = walk(newTarget, null, null); - } else { + } else if (EntryUtil.isFile(entry)) { // erreur, find file in path like '/dir1/dir2/filename/dir3' result = null; + } else { + log.warn("strange case : " + entry); + result = null; } - } else { - result = null; } } return result; @@ -94,44 +124,50 @@ @Override public InputStream read(String path) throws FileNotFoundException, - IOException { + IOException { - String entry = walk(path, null, null); + String entry = walk(path); if (entry == null) { throw new FileNotFoundException(path); } - InputStream result; + InputStream result = null; + if (EntryUtil.isLink(entry)) { + log.info("reading link " + path); String id = EntryUtil.getId(entry); String link = storage.getLink(id); String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); - } else if(EntryUtil.isDirectory(entry)) { + } else if (EntryUtil.isDirectory(entry)) { throw new IOException("target is not a file: " + path); - } else { + } else if (EntryUtil.isFile(entry)) { + log.info("reading file " + path); String id = EntryUtil.getId(entry); result = storage.getFile(id); - } + } return result; } @Override - public void write(String parent, String file, InputStream source) throws IOException { - String entryParent = walk(parent, null, null); + public void write(String parent, String fileName, InputStream source) throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + if (EntryUtil.isDirectory(entryParent)) { - String idParent = EntryUtil.getId(entryParent); - String content = storage.getDirectory(idParent); - String id = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent(content, EntryUtil.TYPE.D, file, id); + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + String newFileId = EntryUtil.generateId(); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.F, fileName, newFileId); // store file before meta info - // FIXME 20100519 bleny this should use putFile - storage.put(id, source); + storage.putFile(newFileId, source); // update meta info directory - InputStream parentContent = new ByteArrayInputStream(newContent.getBytes()); - // FIXME 20100519 bleny this should use putDirectory - storage.put(idParent, parentContent); + storage.putDirectory(parentId, newContent); } else { throw new IOException(parent + " is not a directory"); } @@ -139,7 +175,36 @@ @Override public void mkdir(String path) throws IOException { - throw new UnsupportedOperationException("not yet implemented"); + String parent = EntryUtil.getParentFromPath(path); + String dirName = EntryUtil.getNameFromPath(path); + mkdir(parent, dirName); } + public void mkdir(String parent, String dirName) throws IOException { + + log.info("trying to create directory " + dirName + " in " + parent); + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + String newDirectoryId = EntryUtil.generateId(); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.D, dirName, newDirectoryId); + + // store file before meta info + storage.putDirectory(newDirectoryId, + EntryUtil.newEmptyDirectoryContent()); + // update meta info directory + storage.putDirectory(parentId, newContent); + } else { + throw new IOException(parent + " is not a directory"); + } + } + } Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 13:01:45 UTC (rev 47) @@ -0,0 +1,20 @@ +package org.nuiton.disworkfs; + +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class InMemoryMap extends HashMap<String, byte[]> { + + private final Log log = LogFactory.getLog(InMemoryMap.class); + + @Override + public byte[] put(String key, byte[] value) { + log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); + byte[] valueCopy = value.clone(); + log.info("putting " + valueCopy.length + " bytes"); + return super.put(key, valueCopy); + } +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 13:01:45 UTC (rev 47) @@ -87,11 +87,17 @@ } static public int getTotalSizeFromMetaBlock(String metaBlock) { + // FIXME 20100520 bleny this workaround should not be + if (metaBlock.equals("0")) return 0; String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); return Integer.valueOf(result); } static public String[] getBlockIdsFromMetaBlock(String metaBlock) { + // FIXME 20100520 bleny this workaround should not be + if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { + return new String[0]; + } String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length()); String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); return blockIds; @@ -119,16 +125,23 @@ /** * + */ + static public String newEmptyDirectoryContent() { + return ""; + } + + /** + * * @param content the string content of the directory (all entries before add) * @param type the type of the new entry * @param name the name (not a full path) of the new entry * @param id the id where the content of the entry can be found * @return a String with the new content - */ + */ static public String addEntryToDirectoryContent (String content, TYPE type, String name, String id) { - content += ENTRIES_SEPARATOR + type.name() + ENTRY_SEPARATOR + name + - ENTRY_SEPARATOR + id; + content += type.name() + ENTRY_SEPARATOR + name + + ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR; return content; } @@ -142,6 +155,7 @@ * @return the path to the destination */ static public String resolveLink(String parent, String link) { + /* String result = null; if (link.startsWith("/")) { // destination is absolute, ignore parent @@ -161,6 +175,8 @@ result = FilenameUtils.normalize(parentWithNoFile + link); } return result; + */ + return FilenameUtils.concat(parent, link); } /** @@ -170,7 +186,8 @@ * @return a byte array containing all the bytes in <code>string</code> */ public static byte[] stringToBytes(String string) { - // FIXME 20100519 bleny this code may cause inconsistences with charsets + // FIXME 20100519 bleny this code may cause bad conversion + // due to charsets byte[] bytes = string.getBytes(); return bytes; } @@ -184,4 +201,16 @@ return new String(bytes); } + public static String getParentFromPath(String path) { + String parent = FilenameUtils.getFullPathNoEndSeparator(path); + if (parent.isEmpty()) { + parent = "/"; + } + return parent; + } + + public static String getNameFromPath(String path) { + return FilenameUtils.getName(path); + } + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 13:01:45 UTC (rev 47) @@ -6,9 +6,13 @@ import java.util.Map; import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; public class Storage { + private static final Log log = LogFactory.getLog(Storage.class); + protected class SplitBlocksInputStream extends InputStream { protected Map<String, byte[]> map; @@ -17,8 +21,7 @@ protected String[] blockIds; protected int blockIdsIndex; - protected byte[] currentBlock; - protected int currentBlockIndex; + protected ByteArrayInputStream currentBlock; protected int available; @@ -32,31 +35,43 @@ @Override public int read() throws IOException { checkInitialization(); + + if (blockIdsIndex >= blockIds.length) { + return -1; + } // update current block if needed - if (currentBlock == null || currentBlockIndex >= currentBlock.length) { + if (currentBlock == null || currentBlock.available() == 0) { blockIdsIndex += 1; if (blockIdsIndex < blockIds.length) { - currentBlock = map.get(blockIds[blockIdsIndex]); - currentBlockIndex = 0; + currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex])); + log.debug("new current block (size = " + currentBlock.available() + ")"); } else { return -1; } } - byte result = currentBlock[currentBlockIndex]; - currentBlockIndex += 1; + int result = currentBlock.read(); + /* + log.debug("reading block number " + blockIdsIndex + + " (available = " + available + ")" + + " returns " + result); + */ + available -= 1; return result; } protected void checkInitialization() { - if (blockIds == null) { + if (blockIds == null) { byte[] bytes = map.get(id); String metaBlock = EntryUtil.bytesToString(bytes); blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); blockIdsIndex = -1; available = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + log.debug("initializing stream with meta block \"" + metaBlock + + "\" (" + blockIds.length + " blocks, " + available + + " bytes)"); } } @@ -74,10 +89,14 @@ this.map = map; } - public boolean exists(Object key) { - return map.containsKey(key); + public boolean contains(Object id) { + return map.containsKey(id); } + /** + * @return the content (entries) of the root directory + * @throws IOException + */ public String getRootDirectory() throws IOException { String result = getDirectory("/"); return result; @@ -89,8 +108,9 @@ public String getDirectory(String id) throws IOException { InputStream in = get(id); - String result = IOUtils.toString(in); - return result; + String content = IOUtils.toString(in); + log.debug("getDirectory(\"" + id + "\") returns \n" + content); + return content; } public InputStream getFile(String id) { @@ -99,12 +119,28 @@ } public String getLink(String id) throws IOException { - return EntryUtil.bytesToString(map.get(id)); + String content = EntryUtil.bytesToString(map.get(id)); + log.debug("getDirectory(\"" + id + "\") returns \"" + content + "\""); + return content; } - - // TODO 20100519 bleny write getDirectory, getFile, getLink + public void putDirectory(String id, String content) throws IOException { + log.debug("putDirectory(\"" + id + "\", \"" + content + "\")"); + InputStream value = IOUtils.toInputStream(content); + put(id, value); + } + public void putFile(String id, InputStream content) throws IOException { + put(id, content); + } + + public void putLink(String id, String content) { + log.debug("putLink(\"" + id + "\", \"" + content + "\")"); + byte[] contentAsBytes = EntryUtil.stringToBytes(content); + map.put(id, contentAsBytes); + } + + /** * see {@link #get(String)} */ @@ -119,43 +155,47 @@ * @param value * @throws IOException */ - public void put(String key, InputStream value) throws IOException { + protected void put(String key, InputStream value) throws IOException { // TODO 20100519 bleny deal with copy-on-write - // TODO 20100519 bleny deal with null value properly - + // TODO 20100519 bleny deal with null value properly if (value == null) { value = new ByteArrayInputStream(new byte[0]); } String blocksIds = ""; - + int read = 0; + int totalSize = 0; + // TODO 20100519 bleny the size of blocks should be a config directive byte[] buffer = new byte[10 * 1024 * 1024]; - int read = value.read(buffer); - int totalSize = read; - - while (read == buffer.length) { + while ((read = value.read(buffer)) != -1) { + totalSize += read; + + byte[] newBlock = buffer; + + // if the block is shorter, truncate + if (read < buffer.length) { + newBlock = new byte[read]; + System.arraycopy(buffer, 0, newBlock, 0, read); + } + String id = EntryUtil.generateId(); - map.put(id, buffer); blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + log.debug("saving new block (size = " + newBlock.length + + ") at key " + id); - // for next iteration - read = value.read(buffer); - totalSize += read; + // copy buffer in map + map.put(id, newBlock); + } - byte[] lastBuffer = new byte[read]; - System.arraycopy(buffer, 0, lastBuffer, 0, read); + String metaBlock = totalSize + blocksIds; - String id = EntryUtil.generateId(); - map.put(id, buffer); + log.debug("putting metablock " + metaBlock + " at key " + key); - blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; - - String metaBlock = totalSize + blocksIds; - map.put(key, EntryUtil.stringToBytes(metaBlock)); } Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-21 13:01:45 UTC (rev 47) @@ -5,5 +5,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=TRACE -log4j.logger.org.nuiton.disworkfs.storage=WARN \ No newline at end of file +log4j.logger.org.nuiton.disworkfs=TRACE \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,137 +1,207 @@ -//package org.nuiton.disworkfs; -// -//import static org.junit.Assert.assertFalse; -//import static org.junit.Assert.assertTrue; -// -//import java.io.File; -//import java.io.FileInputStream; -//import java.io.FileNotFoundException; -//import java.io.IOException; -//import java.io.InputStream; -//import java.util.Random; -// -//import org.apache.commons.io.FileUtils; -//import org.apache.commons.io.IOUtils; -//import org.junit.After; -//import org.junit.Before; -//import org.junit.Test; -//import org.nuiton.util.FileUtil; -// -//public class DistributedFileSystemTest { -// -// /** -// * a place to store files for the test it's a subdirectory of the OS temp -// * dir e.g. under linux /tmp/disworkfs/tests/ -// */ -// static protected String tempDirectoryPath = System.getProperty( -// "java.io.tmpdir", ".") -// + "/disworkfs/tests"; -// -// /** -// * We will create a file at this path for test purpose -// */ -// static protected String randomFilePath = tempDirectoryPath + "/randomfile"; -// -// /** -// * The file will have this fixed size -// */ -// static protected int randomFileSize = 3000; -// -// static protected DistributedFileSystem fileSystem; -// -// @Before -// public void setUp() throws Exception { -// File tempDirectory = new File(tempDirectoryPath); -// tempDirectory.mkdir(); -// -// Random random = new Random(); -// // creating random data for the file -// byte[] randomBytes = new byte[randomFileSize]; -// random.nextBytes(randomBytes); -// -// // dumping random data into the file -// File randomFile = new File(randomFilePath); -// FileUtils.writeByteArrayToFile(randomFile, randomBytes); -// -// DisworkConfig disworkConfig = new DisworkConfig(); -// -// fileSystem = new DisworkFileSystem(disworkConfig); -// } -// -// @After -// public void tearDown() throws Exception { -// // cleaning -// fileSystem.close(); -// -// FileUtil.deleteRecursively(tempDirectoryPath); -// } -// -// @Test -// public void testWrite() throws Exception { -// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); -// } -// -// @Test -// public void testExists() throws Exception { -// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); -// assertTrue(fileSystem.exists("/my_file")); -// assertFalse(fileSystem.exists("/my_other_file")); -// } -// -// /** -// * try to read a file that as never been created nor written -// */ -// @Test(expected = FileNotFoundException.class) -// public void testFailAtRead() throws Exception { -// fileSystem.read("/not_existing_file"); -// } -// -// /** -// * writing a file at the root directory and reading it. -// * finally, compare original source and read result -// * byte-to-byte : contents should be equals -// * @throws Exception -// */ -// @Test -// public void testWriteRead() throws Exception { -// -// InputStream source = new FileInputStream(randomFilePath); -// -// fileSystem.write("/my_file", source); -// -// InputStream readResult = fileSystem.read("/my_file"); -// -// source = new FileInputStream(randomFilePath); -// boolean actualContentEquality = -// IOUtils.contentEquals(source, readResult); -// -// assertTrue(actualContentEquality); -// } -// -// @Test -// public void testMkdir() throws Exception { -// fileSystem.mkdir("/my_folder"); -// assertTrue(fileSystem.exists("/my_folder")); -// } -// -// /** -// * this use case should raise an exception because my_folder -// * doesn't exists -// */ -// @Test(expected = IOException.class) -// public void testWriteFail() throws Exception { -// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); -// } -// -// @Test -// public void testWriteInFolder() throws Exception { -// fileSystem.mkdir("/my_folder"); -// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); -// fileSystem.mkdir("/my_folder/my_sub_folder"); -// fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); -// -// ((DisworkFileSystem) fileSystem).dump(); -// } -// -// -//} +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertArrayEquals; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.util.FileUtil; + +import com.sun.org.apache.xml.internal.security.keys.storage.StorageResolverException; +import com.sun.xml.internal.ws.util.ByteArrayBuffer; + + +public class DistributedFileSystemTest { + + private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class); + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = System.getProperty( + "java.io.tmpdir", ".") + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 10000; + + static protected DistributedFileSystem fileSystem; + + @Before + public void setUp() throws Exception { + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + Random random = new Random(); + // creating random data for the file + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + DisworkConfig disworkConfig = new DisworkConfig(); + + fileSystem = new DisworkFileSystem(disworkConfig); + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * testing {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte : contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // to be used for debugging purpose + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + System.out.println("source.available() = " + source.available()); + System.out.println("readResult.available() = " + readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + /* + System.out.println("source.available() = " + source.available()); + System.out.println("readResult.available() = " + readResult.available()); + + System.out.println("source (" + sourceAsBytes.length + ") = " + Arrays.toString(sourceAsBytes)); + System.out.println("result (" + readResultAsBytes.length + ") = " + Arrays.toString(readResultAsBytes)); + */ + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testWriteFail() throws Exception { + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + } + + @Test + public void testMkdir() throws Exception { + fileSystem.mkdir("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.mkdir("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + @Test + public void testWriteInFolder() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + fileSystem.mkdir("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -51,7 +51,7 @@ + "myid3"; static protected String metaBlock = - "12345" + EntryUtil.BLOCKIDS_SEPARATOR + "12345" + EntryUtil.BLOCKIDS_SEPARATOR + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR + "myid3"; @@ -65,20 +65,24 @@ @Test public void testFind() { - String content = directoryEntry + "\n" - + fileEntry + "\n" - + linkEntry; + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - { - // recherche et trouve le nom - String entry = EntryUtil.findEntryInDirectory(content, "myfile"); - assertEquals(fileEntry, entry); - } - { - // recherche et ne trouve pas le nom - String entry = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); - assertEquals(null, entry); - } + // seek and return entry + String findResult; + + findResult = EntryUtil.findEntryInDirectory(content, "mydir"); + assertEquals(directoryEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "myfile"); + assertEquals(fileEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "mylink"); + assertEquals(linkEntry, findResult); + + + // return null if file not found + findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); + assertEquals(null, findResult); } @Test @@ -109,8 +113,6 @@ path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); assertEquals("/anotherdir", path); - path = EntryUtil.resolveLink("/dir/afile", "anotherdir"); - assertEquals("/dir/anotherdir", path); path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); assertEquals("/dir/subdir/anotherdir", path); path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -0,0 +1,37 @@ +package org.nuiton.disworkfs; + +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; + +public class InMemoryMapTest { + private final Log log = LogFactory.getLog(InMemoryMapTest.class); + + + @Before + public void setUp() throws Exception {} + + @After + public void tearDown() throws Exception {} + + @Test + public void testPut() { + InMemoryMap map = new InMemoryMap(); + byte[] expected = {0x1, 0x2}; + + map.put("key", expected); + + expected[0] = 0xf; + + byte[] actual = map.get("key"); + + assertFalse(Arrays.equals(expected, actual)); + } + +}

1 0

r46 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 19 May '10

19 May '10

Author: bleny Date: 2010-05-19 19:21:30 +0200 (Wed, 19 May 2010) New Revision: 46 Url: http://nuiton.org/repositories/revision/diswork/46 Log: cleanup, refactoring in code and tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/ Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) @@ -5,7 +5,7 @@ import java.io.IOException; import java.io.InputStream; -public interface DistributedFileSystem extends Closeable { +public interface DistributedFileSystem { /** * checks the existence of a file on the virtual FS Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-19 17:21:30 UTC (rev 46) @@ -12,6 +12,7 @@ setDefaultOption("storage", "/tmp/disworkfs/storage" + random.nextInt()); // replication strategy... + // TODO 20100519 bleny chunk size parameter } public File getStoragePath() { Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,25 +1,16 @@ package org.nuiton.disworkfs; -import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; -import java.util.UUID; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.nodes.DirectoryNode; -import org.nuiton.disworkfs.nodes.FileNode; -import org.nuiton.disworkfs.nodes.Node; -import org.nuiton.disworkfs.operations.Read; -import org.nuiton.disworkfs.operations.Write; +import org.nuiton.disworkfs.storage.EntryUtil; import org.nuiton.disworkfs.storage.Storage; -import org.nuiton.disworkfs.storage.StorageStub; -import org.nuiton.util.FileUtil; -import sun.management.snmp.jvminstr.JvmThreadInstanceEntryImpl.ThreadStateMap.Byte0; public class DisworkFileSystem implements DistributedFileSystem { @@ -28,11 +19,8 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); public DisworkFileSystem(DisworkConfig disworkConfig) { - storage = new StorageStub(); - - DirectoryNode root = new DirectoryNode(); - storage.put("/", root); - + storage = new Storage(new HashMap<String, byte[]>()); + // TODO 20100519 bleny create root dir } @Override @@ -52,7 +40,7 @@ String result = null; if (content == null) { // start of walk - content = storage.getRoot(); + content = storage.getRootDirectory(); result = walk(path, "/", content); } else if (FilenameUtils.getFullPathNoEndSeparator(path).equals(current)) { // in the last directory @@ -62,7 +50,7 @@ String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); String p = dirNames[0]; - String entry = EntryUtil.find(content, p); + String entry = EntryUtil.findEntryInDirectory(content, p); if (entry != null) { result = entry; } else { @@ -79,7 +67,7 @@ // mise a jour de current current += "/" + p; - String entry = EntryUtil.find(content, p); + String entry = EntryUtil.findEntryInDirectory(content, p); if (entry != null) { if (EntryUtil.isDirectory(entry)) { String id = EntryUtil.getId(entry); @@ -88,7 +76,7 @@ } else if (EntryUtil.isLink(entry)) { String id = EntryUtil.getId(entry); String linkContent = storage.getLink(id); - String newTarget = EntryUtil.getAbsolutePath(current, linkContent); + String newTarget = EntryUtil.resolveLink(current, linkContent); newTarget += path.substring(current.length()); // restart walk from / @@ -117,7 +105,7 @@ if (EntryUtil.isLink(entry)) { String id = EntryUtil.getId(entry); String link = storage.getLink(id); - String newTarget = EntryUtil.getAbsolutePath(path, link); + String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); } else if(EntryUtil.isDirectory(entry)) { throw new IOException("target is not a file: " + path); @@ -135,12 +123,14 @@ String idParent = EntryUtil.getId(entryParent); String content = storage.getDirectory(idParent); String id = EntryUtil.generateId(); - String newContent = EntryUtil.add(content, EntryUtil.TYPE.D, file, id); + String newContent = EntryUtil.addEntryToDirectoryContent(content, EntryUtil.TYPE.D, file, id); // store file before meta info + // FIXME 20100519 bleny this should use putFile storage.put(id, source); // update meta info directory InputStream parentContent = new ByteArrayInputStream(newContent.getBytes()); + // FIXME 20100519 bleny this should use putDirectory storage.put(idParent, parentContent); } else { throw new IOException(parent + " is not a directory"); @@ -148,63 +138,8 @@ } @Override - public void close() throws IOException { - storage.close(); - } - - @Override public void mkdir(String path) throws IOException { - - // First check everything is OK with parent dir - log.info("trying create(" + path + ")"); - - if (exists(path)) { - - throw new IOException("can't create " + path + ", already exists"); - - } else { - - String parentPath = FilenameUtils.getFullPathNoEndSeparator(path); - - // needed if parent dir is root - if (parentPath.equals("")) { - parentPath = "/"; - } - - Node parentNode = storage.get(parentPath); - - if (parentNode == null) { - throw new IOException("can't create " + path - + " directory " + parentPath + " doesn't exists"); - } - - if (parentNode.isDirectory()) { - - DirectoryNode parent = (DirectoryNode) parentNode; - - String fileName = FilenameUtils.getName(path); - - log.info("adding \"" + fileName + "\" to " + parentPath + " (" + parent + ")"); - - parent.add(fileName); - - storage.put(path, new DirectoryNode()); - - } else { - - throw new IOException("can't write : " + parentPath + " is not a directory"); - - } - } - + throw new UnsupportedOperationException("not yet implemented"); } - - public void dump() { - - for (String path : storage.keySet()) { - System.out.println(path + " → " + storage.get(path)); - } - - } } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,108 +0,0 @@ -/* *##% - * Copyright (c) 2010 poussin. All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - *##%*/ - -package org.nuiton.disworkfs; - - -import java.util.UUID; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class EntryUtil { - - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(EntryUtil.class); - - static final public String ENTRY_SEPARATOR = ":"; - - static final int TYPE_LENGTH = 1; - static final int UUID_LENGTH = UUID.randomUUID().toString().length(); - static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); - - static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; - - static public String add(String content, TYPE type, String name, String id) { - content += type.name() + ENTRY_SEPARATOR + name + ENTRY_SEPARATOR + id; - return content; - } - - static public String generateId() { - String result = UUID.randomUUID().toString(); - return result; - } - - static public boolean isDirectory(String entry) { - boolean result = entry.startsWith(TYPE.D.name()); - return result; - } - - static public boolean isLink(String entry) { - boolean result = entry.startsWith(TYPE.L.name()); - return result; - } - - /** - * find file name in directory content description - * - * @param directoryContent - * @param name - * @return null if name is not found - */ - static public String find(String directoryContent, String name) { - int index = directoryContent.indexOf(ENTRY_SEPARATOR + name + ENTRY_SEPARATOR); - - String result = null; - if (index != -1) { - result = directoryContent.substring( - index - TYPE_LENGTH, index + ENTRY_SEPARATOR_LENGTH + name.length()); - } - return result; - } - - static public String getId(String entry) { - int index = entry.lastIndexOf(ENTRY_SEPARATOR); - String result = entry.substring(index); - return result; - } - - static public String getName(String entry) { - int start = entry.indexOf(ENTRY_SEPARATOR); - int last = entry.lastIndexOf(ENTRY_SEPARATOR); - String result = entry.substring(start, last); - return result; - } - - static public TYPE getType(String entry) { - int index = entry.lastIndexOf(ENTRY_SEPARATOR); - String type = entry.substring(0, index); - TYPE result = TYPE.valueOf(type); - return result; - } - - static public String getAbsolutePath(String parent, String link) { - - } - -} Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java (from rev 45, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-19 17:21:30 UTC (rev 46) @@ -0,0 +1,187 @@ +package org.nuiton.disworkfs.storage; + + +import java.util.UUID; + +import org.apache.commons.io.FilenameUtils; + +/** + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class EntryUtil { + + static final public String ENTRY_SEPARATOR = ":"; + static final public String ENTRIES_SEPARATOR = "\n"; + static final public String BLOCKIDS_SEPARATOR = ";"; + + static final int TYPE_LENGTH = 1; + /* + // this line may not be accurate for "/" + static final int UUID_LENGTH = UUID.randomUUID().toString().length(); + */ + static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); + + static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; + + /** + * generate a new Id usage as a new key for the map + * @return a String containing the id + */ + static public String generateId() { + String result = UUID.randomUUID().toString(); + return result; + } + + /** + * @param entry a string taken from the entries of a directory + * @return true if the entry is of type directory, false if it's a file or + * a link + */ + static public boolean isDirectory(String entry) { + boolean result = entry.startsWith(TYPE.D.name()); + return result; + } + + /** + * @param entry a string taken from the entries of a directory + * @return true if the entry is of type link, false if it's a file or + * a directory + */ + static public boolean isLink(String entry) { + boolean result = entry.startsWith(TYPE.L.name()); + return result; + } + + /** + * @param entry a string taken from the entries of a directory + * @return true if the entry is of type file, false if it's a link or + * a directory + */ + static public boolean isFile(String entry) { + boolean result = entry.startsWith(TYPE.F.name()); + return result; + } + + static public TYPE getType(String entry) { + int index = entry.indexOf(ENTRY_SEPARATOR); + String type = entry.substring(0, index); + TYPE result = TYPE.valueOf(type); + return result; + } + + static public String getName(String entry) { + int start = entry.indexOf(ENTRY_SEPARATOR); + int last = entry.lastIndexOf(ENTRY_SEPARATOR); + String result = entry.substring(start + ENTRY_SEPARATOR_LENGTH, last); + return result; + } + + static public String getId(String entry) { + int index = entry.lastIndexOf(ENTRY_SEPARATOR); + String result = entry.substring(index + ENTRY_SEPARATOR_LENGTH); + return result; + } + + static public int getTotalSizeFromMetaBlock(String metaBlock) { + String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); + return Integer.valueOf(result); + } + + static public String[] getBlockIdsFromMetaBlock(String metaBlock) { + String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length()); + String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); + return blockIds; + } + + /** + * find file name in directory content description + * + * @param directoryContent the entries of the directory to search in + * @param name + * @return the entry or null if name is not found + */ + static public String findEntryInDirectory(String directoryContent, + String name) { + int index = directoryContent.indexOf( + ENTRY_SEPARATOR + name + ENTRY_SEPARATOR); + String result = null; + if (index != -1) { + result = directoryContent.substring( + index - TYPE_LENGTH, directoryContent.length()); + result = result.substring(0, result.indexOf(ENTRIES_SEPARATOR)); + } + return result; + } + + /** + * + * @param content the string content of the directory (all entries before add) + * @param type the type of the new entry + * @param name the name (not a full path) of the new entry + * @param id the id where the content of the entry can be found + * @return a String with the new content + */ + static public String addEntryToDirectoryContent + (String content, TYPE type, String name, String id) { + content += ENTRIES_SEPARATOR + type.name() + ENTRY_SEPARATOR + name + + ENTRY_SEPARATOR + id; + return content; + } + + /** + * Resolve a path from a parent directory, + * + * ie resolve /dir/subdir, ../file returns /dir/file + * @param parent the path to the dir where start from compute the path + * @param link the path to follow (may be absolute or relative to + * <code>parent</code>) + * @return the path to the destination + */ + static public String resolveLink(String parent, String link) { + String result = null; + if (link.startsWith("/")) { + // destination is absolute, ignore parent + result = link; + } else { + // destination is relative, so we have to consider parent + // parent may have a trailing file name, let's remove it + String parentWithNoFile = parent; + if (!parent.endsWith("/")) { + // removing the file at the end of the path, it's not a + // directory and thus should not be part of the path + parentWithNoFile = parent.substring(0, parent.lastIndexOf("/") + 1); + } + + // concatenate and making the complete path consistent + // ie /dir/subdir/../file to /dir/file + result = FilenameUtils.normalize(parentWithNoFile + link); + } + return result; + } + + /** + * may be used to store strings in a byte[]. String can be found back + * using {@link #bytesToString(byte[])} + * @param string the string + * @return a byte array containing all the bytes in <code>string</code> + */ + public static byte[] stringToBytes(String string) { + // FIXME 20100519 bleny this code may cause inconsistences with charsets + byte[] bytes = string.getBytes(); + return bytes; + } + + /** + * @see {@link #stringToBytes(String)} + * @param string the string + * @return a byte array containing all the bytes in <code>string</code> + */ + public static String bytesToString(byte[] bytes) { + return new String(bytes); + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,67 +1,73 @@ package org.nuiton.disworkfs.storage; import java.io.ByteArrayInputStream; -import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.util.Collection; import java.util.Map; -import java.util.Set; -import java.util.UUID; import org.apache.commons.io.IOUtils; -import org.nuiton.disworkfs.nodes.Node; -/** - * This is a common memory where all the file system will be - * stored. Keys are paths of the tree and values are nodes - * (directory, file or symlink) you find at this precise path - * - * Exemple : - * <table> - * <tr> - * <th> - * Path (key) - * </th> - * <th> - * Node (value) - * </th> - * </tr> - * <tr> - * <td>"/"</td> - * <td> - * an instance of {@link org.nuiton.disworkfs.nodes.DirectoryNode} - * (content ["my_folder", "my_file"]) - * </td> - * </tr> - * <tr> - * <td>"/my_folder"</td> - * <td> - * an instance of {@link org.nuiton.disworkfs.nodes.DirectoryNode} - * (content ["my_second_file"]) - * </td> - * </tr> - * <tr> - * <td>"/my_file"</td> - * <td>an instance of {@link org.nuiton.disworkfs.nodes.FileNode}</td> - * </tr> - * <tr> - * <td>"/my_folder/my_second_file"</td> - * <td>an instance of {@link org.nuiton.disworkfs.nodes.FileNode}</td> - * </tr> - * </table> - * - * The Map should contains at least one entry : a DirectoryNode with the - * key "/". It will be the root folder. - * - * This interface should be implemented using a DHT or any other way to - * store a map on multiple nodes. - * - * Since this storage may use ressources (sockets or stream on files), it - * has to be closeable. - */ public class Storage { + protected class SplitBlocksInputStream extends InputStream { + + protected Map<String, byte[]> map; + protected String id; + + protected String[] blockIds; + protected int blockIdsIndex; + + protected byte[] currentBlock; + protected int currentBlockIndex; + + protected int available; + + + public SplitBlocksInputStream(Map<String, byte[]> map, String id) { + super(); + this.map = map; + this.id = id; + } + + @Override + public int read() throws IOException { + checkInitialization(); + + // update current block if needed + if (currentBlock == null || currentBlockIndex >= currentBlock.length) { + blockIdsIndex += 1; + if (blockIdsIndex < blockIds.length) { + currentBlock = map.get(blockIds[blockIdsIndex]); + currentBlockIndex = 0; + } else { + return -1; + } + } + + byte result = currentBlock[currentBlockIndex]; + currentBlockIndex += 1; + + return result; + } + + protected void checkInitialization() { + if (blockIds == null) { + byte[] bytes = map.get(id); + String metaBlock = EntryUtil.bytesToString(bytes); + blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); + blockIdsIndex = -1; + available = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + } + } + + @Override + public int available() { + checkInitialization(); + return available; + } + + } + protected Map<String, byte[]> map; public Storage(Map<String, byte[]> map) { @@ -72,81 +78,91 @@ return map.containsKey(key); } - public String getRoot() throws IOException { + public String getRootDirectory() throws IOException { String result = getDirectory("/"); return result; } - public String getDirectory(String uuid) throws IOException { - InputStream in = get(uuid); - // lecture de l'entry pour / - byte[] entryAsByte = new byte[in.available()]; - in.read(entry); - String entry = EntryUtil.byteToSting(entryAsByte); - byte[] contentAsByte = storage.get(entry); - if (EntryUtil.isDirectory(entry)) { - String content = EntryUtil.byteToSting(contentAsByte); - return content; - } else { - throw IOException("Root is not a directory"); - } + + + + + public String getDirectory(String id) throws IOException { + InputStream in = get(id); + String result = IOUtils.toString(in); + return result; } - /** - * return path to target file - * - * @param uuid - * @return - * @throws IOException - */ - public String getLink(String uuid) throws IOException { - InputStream in = get(uuid); - // lecture de l'entry pour / - byte[] entryAsByte = new byte[in.available()]; - in.read(entry); - String entry = EntryUtil.byteToSting(entryAsByte); - return entry; + public InputStream getFile(String id) { + InputStream result = get(id); + return result; } - public InputStream getFile(String uuid) { - InputStream result = get(uuid); - return result; + public String getLink(String id) throws IOException { + return EntryUtil.bytesToString(map.get(id)); } - protected InputStream get(String uuid) { - byte[] uuidAsByte = map.get(key); - String uuid = EntryUtil.byteToSting(uuidAsByte); - byte[] content = map.get(uuid); - // TODO poussin 20100518 create specifique InputStream - // Lorsque le content pourra etre en plusieurs bon, il faudra - // ecrire un InputStream qui permet le chargement au fur et a mesure - // de la demande new InputStream(map, uuid) - ByteArrayInputStream result = new ByteArrayInputStream(content); - return result; + // TODO 20100519 bleny write getDirectory, getFile, getLink + + + /** + * see {@link #get(String)} + */ + protected InputStream get(String key) { + InputStream in = new SplitBlocksInputStream(map, key); + return in; } - public Object put(String key, InputStream value) { - // TODO poussin 20100518 split value in multiple block (see also get) - // il faudra decouper en plusieurs block plutot que de tout lire - // dans ce cas, il faut aussi stocker la liste des UUID genere pour - // chaque bloque - byte[] content = null; - if (value != null) { - content = new byte[value.available()]; - value.read(content); + /** + * a put in the map, this involves split considerations + * @param key + * @param value + * @throws IOException + */ + public void put(String key, InputStream value) throws IOException { + + // TODO 20100519 bleny deal with copy-on-write + // TODO 20100519 bleny deal with null value properly + + if (value == null) { + value = new ByteArrayInputStream(new byte[0]); } - map.put(key, content); + + String blocksIds = ""; + + // TODO 20100519 bleny the size of blocks should be a config directive + byte[] buffer = new byte[10 * 1024 * 1024]; + + int read = value.read(buffer); + int totalSize = read; + + while (read == buffer.length) { + String id = EntryUtil.generateId(); + map.put(id, buffer); + blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + // for next iteration + read = value.read(buffer); + totalSize += read; + } + + byte[] lastBuffer = new byte[read]; + System.arraycopy(buffer, 0, lastBuffer, 0, read); + + String id = EntryUtil.generateId(); + map.put(id, buffer); + + blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + String metaBlock = totalSize + blocksIds; + + map.put(key, EntryUtil.stringToBytes(metaBlock)); } - public Object remove(Object key) { + public void remove(Object key) { // TODO poussin 20100518 mark bloque to remove map.remove(key); } - - public void close() throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,58 +0,0 @@ -package org.nuiton.disworkfs.storage; - -import java.io.IOException; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.nodes.Node; - -/** - * This class is a simple stub of {@link Storage}, it simply uses a HashMap - * in the local memory to store all the data. - * - * All the overridden methods of Map are rewritten only for logging purpose - * - */ -public class StorageStub extends HashMap<String, Node> implements Storage { - - private Log log = LogFactory.getLog(StorageStub.class); - - @Override - public boolean containsKey(Object key) { - log.info("containsKey" + key); - - if (!(key instanceof String)) { - log.warn("trying to look for a wrong-typed key"); - } - - return super.containsKey(key); - } - - @Override - public Node put(String key, Node value) { - log.info("put(" + key + ", " + value + ")"); - - return super.put(key, value); - } - - @Override - public Node get(Object key) { - - if (!(key instanceof String)) { - log.warn("trying to get for a wrong-typed key"); - } - - Node value = super.get(key); - - log.info("get(" + key + ") returns " + value + ")"); - - return value; - } - - @Override - public void close() throws IOException { - // nothing to do - } - -} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,136 +1,137 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nuiton.util.FileUtil; - -public class DistributedFileSystemTest { - - /** - * a place to store files for the test it's a subdirectory of the OS temp - * dir e.g. under linux /tmp/disworkfs/tests/ - */ - static protected String tempDirectoryPath = System.getProperty( - "java.io.tmpdir", ".") - + "/disworkfs/tests"; - - /** - * We will create a file at this path for test purpose - */ - static protected String randomFilePath = tempDirectoryPath + "/randomfile"; - - /** - * The file will have this fixed size - */ - static protected int randomFileSize = 3000; - - static protected DistributedFileSystem fileSystem; - - @Before - public void setUp() throws Exception { - File tempDirectory = new File(tempDirectoryPath); - tempDirectory.mkdir(); - - Random random = new Random(); - // creating random data for the file - byte[] randomBytes = new byte[randomFileSize]; - random.nextBytes(randomBytes); - - // dumping random data into the file - File randomFile = new File(randomFilePath); - FileUtils.writeByteArrayToFile(randomFile, randomBytes); - - DisworkConfig disworkConfig = new DisworkConfig(); - - fileSystem = new DisworkFileSystem(disworkConfig); - } - - @After - public void tearDown() throws Exception { - // cleaning - fileSystem.close(); - - FileUtil.deleteRecursively(tempDirectoryPath); - } - - @Test - public void testWrite() throws Exception { - fileSystem.write("/my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testExists() throws Exception { - fileSystem.write("/my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_file")); - assertFalse(fileSystem.exists("/my_other_file")); - } - - /** - * try to read a file that as never been created nor written - */ - @Test(expected = FileNotFoundException.class) - public void testFailAtRead() throws Exception { - fileSystem.read("/not_existing_file"); - } - - /** - * writing a file at the root directory and reading it. - * finally, compare original source and read result - * byte-to-byte : contents should be equals - * @throws Exception - */ - @Test - public void testWriteRead() throws Exception { - - InputStream source = new FileInputStream(randomFilePath); - - fileSystem.write("/my_file", source); - - InputStream readResult = fileSystem.read("/my_file"); - - source = new FileInputStream(randomFilePath); - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - - assertTrue(actualContentEquality); - } - - @Test - public void testMkdir() throws Exception { - fileSystem.mkdir("/my_folder"); - assertTrue(fileSystem.exists("/my_folder")); - } - - /** - * this use case should raise an exception because my_folder - * doesn't exists - */ - @Test(expected = IOException.class) - public void testWriteFail() throws Exception { - fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testWriteInFolder() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); - fileSystem.mkdir("/my_folder/my_sub_folder"); - fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); - - ((DisworkFileSystem) fileSystem).dump(); - } - -} +//package org.nuiton.disworkfs; +// +//import static org.junit.Assert.assertFalse; +//import static org.junit.Assert.assertTrue; +// +//import java.io.File; +//import java.io.FileInputStream; +//import java.io.FileNotFoundException; +//import java.io.IOException; +//import java.io.InputStream; +//import java.util.Random; +// +//import org.apache.commons.io.FileUtils; +//import org.apache.commons.io.IOUtils; +//import org.junit.After; +//import org.junit.Before; +//import org.junit.Test; +//import org.nuiton.util.FileUtil; +// +//public class DistributedFileSystemTest { +// +// /** +// * a place to store files for the test it's a subdirectory of the OS temp +// * dir e.g. under linux /tmp/disworkfs/tests/ +// */ +// static protected String tempDirectoryPath = System.getProperty( +// "java.io.tmpdir", ".") +// + "/disworkfs/tests"; +// +// /** +// * We will create a file at this path for test purpose +// */ +// static protected String randomFilePath = tempDirectoryPath + "/randomfile"; +// +// /** +// * The file will have this fixed size +// */ +// static protected int randomFileSize = 3000; +// +// static protected DistributedFileSystem fileSystem; +// +// @Before +// public void setUp() throws Exception { +// File tempDirectory = new File(tempDirectoryPath); +// tempDirectory.mkdir(); +// +// Random random = new Random(); +// // creating random data for the file +// byte[] randomBytes = new byte[randomFileSize]; +// random.nextBytes(randomBytes); +// +// // dumping random data into the file +// File randomFile = new File(randomFilePath); +// FileUtils.writeByteArrayToFile(randomFile, randomBytes); +// +// DisworkConfig disworkConfig = new DisworkConfig(); +// +// fileSystem = new DisworkFileSystem(disworkConfig); +// } +// +// @After +// public void tearDown() throws Exception { +// // cleaning +// fileSystem.close(); +// +// FileUtil.deleteRecursively(tempDirectoryPath); +// } +// +// @Test +// public void testWrite() throws Exception { +// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); +// } +// +// @Test +// public void testExists() throws Exception { +// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); +// assertTrue(fileSystem.exists("/my_file")); +// assertFalse(fileSystem.exists("/my_other_file")); +// } +// +// /** +// * try to read a file that as never been created nor written +// */ +// @Test(expected = FileNotFoundException.class) +// public void testFailAtRead() throws Exception { +// fileSystem.read("/not_existing_file"); +// } +// +// /** +// * writing a file at the root directory and reading it. +// * finally, compare original source and read result +// * byte-to-byte : contents should be equals +// * @throws Exception +// */ +// @Test +// public void testWriteRead() throws Exception { +// +// InputStream source = new FileInputStream(randomFilePath); +// +// fileSystem.write("/my_file", source); +// +// InputStream readResult = fileSystem.read("/my_file"); +// +// source = new FileInputStream(randomFilePath); +// boolean actualContentEquality = +// IOUtils.contentEquals(source, readResult); +// +// assertTrue(actualContentEquality); +// } +// +// @Test +// public void testMkdir() throws Exception { +// fileSystem.mkdir("/my_folder"); +// assertTrue(fileSystem.exists("/my_folder")); +// } +// +// /** +// * this use case should raise an exception because my_folder +// * doesn't exists +// */ +// @Test(expected = IOException.class) +// public void testWriteFail() throws Exception { +// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); +// } +// +// @Test +// public void testWriteInFolder() throws Exception { +// fileSystem.mkdir("/my_folder"); +// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); +// fileSystem.mkdir("/my_folder/my_sub_folder"); +// fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); +// +// ((DisworkFileSystem) fileSystem).dump(); +// } +// +// +//} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-19 17:21:30 UTC (rev 46) @@ -18,14 +18,13 @@ package org.nuiton.disworkfs; -import java.util.UUID; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; +import org.junit.Test; +import org.nuiton.disworkfs.storage.EntryUtil; + /** * * @author poussin @@ -36,89 +35,109 @@ */ public class EntryUtilTest { - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(EntryUtilTest.class); - + static protected String directoryEntry = + EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "mydir" + EntryUtil.ENTRY_SEPARATOR + + "myid1"; + + static protected String fileEntry = + EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR + + "myfile" + EntryUtil.ENTRY_SEPARATOR + + "myid2"; + + static protected String linkEntry = + EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR + + "mylink" + EntryUtil.ENTRY_SEPARATOR + + "myid3"; + + static protected String metaBlock = + "12345" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid3"; + @Test public void testIsDirectory() { - String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - assertTrue(EntryUtil.isDirectory(d)); - - String f = EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - assertFalse(EntryUtil.isDirectory(f)); - + assertTrue(EntryUtil.isDirectory(directoryEntry)); + assertFalse(EntryUtil.isDirectory(fileEntry)); } @Test public void testFind() { - String d1 = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - String d2 = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "titi" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); + + String content = directoryEntry + "\n" + + fileEntry + "\n" + + linkEntry; - String f = EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR - + "tata" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - String content = d1 + "\n" + d2 + "\n" + f; - { // recherche et trouve le nom - String entry = EntryUtil.find(content, "titi"); - assertEquals(entry, d2); + String entry = EntryUtil.findEntryInDirectory(content, "myfile"); + assertEquals(fileEntry, entry); } { // recherche et ne trouve pas le nom - String entry = EntryUtil.find(content, "pastrouve"); - assertEquals(entry, null); + String entry = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); + assertEquals(null, entry); } } @Test public void testGetId() { - String id = UUID.randomUUID().toString(); - String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + id; - - String id2 = EntryUtil.getId(d); - assertEquals(id, id2); - + assertEquals("myid1", EntryUtil.getId(directoryEntry)); + assertEquals("myid2", EntryUtil.getId(fileEntry)); + assertEquals("myid3", EntryUtil.getId(linkEntry)); } @Test public void testGetName() { - String id = UUID.randomUUID().toString(); - String name = "truc"; - String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + name + EntryUtil.ENTRY_SEPARATOR - + id; + assertEquals("mydir", EntryUtil.getName(directoryEntry)); + assertEquals("myfile", EntryUtil.getName(fileEntry)); + assertEquals("mylink", EntryUtil.getName(linkEntry)); - String name2 = EntryUtil.getName(d); - assertEquals(name, name2); - } @Test public void testGetType() { - EntryUtil.TYPE type = EntryUtil.TYPE.D; - String id = UUID.randomUUID().toString(); - - String d = type + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + id; + assertEquals(EntryUtil.TYPE.D, EntryUtil.getType(directoryEntry)); + assertEquals(EntryUtil.TYPE.F, EntryUtil.getType(fileEntry)); + assertEquals(EntryUtil.TYPE.L, EntryUtil.getType(linkEntry)); + } + + @Test + public void testResolveLink() { + String path; - EntryUtil.TYPE type2 = EntryUtil.getType(d); - assertEquals(type, type2); + path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); + assertEquals("/anotherdir", path); + path = EntryUtil.resolveLink("/dir/afile", "anotherdir"); + assertEquals("/dir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); + assertEquals("/dir/subdir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); + assertEquals("/dir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/", "subdir/file"); + assertEquals("/dir/subdir/file", path); + } + + @Test + public void testBytesToArray() { + String s = "abcdefg@^:éèvwxyz,!;*$"; + String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); + assertEquals(s, copy); + } + @Test + public void testGetTotalSizeFromMetaBlock() { + assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); } + + @Test + public void testGetBlockIdsFromMetaBlock() { + String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; + String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); + assertEquals(expectedBlocksIds.length, actualBlocksIds.length); + assertEquals(expectedBlocksIds[0], actualBlocksIds[0]); + assertEquals(expectedBlocksIds[1], actualBlocksIds[1]); + assertEquals(expectedBlocksIds[2], actualBlocksIds[2]); + } }

1 0

r45 - trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage
by bpoussin＠users.nuiton.org 18 May '10

18 May '10

Author: bpoussin Date: 2010-05-18 21:44:15 +0200 (Tue, 18 May 2010) New Revision: 45 Url: http://nuiton.org/repositories/revision/diswork/45 Log: support put of null value Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:35:33 UTC (rev 44) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:44:15 UTC (rev 45) @@ -130,8 +130,11 @@ // il faudra decouper en plusieurs block plutot que de tout lire // dans ce cas, il faut aussi stocker la liste des UUID genere pour // chaque bloque - byte[] content = new byte[value.available()]; - value.read(content); + byte[] content = null; + if (value != null) { + content = new byte[value.available()]; + value.read(content); + } map.put(key, content); }

1 0

r44 - trunk/diswork-fs
by bpoussin＠users.nuiton.org 18 May '10

18 May '10

Author: bpoussin Date: 2010-05-18 21:35:33 +0200 (Tue, 18 May 2010) New Revision: 44 Url: http://nuiton.org/repositories/revision/diswork/44 Log: suppression de la dependance peerunit Modified: trunk/diswork-fs/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-05-18 19:34:53 UTC (rev 43) +++ trunk/diswork-fs/pom.xml 2010-05-18 19:35:33 UTC (rev 44) @@ -11,7 +11,20 @@ <packaging>jar</packaging> <name>disworkfs</name> - <dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.0.2</version> + <configuration> + <source>1.6</source> + <target>1.6</target> + </configuration> + </plugin> + </plugins> + </build> + <dependencies> <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> @@ -28,16 +41,18 @@ <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> +  <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> </dependency> - </dependencies> + </dependencies> </project> +

1 0

r43 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bpoussin＠users.nuiton.org 18 May '10

18 May '10

Author: bpoussin Date: 2010-05-18 21:34:53 +0200 (Tue, 18 May 2010) New Revision: 43 Url: http://nuiton.org/repositories/revision/diswork/43 Log: Gros changement, rendre le code compilable est laisse en exercice a Bredan :D Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-18 19:34:40 UTC (rev 42) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-18 19:34:53 UTC (rev 43) @@ -4,7 +4,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.URISyntaxException; public interface DistributedFileSystem extends Closeable { @@ -15,7 +14,8 @@ * a path to a file on the virtual FS * @return true if the file exists, false if not */ - public boolean exists(String path); + public boolean exists(String path) throws IOException; +; /** * write a file @@ -27,7 +27,7 @@ * @throws IOException * if problems occurs while reading the source or writing on VFS */ - public void write(String path, InputStream source) throws IOException; + public void write(String parent, String file, InputStream source) throws IOException; /** * read a file on the VFS, the file is downloaded if needed (may take some Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-18 19:34:40 UTC (rev 42) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-18 19:34:53 UTC (rev 43) @@ -1,10 +1,14 @@ package org.nuiton.disworkfs; +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.UUID; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.nodes.DirectoryNode; @@ -14,6 +18,8 @@ import org.nuiton.disworkfs.operations.Write; import org.nuiton.disworkfs.storage.Storage; import org.nuiton.disworkfs.storage.StorageStub; +import org.nuiton.util.FileUtil; +import sun.management.snmp.jvminstr.JvmThreadInstanceEntryImpl.ThreadStateMap.Byte0; public class DisworkFileSystem implements DistributedFileSystem { @@ -30,104 +36,118 @@ } @Override - public boolean exists(String path) { - boolean exists = storage.containsKey(path); - log.info("exists(" + path + ") returns " + exists); - return exists; + public boolean exists(String path) throws IOException { + String entry = walk(path, null, null); + boolean result = entry != null; + return result; } - @Override - public InputStream read(String path) throws FileNotFoundException, - IOException { + /** + * Return id of the last element in path. + * + * @param path + * @return null if path is not valid + */ + protected String walk(String path, String current, String content) throws IOException { + String result = null; + if (content == null) { + // start of walk + content = storage.getRoot(); + result = walk(path, "/", content); + } else if (FilenameUtils.getFullPathNoEndSeparator(path).equals(current)) { + // in the last directory - log.info("trying read(" + path + ")"); - - if (exists(path)) { - Node target = storage.get(path); - Read read = new Read(storage); - try { - target.operation(read); - return read.getInputStream(); - } catch (Exception eee) { - throw new IOException("diswork intern error", eee); + // recuperation du directory a traiter dans p + String tail = path.substring(current.length()); + String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); + String p = dirNames[0]; + + String entry = EntryUtil.find(content, p); + if (entry != null) { + result = entry; + } else { + result = null; } } else { - log.warn("file not found : " + path); - throw new FileNotFoundException(); - } - } - - public void create(String path) throws IOException { - - // First check everything is OK with parent dir - log.info("trying create(" + path + ")"); - - if (exists(path)) { - - throw new IOException("can't create " + path + ", already exists"); - - } else { - - String parentPath = FilenameUtils.getFullPathNoEndSeparator(path); - - // needed if parent dir is root - if (parentPath.equals("")) { - parentPath = "/"; - } - - Node parentNode = storage.get(parentPath); - - if (parentNode == null) { - throw new IOException("can't create file " + path - + ", " + parentPath + " doesn't exists"); - } - - if (parentNode.isDirectory()) { - - DirectoryNode parent = (DirectoryNode) parentNode; - - String fileName = FilenameUtils.getName(path); + // in middle of path - log.info("adding \"" + fileName + "\" to " + parentPath + " (" + parent + ")"); - - parent.add(fileName); - - storage.put(path, new FileNode(null)); - + // recuperation du directory a traiter dans p + String tail = path.substring(current.length()); + String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); + String p = dirNames[0]; + + // mise a jour de current + current += "/" + p; + + String entry = EntryUtil.find(content, p); + if (entry != null) { + if (EntryUtil.isDirectory(entry)) { + String id = EntryUtil.getId(entry); + content = storage.getDirectory(id); + result = walk(path, current, content); + } else if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getId(entry); + String linkContent = storage.getLink(id); + String newTarget = EntryUtil.getAbsolutePath(current, linkContent); + newTarget += path.substring(current.length()); + + // restart walk from / + result = walk(newTarget, null, null); + } else { + // erreur, find file in path like '/dir1/dir2/filename/dir3' + result = null; + } } else { - throw new IOException("can't write : " + parentPath + "is not a directory"); + result = null; } } - + return result; } @Override - public void write(String path, InputStream source) throws IOException { - write(path, source, true); - } - - public void write(String path, InputStream source, boolean forceCreate) - throws IOException { + public InputStream read(String path) throws FileNotFoundException, + IOException { - if (! exists(path)) { - if (forceCreate) { - create(path); - } else { - throw new IOException("can't write not created file"); - } + String entry = walk(path, null, null); + if (entry == null) { + throw new FileNotFoundException(path); } - try { - Write write = new Write(storage, source); - - Node target = storage.get(path); - target.operation(write); - } catch (Exception eee) { - throw new IOException("diswork intern error", eee); + InputStream result; + if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getId(entry); + String link = storage.getLink(id); + String newTarget = EntryUtil.getAbsolutePath(path, link); + result = read(newTarget); + } else if(EntryUtil.isDirectory(entry)) { + throw new IOException("target is not a file: " + path); + } else { + String id = EntryUtil.getId(entry); + result = storage.getFile(id); } + return result; } @Override + public void write(String parent, String file, InputStream source) throws IOException { + String entryParent = walk(parent, null, null); + if (EntryUtil.isDirectory(entryParent)) { + String idParent = EntryUtil.getId(entryParent); + String content = storage.getDirectory(idParent); + String id = EntryUtil.generateId(); + String newContent = EntryUtil.add(content, EntryUtil.TYPE.D, file, id); + + // store file before meta info + storage.put(id, source); + // update meta info directory + InputStream parentContent = new ByteArrayInputStream(newContent.getBytes()); + storage.put(idParent, parentContent); + } else { + throw new IOException(parent + " is not a directory"); + } + } + + @Override public void close() throws IOException { storage.close(); } Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java 2010-05-18 19:34:53 UTC (rev 43) @@ -0,0 +1,108 @@ +/* *##% + * Copyright (c) 2010 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.nuiton.disworkfs; + + +import java.util.UUID; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class EntryUtil { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(EntryUtil.class); + + static final public String ENTRY_SEPARATOR = ":"; + + static final int TYPE_LENGTH = 1; + static final int UUID_LENGTH = UUID.randomUUID().toString().length(); + static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); + + static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; + + static public String add(String content, TYPE type, String name, String id) { + content += type.name() + ENTRY_SEPARATOR + name + ENTRY_SEPARATOR + id; + return content; + } + + static public String generateId() { + String result = UUID.randomUUID().toString(); + return result; + } + + static public boolean isDirectory(String entry) { + boolean result = entry.startsWith(TYPE.D.name()); + return result; + } + + static public boolean isLink(String entry) { + boolean result = entry.startsWith(TYPE.L.name()); + return result; + } + + /** + * find file name in directory content description + * + * @param directoryContent + * @param name + * @return null if name is not found + */ + static public String find(String directoryContent, String name) { + int index = directoryContent.indexOf(ENTRY_SEPARATOR + name + ENTRY_SEPARATOR); + + String result = null; + if (index != -1) { + result = directoryContent.substring( + index - TYPE_LENGTH, index + ENTRY_SEPARATOR_LENGTH + name.length()); + } + return result; + } + + static public String getId(String entry) { + int index = entry.lastIndexOf(ENTRY_SEPARATOR); + String result = entry.substring(index); + return result; + } + + static public String getName(String entry) { + int start = entry.indexOf(ENTRY_SEPARATOR); + int last = entry.lastIndexOf(ENTRY_SEPARATOR); + String result = entry.substring(start, last); + return result; + } + + static public TYPE getType(String entry) { + int index = entry.lastIndexOf(ENTRY_SEPARATOR); + String type = entry.substring(0, index); + TYPE result = TYPE.valueOf(type); + return result; + } + + static public String getAbsolutePath(String parent, String link) { + + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:34:40 UTC (rev 42) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:34:53 UTC (rev 43) @@ -1,8 +1,15 @@ package org.nuiton.disworkfs.storage; +import java.io.ByteArrayInputStream; import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.apache.commons.io.IOUtils; import org.nuiton.disworkfs.nodes.Node; /** @@ -53,6 +60,90 @@ * Since this storage may use ressources (sockets or stream on files), it * has to be closeable. */ -public interface Storage extends Map<String, Node>, Closeable { +public class Storage { + protected Map<String, byte[]> map; + + public Storage(Map<String, byte[]> map) { + this.map = map; + } + + public boolean exists(Object key) { + return map.containsKey(key); + } + + public String getRoot() throws IOException { + String result = getDirectory("/"); + return result; + } + + public String getDirectory(String uuid) throws IOException { + InputStream in = get(uuid); + // lecture de l'entry pour / + byte[] entryAsByte = new byte[in.available()]; + in.read(entry); + String entry = EntryUtil.byteToSting(entryAsByte); + byte[] contentAsByte = storage.get(entry); + if (EntryUtil.isDirectory(entry)) { + String content = EntryUtil.byteToSting(contentAsByte); + return content; + } else { + throw IOException("Root is not a directory"); + } + } + + /** + * return path to target file + * + * @param uuid + * @return + * @throws IOException + */ + public String getLink(String uuid) throws IOException { + InputStream in = get(uuid); + // lecture de l'entry pour / + byte[] entryAsByte = new byte[in.available()]; + in.read(entry); + String entry = EntryUtil.byteToSting(entryAsByte); + return entry; + } + + public InputStream getFile(String uuid) { + InputStream result = get(uuid); + return result; + } + + protected InputStream get(String uuid) { + byte[] uuidAsByte = map.get(key); + String uuid = EntryUtil.byteToSting(uuidAsByte); + byte[] content = map.get(uuid); + // TODO poussin 20100518 create specifique InputStream + // Lorsque le content pourra etre en plusieurs bon, il faudra + // ecrire un InputStream qui permet le chargement au fur et a mesure + // de la demande new InputStream(map, uuid) + ByteArrayInputStream result = new ByteArrayInputStream(content); + return result; + } + + public Object put(String key, InputStream value) { + // TODO poussin 20100518 split value in multiple block (see also get) + // il faudra decouper en plusieurs block plutot que de tout lire + // dans ce cas, il faut aussi stocker la liste des UUID genere pour + // chaque bloque + byte[] content = new byte[value.available()]; + value.read(content); + map.put(key, content); + } + + public Object remove(Object key) { + // TODO poussin 20100518 mark bloque to remove + map.remove(key); + } + + + + public void close() throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + } Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-18 19:34:53 UTC (rev 43) @@ -0,0 +1,124 @@ +/* *##% + * Copyright (c) 2010 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.nuiton.disworkfs; + + +import java.util.UUID; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class EntryUtilTest { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(EntryUtilTest.class); + + @Test + public void testIsDirectory() { + String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "toto" + EntryUtil.ENTRY_SEPARATOR + + UUID.randomUUID().toString(); + + assertTrue(EntryUtil.isDirectory(d)); + + String f = EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR + + "toto" + EntryUtil.ENTRY_SEPARATOR + + UUID.randomUUID().toString(); + + assertFalse(EntryUtil.isDirectory(f)); + + } + + @Test + public void testFind() { + String d1 = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "toto" + EntryUtil.ENTRY_SEPARATOR + + UUID.randomUUID().toString(); + + String d2 = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "titi" + EntryUtil.ENTRY_SEPARATOR + + UUID.randomUUID().toString(); + + String f = EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR + + "tata" + EntryUtil.ENTRY_SEPARATOR + + UUID.randomUUID().toString(); + + String content = d1 + "\n" + d2 + "\n" + f; + + { + // recherche et trouve le nom + String entry = EntryUtil.find(content, "titi"); + assertEquals(entry, d2); + } + { + // recherche et ne trouve pas le nom + String entry = EntryUtil.find(content, "pastrouve"); + assertEquals(entry, null); + } + } + + @Test + public void testGetId() { + String id = UUID.randomUUID().toString(); + String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "toto" + EntryUtil.ENTRY_SEPARATOR + + id; + + String id2 = EntryUtil.getId(d); + assertEquals(id, id2); + + } + + @Test + public void testGetName() { + String id = UUID.randomUUID().toString(); + String name = "truc"; + String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + name + EntryUtil.ENTRY_SEPARATOR + + id; + + String name2 = EntryUtil.getName(d); + assertEquals(name, name2); + + } + + @Test + public void testGetType() { + EntryUtil.TYPE type = EntryUtil.TYPE.D; + String id = UUID.randomUUID().toString(); + + String d = type + EntryUtil.ENTRY_SEPARATOR + + "toto" + EntryUtil.ENTRY_SEPARATOR + + id; + + EntryUtil.TYPE type2 = EntryUtil.getType(d); + assertEquals(type, type2); + + } +}

1 0

r42 - trunk
by bpoussin＠users.nuiton.org 18 May '10

18 May '10

Author: bpoussin Date: 2010-05-18 21:34:40 +0200 (Tue, 18 May 2010) New Revision: 42 Url: http://nuiton.org/repositories/revision/diswork/42 Log: suppression de peerunit dans les dependances Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-05-18 13:45:05 UTC (rev 41) +++ trunk/pom.xml 2010-05-18 19:34:40 UTC (rev 42) @@ -61,13 +61,14 @@ <type>jar</type> <scope>compile</scope> </dependency> +  <dependency> <groupId>junit</groupId>

1 0

r41 - in trunk/diswork-fs: . src/main/java/org/nuiton/disworkfs src/main/java/org/nuiton/disworkfs/nodes src/main/java/org/nuiton/disworkfs/operations src/main/java/org/nuiton/disworkfs/storage src/main/resources src/test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 18 May '10

18 May '10

Author: bleny Date: 2010-05-18 15:45:05 +0200 (Tue, 18 May 2010) New Revision: 41 Url: http://nuiton.org/repositories/revision/diswork/41 Log: read, write, mkdir, exists Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/DirectoryNode.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/FileNode.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/Node.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/NodeType.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/SymLinkNode.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Operation.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Read.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Write.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/ Modified: trunk/diswork-fs/ trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java Property changes on: trunk/diswork-fs ___________________________________________________________________ Modified: svn:ignore - target *.iml *.ipr *.iws .classpath .project + target *.iml *.ipr *.iws .classpath .project .settings Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-05-17 09:58:49 UTC (rev 40) +++ trunk/diswork-fs/pom.xml 2010-05-18 13:45:05 UTC (rev 41) @@ -21,10 +21,6 @@ <artifactId>log4j</artifactId> </dependency> <dependency> - <groupId>jgroups</groupId> - <artifactId>jgroups</artifactId> - </dependency> - <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-17 09:58:49 UTC (rev 40) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-18 13:45:05 UTC (rev 41) @@ -1,10 +1,12 @@ package org.nuiton.disworkfs; +import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.URISyntaxException; -public interface DistributedFileSystem { +public interface DistributedFileSystem extends Closeable { /** * checks the existence of a file on the virtual FS @@ -43,5 +45,7 @@ public InputStream read(String path) throws FileNotFoundException, IOException; + public void mkdir(String path) throws IOException; + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-17 09:58:49 UTC (rev 40) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-18 13:45:05 UTC (rev 41) @@ -4,40 +4,187 @@ import java.io.IOException; import java.io.InputStream; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.nodes.DirectoryNode; +import org.nuiton.disworkfs.nodes.FileNode; +import org.nuiton.disworkfs.nodes.Node; +import org.nuiton.disworkfs.operations.Read; +import org.nuiton.disworkfs.operations.Write; +import org.nuiton.disworkfs.storage.Storage; +import org.nuiton.disworkfs.storage.StorageStub; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - public class DisworkFileSystem implements DistributedFileSystem { - + protected Storage storage; + private static final Log log = LogFactory.getLog(DisworkFileSystem.class); public DisworkFileSystem(DisworkConfig disworkConfig) { - + storage = new StorageStub(); + + DirectoryNode root = new DirectoryNode(); + storage.put("/", root); + } - + @Override public boolean exists(String path) { - // TODO - throw new NotImplementedException(); + boolean exists = storage.containsKey(path); + log.info("exists(" + path + ") returns " + exists); + return exists; } @Override public InputStream read(String path) throws FileNotFoundException, IOException { - // TODO - throw new NotImplementedException(); + + log.info("trying read(" + path + ")"); + + if (exists(path)) { + Node target = storage.get(path); + Read read = new Read(storage); + try { + target.operation(read); + return read.getInputStream(); + } catch (Exception eee) { + throw new IOException("diswork intern error", eee); + } + } else { + log.warn("file not found : " + path); + throw new FileNotFoundException(); + } } + + public void create(String path) throws IOException { + + // First check everything is OK with parent dir + log.info("trying create(" + path + ")"); + + if (exists(path)) { + + throw new IOException("can't create " + path + ", already exists"); + + } else { + + String parentPath = FilenameUtils.getFullPathNoEndSeparator(path); + + // needed if parent dir is root + if (parentPath.equals("")) { + parentPath = "/"; + } + + Node parentNode = storage.get(parentPath); + + if (parentNode == null) { + throw new IOException("can't create file " + path + + ", " + parentPath + " doesn't exists"); + } + + if (parentNode.isDirectory()) { + + DirectoryNode parent = (DirectoryNode) parentNode; + + String fileName = FilenameUtils.getName(path); + log.info("adding \"" + fileName + "\" to " + parentPath + " (" + parent + ")"); + + parent.add(fileName); + + storage.put(path, new FileNode(null)); + + } else { + throw new IOException("can't write : " + parentPath + "is not a directory"); + } + } + + } + @Override public void write(String path, InputStream source) throws IOException { - // TODO - throw new NotImplementedException(); + write(path, source, true); } + + public void write(String path, InputStream source, boolean forceCreate) + throws IOException { - public void close() { + if (! exists(path)) { + if (forceCreate) { + create(path); + } else { + throw new IOException("can't write not created file"); + } + } + + try { + Write write = new Write(storage, source); + + Node target = storage.get(path); + target.operation(write); + } catch (Exception eee) { + throw new IOException("diswork intern error", eee); + } } + @Override + public void close() throws IOException { + storage.close(); + } + + @Override + public void mkdir(String path) throws IOException { + + // First check everything is OK with parent dir + log.info("trying create(" + path + ")"); + + if (exists(path)) { + + throw new IOException("can't create " + path + ", already exists"); + + } else { + + String parentPath = FilenameUtils.getFullPathNoEndSeparator(path); + + // needed if parent dir is root + if (parentPath.equals("")) { + parentPath = "/"; + } + + Node parentNode = storage.get(parentPath); + + if (parentNode == null) { + throw new IOException("can't create " + path + + " directory " + parentPath + " doesn't exists"); + } + + if (parentNode.isDirectory()) { + + DirectoryNode parent = (DirectoryNode) parentNode; + + String fileName = FilenameUtils.getName(path); + + log.info("adding \"" + fileName + "\" to " + parentPath + " (" + parent + ")"); + + parent.add(fileName); + + storage.put(path, new DirectoryNode()); + + } else { + + throw new IOException("can't write : " + parentPath + " is not a directory"); + + } + } + + } + + public void dump() { + + for (String path : storage.keySet()) { + System.out.println(path + " → " + storage.get(path)); + } + + } + } Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/DirectoryNode.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/DirectoryNode.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/DirectoryNode.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,133 @@ +package org.nuiton.disworkfs.nodes; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; + +import org.nuiton.disworkfs.operations.Operation; + +public class DirectoryNode extends Node implements List<String> { + + protected List<String> childs; + + public DirectoryNode() { + super(NodeType.DIRECTORY); + childs = new ArrayList<String>(); + } + + @Override + public void operation(Operation operation) throws Exception { + operation.operation(this); + } + + // generated delegation code + + public void add(int index, String element) { + childs.add(index, element); + } + + public boolean add(String e) { + return childs.add(e); + } + + public boolean addAll(Collection<? extends String> c) { + return childs.addAll(c); + } + + public boolean addAll(int index, Collection<? extends String> c) { + return childs.addAll(index, c); + } + + public void clear() { + childs.clear(); + } + + public boolean contains(Object o) { + return childs.contains(o); + } + + public boolean containsAll(Collection<?> c) { + return childs.containsAll(c); + } + + public boolean equals(Object o) { + return childs.equals(o); + } + + public String get(int index) { + return childs.get(index); + } + + public int hashCode() { + return childs.hashCode(); + } + + public int indexOf(Object o) { + return childs.indexOf(o); + } + + public boolean isEmpty() { + return childs.isEmpty(); + } + + public Iterator<String> iterator() { + return childs.iterator(); + } + + public int lastIndexOf(Object o) { + return childs.lastIndexOf(o); + } + + public ListIterator<String> listIterator() { + return childs.listIterator(); + } + + public ListIterator<String> listIterator(int index) { + return childs.listIterator(index); + } + + public String remove(int index) { + return childs.remove(index); + } + + public boolean remove(Object o) { + return childs.remove(o); + } + + public boolean removeAll(Collection<?> c) { + return childs.removeAll(c); + } + + public boolean retainAll(Collection<?> c) { + return childs.retainAll(c); + } + + public String set(int index, String element) { + return childs.set(index, element); + } + + public int size() { + return childs.size(); + } + + public List<String> subList(int fromIndex, int toIndex) { + return childs.subList(fromIndex, toIndex); + } + + public Object[] toArray() { + return childs.toArray(); + } + + public <T> T[] toArray(T[] a) { + return childs.toArray(a); + } + + @Override + public String toString() { + return "DirectoryNode (size = " + childs.size() + ")"; + } + + +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/FileNode.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/FileNode.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/FileNode.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,63 @@ +package org.nuiton.disworkfs.nodes; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.operations.Operation; +import org.nuiton.util.StringUtil; + +public class FileNode extends Node { + + protected byte[] data; + protected byte[] checkSum; + + private Log log = LogFactory.getLog(FileNode.class); + + public FileNode(byte[] data) { + super(NodeType.FILE); + this.setData(data); + } + + public byte[] getData() { + return data; + } + + public byte[] getCheckSum() { + return checkSum; + } + + public void setData(byte[] data) { + this.data = data; + + // compute checkSum + if (data != null) { + try { + MessageDigest algorithm; + algorithm = MessageDigest.getInstance("MD5"); + algorithm.reset(); + algorithm.update(data); + this.checkSum = algorithm.digest(); + } catch (NoSuchAlgorithmException eee) { + // silent exception, "MD5" is provided in java.security + log.error("unable compute checksum", eee); + } + } + } + + @Override + public void operation(Operation operation) throws Exception { + operation.operation(this); + } + + @Override + public String toString() { + if (data == null) { + return "FileNode (empty file)"; + } else { + String checkSumString = StringUtil.asHex(checkSum); + return "FileNode (size = " + data.length + ", checksum = " + checkSumString + ")"; + } + } +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/Node.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/Node.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/Node.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,47 @@ +package org.nuiton.disworkfs.nodes; + +import java.io.Serializable; + +import org.nuiton.disworkfs.operations.Operation; + +/** + * This class represent a node in the tree structure of a file system. + * + * Since it's a data that way be distributed on a network, it has to be + * {@link Serializable}. + */ + +public abstract class Node implements Serializable { + + protected NodeType nodeType; + + public NodeType getNodeType() { + return nodeType; + } + + public void setNodeType(NodeType nodeType) { + this.nodeType = nodeType; + } + + protected Node(NodeType nodeType) { + this.nodeType = nodeType; + } + + public boolean isDirectory() { + boolean isDirectory = nodeType == NodeType.DIRECTORY; + return isDirectory; + } + + public boolean isFile() { + boolean isFile = nodeType == NodeType.FILE; + return isFile; + } + + public boolean isLink() { + boolean isLink = nodeType == NodeType.LINK; + return isLink; + } + + public abstract void operation(Operation operation) throws Exception; + +} \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/NodeType.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/NodeType.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/NodeType.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,7 @@ +package org.nuiton.disworkfs.nodes; + +public enum NodeType { + DIRECTORY, + FILE, + LINK +} \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/SymLinkNode.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/SymLinkNode.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/SymLinkNode.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,28 @@ +package org.nuiton.disworkfs.nodes; + +import org.nuiton.disworkfs.operations.Operation; + +public class SymLinkNode extends Node { + + protected String target; + + public SymLinkNode(String target) { + super(NodeType.LINK); + this.target = target; + } + + public String getTarget() { + return target; + } + + @Override + public String toString() { + return "symlink -> " + target.toString(); + } + + @Override + public void operation(Operation operation) throws Exception { + operation.operation(this); + } + +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/package-info.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,37 @@ +/** + * The classes in this package represent the different concept of the content + * of the tree-structure of a file-sytem. + * + * A node of the tree is always a {@link Node}. A node is one of those sub-types + * of Node : + * <dl> + * <dt>{@link DirectoryNode}</dt> + * <dd> + * It's a node that contains other nodes, it can be used as a + * {@link java.util.List} to browse its content. + * </dd> + * <dt>{@link FileNode}</dt> + * <dd> + * A file node, it contains the raw-data of the file (some bytes) + * and a checksum + * </dd> + * <dt>{@link SymLinkNode}</dt> + * <dd> + * It's a + * <a href="http://en.wikipedia.org/wiki/Symbolic_link"> + * symbolic link + * </a> + * in the UNIX way if thinking. + * It's a virtual node which consider another node as a target. Every + * operation done on this symbolic link should target the node behind + * the link. + * </dd> + * </dl> + * + * Note that the Directory and the SymLink do not contains direct references to + * other nodes. Instead, it contains String that should be interpreted as paths + * to those nodes (multiple paths for a directory, the path of the target node + * for the SymLink). + */ + +package org.nuiton.disworkfs.nodes; \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Operation.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Operation.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Operation.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,22 @@ +package org.nuiton.disworkfs.operations; + +import org.nuiton.disworkfs.nodes.DirectoryNode; +import org.nuiton.disworkfs.nodes.FileNode; +import org.nuiton.disworkfs.nodes.SymLinkNode; +import org.nuiton.disworkfs.storage.Storage; + +public abstract class Operation { + + protected Storage storage; + + public Operation(Storage storage) { + this.storage = storage; + } + + public abstract void operation(FileNode fileNode) throws Exception; + + public abstract void operation(DirectoryNode directoryNode) throws Exception; + + public abstract void operation(SymLinkNode symLinkNode) throws Exception; + +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Read.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Read.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Read.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,50 @@ +package org.nuiton.disworkfs.operations; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.nodes.DirectoryNode; +import org.nuiton.disworkfs.nodes.FileNode; +import org.nuiton.disworkfs.nodes.Node; +import org.nuiton.disworkfs.nodes.SymLinkNode; +import org.nuiton.disworkfs.storage.Storage; + +public class Read extends Operation { + + protected InputStream inputStream; + + private Log log = LogFactory.getLog(Read.class); + + public Read(Storage storage) { + super(storage); + } + + @Override + public void operation(FileNode fileNode) throws Exception { + log.info("reading file " + fileNode); + byte[] data = fileNode.getData(); + inputStream = new ByteArrayInputStream(data); + } + + @Override + public void operation(DirectoryNode directoryNode) throws Exception { + log.error("trying to read a file but " + directoryNode + " is a directory"); + throw new IOException(directoryNode + " is not a file but a directory"); + } + + @Override + public void operation(SymLinkNode symLinkNode) throws Exception { + log.info(symLinkNode + " is a link, forwarding"); + String targetPath = symLinkNode.getTarget(); + Node targetNode = storage.get(targetPath); + targetNode.operation(this); + } + + public InputStream getInputStream() { + return inputStream; + } + +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Write.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Write.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/Write.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,52 @@ +package org.nuiton.disworkfs.operations; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.nodes.DirectoryNode; +import org.nuiton.disworkfs.nodes.FileNode; +import org.nuiton.disworkfs.nodes.Node; +import org.nuiton.disworkfs.nodes.SymLinkNode; +import org.nuiton.disworkfs.storage.Storage; + +public class Write extends Operation { + + protected InputStream source; + + private Log log = LogFactory.getLog(Write.class); + + public Write(Storage storage, InputStream source) { + super(storage); + this.source = source; + } + + @Override + public void operation(FileNode fileNode) throws Exception { + if (fileNode == null) { + throw new IOException(fileNode + " has not been created"); + } + + byte[] data = IOUtils.toByteArray(source); + + fileNode.setData(data); + + log.info("writing " + fileNode); + + } + + @Override + public void operation(DirectoryNode directoryNode) throws Exception { + throw new IOException(directoryNode + " is not a file but a directory"); + } + + @Override + public void operation(SymLinkNode symLinkNode) throws Exception { + String targetPath = symLinkNode.getTarget(); + Node targetNode = storage.get(targetPath); + targetNode.operation(this); + } + +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,58 @@ +package org.nuiton.disworkfs.storage; + +import java.io.Closeable; +import java.util.Map; + +import org.nuiton.disworkfs.nodes.Node; + +/** + * This is a common memory where all the file system will be + * stored. Keys are paths of the tree and values are nodes + * (directory, file or symlink) you find at this precise path + * + * Exemple : + * <table> + * <tr> + * <th> + * Path (key) + * </th> + * <th> + * Node (value) + * </th> + * </tr> + * <tr> + * <td>"/"</td> + * <td> + * an instance of {@link org.nuiton.disworkfs.nodes.DirectoryNode} + * (content ["my_folder", "my_file"]) + * </td> + * </tr> + * <tr> + * <td>"/my_folder"</td> + * <td> + * an instance of {@link org.nuiton.disworkfs.nodes.DirectoryNode} + * (content ["my_second_file"]) + * </td> + * </tr> + * <tr> + * <td>"/my_file"</td> + * <td>an instance of {@link org.nuiton.disworkfs.nodes.FileNode}</td> + * </tr> + * <tr> + * <td>"/my_folder/my_second_file"</td> + * <td>an instance of {@link org.nuiton.disworkfs.nodes.FileNode}</td> + * </tr> + * </table> + * + * The Map should contains at least one entry : a DirectoryNode with the + * key "/". It will be the root folder. + * + * This interface should be implemented using a DHT or any other way to + * store a map on multiple nodes. + * + * Since this storage may use ressources (sockets or stream on files), it + * has to be closeable. + */ +public interface Storage extends Map<String, Node>, Closeable { + +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java 2010-05-18 13:45:05 UTC (rev 41) @@ -0,0 +1,58 @@ +package org.nuiton.disworkfs.storage; + +import java.io.IOException; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.nodes.Node; + +/** + * This class is a simple stub of {@link Storage}, it simply uses a HashMap + * in the local memory to store all the data. + * + * All the overridden methods of Map are rewritten only for logging purpose + * + */ +public class StorageStub extends HashMap<String, Node> implements Storage { + + private Log log = LogFactory.getLog(StorageStub.class); + + @Override + public boolean containsKey(Object key) { + log.info("containsKey" + key); + + if (!(key instanceof String)) { + log.warn("trying to look for a wrong-typed key"); + } + + return super.containsKey(key); + } + + @Override + public Node put(String key, Node value) { + log.info("put(" + key + ", " + value + ")"); + + return super.put(key, value); + } + + @Override + public Node get(Object key) { + + if (!(key instanceof String)) { + log.warn("trying to get for a wrong-typed key"); + } + + Node value = super.get(key); + + log.info("get(" + key + ") returns " + value + ")"); + + return value; + } + + @Override + public void close() throws IOException { + // nothing to do + } + +} Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-17 09:58:49 UTC (rev 40) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-18 13:45:05 UTC (rev 41) @@ -6,6 +6,4 @@ log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level log4j.logger.org.nuiton.disworkfs=TRACE -#log4j.logger.org.nuiton.disworkfs.transport=WARN -#log4j.logger.org.nuiton.disworkfs.services.DownloadService=WARN -#log4j.logger.org.nuiton.disworkfs.services.UploadService=WARN +log4j.logger.org.nuiton.disworkfs.storage=WARN \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-17 09:58:49 UTC (rev 40) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-18 13:45:05 UTC (rev 41) @@ -5,6 +5,9 @@ import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; import java.util.Random; import org.apache.commons.io.FileUtils; @@ -34,24 +37,13 @@ */ static protected int randomFileSize = 3000; - static protected String storagePath1; - static protected String storagePath2; - - static protected DisworkConfig disworkConfig1; - static protected DisworkConfig disworkConfig2; - + static protected DistributedFileSystem fileSystem; + @Before public void setUp() throws Exception { File tempDirectory = new File(tempDirectoryPath); tempDirectory.mkdir(); - File storage1 = new File(tempDirectory, "storage1"); - storage1.mkdir(); - storagePath1 = storage1.getAbsolutePath(); - File storage2 = new File(tempDirectory, "storage2"); - storage2.mkdir(); - storagePath2 = storage2.getAbsolutePath(); - Random random = new Random(); // creating random data for the file byte[] randomBytes = new byte[randomFileSize]; @@ -61,88 +53,84 @@ File randomFile = new File(randomFilePath); FileUtils.writeByteArrayToFile(randomFile, randomBytes); - disworkConfig1 = new DisworkConfig(); - disworkConfig1.setOption("storage", storagePath1); + DisworkConfig disworkConfig = new DisworkConfig(); - disworkConfig2 = new DisworkConfig(); - disworkConfig2.setOption("storage", storagePath2); + fileSystem = new DisworkFileSystem(disworkConfig); } @After public void tearDown() throws Exception { // cleaning + fileSystem.close(); + FileUtil.deleteRecursively(tempDirectoryPath); } - - @Test - public void fakeTest() throws Exception { - // TODO 2010-05-17 to be removed when a test is written - } + + @Test + public void testWrite() throws Exception { + fileSystem.write("/my_file", new FileInputStream(randomFilePath)); + } + + @Test + public void testExists() throws Exception { + fileSystem.write("/my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte : contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); - /* - @Test - public void testWrite() throws Exception { + fileSystem.write("/my_file", source); + + InputStream readResult = fileSystem.read("/my_file"); + + source = new FileInputStream(randomFilePath); + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + + assertTrue(actualContentEquality); + } + + @Test + public void testMkdir() throws Exception { + fileSystem.mkdir("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testWriteFail() throws Exception { + fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); + } - DisworkFileSystem disworkFileSystem1 = new DisworkFileSystem( - disworkConfig1); - disworkFileSystem1.write("monfichier", new FileInputStream(randomFilePath)); - disworkFileSystem1.close(); - + @Test + public void testWriteInFolder() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); + fileSystem.mkdir("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); + + ((DisworkFileSystem) fileSystem).dump(); + } - File monfichier = new File(randomFilePath); - File monfichierstorage = new File(tempDirectoryPath - + "/storage1/monfichier"); - - boolean actualContentEquality = IOUtils.contentEquals( - new FileInputStream(monfichier), new FileInputStream( - monfichierstorage)); - - assertTrue("file and copy content should be the same", - actualContentEquality); - } - */ - - /** - * this test run two DistributedFS. A file is written on the first when. We - * try to read from the other FS : since it doesn't own the file, it will - * try a lookup and download the file. - */ - /* - @Test - public void testRead() throws Exception { - - DisworkFileSystem disworkFileSystem1 = new DisworkFileSystem( - disworkConfig1); - DisworkFileSystem disworkFileSystem2 = new DisworkFileSystem( - disworkConfig2); - - disworkFileSystem1.write("mon/chemin/vers/mon/fichier", new FileInputStream( - randomFilePath)); - - boolean existsResult = disworkFileSystem2 - .exists("mon/chemin/vers/mon/fichier"); - assertTrue(existsResult); - - existsResult = disworkFileSystem2.exists("unautrefichierquinexistepas"); - assertFalse(existsResult); - - disworkFileSystem2.read("mon/chemin/vers/mon/fichier"); - - File monfichierstorage1 = new File(tempDirectoryPath - + "/storage1/mon/chemin/vers/mon/fichier"); - File monfichierstorage2 = new File(tempDirectoryPath - + "/storage2/mon/chemin/vers/mon/fichier"); - - boolean actualContentEquality = IOUtils.contentEquals( - new FileInputStream(monfichierstorage1), new FileInputStream( - monfichierstorage2)); - - assertTrue("file and copy content should be the same", - actualContentEquality); - - disworkFileSystem1.close(); - disworkFileSystem2.close(); - } - */ - }

1 0

r40 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/transport test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 17 May '10

17 May '10

Author: bleny Date: 2010-05-17 11:58:49 +0200 (Mon, 17 May 2010) New Revision: 40 Url: http://nuiton.org/repositories/revision/diswork/40 Log: removing code moved to the jgroups branch Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/messages/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Address.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Message.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Receiver.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/jgroups/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/util/ Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-17 09:58:49 UTC (rev 40) @@ -12,9 +12,8 @@ * @param path * a path to a file on the virtual FS * @return true if the file exists, false if not - * @throws InterruptedException */ - public boolean exists(String path) throws InterruptedException; + public boolean exists(String path); /** * write a file @@ -25,10 +24,8 @@ * the file to copy on the VFS * @throws IOException * if problems occurs while reading the source or writing on VFS - * @throws InterruptedException */ - public void write(String path, InputStream source) throws IOException, - InterruptedException; + public void write(String path, InputStream source) throws IOException; /** * read a file on the VFS, the file is downloaded if needed (may take some @@ -42,10 +39,9 @@ * {@link #exists(String) to check before read} * @throws IOException * if a problem occur while reading the local file - * @throws InterruptedException */ public InputStream read(String path) throws FileNotFoundException, - IOException, InterruptedException; + IOException; } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-17 09:58:49 UTC (rev 40) @@ -10,7 +10,6 @@ public DisworkConfig() { Random random = new Random(); setDefaultOption("storage", "/tmp/disworkfs/storage" + random.nextInt()); - setDefaultOption("jgroups.cluster_name", "diswork-fs"); // replication strategy... } @@ -18,8 +17,4 @@ public File getStoragePath() { return getOptionAsFile("storage"); } - - public String getJGroupsClusterName() { - return getOption("jgroups.cluster_name"); - } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-17 09:58:49 UTC (rev 40) @@ -1,135 +1,43 @@ package org.nuiton.disworkfs; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.services.DisworkServicesManager; -import org.nuiton.disworkfs.services.DownloadService; -import org.nuiton.disworkfs.services.LookUpService; -import org.nuiton.disworkfs.services.UploadService; -import org.nuiton.disworkfs.split.FileDescription; -import org.nuiton.disworkfs.util.SimpleDownload; -import org.nuiton.disworkfs.util.SimpleLookUp; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + public class DisworkFileSystem implements DistributedFileSystem { - protected DownloadService downloadService; - protected UploadService uploadService; - protected LookUpService lookUpService; - protected DisworkConfig disworkConfig; - protected DisworkServicesManager disworkServicesManager; private static final Log log = LogFactory.getLog(DisworkFileSystem.class); public DisworkFileSystem(DisworkConfig disworkConfig) { - this.disworkConfig = disworkConfig; - - disworkServicesManager = new DisworkServicesManager(disworkConfig); - uploadService = new UploadService(); - disworkServicesManager.register(uploadService); - downloadService = new DownloadService(); - disworkServicesManager.register(downloadService); - lookUpService = new LookUpService(); - disworkServicesManager.register(lookUpService); - } @Override - public InputStream read(String path) throws InterruptedException, - FileNotFoundException { - - log.info("trying to read " + path); - - File file = new File(disworkConfig.getStoragePath(), path); - log.info("trying at local file system " + file.getAbsolutePath()); - - if (!file.exists()) { - // the file is not available - // let's download it - - SimpleDownload simpleDownload = new SimpleDownload(path, - lookUpService, downloadService); - boolean fileFound = simpleDownload.initiateDownload(); - if (fileFound) - simpleDownload.startDownload(); - else - throw new FileNotFoundException("no look-up response received"); - - } - - return new BufferedInputStream(new FileInputStream(file)); + public boolean exists(String path) { + // TODO + throw new NotImplementedException(); } @Override - public void write(String path, InputStream source) throws IOException, - InterruptedException { - - File target = new File(disworkConfig.getStoragePath(), path); - - // first, check if the already exists locally... - if (target.exists()) { - throw new IOException(target.getAbsolutePath() + " already exists"); - } - - // ... or on another node - if (this.exists(path)) { - throw new IOException(target.getAbsolutePath() + " already exists"); - } - - // the file does not exists, we can write it - // first, prepare all directories and file - target.getParentFile().mkdirs(); - target.createNewFile(); - - // copying data from given source on the virtual FS - BufferedInputStream bufferedSource = null; - BufferedOutputStream bufferedTarget = null; - - try { - bufferedSource = new BufferedInputStream(source); - bufferedTarget = new BufferedOutputStream(new FileOutputStream( - target)); - IOUtils.copy(bufferedSource, bufferedTarget); - } finally { - IOUtils.closeQuietly(bufferedSource); - IOUtils.closeQuietly(bufferedTarget); - } - + public InputStream read(String path) throws FileNotFoundException, + IOException { + // TODO + throw new NotImplementedException(); } @Override - public boolean exists(String path) throws InterruptedException { - - File file = new File(disworkConfig.getStoragePath(), path); - boolean fileExists = false; - - // first, check file exists locally - if (file.exists()) { - // the file is stored on the local FS - fileExists = true; - } else { - // the file is not on the local FS, - // let's check on other nodes - SimpleLookUp simpleLookUp = new SimpleLookUp(path, lookUpService); - FileDescription lookUpResult = simpleLookUp.runLookUp(); - fileExists = (lookUpResult != null); - } - - return fileExists; + public void write(String path, InputStream source) throws IOException { + // TODO + throw new NotImplementedException(); } public void close() { - disworkServicesManager.stop(); } } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Address.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Address.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Address.java 2010-05-17 09:58:49 UTC (rev 40) @@ -1,12 +0,0 @@ -package org.nuiton.disworkfs.transport; - -import java.io.Serializable; - -public abstract class Address implements Serializable { - - /** - * - */ - private static final long serialVersionUID = 4178387970911345672L; - -} Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Message.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Message.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Message.java 2010-05-17 09:58:49 UTC (rev 40) @@ -1,44 +0,0 @@ -package org.nuiton.disworkfs.transport; - -import java.io.Serializable; - -/** - * Use Transport as a factory to create new instances of this class - * - * @author bleny - * - */ -public abstract class Message { - - protected Transport transport; - - /** - * contructor is not available as public. Use - * {@link Transport#newEmptyMessage()} if you want to create a new message - * - * @param transport - */ - protected Message(Transport transport) { - // dependency injection done by the newEmptyMessage factory - this.transport = transport; - } - - public abstract void setSource(Address source); - - public abstract void setDestination(Address destination); - - public abstract void setContent(Serializable content); - - public abstract Address getSource(); - - public abstract Address getDestination(); - - public abstract Serializable getContent(); - - public void send() { - this.transport.send(this); - } - - public abstract Message newReply(); - -} Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Receiver.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Receiver.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Receiver.java 2010-05-17 09:58:49 UTC (rev 40) @@ -1,7 +0,0 @@ -package org.nuiton.disworkfs.transport; - -public interface Receiver { - - public void receiveMessage(Message message); - -} Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java 2010-05-17 09:58:49 UTC (rev 40) @@ -1,51 +0,0 @@ -package org.nuiton.disworkfs.transport; - -public interface Transport { - - /** - * @return true if the message have been sent without error - */ - public boolean send(Message message); - - /** - * Factory method - * - * @return a new empty message, ready to be sent - */ - public Message newEmptyMessage(); - - /** - * Factory method - * - * @return a new empty message, ready to be sent to all nodes - */ - public Message newMulticastMessage(); - - /** - * Use this method to set the receiver for this transport every message - * received by this transport will be sent to this object - * - * @param receiver - */ - public void setReceiver(Receiver receiver); - - /** - * @return an address you should use in {@link Message#setSource(Address)} - */ - public Address getLocalAddress(); - - /** - * if you want to send a message to all nodes, use this method to get the - * address you will use as destination in your message - * - * @return an address you can use in Message.setDestination() - */ - public Address getMulticastAddress(); - - /** - * this call should release all the ressources used by the transport layer - * i.e. close sockets, connections etc. - */ - public void close(); - -} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-17 08:54:18 UTC (rev 39) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-17 09:58:49 UTC (rev 40) @@ -67,13 +67,19 @@ disworkConfig2 = new DisworkConfig(); disworkConfig2.setOption("storage", storagePath2); } - + @After public void tearDown() throws Exception { // cleaning FileUtil.deleteRecursively(tempDirectoryPath); } + + @Test + public void fakeTest() throws Exception { + // TODO 2010-05-17 to be removed when a test is written + } + /* @Test public void testWrite() throws Exception { @@ -94,12 +100,14 @@ assertTrue("file and copy content should be the same", actualContentEquality); } + */ /** * this test run two DistributedFS. A file is written on the first when. We * try to read from the other FS : since it doesn't own the file, it will * try a lookup and download the file. */ + /* @Test public void testRead() throws Exception { @@ -135,5 +143,6 @@ disworkFileSystem1.close(); disworkFileSystem2.close(); } + */ }

1 0

r39 - branches
by bleny＠users.nuiton.org 17 May '10

17 May '10

Author: bleny Date: 2010-05-17 10:54:18 +0200 (Mon, 17 May 2010) New Revision: 39 Url: http://nuiton.org/repositories/revision/diswork/39 Log: new branch based on JGroups Added: branches/diswork-jgroups/

1 0

r38 - trunk
by tchemit＠users.nuiton.org 17 May '10

17 May '10

Author: tchemit Date: 2010-05-17 10:53:33 +0200 (Mon, 17 May 2010) New Revision: 38 Url: http://nuiton.org/repositories/revision/diswork/38 Log: fix scm Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-05-17 08:40:34 UTC (rev 37) +++ trunk/pom.xml 2010-05-17 08:53:33 UTC (rev 38) @@ -147,6 +147,10 @@ <packaging>pom</packaging> <properties> +  + <projectId>diswork</projectId> + +  <locales>en</locales> </properties> @@ -175,9 +179,9 @@  <scm> - <connection>scm:svn:http://svn.nuiton.org/svn/diswork/diswork/trunk</connection> - <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/diswork/trunk</developerConnection> - <url>http://www.nuiton.org/repositories/browse/diswork/diswork/trunk</url> + <connection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</connection> + <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</developerConnection> + <url>http://www.nuiton.org/repositories/browse/diswork/trunk</url> </scm> </project>

1 0