Author: bleny Date: 2010-05-21 18:46:41 +0200 (Fri, 21 May 2010) New Revision: 50 Url: http://nuiton.org/repositories/revision/diswork/50 Log: ls + test Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) @@ -3,13 +3,17 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.List; public interface DistributedFileSystem { public boolean exists(String path) throws IOException; - public void write(String parent, String file, InputStream source) throws IOException; + public void write(String path, InputStream source) throws IOException; + public void write(String parent, String fileName, InputStream source) + throws IOException; + public InputStream read(String path) throws FileNotFoundException, IOException; @@ -19,4 +23,6 @@ public void remove(String path) throws IOException; + public List<String> ls(String path) throws IOException; + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 16:46:41 UTC (rev 50) @@ -1,21 +1,14 @@ package org.nuiton.disworkfs; -import java.io.File; -import java.util.Random; - import org.nuiton.util.ApplicationConfig; public class DisworkConfig extends ApplicationConfig { public DisworkConfig() { - Random random = new Random(); - setDefaultOption("storage", "/tmp/disworkfs/storage" + random.nextInt()); - - // replication strategy... - // TODO 20100519 bleny chunk size parameter + setDefaultOption("blocks_size", "10485760"); // 10 MiB } - public File getStoragePath() { - return getOptionAsFile("storage"); + public int getBlockSize() { + return Integer.parseInt(getOption("blocks_size")); } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) @@ -3,10 +3,13 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.storage.EntryUtil; +import org.nuiton.disworkfs.storage.InMemoryMap; import org.nuiton.disworkfs.storage.Storage; public class DisworkFileSystem implements DistributedFileSystem { @@ -16,7 +19,7 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException { - storage = new Storage(new InMemoryMap()); + storage = new Storage(disworkConfig, new InMemoryMap()); storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); } @@ -40,7 +43,8 @@ } // FIXME 20105021 bleny works fine but is not understandable - protected String walk(String path, String current, String content) throws IOException { + protected String walk(String path, String current, String content) + throws IOException { String result = null; if (path.equals("/")) { @@ -49,6 +53,7 @@ + "/"; } + String parentPath = EntryUtil.getParentFromPath(path); if (content == null) { @@ -150,6 +155,13 @@ } @Override + public void write(String path, InputStream source) throws IOException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + write(parent, name, source); + } + + @Override public void write(String parent, String fileName, InputStream source) throws IOException { String entryParent = walk(parent); @@ -215,6 +227,14 @@ } public void ln(String parent, String name, String target) throws IOException { + + if (target.startsWith("/")) { + // target is absolute + } else { + // target is relative, taking this in consideration + target = EntryUtil.resolveLink(parent, target); + } + if (exists(target)) { String entryParent = walk(parent); @@ -290,4 +310,33 @@ } } + @Override + public List<String> ls(String path) throws IOException { + String entry = walk(path); + List<String> result = null; + + + if (entry == null) { + throw new IOException(path + " doesn't exists"); + } else { + if (EntryUtil.isLink(entry)) { + String target = storage.getLink(EntryUtil.getId(entry)); + entry = walk(target); + } + + if (EntryUtil.isDirectory(entry)) { + String content = storage.getDirectory(EntryUtil.getId(entry)); + String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR); + result = new ArrayList<String>(); + for (String elementEntry : entries) { + result.add(EntryUtil.getName(elementEntry)); + } + } else { + throw new IOException(path + " is not a directory"); + } + } + + return result; + } + } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50) @@ -1,20 +0,0 @@ -package org.nuiton.disworkfs; - -import java.util.Arrays; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class InMemoryMap extends HashMap<String, byte[]> { - - private final Log log = LogFactory.getLog(InMemoryMap.class); - - @Override - public byte[] put(String key, byte[] value) { - log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); - byte[] valueCopy = value.clone(); - log.info("putting " + valueCopy.length + " bytes"); - return super.put(key, valueCopy); - } -} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 16:46:41 UTC (rev 50) @@ -1,6 +1,7 @@ package org.nuiton.disworkfs.storage; +import java.nio.charset.Charset; import java.util.UUID; import org.apache.commons.io.FilenameUtils; @@ -27,6 +28,8 @@ static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; + static protected Charset CHARSET = Charset.forName("UTF-8"); + /** * generate a new Id usage as a new key for the map * @return a String containing the id @@ -87,14 +90,14 @@ } static public int getTotalSizeFromMetaBlock(String metaBlock) { - // FIXME 20100520 bleny this workaround should not be + // dealing with empty meta-block (for empty directory or file) if (metaBlock.equals("0")) return 0; String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); return Integer.valueOf(result); } static public String[] getBlockIdsFromMetaBlock(String metaBlock) { - // FIXME 20100520 bleny this workaround should not be + // dealing with empty meta-block (for empty directory or file) if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { return new String[0]; } @@ -188,7 +191,7 @@ public static byte[] stringToBytes(String string) { // FIXME 20100519 bleny this code may cause bad conversion // due to charsets - byte[] bytes = string.getBytes(); + byte[] bytes = string.getBytes(CHARSET); return bytes; } @@ -198,7 +201,7 @@ * @return a byte array containing all the bytes in <code>string</code> */ public static String bytesToString(byte[] bytes) { - return new String(bytes); + return new String(bytes, CHARSET); } public static String getParentFromPath(String path) { Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java (from rev 47, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50) @@ -0,0 +1,20 @@ +package org.nuiton.disworkfs.storage; + +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class InMemoryMap extends HashMap<String, byte[]> { + + private final Log log = LogFactory.getLog(InMemoryMap.class); + + @Override + public byte[] put(String key, byte[] value) { + log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); + byte[] valueCopy = value.clone(); + log.info("putting " + valueCopy.length + " bytes"); + return super.put(key, valueCopy); + } +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 16:46:41 UTC (rev 50) @@ -8,6 +8,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.DisworkConfig; public class Storage { @@ -84,9 +85,12 @@ } protected Map<String, byte[]> map; + + protected DisworkConfig disworkConfig; - public Storage(Map<String, byte[]> map) { + public Storage(DisworkConfig disworkConfig, Map<String, byte[]> map) { this.map = map; + this.disworkConfig = disworkConfig; } public boolean contains(Object id) { @@ -167,8 +171,8 @@ int read = 0; int totalSize = 0; - // TODO 20100519 bleny the size of blocks should be a config directive - byte[] buffer = new byte[10 * 1024 * 1024]; + int bufferSize = disworkConfig.getBlockSize(); + byte[] buffer = new byte[bufferSize]; while ((read = value.read(buffer)) != -1) { totalSize += read; Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 16:46:41 UTC (rev 50) @@ -11,21 +11,18 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Random; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.nuiton.util.FileUtil; -public class DistributedFileSystemTest { - - private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class); +public class DistributedFileSystemTest { /** * a place to store files for the test it's a subdirectory of the OS temp @@ -240,5 +237,17 @@ // trying to remove a non-empty directory should raise an exception fileSystem.remove("/my_folder"); } + + @Test + public void testLs() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.mkdir("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + fileSystem.ln("/my_folder/my_link", "my_file"); + + // trying to remove a non-empty directory should raise an exception + List<String> lsResult = fileSystem.ls("/my_folder"); + assertEquals(3, lsResult.size()); + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 16:46:41 UTC (rev 50) @@ -4,33 +4,23 @@ import java.util.Arrays; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.After; -import org.junit.Before; import org.junit.Test; +import org.nuiton.disworkfs.storage.InMemoryMap; public class InMemoryMapTest { - private final Log log = LogFactory.getLog(InMemoryMapTest.class); - - - @Before - public void setUp() throws Exception {} - @After - public void tearDown() throws Exception {} - + /** + * this test show that the InMemoryMap put implies a copy of the data. + * After the put, the original value is modified. A final check shows + * there is no side effect + */ @Test public void testPut() { InMemoryMap map = new InMemoryMap(); - byte[] expected = {0x1, 0x2}; - + byte[] expected = {0x1, 0x2}; map.put("key", expected); - expected[0] = 0xf; - byte[] actual = map.get("key"); - assertFalse(Arrays.equals(expected, actual)); }