Author: bleny Date: 2010-06-08 14:13:22 +0200 (Tue, 08 Jun 2010) New Revision: 66 Url: http://nuiton.org/repositories/revision/diswork/66 Log: bugfix; documentation; specif de diswork daemon Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java trunk/src/site/rst/diswork-daemon/how_it_works.rst trunk/src/site/rst/diswork-fs/history.rst trunk/src/site/rst/diswork-fs/how_it_works.rst trunk/src/site/site_en.xml Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-08 12:13:22 UTC (rev 66) @@ -521,9 +521,8 @@ if (EntryUtil.isLink(entry)) { String target = storage.getLink( EntryUtil.getIdFromEntry(entry)); - entry = walk(target); - } - if (EntryUtil.isDirectory(entry)) { + return readDirectory(target); + } else if (EntryUtil.isDirectory(entry)) { result = new ArrayList<String>(); String content = storage.getDirectory( EntryUtil.getIdFromEntry(entry)); @@ -536,8 +535,11 @@ result.add(EntryUtil.getNameFromEntry(elementEntry)); } } + } else if (EntryUtil.isFile(entry)) { + throw new IOException(path + " is not a directory but a file"); } else { - throw new IOException(path + " is not a directory"); + log.warn("strange entry" + entry); + throw new IOException("strange entry" + entry); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-08 12:13:22 UTC (rev 66) @@ -341,6 +341,20 @@ readResult = fileSystem.readDirectory("/otherdir"); assertEquals(1, readResult.size()); assertTrue(readResult.contains("finaldir")); + + // tests that delete remove the link itself and not the target + fileSystem.delete("/link/subsubdirlink"); + assertTrue(fileSystem.exists("/otherdir")); + fileSystem.delete("/link"); + assertTrue(fileSystem.exists("/dir/subdir")); + + // testing the use of multiple links at the end of a path + fileSystem.createSymbolicLink("/link1", "/dir"); + fileSystem.createSymbolicLink("/link2", "/link1"); + fileSystem.createSymbolicLink("/link3", "/link2"); + readResult = fileSystem.readDirectory("/link3"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subdir")); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) @@ -11,7 +11,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.nuiton.disworkfs.DisworkFileSystemConfig; public abstract class AbstractDisworkMapTest { @@ -21,9 +20,13 @@ protected DisworkMap map1; protected DisworkMap map2; + /** + * should be overridden by method that set map1 and map2 with different + * implementation of DisworkMap + * @throws Exception + */ @Before - public void setUp() throws Exception { - + public void setUp() throws Exception { } @After @@ -40,7 +43,7 @@ * there is no side effect */ @Test - public void testPut() { + public void testPutMakeCopy() { byte[] expected = {0x1, 0x2}; map1.put("key", expected); expected[0] = 0xf; @@ -50,11 +53,7 @@ @Test - public void test() throws Exception { - DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); - - map1 = new KademliaDisworkMap(config); - + public void testPutGet() throws Exception { byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; map1.put("test", newBytes); @@ -65,12 +64,7 @@ @Test public void test2nodes() throws Exception { - DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); - map1 = new KademliaDisworkMap(config1); - DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); - map2 = new KademliaDisworkMap(config2); - byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; log.info("trying put test"); Modified: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,4 +1,82 @@ +======================== How Diswork Daemon works ======================== -Diswork Daemon uses Diswork File System. \ No newline at end of file +Overview +-------- + +Diswork Daemon uses Diswork File System. + +Submit a job +------------ + +First, a node, before joining a network generate, for himself, a unique ID. And +then, create a directory "/home/ID". If this is not the first time the node +is run, ID should be reused so jobs can be found back after disconnect/connect. + +In his home directory, a node can put and remove his jobs. + +To create a job, a node should : + +#. Create a directory in home for each of his jobs + /home/ID/job-1, /home/ID/job-2... + +#. In a job directory, put job.sh, a command line to run. + +#. Declare the job as available by adding a symlink in /var/jobs/TODO + with the link named after the current date, the id of the job owner and + the job name. Places possible in /var/jobs/ are : + +#. Check the content of /home/ID/job-i/status + +#. Finally, clean home by deleting the directories of finished jobs + +Perform a job +------------- + +To perform a job, a node should find a job in /var/jobs/, this directory has +6 sub-directories : + +- /var/jobs/TODO + +- /var/jobs/FAILED_1 + +- /var/jobs/FAILED_2 + +- /var/jobs/FAILED_3 + +- /var/jobs/DONE + +TODO, FAILED_1, FAILED_2 all have a sub-directory RUNNING. + +#. Choose a job in one of those directories except DONE and FAILED_3. Use the + data in the link same to know which job should be done first; + +#. Once the job is chosen, move the symlink to the RUNNING subdir where you + found the link. Then, follow the link to get the actual datas needed for + the job; + +#. Run the job; + +#. Now there is two way to proceed : + + - If the job successfully ran (no exception raised and return code is 0) : + + #. write the results in the job directory; + + #. move the link to DONE. + + - If the job has failed (a job taking too much time may be considered as + a failure). move it to according to where the job was found. If + the job was in TODO, move it to FAILED_1 (read "failed once") ; + if the job was in FAILED_1, move it to FAILED_2, if it was it FAILED_2, + move it to FAILED_3. + + In both cases, write what occurred in the job directory in the status.txt + file (append your message, don't remove anything); + +#. Sometimes, while choosing a job, check in the RUNNING directory if any + symlink is not too old (use the name of the link). It may means that a + node crashed while processing this job. Move it to the RUNNING directory + of the next failure level (TODO → FAILED_1 → FAILED_2 → FAILED_3) and try + process it. \ No newline at end of file Modified: trunk/src/site/rst/diswork-fs/history.rst =================================================================== --- trunk/src/site/rst/diswork-fs/history.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-fs/history.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,5 +1,9 @@ +========== +Historique +========== + Premier prototype -================= +----------------- L'objectif était de réaliser un système opportuniste : à chaque fois que des données sont envoyées par un nœud parce qu'un autre nœud les a demandées, Modified: trunk/src/site/rst/diswork-fs/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,6 +1,10 @@ +============================= How Diswork File System works ============================= +Diswork File System uses a map to store every data +-------------------------------------------------- + Diswork File System uses a map to store data persistently. We illustrate the use of a file system by setting the content of "/", @@ -122,4 +126,11 @@ When reading and writing in storage, split is done transparently. When reading, a Stream is returned: it loads data blocks after blocks when needed inner class Storage.SplitBlocksInputStream}). -When writing, data are split in blocks of a maximum configurable size. \ No newline at end of file +When writing, data are split in blocks of a maximum configurable size. + +Diswork File System manage concurrency +-------------------------------------- + +DFS uses copy-on-write to deal with concurrency. Nothing is overridden, when +a file is replaced, the new file is written on the FS without removing the +old file. The old file data is kept and actually removed later. \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/site_en.xml 2010-06-08 12:13:22 UTC (rev 66) @@ -13,26 +13,26 @@ </poweredBy> <body> - + <breadcrumbs> <item name="${project.name}" href="index.html" /> </breadcrumbs> - + <menu name="Users"> <item href="user/install.html" name="Install"/> <item href="user/starting.html" name="Starting"/> </menu> - + <menu name="Developer"> <item href="devel/draft.html" name="First draft"/> </menu> - + <menu name="diswork daemon"> - <item href="diswork-fs/how_it_works.html" name="How it works"/> + <item href="diswork-daemon/how_it_works.html" name="How it works"/> </menu> - + <menu name="diswork FS"> - <item href="diswork-fs/history.html" name="Install"/> + <item href="diswork-fs/history.html" name="History of the project"/> <item href="diswork-fs/how_it_works.html" name="How it works"/> </menu>