Author: bleny Date: 2010-07-01 16:35:38 +0200 (Thu, 01 Jul 2010) New Revision: 97 Url: http://nuiton.org/repositories/revision/diswork/97 Log: documentation, global stats Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/src/site/rst/user/how_to_use.rst Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-01 14:35:38 UTC (rev 97) @@ -107,10 +107,6 @@ setDefaultOption("diswork.total_uptime", "0"); } - public String getTempDirectory() { - return getOption("diswork.temp_directory"); - } - public static DisworkConfig newConfig() { DisworkConfig newConfig = new DisworkConfig(); newConfig.setFileSystemConfig(DisworkFileSystemConfig.newKademliaDisworkConfig()); @@ -186,6 +182,11 @@ + + public String getTempDirectory() { + return getOption("diswork.temp_directory"); + } + public String getOwnerId() { return getOption("diswork.owner"); } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-01 14:35:38 UTC (rev 97) @@ -42,6 +42,7 @@ import org.nuiton.diswork.fs.DisworkFileSystem; import org.nuiton.diswork.fs.DisworkFileSystemConfig; import org.nuiton.diswork.fs.DisworkFileSystemException; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; import org.nuiton.util.FileUtil; /** @@ -230,7 +231,8 @@ // init fileSystem with all needed directories String[] directories = { BIN, HOME, TODO, TODO_RUNNING, FAILED_1, - FAILED_1_RUNNING, FAILED_2, FAILED_2_RUNNING, FAILED_3, DONE}; + FAILED_1_RUNNING, FAILED_2, FAILED_2_RUNNING, FAILED_3, DONE, + "/proc"}; // if HOME exists, we suppose all others exists if (! fileSystem.exists(HOME)) { for (String directory : directories) { @@ -294,6 +296,9 @@ * @throws DisworkException */ protected void initWorkersManager() throws DisworkException { + // cleaning the temp dir + FileUtil.deleteRecursively(config.getTempDirectory()); + // check if config implies to run a worker if (config.getNumberOfWorkers() >= 0) { workers = new WorkersManager(fileSystem, config); @@ -501,12 +506,23 @@ jobDescription.getInputData().get(fileName)); } - // FIXME 20100609 bleny may throws exception if jobs are proposed - // at a same time - // propose job - String linkName = newJobLinkName(); - fileSystem.createSymbolicLink(TODO + "/" + linkName, jobDir); + boolean success = false; + while (!success) { + try { + String linkName = newJobLinkName(); + // there is a risk that jobs be proposed at the same time + fileSystem.createSymbolicLink(TODO + "/" + linkName, jobDir); + success = true; + } catch (DisworkFileSystemException e) { + if (e.getType() == Type.ALREADY_EXISTS) { + // retry + success = false; + } else { + throw e; + } + } + } log.info("job submited"); @@ -620,34 +636,74 @@ * @return * @throws DisworkException */ - public Map<String, Integer> getGlobalStats() throws DisworkException { + public Map<String, String> getGlobalStats() throws DisworkException { + final String globalStatsPath = "/proc/globalstats"; + final int timeBeforeGlobalStatsAreObsolete = 1 * 60 * 60 * 1000; + + // in this file, key and values are split with \t and entries are split + // with \n, if a global-stats file is found, read it. If not, create + // it and date it. If file is obsolete read it and delete it + try { - Map<String, Integer> stats = new HashMap<String, Integer>(); - stats.put("available_processors", 0); - - List<String> homeDirs = fileSystem.readDirectory(HOME); - for (String homeDir : homeDirs) { - String hardinfo = IOUtils.toString(fileSystem.read( - HOME + "/" + homeDir + "/" + HARDINFO_PATH)); - String[] infos = hardinfo.split("\n"); + Long currentTime = System.currentTimeMillis(); + Map<String, String> result = new HashMap<String, String>(); + + if (fileSystem.exists(globalStatsPath)) { + String globalStats = IOUtils.toString(fileSystem.read(globalStatsPath)); + log.debug("global stats file found, reading " + globalStats); + String[] lines = globalStats.split("\n"); + for (String line : lines) { + String[] keyValue = line.split("\t"); + result.put(keyValue[0], keyValue[1]); + } - // reading the OS name - if (!stats.containsKey(infos[0])) { - stats.put(infos[0], 0); + // delete the file if it's too old + Long statsTime = Long.parseLong(result.get("date")); + if (currentTime - statsTime > timeBeforeGlobalStatsAreObsolete) { + log.info("deleting global stats file"); + fileSystem.delete(globalStatsPath); } - // reading the architecture - stats.put(infos[0], stats.get(infos[0]) + 1); - if (!stats.containsKey(infos[1])) { - stats.put(infos[1], 0); + } else { + log.info("global stats file doesn't exists, creating one"); + Map<String, Long> stats = new HashMap<String, Long>(); + Long availableProcessors = 0L; + + List<String> homeDirs = fileSystem.readDirectory(HOME); + for (String homeDir : homeDirs) { + String hardinfo = IOUtils.toString(fileSystem.read( + HOME + "/" + homeDir + "/" + HARDINFO_PATH)); + String[] infos = hardinfo.split("\n"); + + // reading the OS name + if (!stats.containsKey(infos[0])) { + stats.put(infos[0], 0L); + } + stats.put(infos[0], stats.get(infos[0]) + 1); + + // reading the architecture + if (!stats.containsKey(infos[1])) { + stats.put(infos[1], 0L); + } + stats.put(infos[1], stats.get(infos[1]) + 1); + + // reading the number of processors + availableProcessors += Integer.parseInt(infos[2]); } - stats.put(infos[1], stats.get(infos[1]) + 1); - - // reading the number of processors - stats.put("available_processors", - stats.get("available_processors") + - Integer.parseInt(infos[2])); + stats.put("available_processors", availableProcessors); + stats.put("date", currentTime); + + // write the result + String statsFileContent = ""; + for (String key : stats.keySet()) { + result.put(key, stats.get(key).toString()); + statsFileContent += key + "\t" + stats.get(key) + "\n"; + } + + log.debug("writing stats file " + statsFileContent); + fileSystem.write(globalStatsPath, IOUtils.toInputStream(statsFileContent)); } - return stats; + + return result; } catch (DisworkFileSystemException e) { log.error("file system error ", e); throw new DisworkException("file system error ", e); Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-07-01 14:35:38 UTC (rev 97) @@ -58,7 +58,7 @@ userEntry = br.readLine(); if ("stats".equals(userEntry)) { - Map<String, Integer> stats = daemon.getGlobalStats(); + Map<String, String> stats = daemon.getGlobalStats(); for (String key : stats.keySet()) { System.out.println(key + " => " + stats.get(key)); } Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-01 14:35:38 UTC (rev 97) @@ -149,10 +149,17 @@ public void testStats() throws Exception { daemon.getUptimeRatio(); - Map<String, Integer> stats = daemon.getGlobalStats(); - // deamon should read 3 stats : 1 OS, 1 architecture and the number + Map<String, String> stats = daemon.getGlobalStats(); + // deamon should read 3 stats : 1 OS, 1 architecture, the number + // of processors and the date when the stats was computed + assertEquals(4, stats.size()); + + // a second read should return the same data, without re generate them + // check logs + stats = daemon.getGlobalStats(); + // deamon should read 4 stats : 1 OS, 1 architecture and the number // of processors - assertEquals(3, stats.size()); + assertEquals(4, stats.size()); } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-01 14:35:38 UTC (rev 97) @@ -572,149 +572,8 @@ return result; } - - /** - * return the entry of the element at the end of <code>path</code> - * @param path - * @return null if path is not valid - */ - protected String walk(String path) throws DisworkFileSystemException { - String result = walk(path, null, null); - log.info("walking to " + path + " returns " + result); - return result; - } /** - * This method is a recursive function to walk through the tree structure - * of the directories and starting for root directory, following the - * symbolic links to reach the given path - * @param path - * @param current - * @param content - * @return null if path is not valid or the entry corresponding to path - * @throws IOException - */ - protected String walk(String path, String current, String content) - throws DisworkFileSystemException { - // FIXME 20105021 bleny works fine but is not understandable - String result = null; - - // if path is "/", recursion can be initiated with this value : - // (it returns "/" as the id where to get the content of "/" - if (path.equals(EntryUtil.ROOT_DIRECTORY)) { - return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR - + EntryUtil.ROOT_DIRECTORY; - } - - String parentPath = EntryUtil.getParentFromPath(path); - - if (content == null) { - // start the recursion from root directory - content = storage.getRootDirectory(); - result = walk(path, EntryUtil.ROOT_DIRECTORY, content); - } else if (parentPath.equals(current)) { - // we are now in the last directory - // ie if path is a/b/p we are at a/b - - // p is the name of the element in a/b we search for - String tail = path.substring(current.length()); - String p = EntryUtil.getNameFromPath(tail); - - log.info("in final dir " + current + ", looking for " + p); - - String entry = EntryUtil.findEntryInDirectory(content, p); - result = entry; - } else { - // in middle of path - // if path is a/b/c/d/e/f, we may be at b, c, d... - - String tail; // the path still remaining when in current - // ie if we are in c, tail is "d/e/f" - - // if we are at root directory, deal with the "/" - if (current.equals(EntryUtil.ROOT_DIRECTORY)) { - tail = path.substring(current.length()); - } else { - tail = path.substring(current.length() + 1); - } - - log.debug("current = " + current); - log.debug("tail = " + tail); - String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); - String p = elementsNames[0]; - - log.info("in intermediate dir " + current + ", looking for " + p); - - // updating current for recursion - if (current.equals(EntryUtil.ROOT_DIRECTORY)) { - current = ""; // avoid "//path" next line - } - current += EntryUtil.PATH_SEPARATOR + p; - - String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry == null) { - result = null; - } else { - // we have found the entry to call recursion - - if (EntryUtil.isDirectory(entry)) { - String id = EntryUtil.getIdFromEntry(entry); - content = storage.getDirectory(id); - result = walk(path, current, content); - } else if (EntryUtil.isLink(entry)) { - String id = EntryUtil.getIdFromEntry(entry); - String linkContent = storage.getLink(id); - String newTarget = - EntryUtil.resolveLink(current, linkContent); - newTarget += path.substring(current.length()); - - // restart walk from / - result = walk(newTarget, null, null); - } else if (EntryUtil.isFile(entry)) { - // error, found file in path like '/dir1/dir2/filename/dir3' - result = null; - } else { - log.warn("strange case: " + entry); - result = null; - } - } - } - return result; - } - - @Override - public void close() throws IOException { - storage.close(); - } - - protected void setMap(DisworkMap map) { - storage.setMap(map); - } - - /** - * check a path is absolute and syntactically correct, throw exception if - * that's not the case. - */ - protected void checkPathSyntax(String path) - throws DisworkFileSystemException { - if (!path.startsWith(EntryUtil.ROOT_DIRECTORY)) { - throw new DisworkFileSystemException(Type.INVALID_PATH, - "\"" + path + "\" is not correct, all pathes " + - "have to be absolute (thus, starts with)" + - EntryUtil.ROOT_DIRECTORY); - } - - String doubleSeparator = EntryUtil.PATH_SEPARATOR - + EntryUtil.PATH_SEPARATOR; - if (path.contains(doubleSeparator)) { - throw new DisworkFileSystemException(Type.INVALID_PATH, - "\"" + path + "\" is not correct, it contains " - + doubleSeparator); - } - } - - /** * move a file, a directory a link. May be used for renaming purpose * @param path the path to the object to move * @param destination the full path of the target @@ -857,4 +716,150 @@ } } } + + /** + * return the entry of the element at the end of <code>path</code> + * @param path + * @return null if path is not valid + */ + protected String walk(String path) throws DisworkFileSystemException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + /** + * This method is a recursive function to walk through the tree structure + * of the directories and starting for root directory, following the + * symbolic links to reach the given path + * @param path + * @param current + * @param content + * @return null if path is not valid or the entry corresponding to path + * @throws IOException + */ + protected String walk(String path, String current, String content) + throws DisworkFileSystemException { + // FIXME 20105021 bleny works fine but is not understandable + String result = null; + + // if path is "/", recursion can be initiated with this value : + // (it returns "/" as the id where to get the content of "/" + if (path.equals(EntryUtil.ROOT_DIRECTORY)) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY; + } + + String parentPath = EntryUtil.getParentFromPath(path); + + if (content == null) { + // start the recursion from root directory + content = storage.getRootDirectory(); + result = walk(path, EntryUtil.ROOT_DIRECTORY, content); + } else if (parentPath.equals(current)) { + // we are now in the last directory + // ie if path is a/b/p we are at a/b + + // p is the name of the element in a/b we search for + String tail = path.substring(current.length()); + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + + String entry = EntryUtil.findEntryInDirectory(content, p); + result = entry; + } else { + // in middle of path + // if path is a/b/c/d/e/f, we may be at b, c, d... + + String tail; // the path still remaining when in current + // ie if we are in c, tail is "d/e/f" + + // if we are at root directory, deal with the "/" + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + tail = path.substring(current.length()); + } else { + tail = path.substring(current.length() + 1); + } + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); + String p = elementsNames[0]; + + log.info("in intermediate dir " + current + ", looking for " + p); + + // updating current for recursion + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + current = ""; // avoid "//path" next line + } + current += EntryUtil.PATH_SEPARATOR + p; + + String entry = EntryUtil.findEntryInDirectory(content, p); + if (entry == null) { + result = null; + } else { + // we have found the entry to call recursion + + if (EntryUtil.isDirectory(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + content = storage.getDirectory(id); + result = walk(path, current, content); + } else if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + String linkContent = storage.getLink(id); + String newTarget = + EntryUtil.resolveLink(current, linkContent); + newTarget += path.substring(current.length()); + + // restart walk from / + result = walk(newTarget, null, null); + } else if (EntryUtil.isFile(entry)) { + // error, found file in path like '/dir1/dir2/filename/dir3' + result = null; + } else { + log.warn("strange case: " + entry); + result = null; + } + } + } + return result; + } + + @Override + public void close() throws IOException { + storage.close(); + } + + protected void setMap(DisworkMap map) { + storage.setMap(map); + } + + /** + * check a path is absolute and syntactically correct, throw exception if + * that's not the case. + */ + protected void checkPathSyntax(String path) + throws DisworkFileSystemException { + if (path == null) { + throw new DisworkFileSystemException(Type.INVALID_PATH, + new NullPointerException()); + } + + if (!path.startsWith(EntryUtil.ROOT_DIRECTORY)) { + throw new DisworkFileSystemException(Type.INVALID_PATH, + "\"" + path + "\" is not correct, all pathes " + + "have to be absolute (thus, starts with)" + + EntryUtil.ROOT_DIRECTORY); + } + + String doubleSeparator = EntryUtil.PATH_SEPARATOR + + EntryUtil.PATH_SEPARATOR; + if (path.contains(doubleSeparator)) { + throw new DisworkFileSystemException(Type.INVALID_PATH, + "\"" + path + "\" is not correct, it contains " + + doubleSeparator); + } + } } \ No newline at end of file Modified: trunk/src/site/rst/user/how_to_use.rst =================================================================== --- trunk/src/site/rst/user/how_to_use.rst 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/src/site/rst/user/how_to_use.rst 2010-07-01 14:35:38 UTC (rev 97) @@ -8,8 +8,79 @@ How to install a diswork node on my server ------------------------------------------ +:: + + svn checkout http://svn.nuiton.org/svn/diswork/trunk/ diswork + cd diswork/diswork-daemon + mvn assembly:assembly -Dmaven.test.skip -DdescriptorId=jar-with-dependencies + + cd ../.. + # under debian commons-daemon can be found at /usr/share/java/commons-daemon.jar + jsvc -cp /usr/share/java/commons-daemon.jar:diswork/diswork-daemon/target/diswork-daemon-0.0.1-SNAPSHOT-jar-with-dependencies.jar -pidfile ./pid -outfile ./out -errfile ./err DisworkDaemonRunner + + How to make my own application ready for being run by all diswork nodes ----------------------------------------------------------------------- +Applications for diswork are very easy to create. An application is a zip file. +The only specification of the content is that it has to be set considering that +the whole application will be unzipped in the job-specific directory. + +You can put executables in it, a jar file with its dependencies, scripts, etc. + +You have to write the command-line for the job being aware that the application +and all input files will be in the current working directory. + How to make my application able to submit jobs to Diswork and retrieve results ------------------------------------------------------------------------------- \ No newline at end of file +------------------------------------------------------------------------------ + +Your application have to use the Daemon API, here is a sample code : + +:: + + // First, start a daemon + DisworkConfig config = DisworkConfig.newConfig(); + daemon = new DisworkDaemon(config); + + // submit the application + daemon.submitApplication("fake-app", "1.0", new FileInputStream("fake-app-1.0.zip")); + + // now all the nodes will be able to run jobs that need fake-app version 1.0 + + // create a new job + JobDescription job = new JobDescription(); + job.setJobName("My Job"); + job.setApplication("fake-app", "1.0"); + job.setCommandLine("%java -jar fake-app.jar"); + + // now, defining data in input + + // this will be downloaded by the worker before job start + job.addInput("example.com_index", new URL("http://www.example.com/")); + // this will, needed for fake-app will be provided to the worker + job.addInput("input.txt", new FileInputStream("input.txt")); + + // defining expected data in output + job.addOutput("output.txt"); + job.addOutput("example.com_index"); + + // setting standard input and output file + job.setStandardInput("input.txt"); + job.setStandardOutput("output.txt"); + + // submit the job + daemon.submitJob(job); + + // waiting for the job to finish + while(! daemon.isFinished(job)) { + Thread.sleep(5 * 60 * 1000); + } + + // check that job is successful and read the result files + if(daemon.isSuccessful(job)) { + // results will contains keys "output.txt" and "example.com_index" + // values are InputStream on the content of those files + Map<String, InputStream> results = daemon.getResults(job); + } + + \ No newline at end of file