Author: bpoussin Date: 2014-04-10 14:44:04 +0200 (Thu, 10 Apr 2014) New Revision: 471 Url: http://forge.nuiton.org/projects/nuiton-matrix/repository/revisions/471 Log: Add LazyVector to prevent memory allocation when not necessary Added: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/LazyVector.java Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigMappedVector.java trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigVector.java trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleVector.java trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatBigVector.java trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatVector.java trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/MatrixFactory.java trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/Vector.java trunk/nuiton-matrix/src/test/java/org/nuiton/math/matrix/PerfDoubleBigVectorTest.java Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigMappedVector.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigMappedVector.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigMappedVector.java 2014-04-10 12:44:04 UTC (rev 471) @@ -66,6 +66,9 @@ } } + public DoubleBigMappedVector() { + } + /** * Create temp file, this file is deleted when you stop application or * when this instance is no more used. @@ -73,15 +76,26 @@ * @throws IOException */ public DoubleBigMappedVector(int capacity) throws IOException { - this.capacity = capacity; - this.file = File.createTempFile("matrix", ".mapped"); - this.file.deleteOnExit(); - RandomAccessFile raf = new RandomAccessFile(file, "rw"); - data = raf.getChannel() - .map(FileChannel.MapMode.READ_WRITE, 0, capacity * DOUBLE_SIZE) - .asDoubleBuffer(); + init(capacity); } + @Override + public void init(int capacity) { + try { + if (data == null) { + this.capacity = capacity; + this.file = File.createTempFile("matrix", ".mapped"); + this.file.deleteOnExit(); + RandomAccessFile raf = new RandomAccessFile(file, "rw"); + data = raf.getChannel() + .map(FileChannel.MapMode.READ_WRITE, 0, capacity * DOUBLE_SIZE) + .asDoubleBuffer(); + } + } catch (IOException eee) { + throw new MatrixException("Can't init vector", eee); + } + } + /** * Utilise une partie du fichier pour stocker les informations * @param raf le fichier a utiliser Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigVector.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigVector.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleBigVector.java 2014-04-10 12:44:04 UTC (rev 471) @@ -42,11 +42,21 @@ protected double data[] = null; + public DoubleBigVector() { + } + public DoubleBigVector(int capacity) { - data = new double[capacity]; + init(capacity); } @Override + public void init(int capacity) { + if (data == null) { + data = new double[capacity]; + } + } + + @Override public int size() { return data.length; } Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleVector.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleVector.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/DoubleVector.java 2014-04-10 12:44:04 UTC (rev 471) @@ -56,12 +56,14 @@ protected int positionSize = 0; /** contient la valeur de l'element */ - protected ArrayDoubleList data = new ArrayDoubleList(); + protected ArrayDoubleList data; + public DoubleVector() { + } + + public DoubleVector(int capacity) { - this.capacity = capacity; - position = new int[8]; - Arrays.fill(position, Integer.MAX_VALUE); + init(capacity); } public DoubleVector(int capacity, double defaultValue) { @@ -70,6 +72,16 @@ } @Override + public void init(int capacity) { + if (data == null) { + this.capacity = capacity; + this.data = new ArrayDoubleList(); + position = new int[8]; + Arrays.fill(position, Integer.MAX_VALUE); + } + } + + @Override public int size() { return capacity; } Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatBigVector.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatBigVector.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatBigVector.java 2014-04-10 12:44:04 UTC (rev 471) @@ -42,11 +42,21 @@ protected float data[] = null; + public FloatBigVector() { + } + public FloatBigVector(int capacity) { - data = new float[capacity]; + init(capacity); } @Override + public void init(int capacity) { + if (data == null) { + data = new float[capacity]; + } + } + + @Override public int size() { return data.length; } Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatVector.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatVector.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/FloatVector.java 2014-04-10 12:44:04 UTC (rev 471) @@ -56,12 +56,13 @@ protected int positionSize = 0; /** contient la valeur de l'element */ - protected ArrayFloatList data = new ArrayFloatList(); + protected ArrayFloatList data; + public FloatVector() { + } + public FloatVector(int capacity) { - this.capacity = capacity; - position = new int[8]; - Arrays.fill(position, Integer.MAX_VALUE); + init(capacity); } public FloatVector(int capacity, float defaultValue) { @@ -70,6 +71,16 @@ } @Override + public void init(int capacity) { + if (data == null) { + this.capacity = capacity; + position = new int[8]; + data = new ArrayFloatList(); + Arrays.fill(position, Integer.MAX_VALUE); + } + } + + @Override public int size() { return capacity; } Added: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/LazyVector.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/LazyVector.java (rev 0) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/LazyVector.java 2014-04-10 12:44:04 UTC (rev 471) @@ -0,0 +1,168 @@ +package org.nuiton.math.matrix; + + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * + * Cette classe encapsule un Vector et permet de l'initialiser reellement que + * lorsqu'on souhaite modifier une valeur. Cela permet de ne pas initialise + * d'enorme tableau s'il n'y en a pas besoin. + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class LazyVector implements Vector { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(LazyVector.class); + + protected boolean isInitBackend = false; + /** reel backend, not initialized since not necessary */ + protected Vector backend; + /** vector used when paste is called and backend is not initialized */ + protected Vector copy; + protected int capacity; + + public LazyVector(Vector backend, int capacity) { + this.backend = backend; + this.capacity = capacity; + } + + @Override + public void init(int capacity) { + } + + @Override + public double getMaxOccurence() { + return getMaxOccurrence(); + } + + @Override + public double getMaxOccurrence() { + double result = 0; + if (isInitBackend) { + result = backend.getMaxOccurrence(); + } else if (copy != null) { + result = copy.getMaxOccurrence(); + } + return result; + } + + @Override + public double getValue(int pos) { + double result = 0; + if (isInitBackend) { + result = backend.getValue(pos); + } else if (copy != null) { + result = copy.getValue(pos); + } + return result; + } + + @Override + public void setValue(int pos, double value) { + initBackend(); + backend.setValue(pos, value); + } + + @Override + public int size() { + return capacity; + } + + @Override + public boolean isImplementedPaste(Vector v) { + return true; + } + + @Override + public boolean isImplementedAdd(Vector v) { + return backend.isImplementedAdd(v); + } + + @Override + public boolean isImplementedMinus(Vector v) { + return backend.isImplementedMinus(v); + } + + @Override + public boolean isImplementedMap() { + return backend.isImplementedMap(); + } + + @Override + public void paste(Vector source) { + if (!isInitBackend && copy == null) { + copy = source; + } else { + // initialized or already has copy, we need to paste, and not just + // reassign copy variable, in case of source Vector is smaller than copy + initBackend(); + pasteToBackend(source); + } + } + + @Override + public void add(Vector v) { + initBackend(); + backend.add(v); + } + + @Override + public void minus(Vector v) { + initBackend(); + backend.minus(v); + } + + @Override + public void map(MapFunction f) { + initBackend(); + backend.map(f); + } + + @Override + public boolean equals(Object o) { + boolean result = false; + if (o instanceof Vector) { + Vector v = (Vector)o; + if (size() == v.size()) { + if (isInitBackend) { + result = v.equals(backend); + } else if (copy != null) { + result = v.equals(copy); + } else if (v instanceof LazyVector && !((LazyVector)v).isInitBackend && ((LazyVector)v).copy == null) { + result = true; + } else { + result = v.equals(this); + } + } + } + return result; + } + + protected void initBackend() { + if (!isInitBackend) { + backend.init(capacity); + if (copy != null) { + pasteToBackend(copy); + } + isInitBackend = true; + } + } + + protected void pasteToBackend(Vector copy) { + if (backend.isImplementedPaste(copy)) { + backend.paste(copy); + } else { + for (int i=0, max=Math.min(size(), copy.size()); i<max; i++) { + double v = copy.getValue(i); + backend.setValue(i, v); + } + } + } +} Property changes on: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/LazyVector.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/MatrixFactory.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/MatrixFactory.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/MatrixFactory.java 2014-04-10 12:44:04 UTC (rev 471) @@ -43,6 +43,13 @@ */ public class MatrixFactory { // MatrixFactory + /** + * If true, createVector return all time LazyVector to prevent memory + * allocation when not necessary. LazyVector init real vector only when + * necessary + */ + protected static boolean useLazyVector = true; + /** Valeur par defaut si aucun type de Vector n'est donné */ protected static Class<?> defaultVectorClass = DoubleBigVector.class; @@ -235,9 +242,12 @@ protected Vector createVector(int length) { try { - Constructor<?> c = vectorClass - .getConstructor(new Class<?>[] { Integer.TYPE }); - return (Vector) c.newInstance(new Object[] { length }); + Vector result = (Vector)vectorClass.newInstance(); + if (useLazyVector) { + result = new LazyVector(result, length); + } + result.init(length); + return result; } catch (Exception eee) { throw new RuntimeException("Can't create vector", eee); } Modified: trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/Vector.java =================================================================== --- trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/Vector.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/main/java/org/nuiton/math/matrix/Vector.java 2014-04-10 12:44:04 UTC (rev 471) @@ -39,11 +39,24 @@ public interface Vector { // Vector /** + * Init vector, before this method call, vector is in indetermined state. + * multiple call to init method must be permit. Only first call must do + * some work, extra call must do nothing + * + * @param capacity + */ + public void init(int capacity); + + /** * @deprecated since 2.1, use {@link #getMaxOccurrence()} instead */ @Deprecated public double getMaxOccurence(); - + + /** + * Retourne la valeur la plus utilise dans le vector + * @return + */ public double getMaxOccurrence(); public double getValue(int pos); Modified: trunk/nuiton-matrix/src/test/java/org/nuiton/math/matrix/PerfDoubleBigVectorTest.java =================================================================== --- trunk/nuiton-matrix/src/test/java/org/nuiton/math/matrix/PerfDoubleBigVectorTest.java 2014-03-17 16:08:38 UTC (rev 470) +++ trunk/nuiton-matrix/src/test/java/org/nuiton/math/matrix/PerfDoubleBigVectorTest.java 2014-04-10 12:44:04 UTC (rev 471) @@ -137,6 +137,52 @@ } @Test + public void testDoubleVectorMap() throws Exception { + List<String> sem = getSem(); + + // [ 2600 x 2600 x 2600 ] + System.gc(); // force gc now to minimize gc call during timing + long timestart = System.nanoTime(); + MatrixND mat = MatrixFactory.getInstance(getVectorClass()).create("Ma mat", + new List[] { sem, sem, sem }); + + long cpt = 1; + for (MatrixIterator i = mat.iterator(); i.hasNext();) { + i.next(); + i.setValue(cpt++); + } + + long timeinit = System.nanoTime(); + + // application de la fonction + mat.map(new MapFunction() { + long cpt = 1; + public double apply(double value) { + return cpt++; + } + }); + + long timeend = System.nanoTime(); + + long freeMem = Runtime.getRuntime().freeMemory(); + long totalMem = Runtime.getRuntime().totalMemory(); + + long vectorSize = ((MatrixNDImpl)mat).getInternalMatrix().getInternalVector().size(); + + System.out.println( + "Perf " + getVectorClass().getSimpleName() + "(" + + vectorSize + ")" + + "\tMap func\tinit: " + + DurationFormatUtils.formatDuration( + (timeinit - timestart) / 1000000, "s'.'S") + + " fill: " + + DurationFormatUtils.formatDuration( + (timeend - timeinit) / 1000000, "s'.'S") + + " memory: " + StringUtil.convertMemory(totalMem - freeMem) + "/" + + StringUtil.convertMemory(totalMem)); + } + + @Test public void testDoubleVectorSemantic() throws Exception { List<String> sem = getSem(); int size = sem.size();