Merge pull request #33 from picoded/fileWorkspace-keySet-support

File workspace key set support

Author: PicoCreator

src/main/java/picoded/dstack/file/layered/FileLayered_FileWorkspaceMap.java

@@ -3,6 +3,8 @@
 import picoded.dstack.file.simple.FileSimple_FileWorkspaceMap;
 
 import java.io.File;
+import java.util.HashSet;
+import java.util.Set;
 
 public class FileLayered_FileWorkspaceMap extends FileSimple_FileWorkspaceMap {
 
@@ -51,4 +53,64 @@ public File workspaceDirObj(String oid) {
 		return new File(baseDir, workspacePath);
 	}
 
+	//--------------------------------------------------------------------------
+	//
+	// KeySet support
+	//
+	//--------------------------------------------------------------------------
+	
+	/**
+	 * Get and returns all the GUID's, note that due to its
+	 * potential of returning a large data set, production use
+	 * should be avoided.
+	 *
+	 * @return set of keys
+	 **/
+	@Override
+	public Set<String> keySet() {
+		// The return hashset
+		HashSet<String> ret = new HashSet<String>();
+		
+		// List all the files/folders
+		File[] l1_dirList = baseDir.listFiles();
+		
+		// For each one of it, process it!
+		for (File l1_dir : l1_dirList) {
+			// Skip if its not a directory
+			if( !l1_dir.isDirectory() ) {
+				continue;
+			}
+
+			// List all the files/folders
+			File[] l2_dirList = baseDir.listFiles();
+			
+			for(File l2_dir : l2_dirList) {
+				// Skip if its not a directory
+				if( !l2_dir.isDirectory() ) {
+					continue;
+				}
+
+				// Get the oidDirLIst
+				File[] oid_list = l2_dir.listFiles();
+
+				// For each oid dir
+				for(File oid_dir : oid_list) {
+					// Get the presumed oid
+					String oid = oid_dir.getName();
+
+					// Validate the dir name (oid)
+					if( !validateOid(oid) ) {
+						continue;
+					}
+
+					// Add the oid to the ret set
+					ret.add(oid);
+				}
+			}
+		}
+
+		// Return the full keyset
+		return ret;
+	}
+	
 }

src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java

@@ -627,4 +627,48 @@ public Set<String> backend_getFileAndFolderPathSet(final String oid, final Strin
 		return backend_filterPathSet(retSet, "", minDepth, maxDepth, 0);
 	}
 
+	//--------------------------------------------------------------------------
+	//
+	// KeySet support
+	//
+	//--------------------------------------------------------------------------
+	
+	/**
+	 * Get and returns all the GUID's, note that due to its
+	 * potential of returning a large data set, production use
+	 * should be avoided.
+	 *
+	 * @return set of keys
+	 **/
+	@Override
+	public Set<String> keySet() {
+		// The return hashset
+		HashSet<String> ret = new HashSet<String>();
+		
+		// List all the files/folders
+		File[] dirList = baseDir.listFiles();
+		
+		// For each one of it, process it!
+		for (File subFile : dirList) {
+			// Skip if its not a directory
+			if( !subFile.isDirectory() ) {
+				continue;
+			}
+
+			// Get the presumed oid
+			String oid = subFile.getName();
+
+			// Validate the dir name (oid)
+			if( !validateOid(oid) ) {
+				continue;
+			}
+
+			// Add the oid to the ret set
+			ret.add(oid);
+		}
+
+		// Return the full keyset
+		return ret;
+	}
+	
 }

src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java

@@ -293,13 +293,12 @@ public Set<String> keySet() {
 		HashSet<String> ret = new HashSet<String>();
 
 		// Lets fetch everything ... D=
-		FindIterable<Document> search = collection.find();
-		search = search.projection(Projections.include("_oid"));
-		
+		DistinctIterable<String> search = collection.distinct("_oid", String.class);
+
 		// Lets iterate the search
-		try (MongoCursor<Document> cursor = search.iterator()) {
+		try (MongoCursor<String> cursor = search.iterator()) {
 			while (cursor.hasNext()) {
-				ret.add(cursor.next().getString("_oid"));
+				ret.add(cursor.next());
 			}
 		}
 

src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java

@@ -894,4 +894,37 @@ public Set<String> backend_getFileAndFolderPathSet(final String oid, String fold
 		return backend_filterPathSet( ret, folderPath, minDepth, maxDepth, 0);
 	}
 
+	//--------------------------------------------------------------------------
+	//
+	// KeySet support
+	//
+	//--------------------------------------------------------------------------
+	
+	/**
+	 * Get and returns all the GUID's, note that due to its
+	 * potential of returning a large data set, production use
+	 * should be avoided.
+	 *
+	 * @return set of keys
+	 **/
+	@Override
+	public Set<String> keySet() {
+		// The return hashset
+		HashSet<String> ret = new HashSet<String>();
+		
+		// Lets fetch everything ... D=
+		DistinctIterable<String> search = filesCollection.distinct("metadata.oid", String.class);
+
+		// Lets iterate the search
+		try (MongoCursor<String> cursor = search.iterator()) {
+			while (cursor.hasNext()) {
+				ret.add(cursor.next());
+			}
+		}
+		
+		// Return the full keyset
+		return ret;
+	}
+	
+	
 }

src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java

@@ -576,6 +576,64 @@ public Set<String> backend_getFolderPathSet(final String oid, final String folde
 		return queryLayer.backend_getFolderPathSet(oid, folderPath, minDepth, maxDepth);
 	}
 
+	//--------------------------------------------------------------------------
+	//
+	// KeySet support
+	//
+	//--------------------------------------------------------------------------
+	
+	/**
+	 * List all the oid supported in the current backend
+	 */
+	public Set<String> keySet() {
+		return queryLayer.keySet();
+	}
+	
+	/**
+	 * Gets and return a random object ID
+	 *
+	 * @return  Random object ID
+	 **/
+	public String randomObjectID() {
+		return queryLayer.randomObjectID();
+	}
+	
+	/**
+	 * Gets and return the next object ID key for iteration given the current ID,
+	 * null gets the first object in iteration.
+	 *
+	 * It is important to note actual iteration sequence is implementation dependent.
+	 * And does not gurantee that newly added objects, after the iteration started,
+	 * will be part of the chain of results.
+	 *
+	 * Similarly if the currentID was removed midway during iteration, the return
+	 * result is not properly defined, and can either be null, or the closest object matched
+	 * or even a random object.
+	 *
+	 * It is however guranteed, if no changes / writes occurs. A complete iteration
+	 * will iterate all existing objects.
+	 *
+	 * The larger intention of this function, is to allow a background thread to slowly
+	 * iterate across all objects, eventually. With an acceptable margin of loss on,
+	 * recently created/edited object. As these objects will eventually be iterated in
+	 * repeated rounds on subsequent calls.
+	 *
+	 * Due to its roughly random nature in production (with concurrent objects generated)
+	 * and its iterative nature as an eventuality. The phrase looselyIterate was chosen,
+	 * to properly reflect its nature.
+	 *
+	 * Another way to phrase it, in worse case scenerio, its completely random,
+	 * eventually iterating all objects. In best case scenerio, it does proper
+	 * iteration as per normal.
+	 *
+	 * @param   Current object ID, can be NULL
+	 *
+	 * @return  Next object ID, if found
+	 **/
+	public String looselyIterateObjectID(String currentID) {
+		return queryLayer.looselyIterateObjectID(currentID);
+	}
+	
 	//--------------------------------------------------------------------------
 	//
 	// Copy pasta code, I wished could have worked in an interface

src/main/java/picoded/dstack/struct/cache/StructCache_DataObjectMap.java

@@ -146,8 +146,9 @@ public void systemSetup() {
 
 		//
 		// Alright, time to build a new cache
-		// We are in the era of GB ram computing, 100k cache would
-		// be a good sane default in server environment.
+		// We are in the era of GB ram computing, 10k cache would
+		// be a good sane default in server environment. Even if there are 
+		// multiple sets of StructCache, as it would take ~60MB
 		//
 		// to consider : auto detect RAM size in KB - and use that?
 		// a good rough guideline would be 1/4 of free ram space divided by 6kb
@@ -177,13 +178,13 @@ public void systemSetup() {
 		// 5006 bytes : Total bytes per object map
 		// ~ 6 kilo bytes : Rounded up
 		//
-		// # RAM cost for 100k objects
+		// # RAM cost for 10k objects
 		//
-		// 100,000 * 6 KB = 600 MB
+		// 100,000 * 6 KB = 60 MB
 		//
-		// > So yea, we are ok to assume a 100k objects for most parts
+		// > So yea, we are ok to assume a 10k objects for most parts
 		//
-		int capicity = configMap().getInt("capacity", 100000);
+		int capicity = configMap().getInt("capacity", 10000);
 		_valueMap = new Cache2kBuilder<String, Map<String, Object>>() {
 		} //
 		.name(cacheName())//