Merge branch 'mongodb-support' of github.com:picoded/JavaCommons-dstack into mongodb-support

Author: PicoCreator

.github/.codecov.yml

@@ -0,0 +1,15 @@
+coverage:
+  status:
+    project:
+      # Full configuration format is found here :
+      # https://docs.codecov.com/docs/commit-status#section-project-status
+      default:
+        # basic
+        #target: 10%
+        #threshold: 10%
+        paths: 
+          - "src"
+        # advanced settings
+        if_ci_failed: error #success, failure, error, ignore
+        informational: true
+        only_pulls: false

.github/workflows/gradle-build.yml

@@ -6,7 +6,7 @@ on:
     branches: [ master ]
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
     - name: Init submodules

.github/workflows/redis.yml

@@ -0,0 +1,63 @@
+name: Redis support test
+on:
+  push:
+    branches: [ master, "redis*", "*redis*" ]
+  pull_request:
+    branches: [ master ]
+jobs:
+  unit_test:
+    runs-on: ubuntu-latest
+    
+    # Configuration, for us to perform test against multiple version in the future
+    strategy:
+      matrix:
+        redis-version: [7]
+    
+    # The test step
+    steps:
+    # Start Redis 
+    - name: Start Redis
+      uses: supercharge/redis-github-action@1.4.0
+      with:
+        redis-version: ${{ matrix.redis-version }}
+
+    # The usual checkout, and test
+    - uses: actions/checkout@v1
+    - name: Init submodules
+      run: ./git-init-submodules.sh
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Cache Gradle packages
+      uses: actions/cache@v2
+      with:
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/build.gradle') }}
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+    - name: Setup gradle binaries
+      run: ./gradlew
+    - name: Pull dependencies libs, and perform initial compile
+      run: ./gradlew src
+    - name: Run unit tests
+      run: ./gradlew test -Ptest_redis
+    - name: Run jacocoTestReport
+      run: ./gradlew jacocoTestReport
+    - name: Archive code coverage results
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: test-result-report
+        path: |
+          build/reports/**/*
+          build/test-results/**/*
+          build/jacoco/**/*
+    - name: Upload code coverage to codecov
+      uses: codecov/codecov-action@v2
+      if: always()
+      with:
+        # flags: gradle-build # optional
+        # directory: build/jacoco/
+        verbose: true # optional (default = false)
+        fail_ci_if_error: false

JavaCommons-core

@@ -13,29 +13,37 @@ All while maintaining the same interface for the application. Leaving the decisi
 Its most notable claass is `DataObjectMap` which functions as a `SQL` queryable `NoSQL` interface. That functions even against an SQL backend.
 Followed by `DStack` which faciltates the stacking of data backend provider for devops. Such as a distributed cache with an SQL backend.
 
-# Support Matrix
-
-| backend       | status        | notes                                                | DataObjectMap | KeyValueMap | KeyLongMap | FileWorkspaceMap |
-|---------------|---------------|------------------------------------------------------|---------------|-------------|------------|------------------|
-| struct.simple | development   | reference implementation, not for used in production | storage       | storage     | storage    | storage          |
-| struct.cache  | development   |                                                      | storage       |             |            |                  |
-| jsql          | in-production | see SQL support table below for details              | full          | full        | full       | full             |
-| hazelcast     | development   |                                                      | storage       | storage     |            |                  |
-| file.simple   | development   |                                                      |               |             |            | development      |
-| ignite        | development   | roadmap                                              |               |             |            |                  |
-| cockroachdb   | development   | roadmap                                              |               |             |            |                  |
+# Provider Support Matrix
+
+| backend          | status        | notes                                                | DataObjectMap | KeyValueMap | KeyLongMap | FileWorkspaceMap |
+|------------------|---------------|------------------------------------------------------|---------------|-------------|------------|------------------|
+| struct.simple    | in-production | reference implementation, not recommended for use    | storage       | storage     | storage    | storage          |
+| struct.cache     | in-production | local instance caching, useful for WORM data         | storage       | storage     |            |                  |
+| jsql             | in-production | *with limits: see SQL support notes below            | full          | full        | full       | full             |
+| mongodb          | in-production | *with limits: see MongoDB support notes below        | full          | full        | full       | full             |
+| hazelcast.cache  | in-production |                                                      | full          | full        | full       |                  |
+| hazelcast.store  | in-production |                                                      | full          | full        | full       |                  |
+| file.simple      | in-production |                                                      |               |             |            | storage          |
+| file.layered     | in-production |                                                      |               |             |            | storage          |
+| resdisson        | experimental  |                                                      | storage       | storage     |            |                  |
+| ignite           | roadmap       | roadmap                                              |               |             |            |                  |
+| cockroachdb      | roadmap       | roadmap                                              |               |             |            |                  |
+
+**Important notes**
+
+- "full", means it has been optimized for both storage, and query operations.
+- "storage" means does not support optimization for queries, for large queries, this can have detrimental performance implication, as the apistack will need to iterate a large number of data.
+- Hazelcast require a custom build / deployment with the JavaCommons JAR file to support the required functionality
+- MySQL connection / db seems to support only up to 16 digits of accuracy
+- Requires read-after-write consistency, for expected behaviour, use w=majority&readConcernLevel=linearizable
 
 # Data Structures
 
 | DataStructure    | Status        | Description                                                                |
 |------------------|---------------|----------------------------------------------------------------------------|
 | DataObjectMap    | in-production | Map document storage, with SQL query support                               |
 | KeyValueMap      | in-production | High performance key to string value storage                               |
-| KeyLongMap       | experimental  | Varient of KeyValue with atomic long support (if used in single tier mode) |
-| FileWorkspaceMap | development   | File workspace storage support                                             |
+| KeyLongMap       | in-production | Varient of KeyValue with atomic long support (if used in single tier mode) |
+| FileWorkspaceMap | in-production | File workspace storage support                                             |
 | MessageQueue     | road-map      | Message queue                                                              |
 | JobQueue         | road-map      | Job request, response queue                                                |
-
-# Important known issues
-
-- MySQL connection / db seems to support only up to 16 digits of accuracy

README.md

@@ -91,7 +91,7 @@ dependencies {
 	api                "com.oracle.ojdbc:ojdbc8:19.3.0.0"  // for oracle
 
 	// api files("./lib/jsql/ojdbc6.jar")  // for oracle	
-	api "org.xerial:sqlite-jdbc:3.25.2" // for sqlite
+	api "org.xerial:sqlite-jdbc:3.34.0" // for sqlite
 	api "mysql:mysql-connector-java:8.0.26" // for mysql
 
 	// Newer MSSQL connection (THIS DOES NOT WORK)
@@ -113,6 +113,9 @@ dependencies {
 	api 'org.mongodb:mongodb-driver-sync:4.5.1'
 	api 'com.github.luben:zstd-jni:1.5.2-3'
 
+	// Redis
+	api 'org.redisson:redisson:3.17.1'
+
 	// PostgresSQL library
 	api "org.postgresql:postgresql:42.2.5"
 
@@ -279,6 +282,9 @@ test {
 		if(!project.hasProperty("test_mongodb")) {
 			exclude "**/*MongoDB*"
 		}
+		if(!project.hasProperty("test_redis")) {
+			exclude "**/*Redis*"
+		}
 	}
 
 	// Setup test to run on parllel threads = to number of processors by default
@@ -382,6 +388,9 @@ task incrementalTest(type: TestWatcher) {
 		if(!project.hasProperty("test_mongodb")) {
 			exclude "**/*MongoDB*"
 		}
+		if(!project.hasProperty("test_redis")) {
+			exclude "**/*Redis*"
+		}
 	}
 
 	// Setup test to run on parllel threads = to number of processors by default

build.gradle

@@ -312,6 +312,33 @@ default String[] getFromKeyName_id(String keyName, String orderByStr, int offset
 		return retList.toArray(new String[retList.size()]);
 	}
 
+	//
+	// Query count optimization handling
+	//--------------------------------------------------------------------------
+	
+	/**
+	 * Get the size of the current DataObjectMap
+	 * 
+	 * Note that if the dataset size is larger then Integer.MAX_VALUE
+	 * it is clamped accordingly to the MAX_VALUE
+	 **/
+	@Override
+	default int size() {
+		long s = queryCount(null, null);
+		if (s > Integer.MAX_VALUE) {
+			return Integer.MAX_VALUE;
+		}
+		return (int) s;
+	}
+	
+	/**
+	 * Get the size as a long for the DataObjectMap
+	 * This should be used instead of size (to work around int limits)
+	 */
+	default long longSize() {
+		return queryCount(null, null);
+	}
+	
 	//
 	// Get key names handling
 	//--------------------------------------------------------------------------

src/main/java/picoded/dstack/DataObjectMap.java

@@ -123,8 +123,15 @@ default FileWorkspace[] getArrayFromID(String[] idArray, boolean isUnchecked) {
 	 *
 	 * This is equivalent of "TRUNCATE TABLE {TABLENAME}"
 	 **/
+	@Override
 	void clear();
 
+	/**
+	 * Return the current list of ID keys for fileWorkspaceMap
+	 **/
+	@Override
+	Set<String> keySet();
+	
 	// Special iteration support
 	//--------------------------------------------------------------------------
 

src/main/java/picoded/dstack/FileWorkspaceMap.java

@@ -3,6 +3,8 @@
 import picoded.dstack.file.simple.FileSimple_FileWorkspaceMap;
 
 import java.io.File;
+import java.util.HashSet;
+import java.util.Set;
 
 public class FileLayered_FileWorkspaceMap extends FileSimple_FileWorkspaceMap {
 
@@ -51,4 +53,64 @@ public File workspaceDirObj(String oid) {
 		return new File(baseDir, workspacePath);
 	}
 
+	//--------------------------------------------------------------------------
+	//
+	// KeySet support
+	//
+	//--------------------------------------------------------------------------
+	
+	/**
+	 * Get and returns all the GUID's, note that due to its
+	 * potential of returning a large data set, production use
+	 * should be avoided.
+	 *
+	 * @return set of keys
+	 **/
+	@Override
+	public Set<String> keySet() {
+		// The return hashset
+		HashSet<String> ret = new HashSet<String>();
+		
+		// List all the files/folders
+		File[] l1_dirList = baseDir.listFiles();
+		
+		// For each one of it, process it!
+		for (File l1_dir : l1_dirList) {
+			// Skip if its not a directory
+			if (!l1_dir.isDirectory()) {
+				continue;
+			}
+			
+			// List all the files/folders
+			File[] l2_dirList = l1_dir.listFiles();
+			
+			for (File l2_dir : l2_dirList) {
+				// Skip if its not a directory
+				if (!l2_dir.isDirectory()) {
+					continue;
+				}
+				
+				// Get the oidDirLIst
+				File[] oid_list = l2_dir.listFiles();
+				
+				// For each oid dir
+				for (File oid_dir : oid_list) {
+					// Get the presumed oid
+					String oid = oid_dir.getName();
+					
+					// Validate the dir name (oid)
+					if (!validateOid(oid)) {
+						continue;
+					}
+					
+					// Add the oid to the ret set
+					ret.add(oid);
+				}
+			}
+		}
+		
+		// Return the full keyset
+		return ret;
+	}
+	
 }

src/main/java/picoded/dstack/file/layered/FileLayered_FileWorkspaceMap.java

@@ -73,7 +73,7 @@ protected boolean validateOid(String oid) {
 		}
 
 		// Adding safety check for file operation, ensuring oid is alphanumeric
-		if (!oid.matches("[a-zA-Z0-9]+")) {
+		if (!oid.matches("[a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9]+")) {
 			return false;
 		}
 
@@ -147,7 +147,7 @@ public void backend_setupWorkspace(String oid) {
 		if (file == null) {
 			throw new RuntimeException("Invalid OID (unable to setup)");
 		}
-		boolean mkdir = file.mkdirs();
+		file.mkdirs();
 	}
 
 	//--------------------------------------------------------------------------
@@ -627,4 +627,48 @@ public Set<String> backend_getFileAndFolderPathSet(final String oid, final Strin
 		return backend_filterPathSet(retSet, "", minDepth, maxDepth, 0);
 	}
 
+	//--------------------------------------------------------------------------
+	//
+	// KeySet support
+	//
+	//--------------------------------------------------------------------------
+	
+	/**
+	 * Get and returns all the GUID's, note that due to its
+	 * potential of returning a large data set, production use
+	 * should be avoided.
+	 *
+	 * @return set of keys
+	 **/
+	@Override
+	public Set<String> keySet() {
+		// The return hashset
+		HashSet<String> ret = new HashSet<String>();
+		
+		// List all the files/folders
+		File[] dirList = baseDir.listFiles();
+		
+		// For each one of it, process it!
+		for (File subFile : dirList) {
+			// Skip if its not a directory
+			if (!subFile.isDirectory()) {
+				continue;
+			}
+			
+			// Get the presumed oid
+			String oid = subFile.getName();
+			
+			// Validate the dir name (oid)
+			if (!validateOid(oid)) {
+				continue;
+			}
+			
+			// Add the oid to the ret set
+			ret.add(oid);
+		}
+		
+		// Return the full keyset
+		return ret;
+	}
+	
 }