Skip to content

Commit 92ee1af

Browse files
PicoCreatorPicoCreator
authored
Merge pull request #38 from picoded/mongodb-secondary-query
Mongodb secondary query
2 parents ba81f9e + 00d9aff commit 92ee1af

3 files changed

Lines changed: 224 additions & 30 deletions

File tree

‎.github/.codecov.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,8 @@ coverage:
1212
# advanced settings
1313
if_ci_failed: error #success, failure, error, ignore
1414
informational: true
15-
only_pulls: false
15+
only_pulls: false
16+
# Disable patch codecov errors
17+
patch:
18+
default:
19+
enabled: false

‎src/main/java/picoded/dstack/mongodb/MongoDBStack.java‎

Lines changed: 187 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
package picoded.dstack.mongodb;
22

3+
import java.util.ArrayList;
4+
import java.util.Collections;
5+
import java.util.List;
36
import java.util.Map;
47
import java.util.logging.Level;
58
import java.util.logging.Logger;
69

10+
import picoded.core.conv.GenericConvert;
11+
import picoded.core.struct.GenericConvertHashMap;
712
import picoded.core.struct.GenericConvertMap;
813
import picoded.dstack.core.*;
914

@@ -31,14 +36,60 @@ public class MongoDBStack extends CoreStack {
3136
protected MongoClient client_conn = null;
3237
protected MongoDatabase db_conn = null;
3338

39+
/**
40+
* The secondary connetion settings
41+
*/
42+
protected MongoClient sec_client_conn = null;
43+
protected MongoDatabase sec_db_conn = null;
44+
protected String sec_mode = null;
45+
46+
//-------------------------------------------------------------------------
47+
// Connector utilities
48+
//-------------------------------------------------------------------------
49+
50+
/// Default settings option JSON
51+
protected static String defaultOptJson = "{" + //
52+
" \"w\":\"majority\"," + //
53+
" \"retryWrites\":\"true\"," + //
54+
" \"retryReads\":\"true\"," + //
55+
" \"maxPoolSize\":20," + //
56+
" \"compressors\":\"zstd\"" + //
57+
"}";
58+
59+
// "&readPreference=master&readConcernLevel=majority"
60+
// "&readPreference=nearest&readConcernLevel=linearizable"
61+
62+
/**
63+
* Map the option object to a URI parameter string
64+
*/
65+
protected static String mapToOptStr(Map<String, Object> map) {
66+
// Get the sorted list of keys
67+
List<String> keys = (new ArrayList<>(map.keySet()));
68+
Collections.sort(keys);
69+
70+
// The ret stringbuilder
71+
StringBuilder ret = new StringBuilder();
72+
73+
// Lets loop the keys
74+
for (String key : keys) {
75+
if (ret.length() > 0) {
76+
ret.append("&");
77+
}
78+
ret.append(key + "=" + GenericConvert.toString(map.get(key)));
79+
}
80+
81+
// And return the built str
82+
return ret.toString();
83+
}
84+
3485
//-------------------------------------------------------------------------
35-
// Database connection constructor
86+
// Database connection URL constructor
3687
//-------------------------------------------------------------------------
3788

3889
/**
3990
* Given the mongodb config object, get the full_url
4091
*/
41-
public static String getFullConnectionURL(GenericConvertMap<String, Object> config) {
92+
public static String getFullConnectionURL_primary(GenericConvertMap<String, Object> config) {
4293
// Get the DB name (required)
4394
String dbname = config.getString("name", null);
4495
if (dbname == null || dbname.isEmpty()) {
@@ -57,13 +108,14 @@ public static String getFullConnectionURL(GenericConvertMap<String, Object> conf
57108
String pass = config.getString("pass", null);
58109
String host = config.getString("host", "localhost");
59110
int port = config.getInt("port", 27017);
60-
String opts = config.getString("opt_str", "w=majority&retryWrites=true&retryReads=true"
61-
+ "&maxPoolSize=10&compressors=zstd");
62-
// "&readPreference=master&readConcernLevel=majority"
63-
// "&readPreference=nearest&readConcernLevel=linearizable"
111+
112+
// Hanlding of option string
113+
GenericConvertMap<String, Object> optMap = config.getGenericConvertStringMap("opt",
114+
defaultOptJson);
115+
String optStr = config.getString("opt_str", mapToOptStr(optMap));
64116

65117
// Lets do a logging, for missing read concern if its not configured
66-
if (opts.indexOf("readConcernLevel") < 0) {
118+
if (optStr.indexOf("readConcernLevel") < 0) {
67119
//
68120
// readConcernLevel is a complicated topic, do consider reading up
69121
// https://jepsen.io/analyses/mongodb-4.2.6
@@ -75,15 +127,13 @@ public static String getFullConnectionURL(GenericConvertMap<String, Object> conf
75127
// Unless you know what your doing from a performance standpoint, it is strongly recommended to use
76128
// `readConcernLevel=linearizable`
77129
//
78-
LOGGER.warning("MongoDB is configured without readConcernLevel, "
79-
+ "this is alright for a single node, but `readConcernLevel=linearizable`"
80-
+ "or `readPreference=master&readConcernLevel=majority`"
81-
+ "is highly recommended for replica clusters to ensure read after write consistency.");
130+
LOGGER
131+
.warning("MongoDB is configured without readConcernLevel for the primary connection, "
132+
+ "this is alright for a single node, but `readConcernLevel=linearizable`"
133+
+ "or `readPreference=master&readConcernLevel=majority`"
134+
+ "is highly recommended for replica clusters to ensure read after write consistency.");
82135
}
83136

84-
// In the future we may want to support opt_map
85-
// GenericConvertMap<String,Object> optMap = config.getGenericConvertStringMap("opt_map", "{}");
86-
87137
// Lets build the auth str
88138
String authStr = "";
89139
if (user != null && pass != null) {
@@ -93,25 +143,98 @@ public static String getFullConnectionURL(GenericConvertMap<String, Object> conf
93143
// Return the full URL depending on the settings
94144
if (protocol.equals("mongodb+srv")) {
95145
// mongodb+srv does not support the port protocol
96-
return protocol + "://" + authStr + host + "/" + dbname + "?" + opts;
146+
return protocol + "://" + authStr + host + "/" + dbname + "?" + optStr;
97147
}
98-
return protocol + "://" + authStr + host + ":" + port + "/" + dbname + "?" + opts;
148+
return protocol + "://" + authStr + host + ":" + port + "/" + dbname + "?" + optStr;
99149
}
100150

101151
/**
102-
* Given the mongodb config object, get the MongoClient connection
152+
* Given the mongodb config object, get the full_url
103153
*/
104-
public static MongoClient setupFromConfig(GenericConvertMap<String, Object> inConfig) {
105-
// Get the full_url
106-
String full_url = getFullConnectionURL(inConfig);
154+
public static String getFullConnectionURL_secondary(GenericConvertMap<String, Object> config) {
155+
// Null check for secondary connection
156+
String sec_mode = config.getString("sec_mode", null);
157+
if (sec_mode == null) {
158+
return null;
159+
}
107160

108-
// Lets build using the stable API settings
109-
ServerApi serverApi = ServerApi.builder().version(ServerApiVersion.V1).build();
110-
MongoClientSettings settings = MongoClientSettings.builder()
111-
.applyConnectionString(new ConnectionString(full_url)).serverApi(serverApi).build();
161+
// Get the DB name (required)
162+
String dbname = config.getString("name", null);
163+
if (dbname == null || dbname.isEmpty()) {
164+
throw new IllegalArgumentException("Missing database 'name' for mongodb config");
165+
}
166+
167+
// Get the full connection url, and use it if present
168+
String full_url = config.getString("full_url", null);
169+
if (full_url != null) {
170+
return full_url;
171+
}
112172

113-
// Create the client, and return it
114-
return MongoClients.create(settings);
173+
// Lets get the config respectively
174+
String protocol = config.getString("protocol", "mongodb");
175+
String user = config.getString("user", null);
176+
String pass = config.getString("pass", null);
177+
String host = config.getString("host", "localhost");
178+
int port = config.getInt("port", 27017);
179+
180+
// Hanlding of option string, default sec_opt uses `secondaryPreferred`
181+
GenericConvertMap<String, Object> optMap = new GenericConvertHashMap<>();
182+
optMap.putAll(config.getGenericConvertStringMap("opt", defaultOptJson));
183+
optMap.putAll(config.getGenericConvertStringMap("sec_opt", "{ \"readPreference\":\"secondaryPreferred\" }"));
184+
185+
// The opt string overwrite
186+
String optStr = config.getString("sec_opt_str", mapToOptStr(optMap));
187+
188+
// Lets do a logging, for missing read concern if its not configured
189+
if (optStr.indexOf("readConcernLevel") < 0) {
190+
//
191+
// readConcernLevel is a complicated topic, do consider reading up
192+
// https://jepsen.io/analyses/mongodb-4.2.6
193+
// https://www.mongodb.com/blog/post/performance-best-practices-transactions-and-read--write-concerns
194+
// https://www.mongodb.com/docs/manual/reference/read-concern-linearizable/#mongodb-readconcern-readconcern.-linearizable-
195+
//
196+
// This option was removed by default, as an error will be thrown when its applied to single node clusters
197+
//
198+
// Unless you know what your doing from a performance standpoint, it is strongly recommended to use
199+
// `readConcernLevel=linearizable`
200+
//
201+
LOGGER
202+
.warning("MongoDB is configured without readConcernLevel for the secondary connection, "
203+
+ "this is alright for a single node, but `readConcernLevel=linearizable`"
204+
+ "or `readPreference=secondaryPreferred&readConcernLevel=majority`"
205+
+ "is highly recommended for replica clusters to ensure read after write consistency.");
206+
}
207+
208+
// Lets build the auth str
209+
String authStr = "";
210+
if (user != null && pass != null) {
211+
authStr = user + ":" + pass + "@";
212+
}
213+
214+
// Return the full URL depending on the settings
215+
if (protocol.equals("mongodb+srv")) {
216+
// mongodb+srv does not support the port protocol
217+
return protocol + "://" + authStr + host + "/" + dbname + "?" + optStr;
218+
}
219+
return protocol + "://" + authStr + host + ":" + port + "/" + dbname + "?" + optStr;
220+
}
221+
222+
//-------------------------------------------------------------------------
223+
// Database connection setup
224+
//-------------------------------------------------------------------------
225+
226+
/**
227+
* Utility library, used to check that the connection "works"
228+
* @param db_conn
229+
*/
230+
protected void checkMongoDatabaseConnection(MongoDatabase db_conn) {
231+
232+
// Safety check, get the list of connection names
233+
// this should throw an error if its not connected
234+
Iterable<String> list = db_conn.listCollectionNames();
235+
for (String n : list) {
236+
// does nothing
237+
}
115238
}
116239

117240
/**
@@ -129,11 +252,47 @@ public MongoDBStack(GenericConvertMap<String, Object> inConfig) {
129252
"Missing 'mongodb' config object for MongoDB stack provider");
130253
}
131254

255+
// Primary connection
256+
// ------
257+
258+
// Get the full_url
259+
String full_url = getFullConnectionURL_primary(inConfig);
260+
261+
// Lets build using the stable API settings
262+
ServerApi serverApi = ServerApi.builder().version(ServerApiVersion.V1).build();
263+
MongoClientSettings settings = MongoClientSettings.builder()
264+
.applyConnectionString(new ConnectionString(full_url)).serverApi(serverApi).build();
265+
132266
// Get the connection & database
133-
client_conn = setupFromConfig(dbConfig);
267+
client_conn = MongoClients.create(settings);
134268

135-
// Get the DB connection
269+
// Get the DB connection, and validate it
136270
db_conn = client_conn.getDatabase(dbConfig.fetchString("name"));
271+
checkMongoDatabaseConnection(db_conn);
272+
273+
// Secondary connection
274+
// ------
275+
276+
// Null check for secondary connection
277+
String config_sec_mode = config.getString("sec_mode", null);
278+
if (config_sec_mode == null) {
279+
return;
280+
}
281+
sec_mode = config_sec_mode.trim().toUpperCase();
282+
283+
// lets get the secondary connection
284+
full_url = getFullConnectionURL_secondary(inConfig);
285+
serverApi = ServerApi.builder().version(ServerApiVersion.V1).build();
286+
settings = MongoClientSettings.builder()
287+
.applyConnectionString(new ConnectionString(full_url)).serverApi(serverApi).build();
288+
289+
// Get the connection & database
290+
client_conn = MongoClients.create(settings);
291+
292+
// Get the DB connection, and validate it
293+
sec_db_conn = client_conn.getDatabase(dbConfig.fetchString("name"));
294+
checkMongoDatabaseConnection(sec_db_conn);
295+
137296
}
138297

139298
/**

‎src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ public class MongoDB_DataObjectMap extends Core_DataObjectMap {
6060
/** MongoDB instance representing the backend connection */
6161
MongoCollection<Document> collection = null;
6262

63+
/** Secondary connection, and its applicable mode */
64+
MongoCollection<Document> sec_collection = null;
65+
String sec_mode = null;
66+
6367
/**
6468
* Constructor, with name constructor
6569
*
@@ -69,6 +73,12 @@ public class MongoDB_DataObjectMap extends Core_DataObjectMap {
6973
public MongoDB_DataObjectMap(MongoDBStack inStack, String name) {
7074
super();
7175
collection = inStack.db_conn.getCollection(name);
76+
77+
// Get the secondary collection if applicable
78+
if (inStack.sec_db_conn != null) {
79+
sec_collection = inStack.sec_db_conn.getCollection(name);
80+
sec_mode = inStack.sec_mode;
81+
}
7282
}
7383

7484
//--------------------------------------------------------------------------
@@ -418,7 +428,28 @@ public String[] query_id(Query queryClause, String orderByStr, int offset, int l
418428
}
419429

420430
// Lets fetch the data, for the various _oid
421-
FindIterable<Document> search = collection.find(bsonFilter);
431+
FindIterable<Document> search = null;
432+
433+
// Chose the respective serach mode
434+
if (sec_mode == null) {
435+
// NULL safety
436+
search = collection.find(bsonFilter);
437+
} else if (sec_mode.equals("LIKE")) {
438+
// Use secondary connection for LIKE query
439+
if (queryClause.toSqlString().toUpperCase().indexOf("LIKE") > 0) {
440+
search = sec_collection.find(bsonFilter);
441+
}
442+
} else if (sec_mode.equals("QUERY")) {
443+
// Use secondary for all queries
444+
search = sec_collection.find(bsonFilter);
445+
}
446+
447+
// Fallback to main collection query
448+
if (search == null) {
449+
search = collection.find(bsonFilter);
450+
}
451+
452+
// Apply the projection, to only fetch _oid
422453
search = search.projection(Projections.include("_oid"));
423454

424455
// Build the orderBy clause

0 commit comments

Comments
 (0)