Skip to content

Commit e132f4d

Browse files
PicoCreatorPicoCreator
committed
secondary connection setup WIP
1 parent a259480 commit e132f4d

2 files changed

Lines changed: 158 additions & 39 deletions

File tree

‎src/main/java/picoded/dstack/mongodb/MongoDBStack.java‎

Lines changed: 155 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.util.logging.Logger;
99

1010
import picoded.core.conv.GenericConvert;
11+
import picoded.core.struct.GenericConvertHashMap;
1112
import picoded.core.struct.GenericConvertMap;
1213
import picoded.dstack.core.*;
1314

@@ -47,44 +48,48 @@ public class MongoDBStack extends CoreStack {
4748
//-------------------------------------------------------------------------
4849

4950
/// Default settings option JSON
50-
protected static String defaultOptJson = "{"+ //
51-
" \"w\":\"majority\","+ //
52-
" \"retryWrites\":\"true\","+ //
53-
" \"retryReads\":\"true\","+ //
54-
" \"maxPoolSize\":25,"+ //
55-
" \"compressors\":\"zstd\""+ //
56-
"}";
51+
protected static String defaultOptJson = "{" + //
52+
" \"w\":\"majority\"," + //
53+
" \"retryWrites\":\"true\"," + //
54+
" \"retryReads\":\"true\"," + //
55+
" \"maxPoolSize\":10," + //
56+
" \"compressors\":\"zstd\"" + //
57+
"}";
58+
5759
// "&readPreference=master&readConcernLevel=majority"
5860
// "&readPreference=nearest&readConcernLevel=linearizable"
59-
60-
protected static String mapToOptStr(Map<String,Object> map) {
61+
62+
/**
63+
* Map the option object to a URI parameter string
64+
*/
65+
protected static String mapToOptStr(Map<String, Object> map) {
6166
// Get the sorted list of keys
6267
List<String> keys = (new ArrayList<>(map.keySet()));
6368
Collections.sort(keys);
64-
69+
6570
// The ret stringbuilder
6671
StringBuilder ret = new StringBuilder();
67-
72+
6873
// Lets loop the keys
69-
for(String key : keys) {
70-
if( ret.length() > 0 ) {
74+
for (String key : keys) {
75+
if (ret.length() > 0) {
7176
ret.append("&");
7277
}
73-
ret.append(key+"="+GenericConvert.toString(map.get(key)));
78+
ret.append(key + "=" + GenericConvert.toString(map.get(key)));
7479
}
75-
80+
7681
// And return the built str
7782
return ret.toString();
7883
}
79-
84+
8085
//-------------------------------------------------------------------------
81-
// Database connection constructor
86+
// Database connection URL constructor
8287
//-------------------------------------------------------------------------
8388

8489
/**
8590
* Given the mongodb config object, get the full_url
8691
*/
87-
public static String getFullConnectionURL(GenericConvertMap<String, Object> config) {
92+
public static String getFullConnectionURL_primary(GenericConvertMap<String, Object> config) {
8893
// Get the DB name (required)
8994
String dbname = config.getString("name", null);
9095
if (dbname == null || dbname.isEmpty()) {
@@ -103,9 +108,10 @@ public static String getFullConnectionURL(GenericConvertMap<String, Object> conf
103108
String pass = config.getString("pass", null);
104109
String host = config.getString("host", "localhost");
105110
int port = config.getInt("port", 27017);
106-
111+
107112
// Hanlding of option string
108-
GenericConvertMap<String,Object> optMap = config.getGenericConvertStringMap("opt", defaultOptJson);
113+
GenericConvertMap<String, Object> optMap = config.getGenericConvertStringMap("opt",
114+
defaultOptJson);
109115
String optStr = config.getString("opt_str", mapToOptStr(optMap));
110116

111117
// Lets do a logging, for missing read concern if its not configured
@@ -121,10 +127,11 @@ public static String getFullConnectionURL(GenericConvertMap<String, Object> conf
121127
// Unless you know what your doing from a performance standpoint, it is strongly recommended to use
122128
// `readConcernLevel=linearizable`
123129
//
124-
LOGGER.warning("MongoDB is configured without readConcernLevel, "
125-
+ "this is alright for a single node, but `readConcernLevel=linearizable`"
126-
+ "or `readPreference=master&readConcernLevel=majority`"
127-
+ "is highly recommended for replica clusters to ensure read after write consistency.");
130+
LOGGER
131+
.warning("MongoDB is configured without readConcernLevel for the primary connection, "
132+
+ "this is alright for a single node, but `readConcernLevel=linearizable`"
133+
+ "or `readPreference=master&readConcernLevel=majority`"
134+
+ "is highly recommended for replica clusters to ensure read after write consistency.");
128135
}
129136

130137
// In the future we may want to support opt_map
@@ -145,19 +152,95 @@ public static String getFullConnectionURL(GenericConvertMap<String, Object> conf
145152
}
146153

147154
/**
148-
* Given the mongodb config object, get the MongoClient connection
155+
* Given the mongodb config object, get the full_url
149156
*/
150-
public static MongoClient setupFromConfig(GenericConvertMap<String, Object> inConfig) {
151-
// Get the full_url
152-
String full_url = getFullConnectionURL(inConfig);
157+
public static String getFullConnectionURL_secondary(GenericConvertMap<String, Object> config) {
158+
// Null check for secondary connection
159+
String sec_mode = config.getString("sec_mode", null);
160+
if (sec_mode == null) {
161+
return null;
162+
}
153163

154-
// Lets build using the stable API settings
155-
ServerApi serverApi = ServerApi.builder().version(ServerApiVersion.V1).build();
156-
MongoClientSettings settings = MongoClientSettings.builder()
157-
.applyConnectionString(new ConnectionString(full_url)).serverApi(serverApi).build();
164+
// Get the DB name (required)
165+
String dbname = config.getString("name", null);
166+
if (dbname == null || dbname.isEmpty()) {
167+
throw new IllegalArgumentException("Missing database 'name' for mongodb config");
168+
}
158169

159-
// Create the client, and return it
160-
return MongoClients.create(settings);
170+
// Get the full connection url, and use it if present
171+
String full_url = config.getString("full_url", null);
172+
if (full_url != null) {
173+
return full_url;
174+
}
175+
176+
// Lets get the config respectively
177+
String protocol = config.getString("protocol", "mongodb");
178+
String user = config.getString("user", null);
179+
String pass = config.getString("pass", null);
180+
String host = config.getString("host", "localhost");
181+
int port = config.getInt("port", 27017);
182+
183+
// Hanlding of option string
184+
GenericConvertMap<String, Object> optMap = new GenericConvertHashMap<>();
185+
optMap.putAll(config.getGenericConvertStringMap("opt", defaultOptJson));
186+
optMap.putAll(config.getGenericConvertStringMap("sec_opt", "{}"));
187+
188+
// The opt string overwrite
189+
String optStr = config.getString("sec_opt_str", mapToOptStr(optMap));
190+
191+
// Lets do a logging, for missing read concern if its not configured
192+
if (optStr.indexOf("readConcernLevel") < 0) {
193+
//
194+
// readConcernLevel is a complicated topic, do consider reading up
195+
// https://jepsen.io/analyses/mongodb-4.2.6
196+
// https://www.mongodb.com/blog/post/performance-best-practices-transactions-and-read--write-concerns
197+
// https://www.mongodb.com/docs/manual/reference/read-concern-linearizable/#mongodb-readconcern-readconcern.-linearizable-
198+
//
199+
// This option was removed by default, as an error will be thrown when its applied to single node clusters
200+
//
201+
// Unless you know what your doing from a performance standpoint, it is strongly recommended to use
202+
// `readConcernLevel=linearizable`
203+
//
204+
LOGGER
205+
.warning("MongoDB is configured without readConcernLevel for the secondary connection, "
206+
+ "this is alright for a single node, but `readConcernLevel=linearizable`"
207+
+ "or `readPreference=master&readConcernLevel=majority`"
208+
+ "is highly recommended for replica clusters to ensure read after write consistency.");
209+
}
210+
211+
// In the future we may want to support opt_map
212+
// GenericConvertMap<String,Object> optMap = config.getGenericConvertStringMap("opt_map", "{}");
213+
214+
// Lets build the auth str
215+
String authStr = "";
216+
if (user != null && pass != null) {
217+
authStr = user + ":" + pass + "@";
218+
}
219+
220+
// Return the full URL depending on the settings
221+
if (protocol.equals("mongodb+srv")) {
222+
// mongodb+srv does not support the port protocol
223+
return protocol + "://" + authStr + host + "/" + dbname + "?" + optStr;
224+
}
225+
return protocol + "://" + authStr + host + ":" + port + "/" + dbname + "?" + optStr;
226+
}
227+
228+
//-------------------------------------------------------------------------
229+
// Database connection setup
230+
//-------------------------------------------------------------------------
231+
232+
/**
233+
* Utility library, used to check that the connection "works"
234+
* @param db_conn
235+
*/
236+
protected void checkMongoDatabaseConnection(MongoDatabase db_conn) {
237+
238+
// Safety check, get the list of connection names
239+
// this should throw an error if its not connected
240+
Iterable<String> list = db_conn.listCollectionNames();
241+
for (String n : list) {
242+
// does nothing
243+
}
161244
}
162245

163246
/**
@@ -175,11 +258,47 @@ public MongoDBStack(GenericConvertMap<String, Object> inConfig) {
175258
"Missing 'mongodb' config object for MongoDB stack provider");
176259
}
177260

261+
// Primary connection
262+
// ------
263+
264+
// Get the full_url
265+
String full_url = getFullConnectionURL_primary(inConfig);
266+
267+
// Lets build using the stable API settings
268+
ServerApi serverApi = ServerApi.builder().version(ServerApiVersion.V1).build();
269+
MongoClientSettings settings = MongoClientSettings.builder()
270+
.applyConnectionString(new ConnectionString(full_url)).serverApi(serverApi).build();
271+
178272
// Get the connection & database
179-
client_conn = setupFromConfig(dbConfig);
273+
client_conn = MongoClients.create(settings);
180274

181-
// Get the DB connection
275+
// Get the DB connection, and validate it
182276
db_conn = client_conn.getDatabase(dbConfig.fetchString("name"));
277+
checkMongoDatabaseConnection(db_conn);
278+
279+
// Secondary connection
280+
// ------
281+
282+
// Null check for secondary connection
283+
String config_sec_mode = config.getString("sec_mode", null);
284+
if (config_sec_mode == null) {
285+
return;
286+
}
287+
sec_mode = config_sec_mode.trim().toUpperCase();
288+
289+
// lets get the secondary connection
290+
full_url = getFullConnectionURL_secondary(inConfig);
291+
serverApi = ServerApi.builder().version(ServerApiVersion.V1).build();
292+
settings = MongoClientSettings.builder()
293+
.applyConnectionString(new ConnectionString(full_url)).serverApi(serverApi).build();
294+
295+
// Get the connection & database
296+
client_conn = MongoClients.create(settings);
297+
298+
// Get the DB connection, and validate it
299+
sec_db_conn = client_conn.getDatabase(dbConfig.fetchString("name"));
300+
checkMongoDatabaseConnection(sec_db_conn);
301+
183302
}
184303

185304
/**

‎src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,10 +250,10 @@ public void DataObjectRemoteDataMap_update(String _oid, Map<String, Object> full
250250

251251
// If value is a Map, we recast it to avoid referencing and formatting issues
252252
// this also resolves the issue when a Map, contains an array
253-
if( value instanceof Map ) {
254-
value = ConvertJSON.toMap( ConvertJSON.fromObject(value) );
253+
if (value instanceof Map) {
254+
value = ConvertJSON.toMap(ConvertJSON.fromObject(value));
255255
}
256-
256+
257257
// Lets apply the update values
258258
if (updateKeys.contains(key)) {
259259
// Handle NULL values unset

0 commit comments

Comments
 (0)