Skip to content

Commit 0e8368a

Browse files
PicoCreatorPicoCreator
committed
Working mongodb FileWorkspaceMap prototype (no file cleanup yet)
1 parent a9a83b9 commit 0e8368a

4 files changed

Lines changed: 120 additions & 47 deletions

File tree

‎src/main/java/picoded/dstack/core/Core_FileWorkspace.java‎

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -122,38 +122,16 @@ protected void setupUninitializedWorkspace() {
122122
* @param filePath
123123
* @return filePath normalized to remove ending "/"
124124
*/
125-
private static String normalizeFilePathString(final String filePath) {
126-
if (filePath == null) {
127-
throw new IllegalArgumentException("Invalid null filePath");
128-
}
129-
130-
String res = FileUtil.normalize(filePath, true);
131-
if (res.startsWith("/")) {
132-
res = res.substring(1);
133-
}
134-
if (res.endsWith("/")) {
135-
res = res.substring(0, res.length() - 1);
136-
}
137-
return res;
125+
protected static String normalizeFilePathString(final String filePath) {
126+
return Core_FileWorkspaceMap.normalizeFilePathString(filePath);
138127
}
139128

140129
/**
141130
* @param folderPath
142131
* @return folderPath normalized with ending "/"
143132
*/
144-
private static String normalizeFolderPathString(final String folderPath) {
145-
if (folderPath == null || folderPath.length() <= 0) {
146-
return "/";
147-
}
148-
149-
String res = FileUtil.normalize(folderPath, true);
150-
if (res.startsWith("/")) {
151-
res = res.substring(1);
152-
}
153-
if (!res.endsWith("/")) {
154-
res = res + "/";
155-
}
156-
return res;
133+
protected static String normalizeFolderPathString(final String folderPath) {
134+
return Core_FileWorkspaceMap.normalizeFolderPathString(folderPath);
157135
}
158136

159137
// File exists checks

‎src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package picoded.dstack.core;
22

33
import picoded.core.conv.ArrayConv;
4+
import picoded.core.file.FileUtil;
45

56
// Java imports
67

@@ -104,6 +105,48 @@ public void setupWorkspace(String oid) {
104105
*/
105106
abstract public void backend_setupWorkspace(String oid);
106107

108+
//--------------------------------------------------------------------------
109+
// File / Folder string normalization
110+
//--------------------------------------------------------------------------
111+
112+
/**
113+
* @param filePath
114+
* @return filePath normalized to remove ending "/"
115+
*/
116+
protected static String normalizeFilePathString(final String filePath) {
117+
if (filePath == null) {
118+
throw new IllegalArgumentException("Invalid null filePath");
119+
}
120+
121+
String res = FileUtil.normalize(filePath, true);
122+
if (res.startsWith("/")) {
123+
res = res.substring(1);
124+
}
125+
if (res.endsWith("/")) {
126+
res = res.substring(0, res.length() - 1);
127+
}
128+
return res;
129+
}
130+
131+
/**
132+
* @param folderPath
133+
* @return folderPath normalized with ending "/"
134+
*/
135+
protected static String normalizeFolderPathString(final String folderPath) {
136+
if (folderPath == null || folderPath.length() <= 0) {
137+
return "/";
138+
}
139+
140+
String res = FileUtil.normalize(folderPath, true);
141+
if (res.startsWith("/")) {
142+
res = res.substring(1);
143+
}
144+
if (!res.endsWith("/")) {
145+
res = res + "/";
146+
}
147+
return res;
148+
}
149+
107150
//--------------------------------------------------------------------------
108151
//
109152
// Functions, used by FileWorkspace
@@ -364,13 +407,13 @@ public void backend_moveFolderPath(final String oid, final String sourceFolder,
364407
// Lets sync up all the folders first
365408
for(String dir : subPath) {
366409
if(dir.endsWith("/")) {
367-
backend_ensureFolderPath(oid, destinationFolder+subPath);
410+
backend_ensureFolderPath(oid, destinationFolder+dir);
368411
}
369412
}
370413
// Lets sync up all the files next
371414
for(String file : subPath) {
372415
if(!file.endsWith("/")) {
373-
backend_moveFile(oid, sourceFolder+subPath, destinationFolder+subPath);
416+
backend_copyFile(oid, sourceFolder+file, destinationFolder+file);
374417
}
375418
}
376419
// Lets remove the original folders
@@ -431,13 +474,13 @@ public void backend_copyFolderPath(final String oid, final String sourceFolder,
431474
// Lets sync up all the folders first
432475
for(String dir : subPath) {
433476
if(dir.endsWith("/")) {
434-
backend_ensureFolderPath(oid, destinationFolder+subPath);
477+
backend_ensureFolderPath(oid, destinationFolder+dir);
435478
}
436479
}
437480
// Lets sync up all the files next
438481
for(String file : subPath) {
439-
if(!file.endsWith("/")) {
440-
backend_copyFile(oid, sourceFolder+subPath, destinationFolder+subPath);
482+
if(file.endsWith("/") == false) {
483+
backend_copyFile(oid, sourceFolder+file, destinationFolder+file);
441484
}
442485
}
443486
}
@@ -551,12 +594,21 @@ protected Set<String> backend_filterPathSet(final Set<String> rawSet, final Stri
551594
}
552595

553596
// Alrighto - lets check file / folder type - and add it in
597+
598+
// Ignore empty, or root path
599+
if(subPath.isEmpty() || subPath.equals("/")) {
600+
continue;
601+
}
602+
603+
// Expect a folder, reject files
554604
if (pathType == 1) {
555605
if (subPath.endsWith("/")) {
556606
// Not a file - abort!
557607
continue;
558608
}
559609
}
610+
611+
// Expect files, reject folders
560612
if (pathType == 2) {
561613
if (!subPath.endsWith("/")) {
562614
// Not a folder - abort!

‎src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
// MongoDB imports
2929
import org.bson.Document;
3030
import org.bson.types.Binary;
31+
import org.bson.types.ObjectId;
3132
import org.bson.conversions.Bson;
3233
import com.mongodb.client.*;
3334
import com.mongodb.client.gridfs.*;
@@ -110,7 +111,7 @@ public void systemSetup() {
110111
// We insert a "root" object, to ensure the tables are initialized
111112
// ---
112113
if(!fullRawPathExist("root")) {
113-
setupAnchorFile("root", "root", "root");
114+
setupAnchorFile_withFullRawPath("root", "root", "root");
114115
}
115116

116117
// Lets setup the index for the metadata fields (which is not enabled by default)
@@ -172,7 +173,7 @@ public boolean backend_workspaceExist(String oid) {
172173
public void backend_setupWorkspace(String oid) {
173174
// We setup a blank file with type root
174175
if(!fullRawPathExist(oid)) {
175-
setupAnchorFile(oid, oid, "space");
176+
setupAnchorFile_withFullRawPath(oid, oid, "space");
176177
}
177178
}
178179

@@ -219,10 +220,16 @@ protected boolean prefixPathExist(String oid, String path) {
219220
// Lets build the query for the "root file"
220221
Bson query = null;
221222

223+
// Get the full prefixpath
224+
String fullPrefixPath = oid+"/"+path;
225+
222226
// Remove matching path
223-
query = Filters.and(
224-
Filters.eq("metadata.oid", oid),
225-
Filters.regex("filename", "^"+Pattern.quote(oid+"/"+path)+".*")
227+
query = Filters.or(
228+
Filters.eq("filename", fullPrefixPath),
229+
Filters.and(
230+
Filters.eq("metadata.oid", oid),
231+
Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*")
232+
)
226233
);
227234

228235
// Lets prepare the search
@@ -241,8 +248,9 @@ protected boolean prefixPathExist(String oid, String path) {
241248

242249
/**
243250
* Setup an empty file, used for various use cases
251+
* The extended funciton name is intentional to avoid confusion of "full path" with "path"
244252
*/
245-
public void setupAnchorFile(String oid, String fullPath, String type) {
253+
public void setupAnchorFile_withFullRawPath(String oid, String fullPath, String type) {
246254
// In general we will upload a blank file
247255
// with the relevent oid, that can be easily lookedup
248256
//
@@ -255,7 +263,10 @@ public void setupAnchorFile(String oid, String fullPath, String type) {
255263

256264
// Prepare the upload options
257265
GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata);
258-
gridFSBucket.uploadFromStream(fullPath, emptyStream, opt);
266+
ObjectId objID = gridFSBucket.uploadFromStream(fullPath, emptyStream, opt);
267+
268+
// Flush it?
269+
objID.toString();
259270
} catch (IOException e) {
260271
throw new RuntimeException(e);
261272
}
@@ -310,7 +321,6 @@ protected boolean removeFilePath(String oid, String path) {
310321
while (cursor.hasNext()) {
311322
GridFSFile fileObj = cursor.next();
312323
gridFSBucket.delete(fileObj.getId());
313-
314324
rmFlag = true;
315325
}
316326
}
@@ -319,8 +329,39 @@ protected boolean removeFilePath(String oid, String path) {
319329
return rmFlag;
320330
}
321331

322-
protected void performFileCleanup(String oid, String path) {
332+
/**
333+
* Given the current path, enforce the parent pathing dir
334+
* Used mainly to ensure "parent" folder exists on file write/rm
335+
**/
336+
protected void ensureParentPath(String oid, String path) {
337+
// Does nothing if path is empty
338+
if( path == null || path.equals("/") || path.isEmpty() ) {
339+
return;
340+
}
341+
342+
// Cleanup ending slash
343+
if( path.endsWith("/") ) {
344+
path = path.substring(0, path.length() - 1);
345+
}
346+
347+
// Get the parent path
348+
String parPath = normalizeFolderPathString( FileUtil.getParentPath(path) );
349+
350+
// Does nothing if folder path is "blank"
351+
if( parPath == null || parPath.equals("/") || parPath.isEmpty() ) {
352+
return;
353+
}
354+
355+
// Path enforcement
356+
backend_ensureFolderPath(oid, parPath);
357+
}
323358

359+
/**
360+
* Because mongoDB does file versioining on each save, we would need to cleanup
361+
* older file versions where applicable, in a safe way
362+
*/
363+
protected void performVersionedFileCleanup(String oid, String path) {
364+
// @TODO !!!
324365
}
325366

326367
//--------------------------------------------------------------------------
@@ -383,7 +424,8 @@ public void backend_fileWriteInputStream(final String oid, final String filepath
383424

384425
// Prepare the upload options
385426
GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata);
386-
gridFSBucket.uploadFromStream(fullPath, data, opt);
427+
ObjectId objID = gridFSBucket.uploadFromStream(fullPath, data, opt);
428+
objID.toString();
387429
} catch (Exception e) {
388430
throw new RuntimeException(e);
389431
} finally {
@@ -455,6 +497,7 @@ public boolean backend_fileExist(String oid, String filepath) {
455497

456498
@Override
457499
public void backend_removeFile(String oid, String filepath) {
500+
ensureParentPath(oid, filepath);
458501
removeFilePath(oid, filepath);
459502
}
460503

@@ -473,6 +516,7 @@ public void backend_removeFile(String oid, String filepath) {
473516
* @return the stored byte array of the file
474517
**/
475518
public void backend_removeFolderPath(final String oid, final String folderPath) {
519+
ensureParentPath(oid, folderPath);
476520
removeFilePathRecursively(oid, folderPath);
477521
}
478522

@@ -504,8 +548,8 @@ public boolean backend_folderPathExist(final String oid, final String folderPath
504548
public void backend_ensureFolderPath(final String oid, final String folderPath) {
505549
// We setup a blank file with type root, this checks only for the anchor file
506550
// if it does not exists, we will make it
507-
if(!fullRawPathExist(oid+"/"+folderPath)) {
508-
setupAnchorFile(oid, folderPath, "dir");
551+
if(fullRawPathExist(oid+"/"+folderPath) == false) {
552+
setupAnchorFile_withFullRawPath(oid, oid+"/"+folderPath, "dir");
509553
}
510554
}
511555

@@ -599,15 +643,13 @@ public Set<String> backend_getFileAndFolderPathSet(final String oid, String fold
599643
// Query using oid and the path
600644
fullPrefixPath = fullPrefixPath+folderPath;
601645

602-
// Remove matching path
646+
// Filter for matching path
603647
query = Filters.and(
604648
Filters.eq("metadata.oid", oid),
605649
Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*")
606650
);
607651
}
608652

609-
query = Filters.eq("metadata.oid", oid);
610-
611653
// The return set
612654
Set<String> ret = new HashSet<>();
613655

‎src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,10 @@ public void fileWrite_andProperlySetupFolder() {
160160

161161
// Remove and assert
162162
fileWorkspace.removeFolderPath("test/folder");
163+
assertFalse(fileWorkspace.fileExist("test/folder/file.txt"));
163164
assertFalse(fileWorkspace.folderPathExist("test/folder"));
165+
164166
assertTrue(fileWorkspace.folderPathExist("test"));
165-
assertFalse(fileWorkspace.fileExist("test/folder/file.txt"));
166167
}
167168

168169
//-----------------------------------------------------------------------------------

0 commit comments

Comments
 (0)