Added basic save and from_file functions for LSHCache

lsh/__init__.py

@@ -4,6 +4,7 @@
 import sys
 import time
 import logging
+import cPickle as pickle
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -129,13 +130,13 @@ def _insert_lsh(self,lsh,doc_id,date_added):
     def prepare_dup_buckets(cls, buckets, id=None):
         # logging.debug('buckets: %s', buckets)
         all = list(set(reduce(list.__add__, buckets, [])))
-        if id:
+        if id != None:
             all.remove(id)
         return all
 
     # public methods
 
-    def get_dup_buckets(self, doc):
+    def get_dup_buckets(self, doc, id):
         """
         Returns a list of buckets (which are themselves lists) that contain the ids
         of any matching documents.  If the cache was built in chronological order
@@ -179,3 +180,12 @@ def most_recent_insert(self):
     def num_shingles(self):
         return self._counter
 
+    def save(self, filename='lshcache.pkl'):
+        with open(filename, 'wb') as f:
+            pickle.dump(self.__dict__, f)
+
+    def from_file(self, filename):
+        with open(filename, 'rb') as f:
+            class_vars = pickle.load(f)
+        self.__dict__.update(class_vars)
+

lsh/examples/simple.py

@@ -33,3 +33,12 @@
                 print'\tdup : [%d] %s' % (dup, docs[dup])
         else:
             print 'no dups found for doc [%d] : %s' % (i, docs[i])
+
+    cache.save('test.pkl')
+
+    # Make a new cache to try loading from file
+    new_cache = LSHCache()
+
+    new_cache.from_file('test.pkl')
+    print new_cache.num_docs()
+    print new_cache._cache == cache._cache