Reverted changes

data/scripts/migrator/dbsmerge_mongo.py

@@ -1,101 +1,68 @@
-#!/usr/bin/python
+#!/#!/usr/bin/python
 
 # depends:
-#   ^ pymongo # install via: easy_install pymongo
+#   ^ redis # install via easy_install redis
+# asserts:
+#   ^ destination redis is not password protected when connected from source redis server
+#     (https://github.com/antirez/redis/pull/2507)
 # behaviour:
-#   ^ the script will "move" the collections if source and target server are the same
-#     but will "copy" (dump/restore) if source and target servers are different
+#   ^ the script will not overwrite keys on the destination server/database
 
-from_host    = '127.0.0.1'
-from_port    = '27017'
-from_db      = '11'
-from_auth_db = 'cgrates' # Auth db on source server
-from_user    = 'cgrates'
-from_pass    = ''
+from_host   = '127.0.0.1'
+from_port   = 6379
+from_db     = 11
+from_pass   = ''
 
-to_host      = '127.0.0.1'
-to_port      = '27017'
-to_db        = '10'
-to_auth_db   = "cgrates" # Auth db on target server
-to_user      = 'cgrates'
-to_pass      = ''
+to_host     = '127.0.0.1'
+to_port     = 6379
+to_db       = 10
+to_pass     = '' # Not used
 
-ignore_empty_cols = True
-# Do not migrate collections with 0 document count.
-# Works only if from/to is on same host.
+keymask     = '*'
+timeout     = 2000
 
-# Overwrite target collections flag.
-# Works only if from/to is on same host.
-# If from/to hosts are different we use mongorestore which overwrites by default.
-drop_target = False
+import time
+import redis
 
-dump_folder = 'dump'
+from_redis = redis.Redis(host = from_host, port = from_port, password=from_pass, db = from_db)
+to_redis = redis.Redis(host = to_host, port = to_port, db = to_db)
 
-import sys
-from pymongo import MongoClient
-from urllib import quote_plus
-from collections import OrderedDict
+to_keys = to_redis.keys(keymask)
+from_keys = from_redis.keys(keymask)
+print('Found %d keys on source.' % len(from_keys))
+print('Found %d keys on destination.' % len(to_keys))
 
-# same server
-if from_host == to_host and from_port == to_port:
+# keys found
+if len(from_keys) > 0:
+    # same server
+    if from_host == to_host and from_port == to_port:
         print('Migrating on same server...')
-        mongo_from_url = 'mongodb://' + from_user + ':' + quote_plus(from_pass) + '@'+ from_host + ':' + from_port + '/' + from_auth_db
-        if from_pass == '': # disabled auth
-          mongo_from_url = 'mongodb://' + from_host + ':' + from_port + '/' + from_db
-        client = MongoClient(mongo_from_url)
+        i = 0
+        for key in from_keys:
+            i += 1
+            print('Moving key %s (%d of %d)...' % (key, i, len(from_keys)))
+            from_redis.execute_command('MOVE', key, to_db)
 
-        db = client[from_db]
-        cols = db.collection_names()
-
-        # collections found
-        if len(cols) > 0:
-            print('Found %d collections on source. Moving...' % len(cols))
-            i = 0
-            for col in cols:
-                i += 1
-                if not ignore_empty_cols or (ignore_empty_cols and db[col].count() > 0):
-                    print('Moving collection %s (%d of %d)...' % (col, i, len(cols)))
-                    try:
-                        client.admin.command(OrderedDict([('renameCollection', from_db + '.' + col), ('to', to_db + '.' + col), ('dropTarget', drop_target)]))
-                    except:
-                        e = sys.exc_info()[0]
-                        print(e)
-                else:
-                    print('Skipping empty collection %s (%d of %d)...' % (col, i, len(cols)))
-        # no collections found
-        else:
-            print('No collections in source database.')
-
-# different servers
+    # different servers
+    else:
+        print('Migrating between different servers...')
+        i = 0
+        for key in from_keys:
+            i += 1
+            print('Moving key %s (%d of %d)...' % (key, i, len(from_keys))),
+            try:
+                from_redis.execute_command('MIGRATE', to_host, to_port, key, to_db, timeout)
+            except redis.exceptions.ResponseError, e:
+                if not 'ERR Target key name is busy' in str(e):
+                    raise e
+            print('Done.')
+    # done
+    from_keys_after = from_redis.keys(keymask)
+    to_keys_after = to_redis.keys(keymask)
+    print('There are now %d keys on source.' % len(from_keys_after))
+    print('There are now %d keys on destination.' % len(to_keys_after))
+    print('%d keys were moved' % (len(to_keys_after) - len(to_keys)))
+    print('Migration complete.')
+# no keys found
 else:
-    import subprocess
-    import os
-    import shutil
-
-    print('Migrating between different servers...')
-    print('Dumping...')
-    out = subprocess.check_output([
-      'mongodump',
-      '--host', '%s' % from_host,
-      '-u',     '%s' % from_user,
-      '-p',     '%s' % from_pass,
-      '--authenticationDatabase', '%s' % from_auth_db,
-      '--db',     '%s' % from_db,
-      '--port', '%s' % from_port,
-      '-o',     '%s' % dump_folder,
-      ], stderr= subprocess.STDOUT)
-    print('Dump complete.')
-
-    print('Restoring...')
-    out = subprocess.check_output([
-      'mongorestore',
-      '--host', '%s' % to_host,
-      '-u',     '%s' % to_user,
-      '-p',     '%s' % to_pass,
-      '--authenticationDatabase', '%s' % to_auth_db,
-      '--db',   '%s' % to_db,
-      '--port', '%s' % to_port,
-      '--drop', '%s/%s' % (dump_folder, from_db),
-      ], stderr= subprocess.STDOUT)
-    print('Restore complete.')
-print('Migration complete.')
+    print('No keys with keymask %s found in source database' % keymask)