From c00ad5fee3ae7fad8fd1be1677485c174b6bee00 Mon Sep 17 00:00:00 2001
From: alin104n <alin104n@users.noreply.github.com>
Date: Mon, 3 Apr 2017 18:50:26 +0300
Subject: [PATCH] Mongo, Redis migrators

---
 data/scripts/migrator/dbsmerge_mongo.py | 80 +++++++++++++++++++++++++
 data/scripts/migrator/dbsmerge_redis.py | 68 +++++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100755 data/scripts/migrator/dbsmerge_mongo.py
 create mode 100755 data/scripts/migrator/dbsmerge_redis.py

diff --git a/data/scripts/migrator/dbsmerge_mongo.py b/data/scripts/migrator/dbsmerge_mongo.py
new file mode 100755
index 000000000..8c15f33ec
--- /dev/null
+++ b/data/scripts/migrator/dbsmerge_mongo.py
@@ -0,0 +1,80 @@
+#!/usr/bin/python
+
+# depends:
+#   ^ pymongo
+# asserts:
+#   ^ destination mongo database exists and the destination user can auth on it
+# behaviour:
+#   ^ the script will "move" the collections if source and target server are the same
+#     but will "copy" (dump/restore) if source and target servers are different
+
+from_host   = '127.0.0.1'
+from_port   = '27017'
+from_db     = 'cgrates2'
+from_user   = 'cgrates'
+from_pass   = 'CGRateS.org'
+
+to_host     = '127.0.0.1'
+to_port     = '27017'
+to_db       = 'cgrates2'
+to_user     = 'cgrates'
+to_pass     = 'CGRateS.org'
+
+dump_folder = 'dump'
+
+from pymongo import MongoClient
+from urllib import quote_plus
+from collections import OrderedDict
+
+mongo_from_url = 'mongodb://' + from_user + ':' + quote_plus(from_pass) + '@'+ from_host + ':' + from_port + '/' + from_db
+client = MongoClient(mongo_from_url)
+
+db = client[from_db]
+cols = db.collection_names()
+
+# collections found
+if len(cols) > 0:
+    # same server
+    if from_host == to_host and from_port == to_port:
+            print('Migrating on same server...')
+            print('Found %d collections on source. Moving...' % len(cols))
+            i = 0
+            for col in db.collection_names():
+                i += 1
+                print('Moving colection %s (%d of %d)...' % (col, i, len(cols)))
+                client.admin.command(OrderedDict([('renameCollection', from_db + '.' + col), ('to', to_db + '.' + col)]))
+    # different servers
+    else:
+        import subprocess
+        import os
+        import shutil
+
+        print('Migrating between different servers...')
+        print('Dumping...')
+        out = subprocess.check_output([
+          'mongodump',
+          '--host', '%s' % from_host,
+          '-u',     '%s' % from_user,
+          '-p',     '%s' % from_pass,
+          '-d',     '%s' % from_db,
+          '--port', '%s' % from_port,
+          '-o',     '%s' % dump_folder,
+          ], stderr= subprocess.STDOUT)
+        print('Dump complete.')
+
+        print('Restoring...')
+        out = subprocess.check_output([
+          'mongorestore',
+          '--host', '%s' % to_host,
+          '-u',     '%s' % to_user,
+          '-p',     '%s' % to_pass,
+          '--authenticationDatabase', '%s' % to_db,
+          '--db',   '%s' % to_db,
+          '--port', '%s' % to_port,
+          '--drop', '%s/%s' % (dump_folder,from_db),
+          ], stderr= subprocess.STDOUT)
+        print('Restore complete.')
+    print('Migration complete.')
+# no collections found
+else:
+    print('No collections in source database.')
diff --git a/data/scripts/migrator/dbsmerge_redis.py b/data/scripts/migrator/dbsmerge_redis.py
new file mode 100755
index 000000000..ca491d0a0
--- /dev/null
+++ b/data/scripts/migrator/dbsmerge_redis.py
@@ -0,0 +1,68 @@
+#!/usr/bin/python
+
+# depends:
+#   ^ redis
+# asserts:
+#   ^ destination redis is not password protected when connected from source redis server
+#     (https://github.com/antirez/redis/pull/2507)
+# behaviour:
+#   ^ the script will not overwrite keys on the destination server/database
+
+from_host   = '127.0.0.1'
+from_port   = 6379
+from_db     = 0
+from_pass   = ''
+
+to_host     = '127.0.0.1'
+to_port     = 6380
+to_db       = 0
+to_pass     = '' # Not used
+
+keymask     = '*'
+timeout     = 2000
+
+import time
+import redis
+
+from_redis = redis.Redis(host = from_host, port = from_port, password=from_pass, db = from_db)
+to_redis = redis.Redis(host = to_host, port = to_port, db = to_db)
+
+to_keys = to_redis.keys(keymask)
+from_keys = from_redis.keys(keymask)
+print('Found %d keys on source.' % len(from_keys))
+print('Found %d keys on destination.' % len(to_keys))
+
+# keys found
+if len(from_keys) > 0:
+    # same server
+    if from_host == to_host and from_port == to_port:
+        print('Migrating on same server...')
+        i = 0
+        for key in from_keys:
+            i += 1
+            print('Moving key %s (%d of %d)...' % (key, i, len(from_keys)))
+            from_redis.execute_command('MOVE', key, to_db)
+
+    # different servers
+    else:
+        print('Migrating between different servers...')
+        i = 0
+        for key in from_keys:
+            i += 1
+            print('Moving key %s (%d of %d)...' % (key, i, len(from_keys))),
+            try:
+                from_redis.execute_command('MIGRATE', to_host, to_port, key, to_db, timeout)
+            except redis.exceptions.ResponseError, e:
+                if not 'ERR Target key name is busy' in str(e):
+                    raise e
+            print('Done.')
+    # done
+    from_keys_after = from_redis.keys(keymask)
+    to_keys_after = to_redis.keys(keymask)
+    print('There are now %d keys on source.' % len(from_keys_after))
+    print('There are now %d keys on destination.' % len(to_keys_after))
+    print('%d keys were moved' % (len(to_keys_after) - len(to_keys)))
+    print('Migration complete.')
+# no keys found
+else:
+    print('No keys with keymask %s found in source database' % keymask)