diff --git a/data/scripts/migrator/dbsmerge_mongo.py b/data/scripts/migrator/dbsmerge_mongo.py index ddc176fe8..ec3ed3b2a 100755 --- a/data/scripts/migrator/dbsmerge_mongo.py +++ b/data/scripts/migrator/dbsmerge_mongo.py @@ -1,101 +1,68 @@ -#!/usr/bin/python +#!/#!/usr/bin/python # depends: -# ^ pymongo # install via: easy_install pymongo +# ^ redis # install via easy_install redis +# asserts: +# ^ destination redis is not password protected when connected from source redis server +# (https://github.com/antirez/redis/pull/2507) # behaviour: -# ^ the script will "move" the collections if source and target server are the same -# but will "copy" (dump/restore) if source and target servers are different +# ^ the script will not overwrite keys on the destination server/database -from_host = '127.0.0.1' -from_port = '27017' -from_db = '11' -from_auth_db = 'cgrates' # Auth db on source server -from_user = 'cgrates' -from_pass = '' +from_host = '127.0.0.1' +from_port = 6379 +from_db = 11 +from_pass = '' -to_host = '127.0.0.1' -to_port = '27017' -to_db = '10' -to_auth_db = "cgrates" # Auth db on target server -to_user = 'cgrates' -to_pass = '' +to_host = '127.0.0.1' +to_port = 6379 +to_db = 10 +to_pass = '' # Not used -ignore_empty_cols = True -# Do not migrate collections with 0 document count. -# Works only if from/to is on same host. +keymask = '*' +timeout = 2000 -# Overwrite target collections flag. -# Works only if from/to is on same host. -# If from/to hosts are different we use mongorestore which overwrites by default. -drop_target = False +import time +import redis -dump_folder = 'dump' +from_redis = redis.Redis(host = from_host, port = from_port, password=from_pass, db = from_db) +to_redis = redis.Redis(host = to_host, port = to_port, db = to_db) -import sys -from pymongo import MongoClient -from urllib import quote_plus -from collections import OrderedDict +to_keys = to_redis.keys(keymask) +from_keys = from_redis.keys(keymask) +print('Found %d keys on source.' % len(from_keys)) +print('Found %d keys on destination.' % len(to_keys)) -# same server -if from_host == to_host and from_port == to_port: +# keys found +if len(from_keys) > 0: + # same server + if from_host == to_host and from_port == to_port: print('Migrating on same server...') - mongo_from_url = 'mongodb://' + from_user + ':' + quote_plus(from_pass) + '@'+ from_host + ':' + from_port + '/' + from_auth_db - if from_pass == '': # disabled auth - mongo_from_url = 'mongodb://' + from_host + ':' + from_port + '/' + from_db - client = MongoClient(mongo_from_url) + i = 0 + for key in from_keys: + i += 1 + print('Moving key %s (%d of %d)...' % (key, i, len(from_keys))) + from_redis.execute_command('MOVE', key, to_db) - db = client[from_db] - cols = db.collection_names() - - # collections found - if len(cols) > 0: - print('Found %d collections on source. Moving...' % len(cols)) - i = 0 - for col in cols: - i += 1 - if not ignore_empty_cols or (ignore_empty_cols and db[col].count() > 0): - print('Moving collection %s (%d of %d)...' % (col, i, len(cols))) - try: - client.admin.command(OrderedDict([('renameCollection', from_db + '.' + col), ('to', to_db + '.' + col), ('dropTarget', drop_target)])) - except: - e = sys.exc_info()[0] - print(e) - else: - print('Skipping empty collection %s (%d of %d)...' % (col, i, len(cols))) - # no collections found - else: - print('No collections in source database.') - -# different servers + # different servers + else: + print('Migrating between different servers...') + i = 0 + for key in from_keys: + i += 1 + print('Moving key %s (%d of %d)...' % (key, i, len(from_keys))), + try: + from_redis.execute_command('MIGRATE', to_host, to_port, key, to_db, timeout) + except redis.exceptions.ResponseError, e: + if not 'ERR Target key name is busy' in str(e): + raise e + print('Done.') + # done + from_keys_after = from_redis.keys(keymask) + to_keys_after = to_redis.keys(keymask) + print('There are now %d keys on source.' % len(from_keys_after)) + print('There are now %d keys on destination.' % len(to_keys_after)) + print('%d keys were moved' % (len(to_keys_after) - len(to_keys))) + print('Migration complete.') +# no keys found else: - import subprocess - import os - import shutil - - print('Migrating between different servers...') - print('Dumping...') - out = subprocess.check_output([ - 'mongodump', - '--host', '%s' % from_host, - '-u', '%s' % from_user, - '-p', '%s' % from_pass, - '--authenticationDatabase', '%s' % from_auth_db, - '--db', '%s' % from_db, - '--port', '%s' % from_port, - '-o', '%s' % dump_folder, - ], stderr= subprocess.STDOUT) - print('Dump complete.') - - print('Restoring...') - out = subprocess.check_output([ - 'mongorestore', - '--host', '%s' % to_host, - '-u', '%s' % to_user, - '-p', '%s' % to_pass, - '--authenticationDatabase', '%s' % to_auth_db, - '--db', '%s' % to_db, - '--port', '%s' % to_port, - '--drop', '%s/%s' % (dump_folder, from_db), - ], stderr= subprocess.STDOUT) - print('Restore complete.') -print('Migration complete.') + print('No keys with keymask %s found in source database' % keymask) \ No newline at end of file