uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Added script to migrate

Browse Source
This commit is contained in:
edwardro22
2017-09-11 12:17:14 +00:00
parent 5a37c9fa94
commit df31455316
11 changed files with 677 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
data/scripts/migrator/dbsmerge_mongo.py
View File

@@ -1,68 +1,101 @@
#!/#!/usr/bin/python
#!/usr/bin/python
# depends:
# ^ redis # install via easy_install redis
# asserts:
# ^ destination redis is not password protected when connected from source redis server
# (https://github.com/antirez/redis/pull/2507)
# ^ pymongo # install via: easy_install pymongo
# behaviour:
# ^ the script will not overwrite keys on the destination server/database
# ^ the script will "move" the collections if source and target server are the same
# but will "copy" (dump/restore) if source and target servers are different
from_host = '127.0.0.1'
from_port = 6379
from_db = 11
from_pass = ''
from_host = '127.0.0.1'
from_port = '27017'
from_db = '11'
from_auth_db = 'cgrates' # Auth db on source server
from_user = 'cgrates'
from_pass = ''
to_host = '127.0.0.1'
to_port = 6379
to_db = 10
to_pass = '' # Not used
to_host = '127.0.0.1'
to_port = '27017'
to_db = '10'
to_auth_db = "cgrates" # Auth db on target server
to_user = 'cgrates'
to_pass = ''
keymask = '*'
timeout = 2000
ignore_empty_cols = True
# Do not migrate collections with 0 document count.
# Works only if from/to is on same host.
import time
import redis
# Overwrite target collections flag.
# Works only if from/to is on same host.
# If from/to hosts are different we use mongorestore which overwrites by default.
drop_target = False
from_redis = redis.Redis(host = from_host, port = from_port, password=from_pass, db = from_db)
to_redis = redis.Redis(host = to_host, port = to_port, db = to_db)
dump_folder = 'dump'
to_keys = to_redis.keys(keymask)
from_keys = from_redis.keys(keymask)
print('Found %d keys on source.' % len(from_keys))
print('Found %d keys on destination.' % len(to_keys))
import sys
from pymongo import MongoClient
from urllib import quote_plus
from collections import OrderedDict
# keys found
if len(from_keys) > 0:
# same server
if from_host == to_host and from_port == to_port:
# same server
if from_host == to_host and from_port == to_port:
print('Migrating on same server...')
i = 0
for key in from_keys:
i += 1
print('Moving key %s (%d of %d)...' % (key, i, len(from_keys)))
from_redis.execute_command('MOVE', key, to_db)
mongo_from_url = 'mongodb://' + from_user + ':' + quote_plus(from_pass) + '@'+ from_host + ':' + from_port + '/' + from_auth_db
if from_pass == '': # disabled auth
mongo_from_url = 'mongodb://' + from_host + ':' + from_port + '/' + from_db
client = MongoClient(mongo_from_url)
# different servers
else:
print('Migrating between different servers...')
i = 0
for key in from_keys:
i += 1
print('Moving key %s (%d of %d)...' % (key, i, len(from_keys))),
try:
from_redis.execute_command('MIGRATE', to_host, to_port, key, to_db, timeout)
except redis.exceptions.ResponseError, e:
if not 'ERR Target key name is busy' in str(e):
raise e
print('Done.')
# done
from_keys_after = from_redis.keys(keymask)
to_keys_after = to_redis.keys(keymask)
print('There are now %d keys on source.' % len(from_keys_after))
print('There are now %d keys on destination.' % len(to_keys_after))
print('%d keys were moved' % (len(to_keys_after) - len(to_keys)))
print('Migration complete.')
# no keys found
db = client[from_db]
cols = db.collection_names()
# collections found
if len(cols) > 0:
print('Found %d collections on source. Moving...' % len(cols))
i = 0
for col in cols:
i += 1
if not ignore_empty_cols or (ignore_empty_cols and db[col].count() > 0):
print('Moving collection %s (%d of %d)...' % (col, i, len(cols)))
try:
client.admin.command(OrderedDict([('renameCollection', from_db + '.' + col), ('to', to_db + '.' + col), ('dropTarget', drop_target)]))
except:
e = sys.exc_info()[0]
print(e)
else:
print('Skipping empty collection %s (%d of %d)...' % (col, i, len(cols)))
# no collections found
else:
print('No collections in source database.')
# different servers
else:
print('No keys with keymask %s found in source database' % keymask)
import subprocess
import os
import shutil
print('Migrating between different servers...')
print('Dumping...')
out = subprocess.check_output([
'mongodump',
'--host', '%s' % from_host,
'-u', '%s' % from_user,
'-p', '%s' % from_pass,
'--authenticationDatabase', '%s' % from_auth_db,
'--db', '%s' % from_db,
'--port', '%s' % from_port,
'-o', '%s' % dump_folder,
], stderr= subprocess.STDOUT)
print('Dump complete.')
print('Restoring...')
out = subprocess.check_output([
'mongorestore',
'--host', '%s' % to_host,
'-u', '%s' % to_user,
'-p', '%s' % to_pass,
'--authenticationDatabase', '%s' % to_auth_db,
'--db', '%s' % to_db,
'--port', '%s' % to_port,
'--drop', '%s/%s' % (dump_folder, from_db),
], stderr= subprocess.STDOUT)
print('Restore complete.')
print('Migration complete.')

50
data/scripts/migrator/dbsmerge_redis.py
View File

@@ -8,28 +8,24 @@
# behaviour:
# ^ the script will not overwrite keys on the destination server/database
from_host = '127.0.0.1'
from_port = 6379
from_db = 11
from_pass = ''
to_host = '127.0.0.1'
to_port = 6379
to_db = 10
to_pass = '' # Not used
keymask = '*'
timeout = 2000
import time
import redis
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-host", "--host",default="127.0.0.1", help='default: "127.0.0.1"')
parser.add_argument("-port", "--port", type=int ,default=6379, help='default: 6379')
parser.add_argument("-pass", "--password", default="", help='default: ""')
args = parser.parse_args()
from_host = args.host
from_port = args.port
from_pass = args.password
from_redis = redis.Redis(host = from_host, port = from_port, password=from_pass, db = from_db)
to_redis = redis.Redis(host = from_host, port = from_port, db = to_db)
to_redis = redis.Redis(host = to_host, port = to_port, db = to_db)
to_keys = to_redis.keys(keymask)
from_keys = from_redis.keys(keymask)
@@ -38,14 +34,28 @@ print('Found %d keys on destination.' % len(to_keys))
# keys found
if len(from_keys) > 0:
print('Migrating on same server...')
i = 0
for key in from_keys:
i += 1
print('Moving key %s (%d of %d)...' % (key, i, len(from_keys)))
from_redis.execute_command('MOVE', key, to_db)
# same server
if from_host == to_host and from_port == to_port:
print('Migrating on same server...')
i = 0
for key in from_keys:
i += 1
print('Moving key %s (%d of %d)...' % (key, i, len(from_keys)))
from_redis.execute_command('MOVE', key, to_db)
print('Done.')
# different servers
else:
print('Migrating between different servers...')
i = 0
for key in from_keys:
i += 1
print('Moving key %s (%d of %d)...' % (key, i, len(from_keys))),
try:
from_redis.execute_command('MIGRATE', to_host, to_port, key, to_db, timeout)
except redis.exceptions.ResponseError, e:
if not 'ERR Target key name is busy' in str(e):
raise e
print('Done.')
# done
from_keys_after = from_redis.keys(keymask)
to_keys_after = to_redis.keys(keymask)
@@ -55,4 +65,4 @@ if len(from_keys) > 0:
print('Migration complete.')
# no keys found
else:
print('No keys with keymask %s found in source database' % keymask)
print('No keys with keymask %s found in source database' % keymask)