diff --git a/cluster/backup/mysqlbinlogchecker b/cluster/backup/mysqlbinlogchecker new file mode 100755 index 0000000..03aa701 Binary files /dev/null and b/cluster/backup/mysqlbinlogchecker differ diff --git a/cluster/dist.sh b/cluster/dist.sh index 32ea421..9f4c3b4 100755 --- a/cluster/dist.sh +++ b/cluster/dist.sh @@ -18,7 +18,9 @@ for hostitem in $hosts; do hname="${hostitem#*:}" - echo "=========== [`date`] transfer ($from) to $to on $host($hname) ===========" + echo "=== [`date`] on $host ===" + echo "=== transfer ($from) to $to ===" + #echo "=========== [`date`] transfer ($from) to $to on $host($hname) ===========" if `ping -c 2 $host >/dev/null 2>/dev/null`; then : else @@ -27,15 +29,19 @@ for hostitem in $hosts; do fi if test "$SSHPASS" = ""; then - eval scp -r $from $REMOTE_USER@$host:$to || ufail=1 + eval scp -q -r $from $REMOTE_USER@$host:$to || ufail=1 else eval sshpass -p "$REMOTE_PASSWORD" scp -r $from $REMOTE_USER@$host:$to || ufail=1 fi - if test "$ufail" = "1"; then - ffail=1 - echo "!!!FAILURES!!!" - fi + if test "$ufail" = "1"; then + ffail=1 + echo -e "=== \033[31m !!!FAILURES!!! on $host \033[0m ===" + echo " " + else + echo -e "=== \033[32m !!!SUCCESS!!! on $host \033[0m ===" + echo " " + fi done diff --git a/cluster/generate_scripts.py b/cluster/generate_scripts.py index e580b87..76a97dc 100755 --- a/cluster/generate_scripts.py +++ b/cluster/generate_scripts.py @@ -1,4 +1,3 @@ -#!/bin/python2 # Copyright (c) 2019 ZettaDB inc. All rights reserved. # This source code is licensed under Apache 2.0 License, # combined with Common Clause Condition 1.0, as detailed in the NOTICE file. @@ -12,142 +11,157 @@ import os import os.path -defuser=getpass.getuser() -defbase='/kunlun' +defuser = getpass.getuser() +defbase = "/kunlun" + def addIpToMachineMap(map, ip): global defuser global defbase - if not map.has_key(ip): - mac={"ip":ip, "user":defuser, "basedir":defbase} - map[ip] = mac + if not map.__contains__(ip): + mac = {"ip": ip, "user": defuser, "basedir": defbase} + map[ip] = mac + def addMachineToMap(map, ip, user, basedir): - mac={"ip":ip, "user":user, "basedir":basedir} + mac = {"ip": ip, "user": user, "basedir": basedir} map[ip] = mac + def addIpToFilesMap(map, ip, fname, targetdir): - if not map.has_key(ip): - map[ip] = {} + if not map.__contains__(ip): + map[ip] = {} tmap = map[ip] - if not tmap.has_key(fname): - tmap[fname] = targetdir + if not tmap.__contains__(fname): + tmap[fname] = targetdir + def addNodeToFilesMap(map, node, fname, targetdir): - ip = node['ip'] + ip = node["ip"] addIpToFilesMap(map, ip, fname, targetdir) + def addNodeToIpset(set, node): - ip = node['ip'] + ip = node["ip"] set.add(ip) + # Not used currently. def addToCommandsMap(map, ip, targetdir, command): - if not map.has_key(ip): - map[ip] = [] + if not map.__contains__(ip): + map[ip] = [] cmds = map[ip] cmds.append([targetdir, command]) + def addToCommandsList(cmds, ip, targetdir, command): lst = [ip, targetdir, command] cmds.append(lst) + def addToDirMap(map, ip, newdir): - if not map.has_key(ip): - map[ip] = [] + if not map.__contains__(ip): + map[ip] = [] dirs = map[ip] dirs.append(newdir) + def getuuid(): return str(uuid.uuid1()) + def getuuid_from_cnf(cnfpath): cnf = open(cnfpath) for line in cnf.readlines(): - if re.match('group_replication_group_name', line): - line=re.sub(r'\n','',line) - linea=re.sub(r'[ #].*', '',line) - lineb=re.sub('group_replication_group_name=','', linea) - linec=re.sub(r'[\'\"]','',lineb) - return linec + if re.match("group_replication_group_name", line): + line = re.sub(r"\n", "", line) + linea = re.sub(r"[ #].*", "", line) + lineb = re.sub("group_replication_group_name=", "", linea) + linec = re.sub(r"[\'\"]", "", lineb) + return linec return None + def generate_install_scripts(jscfg, installtype): global defuser global defbase - localip = '127.0.0.1' + localip = "127.0.0.1" machines = {} - for mach in jscfg['machines']: - ip=mach['ip'] - user=mach.get('user', defuser) - base=mach.get('basedir', defbase) - addMachineToMap(machines, ip, user, base) + for mach in jscfg["machines"]: + ip = mach["ip"] + user = mach.get("user", defuser) + base = mach.get("basedir", defbase) + addMachineToMap(machines, ip, user, base) filesmap = {} commandslist = [] dirmap = {} - usemgr=True - - cluster = jscfg['cluster'] - cluster_name = cluster['name'] - meta = cluster['meta'] - if not meta.has_key('group_uuid'): - meta['group_uuid'] = getuuid() - my_metaname = 'mysql_meta.json' - metaf = open(r'install/%s' % my_metaname,'w') + usemgr = True + + cluster = jscfg["cluster"] + cluster_name = cluster["name"] + meta = cluster["meta"] + if not meta.__contains__("group_uuid"): + meta["group_uuid"] = getuuid() + my_metaname = "mysql_meta.json" + metaf = open(r"install/%s" % my_metaname, "w") json.dump(meta, metaf, indent=4) metaf.close() # commands like: # sudo --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH python2 \ # install-mysql.py dbcfg=./template.cnf mgr_config=./mysql_meta.json target_node_index=0 - targetdir='percona-8.0.18-bin-rel/dba_tools' - i=0 - secmdlist=[] - for node in meta['nodes']: - addNodeToFilesMap(filesmap, node, my_metaname, targetdir) - addIpToMachineMap(machines, node['ip']) - cmdpat = 'sudo --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH python2 \ - install-mysql.py dbcfg=./template.cnf mgr_config=./%s target_node_index=%d' - if node.get('is_primary', False): - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (my_metaname, i)) - else: - addToCommandsList(secmdlist, node['ip'], targetdir, cmdpat % (my_metaname, i)) - addToDirMap(dirmap, node['ip'], node['data_dir_path']) - addToDirMap(dirmap, node['ip'], node['log_dir_path']) - i+=1 + targetdir = "percona-8.0.18-bin-rel/dba_tools" + i = 0 + secmdlist = [] + for node in meta["nodes"]: + addNodeToFilesMap(filesmap, node, my_metaname, targetdir) + addIpToMachineMap(machines, node["ip"]) + cmdpat = "sudo -E python2 \ + install-mysql.py dbcfg=./template.cnf mgr_config=./%s target_node_index=%d" + if node.get("is_primary", False): + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % (my_metaname, i) + ) + else: + addToCommandsList( + secmdlist, node["ip"], targetdir, cmdpat % (my_metaname, i) + ) + addToDirMap(dirmap, node["ip"], node["data_dir_path"]) + addToDirMap(dirmap, node["ip"], node["log_dir_path"]) + i += 1 commandslist.extend(secmdlist) secmdlist = [] - targetdir='percona-8.0.18-bin-rel/dba_tools' - datas = cluster['data'] - i=1 + targetdir = "percona-8.0.18-bin-rel/dba_tools" + datas = cluster["data"] + i = 1 pries = [] secs = [] for shard in datas: - if not shard.has_key('group_uuid'): - shard['group_uuid'] = getuuid() - my_shardname = "mysql_shard%d.json" % i - shardf = open(r'install/%s' % my_shardname, 'w') - json.dump(shard, shardf, indent=4) - shardf.close() - j = 0 - for node in shard['nodes']: - addNodeToFilesMap(filesmap, node, my_shardname, targetdir) - addIpToMachineMap(machines, node['ip']) - cmdpat = 'sudo --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH python2 \ - install-mysql.py dbcfg=./template.cnf mgr_config=./%s target_node_index=%d' - if node.get('is_primary', False): - pries.append([node['ip'], targetdir, cmdpat % (my_shardname, j)]) - else: - secs.append([node['ip'], targetdir, cmdpat % (my_shardname, j)]) - addToDirMap(dirmap, node['ip'], node['data_dir_path']) - addToDirMap(dirmap, node['ip'], node['log_dir_path']) - j += 1 - if j == 1: - usemgr=False - i+=1 + if not shard.__contains__("group_uuid"): + shard["group_uuid"] = getuuid() + my_shardname = "mysql_shard%d.json" % i + shardf = open(r"install/%s" % my_shardname, "w") + json.dump(shard, shardf, indent=4) + shardf.close() + j = 0 + for node in shard["nodes"]: + addNodeToFilesMap(filesmap, node, my_shardname, targetdir) + addIpToMachineMap(machines, node["ip"]) + cmdpat = "sudo -E python2 \ + install-mysql.py dbcfg=./template.cnf mgr_config=./%s target_node_index=%d" + if node.get("is_primary", False): + pries.append([node["ip"], targetdir, cmdpat % (my_shardname, j)]) + else: + secs.append([node["ip"], targetdir, cmdpat % (my_shardname, j)]) + addToDirMap(dirmap, node["ip"], node["data_dir_path"]) + addToDirMap(dirmap, node["ip"], node["log_dir_path"]) + j += 1 + if j == 1: + usemgr = False + i += 1 extraopt = " " if not usemgr: extraopt = " usemgr=False" @@ -157,917 +171,1153 @@ def generate_install_scripts(jscfg, installtype): addToCommandsList(secmdlist, item[0], item[1], item[2] + extraopt) commandslist.extend(secmdlist) # This only needs to transfered to machine creating the cluster. - pg_metaname = 'postgres_meta.json' - metaf = open(r'install/%s' % pg_metaname, 'w') + pg_metaname = "postgres_meta.json" + metaf = open(r"install/%s" % pg_metaname, "w") objs = [] - for node in meta['nodes']: - obj = {} - obj['ip'] = node['ip'] - obj['port'] = node['port'] - obj['user'] = "pgx" - obj['password'] = "pgx_pwd" - objs.append(obj) + for node in meta["nodes"]: + obj = {} + obj["ip"] = node["ip"] + obj["port"] = node["port"] + obj["user"] = "pgx" + obj["password"] = "pgx_pwd" + objs.append(obj) json.dump(objs, metaf, indent=4) metaf.close() # This only needs to transfered to machine creating the cluster. - pg_shardname = 'postgres_shards.json' - shardf = open(r'install/%s' % pg_shardname, 'w') + pg_shardname = "postgres_shards.json" + shardf = open(r"install/%s" % pg_shardname, "w") shards = [] - i=1 + i = 1 for shard in datas: - obj={'shard_name': "shard%d" % i} - i+=1 - nodes=[] - for node in shard['nodes']: - n={'user':'pgx', 'password':'pgx_pwd'} - n['ip'] = node['ip'] - n['port'] = node['port'] - nodes.append(n) - obj['shard_nodes'] = nodes - shards.append(obj) + obj = {"shard_name": "shard%d" % i} + i += 1 + nodes = [] + for node in shard["nodes"]: + n = {"user": "pgx", "password": "pgx_pwd"} + n["ip"] = node["ip"] + n["port"] = node["port"] + nodes.append(n) + obj["shard_nodes"] = nodes + shards.append(obj) json.dump(shards, shardf, indent=4) shardf.close() - comps = cluster['comp']['nodes'] - pg_compname = 'postgres_comp.json' - compf = open(r'install/%s' % pg_compname, 'w') + comps = cluster["comp"]["nodes"] + pg_compname = "postgres_comp.json" + compf = open(r"install/%s" % pg_compname, "w") json.dump(comps, compf, indent=4) compf.close() # python2 install_pg.py config=docker-comp.json install_ids=1,2,3 - targetdir="postgresql-11.5-rel/scripts" + targetdir = "postgresql-11.5-rel/scripts" for node in comps: - addNodeToFilesMap(filesmap, node, pg_compname, targetdir) - addIpToMachineMap(machines, node['ip']) - cmdpat = r'python2 install_pg.py config=./%s install_ids=%d' - if not usemgr: - cmdpat = cmdpat + " usemgr=False" - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (pg_compname, node['id'])) - addToDirMap(dirmap, node['ip'], node['datadir']) + addNodeToFilesMap(filesmap, node, pg_compname, targetdir) + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"python2 install_pg.py config=./%s install_ids=%d" + if not usemgr: + cmdpat = cmdpat + " usemgr=False" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % (pg_compname, node["id"]) + ) + addToDirMap(dirmap, node["ip"], node["datadir"]) comp1 = comps[0] addNodeToFilesMap(filesmap, comp1, pg_metaname, targetdir) addNodeToFilesMap(filesmap, comp1, pg_shardname, targetdir) resourcedir = "postgresql-11.5-rel/resources" - cmdpat=r'/bin/bash build_driver.sh' - addToCommandsList(commandslist, comp1['ip'], resourcedir, cmdpat) - cmdpat=r'python2 bootstrap.py --config=./%s --bootstrap_sql=./meta_inuse.sql' - addToCommandsList(commandslist, comp1['ip'], targetdir, cmdpat % pg_metaname) - cmdpat='python2 create_cluster.py --shards_config ./%s \ ---comps_config ./%s --meta_config ./%s --cluster_name %s --cluster_owner abc --cluster_biz test' + cmdpat = r"/bin/bash build_driver.sh" + addToCommandsList(commandslist, comp1["ip"], resourcedir, cmdpat) + cmdpat = r"python2 bootstrap.py --config=./%s --bootstrap_sql=./meta_inuse.sql" + addToCommandsList(commandslist, comp1["ip"], targetdir, cmdpat % pg_metaname) + cmdpat = "python2 create_cluster.py --shards_config ./%s \ + --comps_config ./%s --meta_config ./%s --cluster_name %s --cluster_owner abc --cluster_biz test" if not usemgr: cmdpat = cmdpat + " --usemgr False" - addToCommandsList(commandslist, comp1['ip'], targetdir, - cmdpat % (pg_shardname, pg_compname, pg_metaname, cluster_name)) + addToCommandsList( + commandslist, + comp1["ip"], + targetdir, + cmdpat % (pg_shardname, pg_compname, pg_metaname, cluster_name), + ) # bash -x bin/cluster_mgr_safe --debug --pidfile=run.pid clustermgr.cnf >& run.log & run.log & run.log env.sh" ''' - tup=(ip, mach['user'], mach['basedir']) - exttup=(mach['user'], ip, mach['basedir'], mach['basedir']) - comf.write(comstr % tup) - comf.write(extstr % exttup) - comf.write("\n") - - if installtype == 'full': - comstr = "bash remote_run.sh --user=%s %s 'cd %s && source ./env.sh && cd percona-8.0.18-bin-rel/lib && bash ../../process_deps.sh'\n" - comf.write(comstr % (mach['user'], ip, mach['basedir'])) - comstr = "bash remote_run.sh --user=%s %s 'cd %s && source ./env.sh && cd postgresql-11.5-rel/lib && bash ../../process_deps.sh'\n" - comf.write(comstr % (mach['user'], ip, mach['basedir'])) + mach = machines.get(ip) + mkstr = "bash remote_run.sh --user=%s %s 'sudo mkdir -p %s && sudo chown -R %s:`id -gn %s` %s'\n" + tup = ( + mach["user"], + ip, + mach["basedir"], + mach["user"], + mach["user"], + mach["basedir"], + ) + comf.write(mkstr % tup) + # Set up the files + if installtype == "full": + comstr = "bash dist.sh --hosts=%s --user=%s %s %s\n" + comf.write( + comstr + % (ip, mach["user"], "percona-8.0.18-bin-rel.tgz", mach["basedir"]) + ) + comf.write( + comstr % (ip, mach["user"], "postgresql-11.5-rel.tgz", mach["basedir"]) + ) + comf.write( + comstr % (ip, mach["user"], "cluster_mgr_rel.tgz", mach["basedir"]) + ) + extstr = "bash remote_run.sh --user=%s %s 'cd %s && tar -xzf %s'\n" + comf.write( + extstr + % (mach["user"], ip, mach["basedir"], "percona-8.0.18-bin-rel.tgz") + ) + comf.write( + extstr % (mach["user"], ip, mach["basedir"], "postgresql-11.5-rel.tgz") + ) + comf.write( + extstr % (mach["user"], ip, mach["basedir"], "cluster_mgr_rel.tgz") + ) + + # files + fmap = { + "build_driver.sh": "postgresql-11.5-rel/resources", + "process_deps.sh": ".", + } + for fname in fmap: + comstr = "bash dist.sh --hosts=%s --user=%s install/%s %s/%s\n" + tup = (ip, mach["user"], fname, mach["basedir"], fmap[fname]) + comf.write(comstr % tup) + + comstr = "bash remote_run.sh --user=%s %s 'cd %s/postgresql-11.5-rel || exit 1; test -d etc && echo > etc/instances_list.txt 2>/dev/null; exit 0'\n" + comf.write(comstr % (mach["user"], ip, mach["basedir"])) + comstr = "bash remote_run.sh --user=%s %s 'cd %s/percona-8.0.18-bin-rel || exit 1; test -d etc && echo > etc/instances_list.txt 2>/dev/null; exit 0'\n" + comf.write(comstr % (mach["user"], ip, mach["basedir"])) + + # Set up the env.sh + comstr = "bash dist.sh --hosts=%s --user=%s env.sh.template %s\n" + extstr = """ bash remote_run.sh --user=%s %s "cd %s && sed -s 's#KUNLUN_BASEDIR#%s#g' env.sh.template > env.sh" """ + tup = (ip, mach["user"], mach["basedir"]) + exttup = (mach["user"], ip, mach["basedir"], mach["basedir"]) + comf.write(comstr % tup) + comf.write(extstr % exttup) + comf.write("\n") + + if installtype == "full": + comstr = "bash remote_run.sh --user=%s %s 'cd %s && source ./env.sh && cd percona-8.0.18-bin-rel/lib && bash ../../process_deps.sh'\n" + comf.write(comstr % (mach["user"], ip, mach["basedir"])) + comstr = "bash remote_run.sh --user=%s %s 'cd %s && source ./env.sh && cd postgresql-11.5-rel/lib && bash ../../process_deps.sh'\n" + comf.write(comstr % (mach["user"], ip, mach["basedir"])) # dir making for ip in dirmap: - mach = machines.get(ip) - dirs=dirmap[ip] - for d in dirs: - mkstr = "bash remote_run.sh --user=%s %s 'sudo mkdir -p %s && sudo chown -R %s:`id -gn %s` %s'\n" - tup= (mach['user'], ip, d, mach['user'], mach['user'], d) - comf.write(mkstr % tup) + mach = machines.get(ip) + dirs = dirmap[ip] + for d in dirs: + mkstr = "bash remote_run.sh --user=%s %s 'sudo mkdir -p %s && sudo chown -R %s:`id -gn %s` %s'\n" + tup = (mach["user"], ip, d, mach["user"], mach["user"], d) + comf.write(mkstr % tup) # files copy. for ip in filesmap: - mach = machines.get(ip) - # files - fmap = filesmap[ip] - for fname in fmap: - comstr = "bash dist.sh --hosts=%s --user=%s install/%s %s/%s\n" - tup=(ip, mach['user'], fname, mach['basedir'], fmap[fname]) - comf.write(comstr % tup) + mach = machines.get(ip) + # files + fmap = filesmap[ip] + for fname in fmap: + comstr = "bash dist.sh --hosts=%s --user=%s install/%s %s/%s\n" + tup = (ip, mach["user"], fname, mach["basedir"], fmap[fname]) + comf.write(comstr % tup) # The reason for not using commands map is that, # we need to keep the order for the commands. for cmd in commandslist: - ip=cmd[0] - mach = machines[ip] - mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" - tup= (mach['user'], ip, mach['basedir'], cmd[1], cmd[2]) - comf.write(mkstr % tup) + ip = cmd[0] + mach = machines[ip] + mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" + tup = (mach["user"], ip, mach["basedir"], cmd[1], cmd[2]) + comf.write(mkstr % tup) comf.close() + # The order is meta shard -> data shards -> cluster_mgr -> comp nodes def generate_start_scripts(jscfg): global defuser global defbase - localip = '127.0.0.1' + localip = "127.0.0.1" machines = {} - for mach in jscfg['machines']: - ip=mach['ip'] - user=mach.get('user', defuser) - base=mach.get('basedir', defbase) - addMachineToMap(machines, ip, user, base) + for mach in jscfg["machines"]: + ip = mach["ip"] + user = mach.get("user", defuser) + base = mach.get("basedir", defbase) + addMachineToMap(machines, ip, user, base) filesmap = {} commandslist = [] - - cluster = jscfg['cluster'] - meta = cluster['meta'] + + cluster = jscfg["cluster"] + meta = cluster["meta"] # commands like: # bash startmysql.sh [port] - targetdir='percona-8.0.18-bin-rel/dba_tools' - for node in meta['nodes']: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'sudo bash startmysql.sh %s' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) + targetdir = "percona-8.0.18-bin-rel/dba_tools" + for node in meta["nodes"]: + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"sudo bash startmysql.sh %s" + addToCommandsList(commandslist, node["ip"], targetdir, cmdpat % node["port"]) # bash startmysql.sh [port] - targetdir='percona-8.0.18-bin-rel/dba_tools' - datas = cluster['data'] + targetdir = "percona-8.0.18-bin-rel/dba_tools" + datas = cluster["data"] for shard in datas: - for node in shard['nodes']: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'sudo bash startmysql.sh %s' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) - + for node in shard["nodes"]: + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"sudo bash startmysql.sh %s" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["port"] + ) + # bash -x bin/cluster_mgr_safe --debug --pidfile=run.pid clustermgr.cnf >& run.log & run.log & run.log cluster_mgr -> data shards -> meta shard def generate_stop_scripts(jscfg): global defuser global defbase - localip = '127.0.0.1' + localip = "127.0.0.1" machines = {} - for mach in jscfg['machines']: - ip=mach['ip'] - user=mach.get('user', defuser) - base=mach.get('basedir', defbase) - addMachineToMap(machines, ip, user, base) + for mach in jscfg["machines"]: + ip = mach["ip"] + user = mach.get("user", defuser) + base = mach.get("basedir", defbase) + addMachineToMap(machines, ip, user, base) commandslist = [] - cluster = jscfg['cluster'] + cluster = jscfg["cluster"] # pg_ctl -D %s stop" - comps = cluster['comp']['nodes'] - targetdir="postgresql-11.5-rel/scripts" + comps = cluster["comp"]["nodes"] + targetdir = "postgresql-11.5-rel/scripts" for node in comps: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'pg_ctl -D %s stop -m immediate' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['datadir']) + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"pg_ctl -D %s stop -m immediate" + addToCommandsList(commandslist, node["ip"], targetdir, cmdpat % node["datadir"]) # bash -x bin/cluster_mgr_safe --debug --pidfile=run.pid --stop - targetdir="cluster_mgr_rel" + targetdir = "cluster_mgr_rel" cmdpat = r"bash -x bin/cluster_mgr_safe --debug --pidfile=run.pid --stop" - addToCommandsList(commandslist, cluster['clustermgr']['ip'], targetdir, cmdpat) + addToCommandsList(commandslist, cluster["clustermgr"]["ip"], targetdir, cmdpat) # bash stopmysql.sh [port] - targetdir='percona-8.0.18-bin-rel/dba_tools' - datas = cluster['data'] + targetdir = "percona-8.0.18-bin-rel/dba_tools" + datas = cluster["data"] for shard in datas: - for node in shard['nodes']: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'bash stopmysql.sh %d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) - - meta = cluster['meta'] + for node in shard["nodes"]: + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"bash stopmysql.sh %d" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["port"] + ) + + meta = cluster["meta"] # commands like: # mysqladmin --defaults-file=/kunlun/percona-8.0.18-bin-rel/etc/my_6001.cnf -uroot -proot shutdown - targetdir='percona-8.0.18-bin-rel/dba_tools' - for node in meta['nodes']: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'bash stopmysql.sh %d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) - - com_name = 'commands.sh' - os.system('mkdir -p stop') - comf = open(r'stop/%s' % com_name, 'w') - comf.write('#! /bin/bash\n') + targetdir = "percona-8.0.18-bin-rel/dba_tools" + for node in meta["nodes"]: + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"bash stopmysql.sh %d" + addToCommandsList(commandslist, node["ip"], targetdir, cmdpat % node["port"]) + + com_name = "commands.sh" + os.system("mkdir -p stop") + comf = open(r"stop/%s" % com_name, "w") + comf.write("#! /bin/bash\n") for cmd in commandslist: - ip=cmd[0] - mach = machines[ip] - mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" - tup= (mach['user'], ip, mach['basedir'], cmd[1], cmd[2]) - comf.write(mkstr % tup) + ip = cmd[0] + mach = machines[ip] + mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" + tup = (mach["user"], ip, mach["basedir"], cmd[1], cmd[2]) + comf.write(mkstr % tup) comf.close() + # The order is: comp-nodes -> cluster_mgr -> data shards -> meta shard def generate_clean_scripts(jscfg, cleantype): global defuser global defbase - localip = '127.0.0.1' + localip = "127.0.0.1" machines = {} - for mach in jscfg['machines']: - ip=mach['ip'] - user=mach.get('user', defuser) - base=mach.get('basedir', defbase) - addMachineToMap(machines, ip, user, base) + for mach in jscfg["machines"]: + ip = mach["ip"] + user = mach.get("user", defuser) + base = mach.get("basedir", defbase) + addMachineToMap(machines, ip, user, base) commandslist = [] - cluster = jscfg['cluster'] + cluster = jscfg["cluster"] + # pg_ctl -D %s stop" - comps = cluster['comp']['nodes'] - targetdir="postgresql-11.5-rel/scripts" + comps = cluster["comp"]["nodes"] + targetdir = "postgresql-11.5-rel/scripts" for node in comps: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'pg_ctl -D %s stop -m immediate' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['datadir']) - cmdpat = r'sudo rm -fr %s' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['datadir']) + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"pg_ctl -D %s stop -m immediate" + addToCommandsList(commandslist, node["ip"], targetdir, cmdpat % node["datadir"]) + cmdpat = r"sudo rm -fr %s" + addToCommandsList(commandslist, node["ip"], targetdir, cmdpat % node["datadir"]) # bash -x bin/cluster_mgr_safe --debug --pidfile=run.pid --stop - targetdir="cluster_mgr_rel" + targetdir = "cluster_mgr_rel" cmdpat = r"bash -x bin/cluster_mgr_safe --debug --pidfile=run.pid --stop" - addToCommandsList(commandslist, cluster['clustermgr']['ip'], targetdir, cmdpat) + addToCommandsList(commandslist, cluster["clustermgr"]["ip"], targetdir, cmdpat) # bash stopmysql.sh [port] - targetdir='percona-8.0.18-bin-rel/dba_tools' - datas = cluster['data'] + targetdir = "percona-8.0.18-bin-rel/dba_tools" + datas = cluster["data"] for shard in datas: - for node in shard['nodes']: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'bash stopmysql.sh %d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) - cmdpat = r'sudo rm -fr %s' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['log_dir_path']) - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['data_dir_path']) - if node.has_key('innodb_log_dir_path'): - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['innodb_log_dir_path']) - - meta = cluster['meta'] + for node in shard["nodes"]: + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"bash stopmysql.sh %d" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["port"] + ) + cmdpat = r"sudo rm -fr %s" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["log_dir_path"] + ) + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["data_dir_path"] + ) + if node.__contains__("innodb_log_dir_path"): + addToCommandsList( + commandslist, + node["ip"], + targetdir, + cmdpat % node["innodb_log_dir_path"], + ) + + meta = cluster["meta"] # commands like: # mysqladmin --defaults-file=/kunlun/percona-8.0.18-bin-rel/etc/my_6001.cnf -uroot -proot shutdown - targetdir='percona-8.0.18-bin-rel/dba_tools' - for node in meta['nodes']: - addIpToMachineMap(machines, node['ip']) - cmdpat = r'bash stopmysql.sh %d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) - cmdpat = r'sudo rm -fr %s' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['log_dir_path']) - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['data_dir_path']) - if node.has_key('innodb_log_dir_path'): - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['innodb_log_dir_path']) - - if cleantype == 'full': + targetdir = "percona-8.0.18-bin-rel/dba_tools" + for node in meta["nodes"]: + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"bash stopmysql.sh %d" + addToCommandsList(commandslist, node["ip"], targetdir, cmdpat % node["port"]) + cmdpat = r"sudo rm -fr %s" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["log_dir_path"] + ) + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["data_dir_path"] + ) + if node.__contains__("innodb_log_dir_path"): + addToCommandsList( + commandslist, + node["ip"], + targetdir, + cmdpat % node["innodb_log_dir_path"], + ) + + if cleantype == "full": for ip in machines: - mach =machines[ip] - cmdpat = 'sudo rm -fr %s/*' - addToCommandsList(commandslist, ip, "/", cmdpat % mach['basedir']) + mach = machines[ip] + cmdpat = "sudo rm -fr %s/*" + addToCommandsList(commandslist, ip, "/", cmdpat % mach["basedir"]) + + com_name = "commands.sh" + os.system("mkdir -p clean") + comf = open(r"clean/%s" % com_name, "w") + comf.write("#! /bin/bash\n") + + # Set up the env.sh + comstr = "bash dist.sh --hosts=%s --user=%s env.sh.template %s\n" + extstr = """bash remote_run.sh --user=%s %s "cd %s && sed -s 's#KUNLUN_BASEDIR#%s#g' env.sh.template > env.sh" """ + tup = (ip, mach["user"], mach["basedir"]) + exttup = (mach["user"], ip, mach["basedir"], mach["basedir"]) + comf.write(comstr % tup) + comf.write(extstr % exttup) + comf.write("\n") + + # files copy. + for ip in machines: + mach = machines.get(ip) + mkstr = "bash remote_run.sh --user=%s %s 'sudo mkdir -p %s && sudo chown -R %s:`id -gn %s` %s'\n" + tup = ( + mach["user"], + ip, + mach["basedir"], + mach["user"], + mach["user"], + mach["basedir"], + ) + comf.write(mkstr % tup) + # Set up the files + comstr = "bash dist.sh --hosts=%s --user=%s %s %s\n" + comf.write( + comstr + % (ip, mach["user"], "percona-8.0.18-bin-rel.tgz", mach["basedir"]) + ) + comf.write( + comstr % (ip, mach["user"], "postgresql-11.5-rel.tgz", mach["basedir"]) + ) + comf.write( + comstr % (ip, mach["user"], "cluster_mgr_rel.tgz", mach["basedir"]) + ) + extstr = "bash remote_run.sh --user=%s %s 'cd %s && tar -xzf %s'\n" + comf.write( + extstr + % (mach["user"], ip, mach["basedir"], "percona-8.0.18-bin-rel.tgz") + ) + comf.write( + extstr % (mach["user"], ip, mach["basedir"], "postgresql-11.5-rel.tgz") + ) + comf.write( + extstr % (mach["user"], ip, mach["basedir"], "cluster_mgr_rel.tgz") + ) - com_name = 'commands.sh' - os.system('mkdir -p clean') - comf = open(r'clean/%s' % com_name, 'w') - comf.write('#! /bin/bash\n') for cmd in commandslist: - ip=cmd[0] - mach = machines[ip] - mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" - tup= (mach['user'], ip, mach['basedir'], cmd[1], cmd[2]) - comf.write(mkstr % tup) + ip = cmd[0] + mach = machines[ip] + mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" + tup = (mach["user"], ip, mach["basedir"], cmd[1], cmd[2]) + comf.write(mkstr % tup) comf.close() + def addDefFilesForBackup(filesmap, ip): - targetdir='backup' - files=['backupmysql.sh', 'backuppostgresql.sh', 'check_gtid.py'] + targetdir = "backup" + files = ["backupmysql.sh", "backuppostgresql.sh", "check_gtid.py", "mysqlbinlogchecker"] for f in files: - addIpToFilesMap(filesmap, ip, f, targetdir) + addIpToFilesMap(filesmap, ip, f, targetdir) + def generate_backup_init(jscfg, fulldir, incrdir): global defuser global defbase - localip = '127.0.0.1' + localip = "127.0.0.1" machines = {} - for mach in jscfg['machines']: - ip=mach['ip'] - user=mach.get('user', defuser) - base=mach.get('basedir', defbase) - addMachineToMap(machines, ip, user, base) + for mach in jscfg["machines"]: + ip = mach["ip"] + user = mach.get("user", defuser) + base = mach.get("basedir", defbase) + addMachineToMap(machines, ip, user, base) filesmap = {} commandslist = [] dirmap = {} - - cluster = jscfg['cluster'] - meta = cluster['meta'] - - com_name = 'commands.sh' - comf = open(r'backup/%s' % com_name, 'w') - comf.write('#! /bin/bash\n') - - metafulldir="%s/meta" % fulldir - metaincrdir="%s/meta" % incrdir - comf.write('mkdir -p %s/meta\n' % metafulldir) - comf.write('mkdir -p %s/meta\n' % metaincrdir) - for node in meta['nodes']: - # All the log/dir should exists and has data, so no need to create. - addIpToMachineMap(machines, node['ip']) - addToDirMap(dirmap, node['ip'], node['backupdir']) - addDefFilesForBackup(filesmap, node['ip']) - - datas = cluster['data'] - i=1 + + cluster = jscfg["cluster"] + meta = cluster["meta"] + + com_name = "commands.sh" + comf = open(r"backup/%s" % com_name, "w") + comf.write("#! /bin/bash\n") + + metafulldir = "%s/meta" % fulldir + metaincrdir = "%s/meta" % incrdir + comf.write("mkdir -p %s/meta\n" % metafulldir) + comf.write("mkdir -p %s/meta\n" % metaincrdir) + for node in meta["nodes"]: + # All the log/dir should exists and has data, so no need to create. + addIpToMachineMap(machines, node["ip"]) + addToDirMap(dirmap, node["ip"], node["backupdir"]) + addDefFilesForBackup(filesmap, node["ip"]) + + datas = cluster["data"] + i = 1 for shard in datas: - shardname="shard%d" % i - shardfulldir="%s/%s" % (fulldir, shardname) - shardincrdir="%s/%s" % (incrdir, shardname) - comf.write('mkdir -p %s\n' % shardfulldir) - comf.write('mkdir -p %s\n' % shardincrdir) - for node in shard['nodes']: - # All the log/dir should exists and has data, so no need to create. - addIpToMachineMap(machines, node['ip']) - addToDirMap(dirmap, node['ip'], node['backupdir']) - addDefFilesForBackup(filesmap, node['ip']) - i+=1 - + shardname = "shard%d" % i + shardfulldir = "%s/%s" % (fulldir, shardname) + shardincrdir = "%s/%s" % (incrdir, shardname) + comf.write("mkdir -p %s\n" % shardfulldir) + comf.write("mkdir -p %s\n" % shardincrdir) + for node in shard["nodes"]: + # All the log/dir should exists and has data, so no need to create. + addIpToMachineMap(machines, node["ip"]) + addToDirMap(dirmap, node["ip"], node["backupdir"]) + addDefFilesForBackup(filesmap, node["ip"]) + i += 1 + # For comp nodes, we always do full backup using pg_dump - comps = cluster['comp']['nodes'] - compfulldir="%s/comp" % fulldir - comf.write('mkdir -p %s\n' % compfulldir) + comps = cluster["comp"]["nodes"] + compfulldir = "%s/comp" % fulldir + comf.write("mkdir -p %s\n" % compfulldir) for node in comps: - addIpToMachineMap(machines, node['ip']) - addToDirMap(dirmap, node['ip'], node['backupdir']) - addDefFilesForBackup(filesmap, node['ip']) + addIpToMachineMap(machines, node["ip"]) + addToDirMap(dirmap, node["ip"], node["backupdir"]) + addDefFilesForBackup(filesmap, node["ip"]) # dir making for ip in dirmap: - mach = machines.get(ip) - dirs=dirmap[ip] - backupdir = mach['basedir']+"/backup" - dirs.append(backupdir) - for d in dirs: - mkstr = "bash remote_run.sh --user=%s %s 'sudo mkdir -p %s && sudo chown -R %s:`id -gn %s` %s'\n" - tup= (mach['user'], ip, d, mach['user'], mach['user'], d) - comf.write(mkstr % tup) + mach = machines.get(ip) + dirs = dirmap[ip] + backupdir = mach["basedir"] + "/backup" + dirs.append(backupdir) + for d in dirs: + mkstr = "bash remote_run.sh --user=%s %s 'sudo mkdir -p %s && sudo chown -R %s:`id -gn %s` %s'\n" + tup = (mach["user"], ip, d, mach["user"], mach["user"], d) + comf.write(mkstr % tup) # copy files to remote machines. for ip in filesmap: - mach = machines.get(ip) - fmap = filesmap[ip] - for fname in fmap: - comstr = "bash dist.sh --hosts=%s --user=%s backup/%s %s/%s\n" - tup=(ip, mach['user'], fname, mach['basedir'], fmap[fname]) - comf.write(comstr % tup) + mach = machines.get(ip) + fmap = filesmap[ip] + for fname in fmap: + comstr = "bash dist.sh --hosts=%s --user=%s backup/%s %s/%s\n" + tup = (ip, mach["user"], fname, mach["basedir"], fmap[fname]) + comf.write(comstr % tup) comf.close() + def generate_backup_scripts(jscfg, backuptype, fulldir, incrdir): global defuser global defbase - localip = '127.0.0.1' + localip = "127.0.0.1" - curtime = str(long(time.time())) + curtime = str(int(time.time())) machines = {} - for mach in jscfg['machines']: - ip=mach['ip'] - user=mach.get('user', defuser) - base=mach.get('basedir', defbase) - addMachineToMap(machines, ip, user, base) + for mach in jscfg["machines"]: + ip = mach["ip"] + user = mach.get("user", defuser) + base = mach.get("basedir", defbase) + addMachineToMap(machines, ip, user, base) filesmap = {} commandslist = [] - mysqlbackups=[] - - cluster = jscfg['cluster'] - meta = cluster['meta'] + mysqlbackups = [] - com_name = 'commands.sh' - comf = open(r'backup/%s' % com_name, 'w') - comf.write('#! /bin/bash\n') + cluster = jscfg["cluster"] + meta = cluster["meta"] - metafulldir="%s/meta" % fulldir - metaincrdir="%s/meta" % incrdir + com_name = "commands.sh" + comf = open(r"backup/%s" % com_name, "w") + comf.write("#! /bin/bash\n") + + metafulldir = "%s/meta" % fulldir + metaincrdir = "%s/meta" % incrdir metagtid = "0" - gtidfile ="%s/gtid" % metaincrdir; - if backuptype == 'incremental' and os.access(gtidfile, os.R_OK): - with open(gtidfile, 'r') as f: - metagtid=f.read() - if metagtid == '': - metagtid='0' - isfull=False - if metagtid == '0': - isfull=True - targetdir=metafulldir + gtidfile = "%s/gtid" % metaincrdir + if backuptype == "incremental" and os.access(gtidfile, os.R_OK): + with open(gtidfile, "r") as f: + metagtid = f.read() + if metagtid == "": + metagtid = "0" + isfull = False + if metagtid == "0": + isfull = True + targetdir = "%s/%s" % (metafulldir,curtime) if not isfull: - targetdir="%s/%s" % (metaincrdir, curtime) - comf.write("mkdir -p %s\n" % targetdir) - mysqlbackups.append([targetdir, gtidfile]) - for node in meta['nodes']: - # All the log/dir should exists and has data, so no need to create. - addIpToMachineMap(machines, node['ip']) - mach = machines.get(node['ip']) - backupdir = mach['basedir']+"/backup" - # data_dir_path and user is not used currently. - cmdpat = r'bash backupmysql.sh %d %s %s %s' - addToCommandsList(commandslist, node['ip'], backupdir, cmdpat % (node['port'], - node['log_dir_path'], node['backupdir'], metagtid)) - fname = "%s/backup.tar.gz" % node['backupdir'] - addIpToFilesMap(filesmap, node['ip'], fname, targetdir) + targetdir = "%s/%s" % (metaincrdir, curtime) + comf.write("mkdir -p %s\n" % targetdir) + mysqlbackups.append([targetdir, gtidfile, metaincrdir]) + for node in meta["nodes"]: + # All the log/dir should exists and has data, so no need to create. + addIpToMachineMap(machines, node["ip"]) + mach = machines.get(node["ip"]) + backupdir = mach["basedir"] + "/backup" + # data_dir_path and user is not used currently. + cmdpat = r"bash backupmysql.sh %d %s %s %s %s" + addToCommandsList( + commandslist, + node["ip"], + backupdir, + cmdpat % (node["port"], node["log_dir_path"], node["data_dir_path"], node["backupdir"], metagtid), + ) + fname = "%s/backup.tar.gz" % node["backupdir"] + addIpToFilesMap(filesmap, node["ip"], fname, targetdir) i = 1 - datas = cluster['data'] + datas = cluster["data"] for shard in datas: - shardname="shard%d" % i - shardfulldir="%s/%s" % (fulldir, shardname) - shardincrdir="%s/%s" % (incrdir, shardname) - gtid="0" - gtidfile ="%s/gtid" % shardincrdir; - if backuptype == 'incremental' and os.access(gtidfile, os.R_OK): - with open(gtidfile, 'r') as f: - gtid=f.read() - if gtid == "": - gtid="0" - isfull = False - if gtid == '0': - isfull=True - targetdir=shardfulldir - if not isfull: - targetdir="%s/%s" % (shardincrdir, curtime) - comf.write("mkdir -p %s\n" % targetdir) - mysqlbackups.append([targetdir, gtidfile]) - for node in shard['nodes']: - # All the log/dir should exists and has data, so no need to create. - addIpToMachineMap(machines, node['ip']) - mach = machines.get(node['ip']) - backupdir = mach['basedir']+"/backup" - cmdpat = r'bash backupmysql.sh %d %s %s %s' - addToCommandsList(commandslist, node['ip'], backupdir, cmdpat % (node['port'], - node['log_dir_path'], node['backupdir'], gtid)) - fname = "%s/backup.tar.gz" % node['backupdir'] - addIpToFilesMap(filesmap, node['ip'], fname, targetdir) - i+=1 - - compfulldir="%s/comp" % fulldir - targetdir="%s/%s" % (compfulldir, curtime) + shardname = "shard%d" % i + shardfulldir = "%s/%s" % (fulldir, shardname) + shardincrdir = "%s/%s" % (incrdir, shardname) + gtid = "0" + gtidfile = "%s/gtid" % shardincrdir + if backuptype == "incremental" and os.access(gtidfile, os.R_OK): + + with open(gtidfile, "r") as f: + gtid = f.read() + if gtid == "": + gtid = "0" + isfull = False + if gtid == "0": + isfull = True + targetdir = "%s/%s" % (shardfulldir,curtime) + if not isfull: + targetdir = "%s/%s" % (shardincrdir, curtime) + comf.write("mkdir -p %s\n" % targetdir) + mysqlbackups.append([targetdir, gtidfile, shardincrdir]) + for node in shard["nodes"]: + # All the log/dir should exists and has data, so no need to create. + addIpToMachineMap(machines, node["ip"]) + mach = machines.get(node["ip"]) + backupdir = mach["basedir"] + "/backup" + cmdpat = r"bash backupmysql.sh %d %s %s %s %s" + addToCommandsList( + commandslist, + node["ip"], + backupdir, + cmdpat % (node["port"], node["log_dir_path"], node["data_dir_path"], node["backupdir"], gtid), + ) + fname = "%s/backup.tar.gz" % node["backupdir"] + addIpToFilesMap(filesmap, node["ip"], fname, targetdir) + i += 1 + + compfulldir = "%s/comp" % fulldir + targetdir = "%s/%s" % (compfulldir, curtime) comf.write("mkdir -p %s\n" % targetdir) - comps = cluster['comp']['nodes'] + comps = cluster["comp"]["nodes"] for node in comps: - addIpToMachineMap(machines, node['ip']) - mach = machines.get(node['ip']) - backupdir = mach['basedir']+"/backup" - cmdpat = r'bash backuppostgresql.sh %d %s' - addToCommandsList(commandslist, node['ip'], backupdir, cmdpat % (node['port'], node['backupdir'])) - fname = "%s/backup.tar.gz" % node['backupdir'] - addIpToFilesMap(filesmap, node['ip'], fname, targetdir) + addIpToMachineMap(machines, node["ip"]) + mach = machines.get(node["ip"]) + backupdir = mach["basedir"] + "/backup" + cmdpat = r"bash backuppostgresql.sh %d %s" + addToCommandsList( + commandslist, + node["ip"], + backupdir, + cmdpat % (node["port"], node["backupdir"]), + ) + fname = "%s/backup.tar.gz" % node["backupdir"] + addIpToFilesMap(filesmap, node["ip"], fname, targetdir) for cmd in commandslist: - ip=cmd[0] - mach = machines[ip] - mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" - tup= (mach['user'], ip, mach['basedir'], cmd[1], cmd[2]) - comf.write(mkstr % tup) + ip = cmd[0] + mach = machines[ip] + mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" + tup = (mach["user"], ip, mach["basedir"], cmd[1], cmd[2]) + comf.write(mkstr % tup) # copy files back. for ip in filesmap: - mach = machines.get(ip) - fmap = filesmap[ip] - for fname in fmap: - name = os.path.basename(fname) - comstr = "test -f %s/%s || bash getback.sh --hosts=%s --user=%s %s %s\n" - tup=(fmap[fname], name, ip, mach['user'], fname, fmap[fname]) - comf.write(comstr % tup) + mach = machines.get(ip) + fmap = filesmap[ip] + for fname in fmap: + name = os.path.basename(fname) + comstr = "test -f %s/%s || bash getback.sh --hosts=%s --user=%s %s %s\n" + tup = (fmap[fname], name, ip, mach["user"], fname, fmap[fname]) + comf.write(comstr % tup) for ba in mysqlbackups: - tdir=ba[0] - gtidpath=ba[1] - comf.write("cd %s && tar -xzf backup.tar.gz && cd - && cp -fr %s/gtid %s\n" % (tdir, tdir, gtidpath)) + tdir = ba[0] + gtidpath = ba[1] + incrdir = ba[2] + if backuptype == "incremental": + comf.write( + "cd %s && tar -xzf backup.tar.gz && cd - && cp -fr %s/gtid %s && cp -fr %s/backup.tar.gz %s\n" + % (tdir, tdir, gtidpath, tdir, incrdir)) + else: + comf.write( + "cd %s && tar -xzf backup.tar.gz && cd - && cp -fr %s/gtid %s && cp -fr %s/backup.tar.gz %s\n" + % (tdir, tdir, gtidpath, tdir, fulldir)) + + comf.close() + def generate_restore_scripts(jscfg): global defuser global defbase - localip = '127.0.0.1' + localip = "127.0.0.1" + stop_time = jscfg["restore_time"] machines = {} - for mach in jscfg['machines']: - ip=mach['ip'] - user=mach.get('user', defuser) - base=mach.get('basedir', defbase) - addMachineToMap(machines, ip, user, base) + for mach in jscfg["machines"]: + ip = mach["ip"] + user = mach.get("user", defuser) + base = mach.get("basedir", defbase) + addMachineToMap(machines, ip, user, base) filesmap = {} commandslist = [] dirmap = {} - usemgr=True - - cluster = jscfg['cluster'] - cluster_name = cluster['name'] - meta = cluster['meta'] - metafulldir = meta['fullbackupdir'] - metaincrdir = meta['incrbackupdir'] - del meta['fullbackupdir'] - del meta['incrbackupdir'] + usemgr = True + + cluster = jscfg["cluster"] + cluster_name = cluster["name"] + meta = cluster["meta"] + metafulldir = meta["fullbackupdir"] + metaincrdir = meta["incrbackupdir"] + del meta["fullbackupdir"] + del meta["incrbackupdir"] metauuid = getuuid_from_cnf("%s/my.cnf" % metafulldir) - meta['group_uuid'] = metauuid; - my_metaname = 'mysql_meta.json' - metaf = open(r'restore/%s' % my_metaname,'w') + meta["group_uuid"] = metauuid + my_metaname = "mysql_meta.json" + metaf = open(r"restore/%s" % my_metaname, "w") json.dump(meta, metaf, indent=4) metaf.close() # commands like: # sudo --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH python2 \ # install-mysql.py dbcfg=./template.cnf mgr_config=./mysql_meta.json target_node_index=0 - targetdir='percona-8.0.18-bin-rel/dba_tools' - i=0 - pricmdlist=[] - secmdlist=[] - cleanlist=[] - for node in meta['nodes']: - name = 'meta'; - addNodeToFilesMap(filesmap, node, "restore/%s" % my_metaname, targetdir) - addIpToMachineMap(machines, node['ip']) - addToDirMap(dirmap, node['ip'], node['data_dir_path']) - addToDirMap(dirmap, node['ip'], node['log_dir_path']) - mach = machines.get(node['ip']) - backuptargetd = "restore/meta" - addToDirMap(dirmap, node['ip'], "%s/%s" % (mach['basedir'], backuptargetd)) - addNodeToFilesMap(filesmap, node, "%s/backup.tar.gz" % metafulldir, backuptargetd) - cmdpat = r'tar -xzf backup.tar.gz' - addToCommandsList(commandslist, node['ip'], backuptargetd, cmdpat) - cmdpat = r'xtrabackup --prepare --target-dir=base > prepare.out' - addToCommandsList(commandslist, node['ip'], backuptargetd, cmdpat) - cmdpat = r'python2 restore-mysql.py dbcfg=./template.cnf config=%s target_node_index=%d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (my_metaname, i)) - cnfpath='%s/percona-8.0.18-bin-rel/etc/my_%d.cnf' % (mach['basedir'], node['port']) - cmdpat = r'xtrabackup --defaults-file=%s --copy-back --target-dir=base > copyback.out' % cnfpath - addToCommandsList(commandslist, node['ip'], backuptargetd, cmdpat) - cmdpat = r'sudo bash startmysql.sh %d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) - cmdpat = r'bash wait_mysqlup.sh %d' - addToCommandsList(commandslist, node['ip'], ".", cmdpat % node['port']) - cmdpat = r'bash start_mgr.sh %d %s' - if node.get('is_primary', False): - addToCommandsList(pricmdlist, node['ip'], ".", cmdpat % (node['port'], 'true')) - cleanpat = "mysql --defaults-file=%s -uroot -proot -e 'use Kunlun_Metadata_DB; drop table if exists commit_log_%s;'" - addToCommandsList(cleanlist, node['ip'], ".", cleanpat % (cnfpath, cluster_name)) - else: - addToCommandsList(secmdlist, node['ip'], ".", cmdpat % (node['port'], 'false')) - i+=1 - - targetdir='percona-8.0.18-bin-rel/dba_tools' - datas = cluster['data'] - i=1 + targetdir = "percona-8.0.18-bin-rel/dba_tools" + i = 0 + pricmdlist = [] + secmdlist = [] + cleanlist = [] + for node in meta["nodes"]: + name = "meta" + addNodeToFilesMap(filesmap, node, "restore/%s" % my_metaname, targetdir) + addIpToMachineMap(machines, node["ip"]) + addToDirMap(dirmap, node["ip"], node["data_dir_path"]) + addToDirMap(dirmap, node["ip"], node["log_dir_path"]) + mach = machines.get(node["ip"]) + backuptargetd = "restore/meta" + backuptargetd_binlog = "restore/meta/binlog" + addToDirMap(dirmap, node["ip"], "%s/%s" % (mach["basedir"], backuptargetd)) + addToDirMap(dirmap, node["ip"], "%s/%s" % (mach["basedir"], backuptargetd_binlog)) + addNodeToFilesMap( + filesmap, node, "%s/backup.tar.gz" % metafulldir, backuptargetd + ) + addNodeToFilesMap( + filesmap, node, "%s/backup.tar.gz" % metaincrdir, backuptargetd_binlog + ) + cmdpat = r"tar -xzf backup.tar.gz;mv base/backup-my.cnf base/.backup-my.cnf;" + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat) + cmdpat = r"cp -fr base/xtrabackup_binlog_info binlog/gtid" + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat ) + cmdpat = r"xtrabackup --prepare --target-dir=base > prepare.out" + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat) + cmdpat = r"python2 restore-mysql.py dbcfg=./template.cnf config=%s target_node_index=%d" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % (my_metaname, i) + ) + cnfpath = "%s/%d/my_%d.cnf" % ( + node["data_dir_path"], + node["port"], + node["port"], + ) + cmdpat = ( + r"xtrabackup --defaults-file=%s --copy-back --target-dir=base > copyback.out" + % cnfpath + ) + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat) + cmdpat = r"sudo bash startmysql.sh %d" + addToCommandsList(commandslist, node["ip"], targetdir, cmdpat % node["port"]) + cmdpat = r"bash wait_mysqlup.sh %d %s" + addToCommandsList(commandslist, node["ip"], ".", cmdpat % (node["port"],cnfpath)) + + # Process the incremental binlog backup and do the fast_apply_binlog + if len(stop_time) != 0: + cmdpat = r"python3 ./apply_binlog_fast.py binlogBackupPath=%s etcfile=%s gtidinfo=%s/gtid stoptime=\'%s\'"\ + % (backuptargetd_binlog,cnfpath,backuptargetd_binlog,stop_time) + else: + cmdpat = r"python3 ./apply_binlog_fast.py binlogBackupPath=%s etcfile=%s gtidinfo=%s/gtid"\ + % (backuptargetd_binlog,cnfpath,backuptargetd_binlog) + + addToCommandsList(commandslist, node["ip"], ".", cmdpat) + + cmdpat = r"bash start_mgr.sh %d %s %s" + if node.get("is_primary", False): + addToCommandsList( + pricmdlist, node["ip"], ".", cmdpat % (node["port"], "true", cnfpath) + ) + cleanpat = "mysql --defaults-file=%s -uroot -proot -e \\'use Kunlun_Metadata_DB; drop table if exists commit_log_%s;\\'" + addToCommandsList( + cleanlist, node["ip"], ".", cleanpat % (cnfpath, cluster_name) + ) + else: + addToCommandsList( + secmdlist, node["ip"], ".", cmdpat % (node["port"], "false", cnfpath) + ) + i += 1 + + targetdir = "percona-8.0.18-bin-rel/dba_tools" + datas = cluster["data"] + i = 1 for shard in datas: - fulldir = shard['fullbackupdir'] - incrdir = shard['incrbackupdir'] - del shard['fullbackupdir'] - del shard['incrbackupdir'] - shard['group_uuid'] = getuuid_from_cnf("%s/my.cnf" % fulldir) - my_shardname = "mysql_shard%d.json" % i - shardf = open(r'restore/%s' % my_shardname, 'w') - json.dump(shard, shardf, indent=4) - shardf.close() - shardname="shard%d" % i - j = 0 - for node in shard['nodes']: - addNodeToFilesMap(filesmap, node, "restore/%s" % my_shardname, targetdir) - addIpToMachineMap(machines, node['ip']) - addToDirMap(dirmap, node['ip'], node['data_dir_path']) - addToDirMap(dirmap, node['ip'], node['log_dir_path']) - mach = machines.get(node['ip']) - backuptargetd = "restore/%s" % shardname - addToDirMap(dirmap, node['ip'], "%s/%s" % (mach['basedir'], backuptargetd)) - addNodeToFilesMap(filesmap, node, "%s/backup.tar.gz" % fulldir, backuptargetd) - cmdpat = r'tar -xzf backup.tar.gz' - addToCommandsList(commandslist, node['ip'], backuptargetd, cmdpat) - cmdpat = r'xtrabackup --prepare --target-dir=base > prepare.out' - addToCommandsList(commandslist, node['ip'], backuptargetd, cmdpat) - cmdpat = r'python2 restore-mysql.py dbcfg=./template.cnf config=%s target_node_index=%d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (my_shardname, j)) - cnfpath='%s/percona-8.0.18-bin-rel/etc/my_%d.cnf' % (mach['basedir'], node['port']) - cmdpat = r'xtrabackup --defaults-file=%s --copy-back --target-dir=base > copyback.out' % cnfpath - addToCommandsList(commandslist, node['ip'], backuptargetd, cmdpat) - cmdpat = r'sudo bash startmysql.sh %d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port']) - cmdpat = r'bash wait_mysqlup.sh %d' - addToCommandsList(commandslist, node['ip'], ".", cmdpat % node['port']) - cmdpat = r'bash start_mgr.sh %d %s' - if node.get('is_primary', False): - addToCommandsList(pricmdlist, node['ip'], ".", cmdpat % (node['port'], 'true')) - else: - addToCommandsList(secmdlist, node['ip'], ".", cmdpat % (node['port'], 'false')) - j += 1 - if j == 1: - usemgr=False - i+=1 + fulldir = shard["fullbackupdir"] + incrdir = shard["incrbackupdir"] + del shard["fullbackupdir"] + del shard["incrbackupdir"] + shard["group_uuid"] = getuuid_from_cnf("%s/my.cnf" % fulldir) + my_shardname = "mysql_shard%d.json" % i + shardf = open(r"restore/%s" % my_shardname, "w") + json.dump(shard, shardf, indent=4) + shardf.close() + shardname = "shard%d" % i + j = 0 + for node in shard["nodes"]: + addNodeToFilesMap(filesmap, node, "restore/%s" % my_shardname, targetdir) + addIpToMachineMap(machines, node["ip"]) + addToDirMap(dirmap, node["ip"], node["data_dir_path"]) + addToDirMap(dirmap, node["ip"], node["log_dir_path"]) + mach = machines.get(node["ip"]) + backuptargetd = "restore/%s" % shardname + backuptargetd_binlog = "restore/%s/binlog" % shardname + addToDirMap(dirmap, node["ip"], "%s/%s" % (mach["basedir"], backuptargetd)) + addToDirMap(dirmap, node["ip"], "%s/%s" % (mach["basedir"], backuptargetd_binlog)) + addNodeToFilesMap( + filesmap, node, "%s/backup.tar.gz" % fulldir, backuptargetd + ) + addNodeToFilesMap( + filesmap, node, "%s/backup.tar.gz" % incrdir, backuptargetd_binlog + ) + cmdpat = r"tar -xzf backup.tar.gz;mv base/backup-my.cnf base/.backup-my.cnf;" + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat) + cmdpat = r"cp -fr base/xtrabackup_binlog_info binlog/gtid" + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat) + cmdpat = r"xtrabackup --prepare --target-dir=base > prepare.out" + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat) + cmdpat = r"python2 restore-mysql.py dbcfg=./template.cnf config=%s target_node_index=%d usemgr=False" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % (my_shardname, j) + ) + cnfpath = "%s/%d/my_%d.cnf" % ( + node["data_dir_path"], + node["port"], + node["port"], + ) + cmdpat = ( + r"xtrabackup --defaults-file=%s --copy-back --target-dir=base > copyback.out" + % cnfpath + ) + addToCommandsList(commandslist, node["ip"], backuptargetd, cmdpat) + cmdpat = r"sudo bash startmysql.sh %d" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % node["port"] + ) + cmdpat = r"bash wait_mysqlup.sh %d %s" + addToCommandsList(commandslist, node["ip"], ".", cmdpat % (node["port"],cnfpath)) + + # Process the incremental binlog backup and do the fast_apply_binlog + if len(stop_time) !=0: + cmdpat = r"python3 ./apply_binlog_fast.py binlogBackupPath=%s etcfile=%s gtidinfo=%s/gtid stoptime=\'%s\'"\ + % (backuptargetd_binlog,cnfpath,backuptargetd_binlog,stop_time) + else: + cmdpat = r"python3 ./apply_binlog_fast.py binlogBackupPath=%s etcfile=%s gtidinfo=%s/gtid"\ + % (backuptargetd_binlog,cnfpath,backuptargetd_binlog) + + addToCommandsList(commandslist, node["ip"], ".", cmdpat) + + if False: + cmdpat = r"bash start_mgr.sh %d %s %s" + if node.get("is_primary", False): + addToCommandsList( + pricmdlist, node["ip"], ".", cmdpat % (node["port"], "true",cnfpath) + ) + else: + addToCommandsList( + secmdlist, node["ip"], ".", cmdpat % (node["port"], "false",cnfpath) + ) + j += 1 + if j == 1: + usemgr = False + i += 1 commandslist.extend(pricmdlist) commandslist.extend(secmdlist) commandslist.extend(cleanlist) # This only needs to transfered to machine creating the cluster. - pg_metaname = 'postgres_meta.json' - metaf = open(r'restore/%s' % pg_metaname, 'w') + pg_metaname = "postgres_meta.json" + metaf = open(r"restore/%s" % pg_metaname, "w") objs = [] - for node in meta['nodes']: - obj = {} - obj['ip'] = node['ip'] - obj['port'] = node['port'] - obj['user'] = "pgx" - obj['password'] = "pgx_pwd" - objs.append(obj) + for node in meta["nodes"]: + obj = {} + obj["ip"] = node["ip"] + obj["port"] = node["port"] + obj["user"] = "pgx" + obj["password"] = "pgx_pwd" + objs.append(obj) json.dump(objs, metaf, indent=4) metaf.close() # This only needs to transfered to machine creating the cluster. - pg_shardname = 'postgres_shards.json' - shardf = open(r'restore/%s' % pg_shardname, 'w') + pg_shardname = "postgres_shards.json" + shardf = open(r"restore/%s" % pg_shardname, "w") shards = [] - i=1 + i = 1 for shard in datas: - obj={'shard_name': "shard%d" % i} - i+=1 - nodes=[] - for node in shard['nodes']: - n={'user':'pgx', 'password':'pgx_pwd'} - n['ip'] = node['ip'] - n['port'] = node['port'] - nodes.append(n) - obj['shard_nodes'] = nodes - shards.append(obj) + obj = {"shard_name": "shard%d" % i} + i += 1 + nodes = [] + for node in shard["nodes"]: + n = {"user": "pgx", "password": "pgx_pwd"} + n["ip"] = node["ip"] + n["port"] = node["port"] + nodes.append(n) + obj["shard_nodes"] = nodes + shards.append(obj) json.dump(shards, shardf, indent=4) shardf.close() - comps = cluster['comp']['nodes'] - pg_compname = 'postgres_comp.json' - compf = open(r'restore/%s' % pg_compname, 'w') + comps = cluster["comp"]["nodes"] + pg_compname = "postgres_comp.json" + compf = open(r"restore/%s" % pg_compname, "w") json.dump(comps, compf, indent=4) compf.close() # python2 install_pg.py config=docker-comp.json install_ids=1,2,3 - compfulldir=cluster['comp']['fullbackupdir'] - targetdir="postgresql-11.5-rel/scripts" + compfulldir = cluster["comp"]["fullbackupdir"] + targetdir = "postgresql-11.5-rel/scripts" for node in comps: - addNodeToFilesMap(filesmap, node, "restore/%s" % pg_compname, targetdir) - addIpToMachineMap(machines, node['ip']) - cmdpat = r'python2 install_pg.py config=./%s install_ids=%d' - addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (pg_compname, node['id'])) - addToDirMap(dirmap, node['ip'], node['datadir']) - mach = machines.get(node['ip']) - addToDirMap(dirmap, node['ip'], "%s/restore/comp" % mach['basedir']) + addNodeToFilesMap(filesmap, node, "restore/%s" % pg_compname, targetdir) + addIpToMachineMap(machines, node["ip"]) + cmdpat = r"python2 install_pg.py config=./%s install_ids=%d" + if usemgr is False: + cmdpat = cmdpat + " usemgr=False" + addToCommandsList( + commandslist, node["ip"], targetdir, cmdpat % (pg_compname, node["id"]) + ) + addToDirMap(dirmap, node["ip"], node["datadir"]) + mach = machines.get(node["ip"]) + addToDirMap(dirmap, node["ip"], "%s/restore/comp" % mach["basedir"]) comp1 = comps[0] addNodeToFilesMap(filesmap, comp1, "restore/%s" % pg_metaname, targetdir) addNodeToFilesMap(filesmap, comp1, "restore/%s" % pg_shardname, targetdir) resourcedir = "postgresql-11.5-rel/resources" - cmdpat=r'/bin/bash build_driver.sh' - addToCommandsList(commandslist, comp1['ip'], resourcedir, cmdpat) - cmdpat=r'python2 bootstrap.py --config=./%s --bootstrap_sql=./clean_meta.sql' - addToCommandsList(commandslist, comp1['ip'], targetdir, cmdpat % pg_metaname) - cmdpat='python2 create_cluster.py --shards_config ./%s \ ---comps_config ./%s --meta_config ./%s --cluster_name %s --cluster_owner abc --cluster_biz test' - addToCommandsList(commandslist, comp1['ip'], targetdir, - cmdpat % (pg_shardname, pg_compname, pg_metaname, cluster_name)) + cmdpat = r"/bin/bash build_driver.sh" + addToCommandsList(commandslist, comp1["ip"], resourcedir, cmdpat) + cmdpat = r"python2 bootstrap.py --config=./%s --bootstrap_sql=./clean_meta.sql" + addToCommandsList(commandslist, comp1["ip"], targetdir, cmdpat % pg_metaname) + cmdpat = "python2 create_cluster.py --shards_config ./%s \ + --comps_config ./%s --meta_config ./%s --cluster_name %s --cluster_owner abc --cluster_biz test" + if usemgr is False: + cmdpat = cmdpat + " --usemgr=False" + addToCommandsList( + commandslist, + comp1["ip"], + targetdir, + cmdpat % (pg_shardname, pg_compname, pg_metaname, cluster_name), + ) # bash -x bin/cluster_mgr_safe --debug --pidfile=run.pid clustermgr.cnf >& run.log & run.log & run.log env.sh" ''' - tup=(ip, mach['user'], mach['basedir']) - exttup=(mach['user'], ip, mach['basedir'], mach['basedir']) - comf.write(comstr % tup) - comf.write(extstr % exttup) - comf.write("\n") + mach = machines.get(ip) + + # Set up the files + comstr = "bash dist.sh --hosts=%s --user=%s %s %s\n" + comf.write( + comstr % (ip, mach["user"], "percona-8.0.18-bin-rel.tgz", mach["basedir"]) + ) + comf.write( + comstr % (ip, mach["user"], "postgresql-11.5-rel.tgz", mach["basedir"]) + ) + comf.write(comstr % (ip, mach["user"], "cluster_mgr_rel.tgz", mach["basedir"])) + extstr = "bash remote_run.sh --user=%s %s 'cd %s && tar -xzf %s'\n" + comf.write( + extstr % (mach["user"], ip, mach["basedir"], "percona-8.0.18-bin-rel.tgz") + ) + comf.write( + extstr % (mach["user"], ip, mach["basedir"], "postgresql-11.5-rel.tgz") + ) + comf.write(extstr % (mach["user"], ip, mach["basedir"], "cluster_mgr_rel.tgz")) + + # files + fmap = filesmap[ip] + fmap["restore/restore-mysql.py"] = "percona-8.0.18-bin-rel/dba_tools" + fmap["restore/build_driver.sh"] = "postgresql-11.5-rel/resources" + fmap["restore/clean_meta.sql"] = "postgresql-11.5-rel/scripts" + fmap["restore/merge_incr.py"] = "." + fmap["restore/wait_mysqlup.sh"] = "." + fmap["restore/wait_pgup.sh"] = "." + fmap["restore/start_mgr.sh"] = "." + fmap["restore/mysqlbinlog"] = "." + fmap["restore/apply_binlog_fast.py"] = "." + for fname in fmap: + comstr = "bash dist.sh --hosts=%s --user=%s %s %s/%s\n" + tup = (ip, mach["user"], fname, mach["basedir"], fmap[fname]) + comf.write(comstr % tup) + + comstr = "bash remote_run.sh --user=%s %s 'cd %s/postgresql-11.5-rel || exit 1; test -d etc && echo > etc/instances_list.txt 2>/dev/null'\n" + comf.write(comstr % (mach["user"], ip, mach["basedir"])) + comstr = "bash remote_run.sh --user=%s %s 'cd %s/percona-8.0.18-bin-rel || exit 1; test -d etc && echo > etc/instances_list.txt 2>/dev/null'\n" + comf.write(comstr % (mach["user"], ip, mach["basedir"])) + + # Set up the env.sh + comstr = "bash dist.sh --hosts=%s --user=%s env.sh.template %s\n" + extstr = """ bash remote_run.sh --user=%s %s "cd %s && sed -s 's#KUNLUN_BASEDIR#%s#g' env.sh.template > env.sh" """ + tup = (ip, mach["user"], mach["basedir"]) + exttup = (mach["user"], ip, mach["basedir"], mach["basedir"]) + comf.write(comstr % tup) + comf.write(extstr % exttup) + comf.write("\n") # The reason for not using commands map is that, # we need to keep the order for the commands. for cmd in commandslist: - ip=cmd[0] - mach = machines[ip] - mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" - tup= (mach['user'], ip, mach['basedir'], cmd[1], cmd[2]) - comf.write(mkstr % tup) + ip = cmd[0] + mach = machines[ip] + mkstr = "bash remote_run.sh --user=%s %s $'cd %s && source ./env.sh && cd %s || exit 1; %s'\n" + tup = (mach["user"], ip, mach["basedir"], cmd[1], cmd[2]) + comf.write(mkstr % tup) comf.close() + def checkdirs(dirs): for d in dirs: - if not os.path.exists(d): - os.mkdir(d) + if not os.path.exists(d): + os.mkdir(d) + def usage(): - print 'Usage: generate-scripts.py action=install|start|stop|clean|backup|restore\n\ + print ("Usage: generate-scripts.py action=install|start|stop|clean|backup|restore\n\ config=/path/to/confile/file defuser=default_user defbase=default_base\n\ For install: installtype=full|cluster - default is full\n\ For clean: cleantype=full|cluster - default is full\n\ -For backup: backuptype=full|incremental|init fulldir=path/to/full_backup incrdir=path/to/incr_backup' +For backup: backuptype=full|incremental|init fulldir=path/to/full_backup incrdir=path/to/incr_backup") + + -if __name__ == '__main__': - args = dict([arg.split('=') for arg in sys.argv[1:]]) - action='install' - if args.has_key('action'): - action=args['action'] +if __name__ == "__main__": + args = dict([arg.split("=") for arg in sys.argv[1:]]) + action = "install" + if args.__contains__("action"): + action = args["action"] else: - args['action']=action + args["action"] = action - if args.has_key('defuser'): - defuser=args['defuser'] - if args.has_key('defbase'): - defbase=args['defbase'] + if args.__contains__("defuser"): + defuser = args["defuser"] + if args.__contains__("defbase"): + defbase = args["defbase"] - if not args.has_key('config'): - usage() - sys.exit(1) + if not args.__contains__("config"): + usage() + sys.exit(1) - actions=["install", "start", "stop", "clean"] + actions = ["install", "start", "stop", "clean"] checkdirs(actions) - print str(args) + print (str(args)) - jsconf = open(args['config']) + jsconf = open(args["config"]) jstr = jsconf.read() jscfg = json.loads(jstr) # print str(jscfg) - if action == 'install': - installtype=args.get('installtype', 'full') - if installtype not in ['full', 'cluster']: - usage() - sys.exit(1) - generate_install_scripts(jscfg, installtype) - elif action == 'start': - generate_start_scripts(jscfg) - elif action == 'stop': - generate_stop_scripts(jscfg) - elif action == 'clean': - cleantype=args.get('cleantype', 'full') - if cleantype not in ['full', 'cluster']: - usage() - sys.exit(1) - generate_clean_scripts(jscfg, cleantype) - elif action == 'backup': - if not args.has_key('fulldir'): - usage() - sys.exit(1) - if not args.has_key('incrdir'): - usage() - sys.exit(1) - if not args.has_key('backuptype'): - usage() - sys.exit(1) - if args['backuptype'] == 'init': - generate_backup_init(jscfg, args['fulldir'], args['incrdir']) - else: - generate_backup_scripts(jscfg, args['backuptype'], args['fulldir'], args['incrdir']) - elif action == 'restore': - generate_restore_scripts(jscfg) - else : - usage() - sys.exit(1) + if action == "install": + installtype = args.get("installtype", "full") + if installtype not in ["full", "cluster"]: + usage() + sys.exit(1) + generate_install_scripts(jscfg, installtype) + elif action == "start": + generate_start_scripts(jscfg) + elif action == "stop": + generate_stop_scripts(jscfg) + elif action == "clean": + cleantype = args.get("cleantype", "full") + if cleantype not in ["full", "cluster"]: + usage() + sys.exit(1) + generate_clean_scripts(jscfg, cleantype) + elif action == "backup": + if not args.__contains__("fulldir"): + usage() + sys.exit(1) + if not args.__contains__("incrdir"): + usage() + sys.exit(1) + if not args.__contains__("backuptype"): + usage() + sys.exit(1) + if args["backuptype"] == "init": + generate_backup_init(jscfg, args["fulldir"], args["incrdir"]) + else: + generate_backup_scripts( + jscfg, args["backuptype"], args["fulldir"], args["incrdir"] + ) + elif action == "restore": + generate_restore_scripts(jscfg) + else: + usage() + sys.exit(1) diff --git a/cluster/getback.sh b/cluster/getback.sh new file mode 100755 index 0000000..89abc3e --- /dev/null +++ b/cluster/getback.sh @@ -0,0 +1,44 @@ +#! /bin/bash + +script_dir="`dirname $0`" +. $script_dir/common.sh + +# Use \$host to represent hostip or hostname value. + +from="$1" +to="$2" + +ffail=0 +for hostitem in $hosts; do + ufail=0 + host="${hostitem%:*}" + hname="${hostitem#*:}" + + echo "=== [`date`] on $host ===" + echo "=== fetch ($from) to $to ===" + #echo "=========== [`date`] fetch ($from) on $host($hname) to $to ===========" + if `ping -c 2 $host >/dev/null 2>/dev/null`; then + : + else + echo "Unable to connect $host($hname) !" + continue + fi + + if test "$SSHPASS" = ""; then + eval scp -q -r $REMOTE_USER@$host:$from $to || ufail=1 + else + eval sshpass -p "$REMOTE_PASSWORD" scp -r $REMOTE_USER@$host:$from $to || ufail=1 + fi + + if test "$ufail" = "1"; then + ffail=1 + echo -e "=== \033[31m !!!FAILURES!!! on $host \033[0m ===" + echo " " + else + echo -e "=== \033[32m !!!SUCCESS!!! on $host \033[0m ===" + echo " " + fi + +done + +exit $ffail diff --git a/cluster/remote_run.sh b/cluster/remote_run.sh index 98b5a02..33cd0ce 100755 --- a/cluster/remote_run.sh +++ b/cluster/remote_run.sh @@ -18,7 +18,8 @@ ffail=0 host="${hostitem%:*}" hname="${hostitem#*:}" -echo "=========== [`date`] execute ($cmd) on $host($hname) ===========" +echo "=== [`date`] on $host ===" +echo "=== execute ($cmd) ===" if `ping -c 2 $host >/dev/null 2>/dev/null`; then : else @@ -31,7 +32,7 @@ sed -i "s#HOST_IP_ADDR#$host#g" $tmpscript sed -i "s#HOST_NAME#$hname#g" "$tmpscript" if test "$SSHPASS" = ""; then - scp $tmpscript $REMOTE_USER@$host:/tmp + scp -q $tmpscript $REMOTE_USER@$host:/tmp ssh $REMOTE_USER@$host "bash $tmpscript" < /dev/null || ufail=1 test "$clear" = "true" && ssh $REMOTE_USER@$host "rm -f $tmpscript" else @@ -42,7 +43,11 @@ fi if test "$ufail" = "1"; then ffail=1 - echo "!!!FAILURES!!!" + echo -e "=== \033[31m !!!FAILURES!!! on $host \033[0m ===" + echo " " +else + echo -e "=== \033[32m !!!SUCCESS!!! on $host \033[0m ===" + echo " " fi rm -f $tmpscript diff --git a/cluster/restore/apply_binlog_fast.py b/cluster/restore/apply_binlog_fast.py new file mode 100755 index 0000000..24f17a4 --- /dev/null +++ b/cluster/restore/apply_binlog_fast.py @@ -0,0 +1,425 @@ +import os +import sys +import tarfile +import glob +import time +import shutil +import json +import subprocess +import mysql.connector +from distutils.util import strtobool +import configparser + + +# Below is the real story +class AnonymousArgs: + backup_file_name = "backup.tar.gz" + binlog_backup_path = None + etcfile = None + relayLogPath = None + mysqluser = "root" + mysqlpwd = "root" + mysqlcnx = None + cursor = None + channel_name = "fast_apply_channel" + gtidset = None + start_gtid = None + start_relay_log_name = None + start_relay_log_pos = None + mysqlbinlog_util_abspath = None + stop_time = None + + @classmethod + def init_mysql_cnx(self): + try: + # initialize the mysql connection + self.mysqlcnx = mysql.connector.connect( + option_files=self.etcfile, + user=self.mysqluser, + password=self.mysqlpwd, + auth_plugin='mysql_native_password' + ) + if self.mysqlcnx.is_connected() is not True: + raise mysql.connector.Error + + # initialize the cursor with dictionary enabled + self.cursor = self.mysqlcnx.cursor(dictionary=True) + + except mysql.connector.Error as err: + print ("MySQL connect failed: {}\n".format(err)) + raise Exception(err) + else: + print ("MySQL connect successfully!\n") + return True + + @classmethod + def parse_relaypath_from_etc(self): + try: + config_file_parser = configparser.RawConfigParser(allow_no_value=True) + config_file_parser.read(self.etcfile) + self.relayLogPath = os.path.join( + os.path.dirname(config_file_parser.get("mysqld","relay-log")),"") + except Exception as e: + raise Exception(e) + + @classmethod + def pick_bounderay_from_gtid_set(self): + try: + #parse the gtid str from the given gtid info file + fd = open(self.gtidinfo,'r') + info = fd.readlines() + for line in info: + self.gtidset = line.split()[2] + break; + + # 2209fde0-27fd-11ec-a801-7c8ae18d3c61:1,c7cbb8de-2706-11ec-9057-7c8ae18d3c61:1-12483 + sets = self.gtidset.split(",") + sid = None + gid = None + for set_l in sets: + sid = set_l.split(":")[0] + gid_range = set_l.split(":")[1] + if len(gid_range.split("-")) < 2: + continue; + gid = gid_range.split("-")[1] + gid = int(gid)+1; + + self.start_gtid = "%s:%d" % (sid,gid) + print ("start gtid is '%s'" % self.start_gtid); + except Exception as e: + raise Exception(e) + + @classmethod + def truncate_file_by_stoptime(self,index_file_name): + if self.stop_time is None: + return + try: + util_name=self.mysqlbinlog_util_abspath + input_args1="--binlog-index-file=%s" % index_file_name + input_args2="--stop-datetime=%s" % self.stop_time; + input_args3="--truncate-file-by-stoptime" + result=subprocess.check_output(\ + [util_name, input_args1, input_args2, input_args3],\ + shell=False) + except subprocess.CalledProcessError as err: + result = err.output + print (result) + + + @classmethod + def get_filepos_by_start_gtid(self,index_file_name): + try: + util_name=self.mysqlbinlog_util_abspath + input_args1="--binlog-index-file=%s" % index_file_name + input_args2="--gtid-to-filepos=%s" % self.start_gtid + result=subprocess.check_output([util_name, input_args1, input_args2],\ + shell=False) + result_dict = json.loads(result); + self.start_relay_log_name = result_dict['filename']; + self.start_relay_log_pos = result_dict['pos']; + except subprocess.CalledProcessError as e: + # can't find the specified gtid in given binlog. + # just assign the first binlog name and pos to the + # start_* info variables. + self.start_relay_log_pos = "4"; + fd = open(index_file_name,"r"); + filenames = fd.readlines(); + for filename in filenames: + self.start_relay_log_name = filename.strip(); + # just assign the first file + break; + +def sort_by_value(str_array): + dic_by_value = {} + for item in str_array: + dic_by_value[int(item)] = item + keys = dic_by_value.keys() + keys = sorted(keys) + return [dic_by_value[key] for key in keys] + + +def extract_tar(file_path, target_path): + try: + tar = tarfile.open(file_path, "r:gz") + file_names = tar.getnames() + for file_name in file_names: + tar.extract(file_name, target_path) + tar.close() + except Exception as e: + raise Exception(e) + + +def transfer_backup(backuppath, relaypath): + + backup_file = AnonymousArgs.backup_file_name; + os.system("mkdir -p %s" % relaypath) + shutil.copy(backuppath+backup_file,relaypath) + # TODO + pass + + +def rename_binlog_backup_to_relay(relay_log_path): + channel_name = AnonymousArgs.channel_name + try: + os.chdir(relay_log_path) + # Attention : cwd have already changed to relay_log_path + + extract_tar(AnonymousArgs.backup_file_name, "./") + ls_result = glob.glob("./logfiles/binlog.*") + + binlog_indexs = [] + for names in ls_result: + # trim the path to get the pure binlog index + index_str = os.path.basename(names).split(".")[1:][0] + binlog_indexs.append(index_str) + + # sort the array + binlog_indexs = sort_by_value(binlog_indexs) + + src_file_prefix = "./logfiles/binlog." + dst_file_prefix = "./logfiles/relay-" + channel_name + "." + + relay_index_file_name = "./logfiles/relay-%s.index" % channel_name + file_hd = open(relay_index_file_name, "w") + + for index in binlog_indexs: + os.rename(src_file_prefix + index, dst_file_prefix + index) + file_hd.write(AnonymousArgs.relayLogPath + "relay-" + channel_name + "." + index + "\n") + file_hd.close() + + # TODO + # all the processed relay log and index file is in ./logfiles dirctory + cmd = "mv %s/logfiles/* %s/" % (AnonymousArgs.relayLogPath,AnonymousArgs.relayLogPath) + print (cmd) + os.system(cmd) + + # Here to identify the file and pos info to feed the change + # master statment to generate the virtual slave channel to + # apply the relay(backup-binlog) to the new server. + index_file_name = "%s/relay-%s.index" % (AnonymousArgs.relayLogPath,channel_name) + relay_index_file_name = os.path.abspath(index_file_name) + AnonymousArgs.get_filepos_by_start_gtid(index_file_name) + AnonymousArgs.truncate_file_by_stoptime(index_file_name) + + + except Exception as e: + raise Exception(e) + + +def prepare_binlog_from_backup_path(): + try: + # localize the variable from global argument container + binlog_backup_path = AnonymousArgs.binlog_backup_path + relay_log_path = AnonymousArgs.relayLogPath + + # move backup.tgz to the relay path + transfer_backup(binlog_backup_path, relay_log_path) + + # do rename staff + rename_binlog_backup_to_relay(relay_log_path) + except Exception as e: + raise Exception(e) + + +def generate_fake_replica_channel(): + try: + # localize the variable from global argument container + channel_name = AnonymousArgs.channel_name + mysql_cnx = AnonymousArgs.mysqlcnx + start_relay_log_name = AnonymousArgs.start_relay_log_name + start_relay_log_pos = AnonymousArgs.start_relay_log_pos + + # initialize the sql statement + do_change_master = \ + "change master to " +\ + "master_host='2.3.4.158'," +\ + "master_port=7890, " +\ + "master_user='repl'," +\ + "master_password='repl'," +\ + "relay_log_file='%s', relay_log_pos=%s for channel '%s'" \ + % (start_relay_log_name,start_relay_log_pos,channel_name) + print (do_change_master) + + + try: + cursor = mysql_cnx.cursor() + # execute the change master statement + cursor.execute(do_change_master) + + except mysql.connector.Error as err: + print ("MySQL operation failed: {}\n".format(err)) + raise Exception(err) + else: + print ("change master successfully!\n") + return True + + except Exception as e: + raise Exception(e) + +def start_slave_sql_thread(): + try: + # localize the variable from global argument container + channel_name = AnonymousArgs.channel_name + mysql_cnx = AnonymousArgs.mysqlcnx + + # initialize the sql statement + # TODO + # Specify the start point from the xtrabackup metadata point + do_start_thread = "start slave sql_thread for channel '%s'" % channel_name + + try: + # execute the change master statement + cursor = mysql_cnx.cursor() + cursor.execute(do_start_thread) + + except mysql.connector.Error as err: + print ("MySQL operation failed: {}\n".format(err)) + raise Exception(err) + else: + print ("Do start sql_thread finish!\n") + return True + + except Exception as e: + raise Exception(e) + +def check_sql_thread_state(): + + # localize the variable from global argument container + channel_name = AnonymousArgs.channel_name + cursor = AnonymousArgs.cursor + + # initialize the SQL statement + show_slave_status_sql = "show slave status for channel '%s' " % channel_name + + try: + # execute the show SQL + cursor.execute(show_slave_status_sql) + result = cursor.fetchall() + for rows in result: + Slave_SQL_Running = rows['Slave_SQL_Running'] + Slave_SQL_Running_State = rows['Slave_SQL_Running_State'] + + if Slave_SQL_Running == 'No': + if rows['Last_SQL_Errno'] == 0: + errinfo = "Slave_SQL is not running, but Last_SQL_Errno is 0, " +\ + "maybe the sql_thread is failed, try restart.\n" + else: + errinfo = "Slave_SQL is not running, Last_SQL_Errno is %s, Last_SQL_Error is %s"\ + % (rows['Last_SQL_Errno'],rows['Last_SQL_Error']) + return False, errinfo + + Slave_SQL_Running_State = rows['Slave_SQL_Running_State'] + + if Slave_SQL_Running_State != "Slave has read all relay log; waiting for more updates": + print ("Fast_apply_binlog Not finished yet, current Retrieved_Gtid_Set is %s, Executed_Gtid_Set is %s" \ + % (rows['Retrieved_Gtid_Set'],rows['Executed_Gtid_Set'])) + time.sleep(1) + return False, None + else: + print ("Fast_apply_binlog finished successfully\n") + return True, None + except mysql.connector.Error as err: + return False, mysql.connector.Error(err) + except Exception as err: + return Exception(err) + + +def sync_confirm_relay_apply(): + try: + retval = False + while retval == False: + retval,errinfo = check_sql_thread_state() + if errinfo != None: + print (errinfo) + return False + except Exception as e: + raise Exception(e) + else: + return True + +def do_finish_ops(): + # localize the variable from global argument container + channel_name = AnonymousArgs.channel_name + cursor = AnonymousArgs.cursor + + # initialize the SQL statement + stop_slave_sql = "stop slave for channel '%s' " % channel_name + + try: + # execute the show SQL + cursor.execute(stop_slave_sql) + except mysql.connector.Error as err: + return False, mysql.connector.Error(err) + except Exception as err: + return Exception(err) + + + +def start_fast_apply(): + + # 1.Prepare the binlog from backup , rename the binlog to relay + prepare_binlog_from_backup_path() + + # 2.Generate the fake slave channel + # TODO + generate_fake_replica_channel() + + # 3.start slave sql_thread + start_slave_sql_thread() + + # 4.confirm whether the apply finished + retval = sync_confirm_relay_apply() + + # 5.clean the replica info. + do_finish_ops() + + # return as the unix-like error number style + if retval: + sys.exit(0); + + sys.exit(1) + + + +def print_usage(): + print ("Usage: apply_binlog_fast.py binlogBackupPath=/path/to/binlog/backup/ "\ + +"etcfile=mysql-config-file gtidinfo=/gtid/info/file/path"\ + +"stoptime=datetime_str") + + +if __name__ == "__main__": + + args = dict([arg.split("=") for arg in sys.argv[1:]]) + + if not args.__contains__("binlogBackupPath") \ + or not args.__contains__("etcfile") \ + or not args.__contains__("gtidinfo"): + print_usage() + raise RuntimeError("see help info.") + + binlog_backup_path = args["binlogBackupPath"] + etcfile = args["etcfile"] + gtidinfo = args["gtidinfo"] + + if not os.path.exists(binlog_backup_path): + raise ValueError( + "Binlog backup path {} doesn't exist!".format(binlog_backup_path) + ) + if not os.path.exists(etcfile): + raise ValueError("DB config file {} doesn't exist!".format(etcfile)) + + # init the AnonymousArgs for global use + if args.__contains__("stoptime"): + stop_time = args["stoptime"] + AnonymousArgs.stop_time = stop_time + AnonymousArgs.binlog_backup_path = os.path.join(os.path.abspath(binlog_backup_path),"") + AnonymousArgs.etcfile = os.path.abspath(etcfile) + AnonymousArgs.gtidinfo = os.path.abspath(gtidinfo) + AnonymousArgs.mysqlbinlog_util_abspath = os.path.abspath("./mysqlbinlog") + AnonymousArgs.parse_relaypath_from_etc() + AnonymousArgs.pick_bounderay_from_gtid_set(); + AnonymousArgs.init_mysql_cnx() + + + # entry of the fast apply binlog. + start_fast_apply() diff --git a/cluster/restore/merge_incr.py b/cluster/restore/merge_incr.py index 4055735..e719b1e 100755 --- a/cluster/restore/merge_incr.py +++ b/cluster/restore/merge_incr.py @@ -1,12 +1,10 @@ -#! /usr/bin/python - import os import sys import re def addLogToMap(lmap, logf, dirf): - numdir = long(dirf) - if not lmap.has_key(logf) or numdir > lmap[logf]: + numdir = int(dirf) + if not lmap.__contains__(logf) or numdir > lmap[logf]: lmap[logf] = numdir # if not lmap.has_key(logf): # lmap[logf] = [numdir] @@ -24,8 +22,8 @@ def merge_binlog(incrdir, binlogpref, targetd): addLogToMap(logmap, l, d) for f in logmap: - print "%s %d" % (f, logmap[f]) - os.system("cp -f %s/%d/logfiles/%s %s" % (incrdir, logmap[f], f, targetd)) + print ("%s %d" % (f, logmap[f])) + #os.system("cp -f %s/%d/logfiles/%s %s" % (incrdir, logmap[f], f, targetd)) if __name__ == '__main__': incrdir=sys.argv[1] diff --git a/cluster/restore/mysqlbinlog b/cluster/restore/mysqlbinlog new file mode 100755 index 0000000..6399098 Binary files /dev/null and b/cluster/restore/mysqlbinlog differ diff --git a/cluster/restore/restore-mysql.py b/cluster/restore/restore-mysql.py index ec78372..62f9437 100755 --- a/cluster/restore/restore-mysql.py +++ b/cluster/restore/restore-mysql.py @@ -11,13 +11,14 @@ import subprocess import json import shlex +from distutils.util import strtobool def param_replace(string, rep_dict): pattern = re.compile("|".join([re.escape(k) for k in rep_dict.keys()]), re.M) return pattern.sub(lambda x: rep_dict[x.group(0)], string) -def make_mgr_args(config_path, replace_items, target_node_index): +def make_mgr_args(config_path, replace_items, target_node_index, usemgr): jsconf = open(config_path) jstr = jsconf.read() jscfg = json.loads(jstr) @@ -85,12 +86,13 @@ def make_mgr_args(config_path, replace_items, target_node_index): log_arch = log_path + "/dblogs/arch" replace_items["place_holder_ip"] = local_ip - replace_items["place_holder_mgr_recovery_retry_count"] = str(mgr_num_nodes*100) - replace_items["place_holder_mgr_local_address"] = local_addr - replace_items["place_holder_mgr_seeds"] = seeds - replace_items["place_holder_mgr_whitelist"] = white_list - replace_items["place_holder_mgr_member_weight"] = str(weight) - replace_items["place_holder_mgr_group_name"] = group_uuid + if usemgr: + replace_items["place_holder_mgr_recovery_retry_count"] = str(mgr_num_nodes*100) + replace_items["place_holder_mgr_local_address"] = local_addr + replace_items["place_holder_mgr_seeds"] = seeds + replace_items["place_holder_mgr_whitelist"] = white_list + replace_items["place_holder_mgr_member_weight"] = str(weight) + replace_items["place_holder_mgr_group_name"] = group_uuid replace_items["prod_dir"] = prod_dir replace_items["data_dir"] = data_dir replace_items["innodb_dir"] = innodb_dir @@ -115,27 +117,30 @@ def make_mgr_args(config_path, replace_items, target_node_index): jsconf.close() return is_master_node, server_port, data_path, log_path, log_dir, db_inst_user -def generate_cnf_file(config_template_file, install_path, server_id, cluster_id, shard_id, config_path, target_node_index): +def generate_cnf_file(config_template_file, install_path, server_id, cluster_id, shard_id, config_path, target_node_index, usemgr): replace_items = { "base_dir": install_path, "place_holder_server_id": str(server_id), "place_holder_shard_id": str(shard_id), "place_holder_cluster_id": str(cluster_id), } - is_master, server_port, data_path, log_path, log_dir, user = make_mgr_args(config_path, replace_items, target_node_index) + is_master, server_port, data_path, log_path, log_dir, user = make_mgr_args(config_path, replace_items, target_node_index, usemgr) config_template = open(config_template_file, 'r').read() conf = param_replace(config_template, replace_items) etc_path = install_path + "/etc" if not os.path.exists(etc_path): os.mkdir(etc_path) - cnf_file_path = etc_path+"/my_"+ str(server_port) +".cnf" + conf_list_file = etc_path+"/instances_list.txt" + cnf_file_path = data_path +"/my_"+ str(server_port) +".cnf" cnf_file = open(cnf_file_path, 'w') cnf_file.write(conf) cnf_file.close() os.system("sed -e 's/#skip_name_resolve=on/skip_name_resolve=on/' -i " + cnf_file_path) - os.system("sed -e 's/#super_read_only=OFF/super_read_only=ON/' -i " + cnf_file_path) - os.system("sed -e 's/^#group_replication_/group_replication_/' -i " + cnf_file_path) - os.system("sed -e 's/^#clone_/clone_/' -i " + cnf_file_path) + if usemgr: + os.system("sed -e 's/#super_read_only=OFF/super_read_only=ON/' -i " + cnf_file_path) + os.system("sed -e 's/^#group_replication_/group_replication_/' -i " + cnf_file_path) + os.system("sed -e 's/^#clone_/clone_/' -i " + cnf_file_path) + os.system("echo \"" + str(server_port) + "==>" + cnf_file_path + "\" >> " + conf_list_file) def print_usage(): print 'Usage: restore-mysql.py config=/path/of/config/file target_node_index=idx [dbcfg=/db/config/template/path/template.cnf] [cluster_id=ID] [shard_id=N] [server_id=N]' @@ -159,15 +164,16 @@ def print_usage(): server_id = int(args['server_id']) else: server_id = str(random.randint(1,65535)) - + usemgr=args.get('usemgr', 'True') + usemgr=strtobool(usemgr) if args.has_key('dbcfg') : config_template_file = args['dbcfg'] else: config_template_file = "./template.cnf" if not os.path.exists(config_template_file): raise ValueError("DB config template file {} doesn't exist!".format(config_template_file)) - install_path = os.getcwd()[:-8] - generate_cnf_file(config_template_file, install_path, server_id, cluster_id, shard_id, config_path, target_node_index) + install_path = os.getcwd()[:-9] + generate_cnf_file(config_template_file, install_path, server_id, cluster_id, shard_id, config_path, target_node_index, usemgr) except KeyError, e: print_usage() print e diff --git a/cluster/restore/start_mgr.sh b/cluster/restore/start_mgr.sh index a5d9dc2..49e816f 100755 --- a/cluster/restore/start_mgr.sh +++ b/cluster/restore/start_mgr.sh @@ -25,4 +25,4 @@ fi echo "start mgr sql: $mgrsql" -mysql --defaults-file=$KUNLUNBASE/percona-8.0.18-bin-rel/etc/my_$port.cnf -uroot -proot -e "$mgrsql" +mysql --defaults-file=$3 -uroot -proot -e "$mgrsql" diff --git a/cluster/restore/wait_mysqlup.sh b/cluster/restore/wait_mysqlup.sh index afd7536..a7691a9 100755 --- a/cluster/restore/wait_mysqlup.sh +++ b/cluster/restore/wait_mysqlup.sh @@ -3,7 +3,7 @@ port="${1:-3306}" while true; do - mysql --defaults-file=$KUNLUNBASE/percona-8.0.18-bin-rel/etc/my_$port.cnf -uroot -proot -e 'select now()' >& /dev/null + mysql --defaults-file=$2 -uroot -proot -e 'select now()' >& /dev/null if test "$?" = "0"; then break; fi