From 732956256e8c630dc59b0be2a12e09d7b3bdb6c3 Mon Sep 17 00:00:00 2001 From: Alex V Date: Thu, 3 Aug 2017 11:25:29 +0200 Subject: [PATCH 1/4] Implement a basic diff command --- marty/commands/mount.py | 33 +++++++++++++++++++++++++++++++++ setup.py | 3 ++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/marty/commands/mount.py b/marty/commands/mount.py index 94569a0..9d558ed 100644 --- a/marty/commands/mount.py +++ b/marty/commands/mount.py @@ -3,6 +3,7 @@ import os.path import subprocess +from marty.printer import printer from marty.commands import Command from marty.operations.fs import MartyFS @@ -51,3 +52,35 @@ def run(self, args, config, storage, remotes): pshell = subprocess.Popen([shell], cwd=fullname) pshell.wait() fs.umount() + + +class Diff(Command): + + """ Make a diff of two backups or tree for a remote. + """ + + help = 'Make a diff of two backups or tree for a remote' + + def prepare(self): + self._aparser.add_argument('remote', nargs='?') + self._aparser.add_argument('names', nargs=2) + + def run(self, args, config, storage, remotes): + names = ['%s/%s' % (args.remote, name) if args.remote else name for name in args.names] + trees = [storage.get_tree(name) for name in names] + + with tempfile.TemporaryDirectory() as fullname1, tempfile.TemporaryDirectory() as fullname2: + fs1 = MartyFS(storage, trees[0], fullname1) + fs2 = MartyFS(storage, trees[1], fullname2) + fs1.mount() + fs2.mount() + pshell = subprocess.Popen(['diff', '-q', '-r', '--no-dereference', '--suppress-common-lines', + fullname1, fullname2], stdout=subprocess.PIPE) + pshell.wait() + output = pshell.stdout.read().decode('utf8') + fs2.umount() + fs1.umount() + names_filenames = zip((fullname1, fullname2), names) + for name_filename in names_filenames: + output = output.replace(*name_filename) + printer.p('Diff output betwwen %s and %s:\n\n%s' % (*names, output)) diff --git a/setup.py b/setup.py index 228a436..f083fd4 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,8 @@ 'restore = marty.commands.restore:Restore', 'check = marty.commands.check:Check', 'mount = marty.commands.mount:Mount', - 'explore = marty.commands.mount:Explore'], + 'explore = marty.commands.mount:Explore', + 'diff = marty.commands.mount:Diff'], 'marty.storages': ['filesystem = marty.storages.filesystem:Filesystem'], 'marty.remotemethods': ['local = marty.remotemethods.local:Local', 'ssh = marty.remotemethods.ssh:SSH', From 8118d7fce32c035e5c4509eabad3bbcc76ec1f17 Mon Sep 17 00:00:00 2001 From: Alex V Date: Fri, 4 Aug 2017 12:51:50 +0200 Subject: [PATCH 2/4] Implement diff using Tree --- marty/commands/mount.py | 83 +++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/marty/commands/mount.py b/marty/commands/mount.py index 9d558ed..98ab819 100644 --- a/marty/commands/mount.py +++ b/marty/commands/mount.py @@ -63,24 +63,69 @@ class Diff(Command): def prepare(self): self._aparser.add_argument('remote', nargs='?') - self._aparser.add_argument('names', nargs=2) + self._aparser.add_argument('ref_name') + self._aparser.add_argument('name') + + def _print_diff_tree(self, storage, ref_tree, tree, level=()): + last = False + next_level = level + (True,) + + ref_tree_set = set(ref_tree.names()) if ref_tree else set() + tree_set = set(tree.names()) + adds = tree_set.difference(ref_tree_set) + deletions = ref_tree_set.difference(tree_set) + + all_items = tree_set.union(ref_tree_set) + + for i, (name) in enumerate(sorted(all_items)): + if len(tree) == i + 1: + last = True + next_level = level + (False,) + header = ''.join([u'│ ' if x else ' ' for x in level]) + if last: + header += '└── ' + else: + header += '├── ' + + # setup filename colors + color = 'yellow' + ref_item = None + item = None + if name in adds: + item = tree[name] + color = 'green' + elif name in deletions: + item = ref_tree[name] + ref_item = ref_tree[name] + color = 'red' + else: + item = tree[name] + ref_item = ref_tree[name] + + filename = name.decode('utf-8', 'replace') + # only print filename when there is a diff + if name in adds or name in deletions or item.ref != ref_item.ref: + if item.type == 'tree': + printer.p('{h}{f}', + h=header, f=filename, color=color) + ref_object = storage.get_tree(ref_item.ref) if ref_item else None + self._print_diff_tree(storage, ref_object, + storage.get_tree(item.ref), level=next_level) + elif item.get('filetype') == 'link': + printer.p('{h}{f} -> {l}', + h=header, f=filename, l=item.get('link', '?'), + color=color) + else: + printer.p('{h}{f}', + h=header, color=color, f=filename) def run(self, args, config, storage, remotes): - names = ['%s/%s' % (args.remote, name) if args.remote else name for name in args.names] - trees = [storage.get_tree(name) for name in names] - - with tempfile.TemporaryDirectory() as fullname1, tempfile.TemporaryDirectory() as fullname2: - fs1 = MartyFS(storage, trees[0], fullname1) - fs2 = MartyFS(storage, trees[1], fullname2) - fs1.mount() - fs2.mount() - pshell = subprocess.Popen(['diff', '-q', '-r', '--no-dereference', '--suppress-common-lines', - fullname1, fullname2], stdout=subprocess.PIPE) - pshell.wait() - output = pshell.stdout.read().decode('utf8') - fs2.umount() - fs1.umount() - names_filenames = zip((fullname1, fullname2), names) - for name_filename in names_filenames: - output = output.replace(*name_filename) - printer.p('Diff output betwwen %s and %s:\n\n%s' % (*names, output)) + ref_name = '%s/%s' % (args.remote, args.ref_name) if args.remote else args.ref_name + other_name = '%s/%s' % (args.remote, args.name) if args.remote else args.name + ref_backup = storage.get_backup(ref_name) + ref_tree = storage.get_tree(ref_name) + other_tree = storage.get_tree(other_name) + printer.p('Backup reference date: {backup_date}', + backup_date=ref_backup.start_date.format('DD/MM/YYYY HH:mm:ss')) + printer.p('Backup reference root: {backup_root}\n', backup_root=ref_backup.root) + self._print_diff_tree(storage, ref_tree, other_tree) From c70f0d34208085334a7824dbc98b3b5f8811f5e4 Mon Sep 17 00:00:00 2001 From: Alex V Date: Wed, 27 Sep 2017 14:06:14 +0200 Subject: [PATCH 3/4] Remove backup reference --- marty/commands/mount.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/marty/commands/mount.py b/marty/commands/mount.py index 98ab819..55b1ee2 100644 --- a/marty/commands/mount.py +++ b/marty/commands/mount.py @@ -122,10 +122,6 @@ def _print_diff_tree(self, storage, ref_tree, tree, level=()): def run(self, args, config, storage, remotes): ref_name = '%s/%s' % (args.remote, args.ref_name) if args.remote else args.ref_name other_name = '%s/%s' % (args.remote, args.name) if args.remote else args.name - ref_backup = storage.get_backup(ref_name) ref_tree = storage.get_tree(ref_name) other_tree = storage.get_tree(other_name) - printer.p('Backup reference date: {backup_date}', - backup_date=ref_backup.start_date.format('DD/MM/YYYY HH:mm:ss')) - printer.p('Backup reference root: {backup_root}\n', backup_root=ref_backup.root) self._print_diff_tree(storage, ref_tree, other_tree) From 269332d9d6135543dea5d5959ff55cf05d6e05f3 Mon Sep 17 00:00:00 2001 From: Alex V Date: Wed, 18 Oct 2017 08:47:20 +0200 Subject: [PATCH 4/4] Put the diff command in a diff module --- marty/commands/diff.py | 77 +++++++++++++++++++++++++++++++++++++++++ marty/commands/mount.py | 74 --------------------------------------- setup.py | 2 +- 3 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 marty/commands/diff.py diff --git a/marty/commands/diff.py b/marty/commands/diff.py new file mode 100644 index 0000000..e189170 --- /dev/null +++ b/marty/commands/diff.py @@ -0,0 +1,77 @@ +from marty.printer import printer +from marty.commands import Command + + +class Diff(Command): + + """ Make a diff of two backups or tree for a remote. + """ + + help = 'Make a diff of two backups or tree for a remote' + + def prepare(self): + self._aparser.add_argument('remote', nargs='?') + self._aparser.add_argument('ref_name') + self._aparser.add_argument('name') + + def _print_diff_tree(self, storage, ref_tree, tree, level=()): + last = False + next_level = level + (True,) + + ref_tree_set = set(ref_tree.names()) if ref_tree else set() + tree_set = set(tree.names()) + adds = tree_set.difference(ref_tree_set) + deletions = ref_tree_set.difference(tree_set) + + all_items = tree_set.union(ref_tree_set) + + for i, (name) in enumerate(sorted(all_items)): + if len(tree) == i + 1: + last = True + next_level = level + (False,) + header = ''.join([u'│ ' if x else ' ' for x in level]) + if last: + header += '└── ' + else: + header += '├── ' + + # setup filename colors + color = 'yellow' + ref_item = None + item = None + if name in adds: + item = tree[name] + color = 'green' + elif name in deletions: + item = ref_tree[name] + ref_item = ref_tree[name] + color = 'red' + else: + item = tree[name] + ref_item = ref_tree[name] + + filename = name.decode('utf-8', 'replace') + # only print filename when there is a diff + if name in adds or name in deletions or item.ref != ref_item.ref: + if item.type == 'tree': + printer.p('{h}{f}', + h=header, f=filename, color=color) + ref_object = storage.get_tree(ref_item.ref) if ref_item else None + self._print_diff_tree(storage, ref_object, + storage.get_tree(item.ref), + level=next_level) + elif item.get('filetype') == 'link': + link = item.get('link', '?').decode('utf-8', 'replace') + printer.p('{h}{f} -> {l}', + h=header, f=filename, l=link, + color=color) + else: + printer.p('{h}{f}', + h=header, color=color, f=filename) + + def run(self, args, config, storage, remotes): + ref_name = '%s/%s' % (args.remote, args.ref_name) if args.remote else args.ref_name + other_name = '%s/%s' % (args.remote, args.name) if args.remote else args.name + ref_tree = storage.get_tree(ref_name) + other_tree = storage.get_tree(other_name) + self._print_diff_tree(storage, ref_tree, other_tree) diff --git a/marty/commands/mount.py b/marty/commands/mount.py index 55b1ee2..94569a0 100644 --- a/marty/commands/mount.py +++ b/marty/commands/mount.py @@ -3,7 +3,6 @@ import os.path import subprocess -from marty.printer import printer from marty.commands import Command from marty.operations.fs import MartyFS @@ -52,76 +51,3 @@ def run(self, args, config, storage, remotes): pshell = subprocess.Popen([shell], cwd=fullname) pshell.wait() fs.umount() - - -class Diff(Command): - - """ Make a diff of two backups or tree for a remote. - """ - - help = 'Make a diff of two backups or tree for a remote' - - def prepare(self): - self._aparser.add_argument('remote', nargs='?') - self._aparser.add_argument('ref_name') - self._aparser.add_argument('name') - - def _print_diff_tree(self, storage, ref_tree, tree, level=()): - last = False - next_level = level + (True,) - - ref_tree_set = set(ref_tree.names()) if ref_tree else set() - tree_set = set(tree.names()) - adds = tree_set.difference(ref_tree_set) - deletions = ref_tree_set.difference(tree_set) - - all_items = tree_set.union(ref_tree_set) - - for i, (name) in enumerate(sorted(all_items)): - if len(tree) == i + 1: - last = True - next_level = level + (False,) - header = ''.join([u'│ ' if x else ' ' for x in level]) - if last: - header += '└── ' - else: - header += '├── ' - - # setup filename colors - color = 'yellow' - ref_item = None - item = None - if name in adds: - item = tree[name] - color = 'green' - elif name in deletions: - item = ref_tree[name] - ref_item = ref_tree[name] - color = 'red' - else: - item = tree[name] - ref_item = ref_tree[name] - - filename = name.decode('utf-8', 'replace') - # only print filename when there is a diff - if name in adds or name in deletions or item.ref != ref_item.ref: - if item.type == 'tree': - printer.p('{h}{f}', - h=header, f=filename, color=color) - ref_object = storage.get_tree(ref_item.ref) if ref_item else None - self._print_diff_tree(storage, ref_object, - storage.get_tree(item.ref), level=next_level) - elif item.get('filetype') == 'link': - printer.p('{h}{f} -> {l}', - h=header, f=filename, l=item.get('link', '?'), - color=color) - else: - printer.p('{h}{f}', - h=header, color=color, f=filename) - - def run(self, args, config, storage, remotes): - ref_name = '%s/%s' % (args.remote, args.ref_name) if args.remote else args.ref_name - other_name = '%s/%s' % (args.remote, args.name) if args.remote else args.name - ref_tree = storage.get_tree(ref_name) - other_tree = storage.get_tree(other_name) - self._print_diff_tree(storage, ref_tree, other_tree) diff --git a/setup.py b/setup.py index f083fd4..9062d15 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ 'check = marty.commands.check:Check', 'mount = marty.commands.mount:Mount', 'explore = marty.commands.mount:Explore', - 'diff = marty.commands.mount:Diff'], + 'diff = marty.commands.diff:Diff'], 'marty.storages': ['filesystem = marty.storages.filesystem:Filesystem'], 'marty.remotemethods': ['local = marty.remotemethods.local:Local', 'ssh = marty.remotemethods.ssh:SSH',