From 83ac437a261825854cc546feb990040a2c520681 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 29 Dec 2019 23:26:50 +0200 Subject: [PATCH 01/11] Create README.md --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b01069 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +# DropboxEncryptedUploader +Synchronizes local folder to Dropbox encrypting each file to zip archive in memory without storing to disk +## Usage +DropboxEncryptedUploader.exe dropbox-app-token path-to-folder dropbox-folder-name encryption-password +### Example +"asdlakdfkfrefggfdgdfg-rgedfgd-adfsfdf3e" "d:\Backups" "/Backups" "password" + +## How to get Dropbox token +http://99rabbits.com/get-dropbox-access-token/ From 32d598408033c15762492d245a95bc0da5e14d20 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 29 Dec 2019 23:27:18 +0200 Subject: [PATCH 02/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b01069..b64dd30 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Synchronizes local folder to Dropbox encrypting each file to zip archive in memo ## Usage DropboxEncryptedUploader.exe dropbox-app-token path-to-folder dropbox-folder-name encryption-password ### Example -"asdlakdfkfrefggfdgdfg-rgedfgd-adfsfdf3e" "d:\Backups" "/Backups" "password" +DropboxEncryptedUploader.exe "asdlakdfkfrefggfdgdfg-rgedfgd-adfsfdf3e" "d:\Backups" "/Backups" "password" ## How to get Dropbox token http://99rabbits.com/get-dropbox-access-token/ From 702c1f0603bfbf08782d0647b5eecad80a9cd090 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 29 Dec 2019 23:29:49 +0200 Subject: [PATCH 03/11] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1f2ff27 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Vladyslav Taranov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 1807afa32bc23d6a569b973f40c31f651bec6397 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 29 Dec 2019 23:40:11 +0200 Subject: [PATCH 04/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b64dd30..e94234c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # DropboxEncryptedUploader -Synchronizes local folder to Dropbox encrypting each file to zip archive in memory without storing to disk +Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk, which means that each file is devided by 140mb blocks and each block is encrypted on the fly, immediately sent to Dropbox servers and then thrown away so the whole encrypted archive (be it even 300 GB) will not take much space on your machine. ## Usage DropboxEncryptedUploader.exe dropbox-app-token path-to-folder dropbox-folder-name encryption-password ### Example From a407ac340f946d91f29d3061eaf6b955b0add682 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 29 Dec 2019 23:40:35 +0200 Subject: [PATCH 05/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e94234c..7293429 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # DropboxEncryptedUploader -Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk, which means that each file is devided by 140mb blocks and each block is encrypted on the fly, immediately sent to Dropbox servers and then thrown away so the whole encrypted archive (be it even 300 GB) will not take much space on your machine. +Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk. Each file is devided by 140mb blocks and each block is encrypted on the fly, immediately sent to Dropbox servers and then thrown away so the whole encrypted archive (be it even 300 GB) will not take much space on your machine. ## Usage DropboxEncryptedUploader.exe dropbox-app-token path-to-folder dropbox-folder-name encryption-password ### Example From 94310cd6deacf923b492afb274db897bac56d344 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 29 Dec 2019 23:41:34 +0200 Subject: [PATCH 06/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7293429..2c32454 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # DropboxEncryptedUploader -Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk. Each file is devided by 140mb blocks and each block is encrypted on the fly, immediately sent to Dropbox servers and then thrown away so the whole encrypted archive (be it even 300 GB) will not take much space on your machine. +Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk. Each file is devided by 140mb blocks, each block is encrypted on the fly and immediately sent to Dropbox servers so the whole encrypted archive (be it even 300 GB) will not take more than just 140mb of RAM. ## Usage DropboxEncryptedUploader.exe dropbox-app-token path-to-folder dropbox-folder-name encryption-password ### Example From 89b9ea62618e401bb0be024da5e16bd29bcf0e77 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 29 Dec 2019 23:42:50 +0200 Subject: [PATCH 07/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c32454..2c21e1d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # DropboxEncryptedUploader -Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk. Each file is devided by 140mb blocks, each block is encrypted on the fly and immediately sent to Dropbox servers so the whole encrypted archive (be it even 300 GB) will not take more than just 140mb of RAM. +Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk. Each file is devided by 140mb blocks, each block is encrypted on the fly and immediately sent to Dropbox servers so no big archive file is kept neither in RAM nor on your disk. ## Usage DropboxEncryptedUploader.exe dropbox-app-token path-to-folder dropbox-folder-name encryption-password ### Example From e39d2f0a26c9e0256d7d33c05bb924660221c83c Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Mon, 30 Dec 2019 13:11:18 +0200 Subject: [PATCH 08/11] fix --- Program.cs | 2 +- Properties/AssemblyInfo.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Program.cs b/Program.cs index b9d8c05..acb1944 100644 --- a/Program.cs +++ b/Program.cs @@ -54,7 +54,7 @@ static async Task Main(string[] args) if (newFiles.Contains(withoutZip)) { var info = new FileInfo(Path.Combine(localDirectory, withoutZip)); - if (info.LastWriteTimeUtc == entry.AsFile.ClientModified) + if ((info.LastWriteTimeUtc - entry.AsFile.ClientModified).TotalSeconds < 1f) newFiles.Remove(withoutZip); } else diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index 51cc94d..3058d36 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.26")] -[assembly: AssemblyFileVersion("1.0.0.25")] +[assembly: AssemblyVersion("1.0.0.28")] +[assembly: AssemblyFileVersion("1.0.0.27")] From 6beb83c399c6741fd2a91542df8b7ed670ba0c2a Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Thu, 23 Jan 2020 01:21:02 +0200 Subject: [PATCH 09/11] fixed hiding exception inside zipWriter using --- Program.cs | 251 ++++++++++++++++++++----------------- Properties/AssemblyInfo.cs | 4 +- 2 files changed, 139 insertions(+), 116 deletions(-) diff --git a/Program.cs b/Program.cs index acb1944..6f6a6b4 100644 --- a/Program.cs +++ b/Program.cs @@ -11,138 +11,161 @@ namespace DropboxEncrypedUploader { internal class Program { - static readonly char[] DirectorySeparatorChars = new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; - static async Task Main(string[] args) { - var token = args[0]; - var localDirectory = Path.GetFullPath(args[1]); - if (!IsEndingWithSeparator(localDirectory)) - localDirectory += Path.DirectorySeparatorChar; - var dropboxDirectory = args[2]; - if (!IsEndingWithSeparator(dropboxDirectory)) - dropboxDirectory += Path.AltDirectorySeparatorChar; - string password = args[3]; - - var newFiles = new HashSet( - Directory.GetFiles(localDirectory, "*", SearchOption.AllDirectories) - .Select(f => f.Substring(localDirectory.Length)), StringComparer.OrdinalIgnoreCase); - - var filesToDelete = new HashSet(); - - using (var dropbox = new DropboxClient(token)) + try { - try - { - await dropbox.Files.CreateFolderV2Async(dropboxDirectory.TrimEnd('/')); - } - catch - { - } - - - for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000); - list != null; - list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) + var token = args[0]; + var localDirectory = Path.GetFullPath(args[1]); + if (!IsEndingWithSeparator(localDirectory)) + localDirectory += Path.DirectorySeparatorChar; + var dropboxDirectory = args[2]; + if (!IsEndingWithSeparator(dropboxDirectory)) + dropboxDirectory += Path.AltDirectorySeparatorChar; + string password = args[3]; + + var newFiles = new HashSet( + Directory.GetFiles(localDirectory, "*", SearchOption.AllDirectories) + .Select(f => f.Substring(localDirectory.Length)), StringComparer.OrdinalIgnoreCase); + + var filesToDelete = new HashSet(); + + using (var dropbox = new DropboxClient(token)) { - foreach (var entry in list.Entries) + try + { + await dropbox.Files.CreateFolderV2Async(dropboxDirectory.TrimEnd('/')); + } + catch { - if (!entry.IsFile) continue; - var relativePath = entry.PathLower.Substring(dropboxDirectory.Length); - if (!relativePath.EndsWith(".zip")) continue; - var withoutZip = relativePath.Substring(0, relativePath.Length - 4).Replace("/", Path.DirectorySeparatorChar + ""); - if (newFiles.Contains(withoutZip)) - { - var info = new FileInfo(Path.Combine(localDirectory, withoutZip)); - if ((info.LastWriteTimeUtc - entry.AsFile.ClientModified).TotalSeconds < 1f) - newFiles.Remove(withoutZip); - } - else - filesToDelete.Add(entry.PathLower); } - } - if (filesToDelete.Count > 0) - { - Console.WriteLine($"Deleting files: \n{string.Join("\n", filesToDelete)}"); - await dropbox.Files.DeleteBatchAsync(filesToDelete.Select(x => new DeleteArg(x))); - } - if (newFiles.Count == 0) return; - Console.WriteLine($"Uploading files: {newFiles.Count}"); - ZipStrings.UseUnicode = true; - ZipStrings.CodePage = 65001; - var entryFactory = new ZipEntryFactory(); - byte[] msBuffer = new byte[1000 * 1000 * 150]; - int bufferSize = 1000 * 1000 * 140; - using var reader = new AsyncMultiFileReader(bufferSize, (f, t) => new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, true)); - var newFilesList = newFiles.ToList(); - for (int i = 0; i < newFilesList.Count; i++) - { - var relativePath = newFilesList[i]; - Console.Write($" {relativePath}"); - string fullPath = Path.Combine(localDirectory, relativePath); - reader.NextFile = (fullPath, null); - reader.OpenNextFile(); - if (i < newFilesList.Count - 1) - reader.NextFile = (Path.Combine(localDirectory, newFilesList[i + 1]), null); - - var info = new FileInfo(fullPath); - var clientModifiedAt = info.LastWriteTimeUtc; - using (var zipOutputStream = new CopyStream()) - using (var zipInputStream = new ZipOutputStream(zipOutputStream, bufferSize) { IsStreamOwner = false, Password = password, UseZip64 = UseZip64.On }) + for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000); + list != null; + list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) { - var bufferStream = new MemoryStream(msBuffer); - bufferStream.SetLength(0); - zipOutputStream.CopyTo = bufferStream; - zipInputStream.SetLevel(0); - var entry = entryFactory.MakeFileEntry(fullPath, '/' + Path.GetFileName(relativePath), true); - entry.AESKeySize = 256; - zipInputStream.PutNextEntry(entry); - UploadSessionStartResult session = null; - - long offset = 0; - int read; - while ((read = reader.ReadNextBlock()) > 0) + foreach (var entry in list.Entries) { - Console.Write($"\r {relativePath} {offset / (double) info.Length * 100:F0}%"); - zipInputStream.Write(reader.CurrentBuffer, 0, read); - zipInputStream.Flush(); - bufferStream.Position = 0; - var length = bufferStream.Length; - if (session == null) - session = await dropbox.Files.UploadSessionStartAsync(new UploadSessionStartArg(), bufferStream); + if (!entry.IsFile) continue; + var relativePath = entry.PathLower.Substring(dropboxDirectory.Length); + if (!relativePath.EndsWith(".zip")) continue; + var withoutZip = relativePath.Substring(0, relativePath.Length - 4).Replace("/", Path.DirectorySeparatorChar + ""); + if (newFiles.Contains(withoutZip)) + { + var info = new FileInfo(Path.Combine(localDirectory, withoutZip)); + if ((info.LastWriteTimeUtc - entry.AsFile.ClientModified).TotalSeconds < 1f) + newFiles.Remove(withoutZip); + } else - await dropbox.Files.UploadSessionAppendV2Async(new UploadSessionCursor(session.SessionId, (ulong) offset), false, bufferStream); - offset += length; - zipOutputStream.CopyTo = bufferStream = new MemoryStream(msBuffer); - bufferStream.SetLength(0); + filesToDelete.Add(entry.PathLower); } + } + + if (filesToDelete.Count > 0) + { + Console.WriteLine($"Deleting files: \n{string.Join("\n", filesToDelete)}"); + await dropbox.Files.DeleteBatchAsync(filesToDelete.Select(x => new DeleteArg(x))); + } - Console.Write($"\r {relativePath} 100%"); - Console.WriteLine(); - - zipInputStream.CloseEntry(); - zipInputStream.Finish(); - zipInputStream.Close(); - - bufferStream.Position = 0; - var commitInfo = new CommitInfo(Path.Combine(dropboxDirectory, relativePath.Replace("\\", "/")) + ".zip", - WriteMode.Overwrite.Instance, - false, - clientModifiedAt); - - if (session == null) - await dropbox.Files.UploadAsync(commitInfo, bufferStream); - else + if (newFiles.Count > 0) + { + Console.WriteLine($"Uploading files: {newFiles.Count}"); + ZipStrings.UseUnicode = true; + ZipStrings.CodePage = 65001; + var entryFactory = new ZipEntryFactory(); + byte[] msBuffer = new byte[1000 * 1000 * 150]; + int bufferSize = 1000 * 1000 * 140; + using var reader = new AsyncMultiFileReader(bufferSize, (f, t) => new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, true)); + var newFilesList = newFiles.ToList(); + for (int i = 0; i < newFilesList.Count; i++) { - await dropbox.Files.UploadSessionFinishAsync(new UploadSessionCursor(session.SessionId, (ulong) offset), commitInfo, bufferStream); + var relativePath = newFilesList[i]; + Console.Write($" {relativePath}"); + string fullPath = Path.Combine(localDirectory, relativePath); + reader.NextFile = (fullPath, null); + reader.OpenNextFile(); + if (i < newFilesList.Count - 1) + reader.NextFile = (Path.Combine(localDirectory, newFilesList[i + 1]), null); + + var info = new FileInfo(fullPath); + var clientModifiedAt = info.LastWriteTimeUtc; + using (var zipWriterUnderlyingStream = new CopyStream()) + { + var bufferStream = new MemoryStream(msBuffer); + bufferStream.SetLength(0); + + UploadSessionStartResult session = null; + long offset = 0; + + using (var zipWriter = new ZipOutputStream(zipWriterUnderlyingStream, bufferSize) { IsStreamOwner = false, Password = password, UseZip64 = UseZip64.On }) + { + try + { + zipWriterUnderlyingStream.CopyTo = bufferStream; + zipWriter.SetLevel(0); + var entry = entryFactory.MakeFileEntry(fullPath, '/' + Path.GetFileName(relativePath), true); + entry.AESKeySize = 256; + zipWriter.PutNextEntry(entry); + + int read; + while ((read = reader.ReadNextBlock()) > 0) + { + Console.Write($"\r {relativePath} {offset / (double) info.Length * 100:F0}%"); + zipWriter.Write(reader.CurrentBuffer, 0, read); + zipWriter.Flush(); + bufferStream.Position = 0; + var length = bufferStream.Length; + if (session == null) + session = await dropbox.Files.UploadSessionStartAsync(new UploadSessionStartArg(), bufferStream); + else + await dropbox.Files.UploadSessionAppendV2Async(new UploadSessionCursor(session.SessionId, (ulong) offset), false, bufferStream); + offset += length; + zipWriterUnderlyingStream.CopyTo = bufferStream = new MemoryStream(msBuffer); + bufferStream.SetLength(0); + } + + Console.Write($"\r {relativePath} 100%"); + Console.WriteLine(); + + zipWriter.CloseEntry(); + zipWriter.Finish(); + zipWriter.Close(); + } + catch + { + // disposing ZipOutputStream causes writing to bufferStream + if (!bufferStream.CanRead && !bufferStream.CanWrite) + zipWriterUnderlyingStream.CopyTo = bufferStream = new MemoryStream(msBuffer); + throw; + } + } + + bufferStream.Position = 0; + var commitInfo = new CommitInfo(Path.Combine(dropboxDirectory, relativePath.Replace("\\", "/")) + ".zip", + WriteMode.Overwrite.Instance, + false, + clientModifiedAt); + + if (session == null) + await dropbox.Files.UploadAsync(commitInfo, bufferStream); + else + { + await dropbox.Files.UploadSessionFinishAsync(new UploadSessionCursor(session.SessionId, (ulong) offset), commitInfo, bufferStream); + } + } } } } - } - Console.WriteLine("All done"); + Console.WriteLine("All done"); + } + catch (Exception e) + { + // redirecting error to normal output + Console.WriteLine(e); + throw; + } } static bool IsEndingWithSeparator(string s) diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index 3058d36..90ba907 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.28")] -[assembly: AssemblyFileVersion("1.0.0.27")] +[assembly: AssemblyVersion("1.0.0.32")] +[assembly: AssemblyFileVersion("1.0.0.31")] From c0fa859408ab493eddc5f9af2a4090aa4c478a90 Mon Sep 17 00:00:00 2001 From: Vladyslav Taranov Date: Sun, 1 Mar 2020 16:23:23 +0200 Subject: [PATCH 10/11] hacking dropbox to endlessly store my backups --- Program.cs | 100 +++++++++++++++++++++++++++++++++++-- Properties/AssemblyInfo.cs | 4 +- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/Program.cs b/Program.cs index 6f6a6b4..902259d 100644 --- a/Program.cs +++ b/Program.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Dropbox.Api; using Dropbox.Api.Files; @@ -41,13 +42,24 @@ static async Task Main(string[] args) } + var existingFiles = new HashSet(StringComparer.InvariantCultureIgnoreCase); + var existingFolders = new HashSet(StringComparer.InvariantCultureIgnoreCase); + existingFolders.Add(""); + for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000); list != null; list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) { foreach (var entry in list.Entries) { - if (!entry.IsFile) continue; + if (!entry.IsFile) + { + if (entry.IsFolder) + existingFolders.Add(entry.AsFolder.PathLower); + continue; + } + + existingFiles.Add(entry.PathLower); var relativePath = entry.PathLower.Substring(dropboxDirectory.Length); if (!relativePath.EndsWith(".zip")) continue; var withoutZip = relativePath.Substring(0, relativePath.Length - 4).Replace("/", Path.DirectorySeparatorChar + ""); @@ -62,10 +74,30 @@ static async Task Main(string[] args) } } - if (filesToDelete.Count > 0) + await DeleteFilesBatchAsync(); + + ulong deletingAccumulatedSize = 0; + + async Task DeleteFilesBatchAsync() { - Console.WriteLine($"Deleting files: \n{string.Join("\n", filesToDelete)}"); - await dropbox.Files.DeleteBatchAsync(filesToDelete.Select(x => new DeleteArg(x))); + if (filesToDelete.Count > 0) + { + Console.WriteLine($"Deleting files: \n{string.Join("\n", filesToDelete)}"); + var j = await dropbox.Files.DeleteBatchAsync(filesToDelete.Select(x => new DeleteArg(x))); + if (j.IsAsyncJobId) + { + + for (DeleteBatchJobStatus r = await dropbox.Files.DeleteBatchCheckAsync(j.AsAsyncJobId.Value); + r.IsInProgress; + r = await dropbox.Files.DeleteBatchCheckAsync(j.AsAsyncJobId.Value)) + { + Thread.Sleep(5000); + } + } + + filesToDelete.Clear(); + deletingAccumulatedSize = 0; + } } if (newFiles.Count > 0) @@ -156,6 +188,66 @@ static async Task Main(string[] args) } } } + + Console.WriteLine("Recycling deleted files for endless storage"); + + const ulong deletingBatchSize = 1024UL * 1024 * 1024 * 32; + + for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000, includeDeleted: true); + list != null; + list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) + { + foreach (var entry in list.Entries) + { + if (!entry.IsDeleted || existingFiles.Contains(entry.PathLower)) continue; + + var parentFolder = entry.PathLower; + int lastSlash = parentFolder.LastIndexOf('/'); + if (lastSlash == -1) continue; + parentFolder = parentFolder.Substring(0, lastSlash); + if (!existingFolders.Contains(parentFolder)) continue; + + ListRevisionsResult rev; + try + { + rev = await dropbox.Files.ListRevisionsAsync(entry.AsDeleted.PathLower, ListRevisionsMode.Path.Instance, 1); + + } + catch + { + // get revisions doesn't work for folders but no way to check if it's a folder beforehand + continue; + } + + if (!(DateTime.UtcNow - rev.ServerDeleted >= TimeSpan.FromDays(15)) || (DateTime.UtcNow - rev.ServerDeleted > TimeSpan.FromDays(29))) + { + // don't need to restore too young + // can't restore too old + continue; + } + + Console.WriteLine("Restoring " + entry.PathDisplay); + var restored = await dropbox.Files.RestoreAsync(entry.PathLower, rev.Entries.First().Rev); + + if (restored.AsFile.Size >= deletingBatchSize && filesToDelete.Count == 0) + { + Console.WriteLine("Deleting " + entry.PathDisplay); + await dropbox.Files.DeleteV2Async(restored.PathLower, restored.Rev); + } + else + { + // warning: rev not included, concurrent modification changes may be lost + filesToDelete.Add(restored.PathLower); + deletingAccumulatedSize += restored.Size; + + if (deletingAccumulatedSize >= deletingBatchSize) + await DeleteFilesBatchAsync(); + } + + } + } + await DeleteFilesBatchAsync(); + } Console.WriteLine("All done"); diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index 90ba907..29a70f8 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.32")] -[assembly: AssemblyFileVersion("1.0.0.31")] +[assembly: AssemblyVersion("1.0.0.46")] +[assembly: AssemblyFileVersion("1.0.0.45")] From 780dd5739bf8138b326de8a42fac6312a2c3fc07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Feb 2022 16:25:36 +0000 Subject: [PATCH 11/11] Bump SharpZipLib from 1.2.0 to 1.3.3 Bumps [SharpZipLib](https://github.com/icsharpcode/SharpZipLib) from 1.2.0 to 1.3.3. - [Release notes](https://github.com/icsharpcode/SharpZipLib/releases) - [Changelog](https://github.com/icsharpcode/SharpZipLib/blob/master/docs/Changes.txt) - [Commits](https://github.com/icsharpcode/SharpZipLib/compare/v1.2.0...v1.3.3) --- updated-dependencies: - dependency-name: SharpZipLib dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- packages.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.config b/packages.config index 480820a..b0c7381 100644 --- a/packages.config +++ b/packages.config @@ -2,5 +2,5 @@ - + \ No newline at end of file