diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1f2ff27 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Vladyslav Taranov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Program.cs b/Program.cs index b9d8c05..902259d 100644 --- a/Program.cs +++ b/Program.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Dropbox.Api; using Dropbox.Api.Files; @@ -11,138 +12,252 @@ namespace DropboxEncrypedUploader { internal class Program { - static readonly char[] DirectorySeparatorChars = new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; - static async Task Main(string[] args) { - var token = args[0]; - var localDirectory = Path.GetFullPath(args[1]); - if (!IsEndingWithSeparator(localDirectory)) - localDirectory += Path.DirectorySeparatorChar; - var dropboxDirectory = args[2]; - if (!IsEndingWithSeparator(dropboxDirectory)) - dropboxDirectory += Path.AltDirectorySeparatorChar; - string password = args[3]; - - var newFiles = new HashSet( - Directory.GetFiles(localDirectory, "*", SearchOption.AllDirectories) - .Select(f => f.Substring(localDirectory.Length)), StringComparer.OrdinalIgnoreCase); - - var filesToDelete = new HashSet(); - - using (var dropbox = new DropboxClient(token)) + try { - try - { - await dropbox.Files.CreateFolderV2Async(dropboxDirectory.TrimEnd('/')); - } - catch - { - } + var token = args[0]; + var localDirectory = Path.GetFullPath(args[1]); + if (!IsEndingWithSeparator(localDirectory)) + localDirectory += Path.DirectorySeparatorChar; + var dropboxDirectory = args[2]; + if (!IsEndingWithSeparator(dropboxDirectory)) + dropboxDirectory += Path.AltDirectorySeparatorChar; + string password = args[3]; + + var newFiles = new HashSet( + Directory.GetFiles(localDirectory, "*", SearchOption.AllDirectories) + .Select(f => f.Substring(localDirectory.Length)), StringComparer.OrdinalIgnoreCase); + var filesToDelete = new HashSet(); - for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000); - list != null; - list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) + using (var dropbox = new DropboxClient(token)) { - foreach (var entry in list.Entries) + try + { + await dropbox.Files.CreateFolderV2Async(dropboxDirectory.TrimEnd('/')); + } + catch { - if (!entry.IsFile) continue; - var relativePath = entry.PathLower.Substring(dropboxDirectory.Length); - if (!relativePath.EndsWith(".zip")) continue; - var withoutZip = relativePath.Substring(0, relativePath.Length - 4).Replace("/", Path.DirectorySeparatorChar + ""); - if (newFiles.Contains(withoutZip)) + } + + + var existingFiles = new HashSet(StringComparer.InvariantCultureIgnoreCase); + var existingFolders = new HashSet(StringComparer.InvariantCultureIgnoreCase); + existingFolders.Add(""); + + for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000); + list != null; + list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) + { + foreach (var entry in list.Entries) { - var info = new FileInfo(Path.Combine(localDirectory, withoutZip)); - if (info.LastWriteTimeUtc == entry.AsFile.ClientModified) - newFiles.Remove(withoutZip); + if (!entry.IsFile) + { + if (entry.IsFolder) + existingFolders.Add(entry.AsFolder.PathLower); + continue; + } + + existingFiles.Add(entry.PathLower); + var relativePath = entry.PathLower.Substring(dropboxDirectory.Length); + if (!relativePath.EndsWith(".zip")) continue; + var withoutZip = relativePath.Substring(0, relativePath.Length - 4).Replace("/", Path.DirectorySeparatorChar + ""); + if (newFiles.Contains(withoutZip)) + { + var info = new FileInfo(Path.Combine(localDirectory, withoutZip)); + if ((info.LastWriteTimeUtc - entry.AsFile.ClientModified).TotalSeconds < 1f) + newFiles.Remove(withoutZip); + } + else + filesToDelete.Add(entry.PathLower); } - else - filesToDelete.Add(entry.PathLower); } - } - if (filesToDelete.Count > 0) - { - Console.WriteLine($"Deleting files: \n{string.Join("\n", filesToDelete)}"); - await dropbox.Files.DeleteBatchAsync(filesToDelete.Select(x => new DeleteArg(x))); - } + await DeleteFilesBatchAsync(); - if (newFiles.Count == 0) return; - Console.WriteLine($"Uploading files: {newFiles.Count}"); - ZipStrings.UseUnicode = true; - ZipStrings.CodePage = 65001; - var entryFactory = new ZipEntryFactory(); - byte[] msBuffer = new byte[1000 * 1000 * 150]; - int bufferSize = 1000 * 1000 * 140; - using var reader = new AsyncMultiFileReader(bufferSize, (f, t) => new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, true)); - var newFilesList = newFiles.ToList(); - for (int i = 0; i < newFilesList.Count; i++) - { - var relativePath = newFilesList[i]; - Console.Write($" {relativePath}"); - string fullPath = Path.Combine(localDirectory, relativePath); - reader.NextFile = (fullPath, null); - reader.OpenNextFile(); - if (i < newFilesList.Count - 1) - reader.NextFile = (Path.Combine(localDirectory, newFilesList[i + 1]), null); - - var info = new FileInfo(fullPath); - var clientModifiedAt = info.LastWriteTimeUtc; - using (var zipOutputStream = new CopyStream()) - using (var zipInputStream = new ZipOutputStream(zipOutputStream, bufferSize) { IsStreamOwner = false, Password = password, UseZip64 = UseZip64.On }) + ulong deletingAccumulatedSize = 0; + + async Task DeleteFilesBatchAsync() { - var bufferStream = new MemoryStream(msBuffer); - bufferStream.SetLength(0); - zipOutputStream.CopyTo = bufferStream; - zipInputStream.SetLevel(0); - var entry = entryFactory.MakeFileEntry(fullPath, '/' + Path.GetFileName(relativePath), true); - entry.AESKeySize = 256; - zipInputStream.PutNextEntry(entry); - UploadSessionStartResult session = null; - - long offset = 0; - int read; - while ((read = reader.ReadNextBlock()) > 0) + if (filesToDelete.Count > 0) { - Console.Write($"\r {relativePath} {offset / (double) info.Length * 100:F0}%"); - zipInputStream.Write(reader.CurrentBuffer, 0, read); - zipInputStream.Flush(); - bufferStream.Position = 0; - var length = bufferStream.Length; - if (session == null) - session = await dropbox.Files.UploadSessionStartAsync(new UploadSessionStartArg(), bufferStream); - else - await dropbox.Files.UploadSessionAppendV2Async(new UploadSessionCursor(session.SessionId, (ulong) offset), false, bufferStream); - offset += length; - zipOutputStream.CopyTo = bufferStream = new MemoryStream(msBuffer); - bufferStream.SetLength(0); + Console.WriteLine($"Deleting files: \n{string.Join("\n", filesToDelete)}"); + var j = await dropbox.Files.DeleteBatchAsync(filesToDelete.Select(x => new DeleteArg(x))); + if (j.IsAsyncJobId) + { + + for (DeleteBatchJobStatus r = await dropbox.Files.DeleteBatchCheckAsync(j.AsAsyncJobId.Value); + r.IsInProgress; + r = await dropbox.Files.DeleteBatchCheckAsync(j.AsAsyncJobId.Value)) + { + Thread.Sleep(5000); + } + } + + filesToDelete.Clear(); + deletingAccumulatedSize = 0; } + } - Console.Write($"\r {relativePath} 100%"); - Console.WriteLine(); - - zipInputStream.CloseEntry(); - zipInputStream.Finish(); - zipInputStream.Close(); - - bufferStream.Position = 0; - var commitInfo = new CommitInfo(Path.Combine(dropboxDirectory, relativePath.Replace("\\", "/")) + ".zip", - WriteMode.Overwrite.Instance, - false, - clientModifiedAt); - - if (session == null) - await dropbox.Files.UploadAsync(commitInfo, bufferStream); - else + if (newFiles.Count > 0) + { + Console.WriteLine($"Uploading files: {newFiles.Count}"); + ZipStrings.UseUnicode = true; + ZipStrings.CodePage = 65001; + var entryFactory = new ZipEntryFactory(); + byte[] msBuffer = new byte[1000 * 1000 * 150]; + int bufferSize = 1000 * 1000 * 140; + using var reader = new AsyncMultiFileReader(bufferSize, (f, t) => new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, true)); + var newFilesList = newFiles.ToList(); + for (int i = 0; i < newFilesList.Count; i++) { - await dropbox.Files.UploadSessionFinishAsync(new UploadSessionCursor(session.SessionId, (ulong) offset), commitInfo, bufferStream); + var relativePath = newFilesList[i]; + Console.Write($" {relativePath}"); + string fullPath = Path.Combine(localDirectory, relativePath); + reader.NextFile = (fullPath, null); + reader.OpenNextFile(); + if (i < newFilesList.Count - 1) + reader.NextFile = (Path.Combine(localDirectory, newFilesList[i + 1]), null); + + var info = new FileInfo(fullPath); + var clientModifiedAt = info.LastWriteTimeUtc; + using (var zipWriterUnderlyingStream = new CopyStream()) + { + var bufferStream = new MemoryStream(msBuffer); + bufferStream.SetLength(0); + + UploadSessionStartResult session = null; + long offset = 0; + + using (var zipWriter = new ZipOutputStream(zipWriterUnderlyingStream, bufferSize) { IsStreamOwner = false, Password = password, UseZip64 = UseZip64.On }) + { + try + { + zipWriterUnderlyingStream.CopyTo = bufferStream; + zipWriter.SetLevel(0); + var entry = entryFactory.MakeFileEntry(fullPath, '/' + Path.GetFileName(relativePath), true); + entry.AESKeySize = 256; + zipWriter.PutNextEntry(entry); + + int read; + while ((read = reader.ReadNextBlock()) > 0) + { + Console.Write($"\r {relativePath} {offset / (double) info.Length * 100:F0}%"); + zipWriter.Write(reader.CurrentBuffer, 0, read); + zipWriter.Flush(); + bufferStream.Position = 0; + var length = bufferStream.Length; + if (session == null) + session = await dropbox.Files.UploadSessionStartAsync(new UploadSessionStartArg(), bufferStream); + else + await dropbox.Files.UploadSessionAppendV2Async(new UploadSessionCursor(session.SessionId, (ulong) offset), false, bufferStream); + offset += length; + zipWriterUnderlyingStream.CopyTo = bufferStream = new MemoryStream(msBuffer); + bufferStream.SetLength(0); + } + + Console.Write($"\r {relativePath} 100%"); + Console.WriteLine(); + + zipWriter.CloseEntry(); + zipWriter.Finish(); + zipWriter.Close(); + } + catch + { + // disposing ZipOutputStream causes writing to bufferStream + if (!bufferStream.CanRead && !bufferStream.CanWrite) + zipWriterUnderlyingStream.CopyTo = bufferStream = new MemoryStream(msBuffer); + throw; + } + } + + bufferStream.Position = 0; + var commitInfo = new CommitInfo(Path.Combine(dropboxDirectory, relativePath.Replace("\\", "/")) + ".zip", + WriteMode.Overwrite.Instance, + false, + clientModifiedAt); + + if (session == null) + await dropbox.Files.UploadAsync(commitInfo, bufferStream); + else + { + await dropbox.Files.UploadSessionFinishAsync(new UploadSessionCursor(session.SessionId, (ulong) offset), commitInfo, bufferStream); + } + } + } + } + + Console.WriteLine("Recycling deleted files for endless storage"); + + const ulong deletingBatchSize = 1024UL * 1024 * 1024 * 32; + + for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000, includeDeleted: true); + list != null; + list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) + { + foreach (var entry in list.Entries) + { + if (!entry.IsDeleted || existingFiles.Contains(entry.PathLower)) continue; + + var parentFolder = entry.PathLower; + int lastSlash = parentFolder.LastIndexOf('/'); + if (lastSlash == -1) continue; + parentFolder = parentFolder.Substring(0, lastSlash); + if (!existingFolders.Contains(parentFolder)) continue; + + ListRevisionsResult rev; + try + { + rev = await dropbox.Files.ListRevisionsAsync(entry.AsDeleted.PathLower, ListRevisionsMode.Path.Instance, 1); + + } + catch + { + // get revisions doesn't work for folders but no way to check if it's a folder beforehand + continue; + } + + if (!(DateTime.UtcNow - rev.ServerDeleted >= TimeSpan.FromDays(15)) || (DateTime.UtcNow - rev.ServerDeleted > TimeSpan.FromDays(29))) + { + // don't need to restore too young + // can't restore too old + continue; + } + + Console.WriteLine("Restoring " + entry.PathDisplay); + var restored = await dropbox.Files.RestoreAsync(entry.PathLower, rev.Entries.First().Rev); + + if (restored.AsFile.Size >= deletingBatchSize && filesToDelete.Count == 0) + { + Console.WriteLine("Deleting " + entry.PathDisplay); + await dropbox.Files.DeleteV2Async(restored.PathLower, restored.Rev); + } + else + { + // warning: rev not included, concurrent modification changes may be lost + filesToDelete.Add(restored.PathLower); + deletingAccumulatedSize += restored.Size; + + if (deletingAccumulatedSize >= deletingBatchSize) + await DeleteFilesBatchAsync(); + } + } } + await DeleteFilesBatchAsync(); + } - } - Console.WriteLine("All done"); + Console.WriteLine("All done"); + } + catch (Exception e) + { + // redirecting error to normal output + Console.WriteLine(e); + throw; + } } static bool IsEndingWithSeparator(string s) diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index 51cc94d..29a70f8 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.26")] -[assembly: AssemblyFileVersion("1.0.0.25")] +[assembly: AssemblyVersion("1.0.0.46")] +[assembly: AssemblyFileVersion("1.0.0.45")] diff --git a/README.md b/README.md new file mode 100644 index 0000000..2c21e1d --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +# DropboxEncryptedUploader +Synchronizes local folder to Dropbox encrypting each file to a separate zip archive. Works in memory without storing to disk. Each file is devided by 140mb blocks, each block is encrypted on the fly and immediately sent to Dropbox servers so no big archive file is kept neither in RAM nor on your disk. +## Usage +DropboxEncryptedUploader.exe dropbox-app-token path-to-folder dropbox-folder-name encryption-password +### Example +DropboxEncryptedUploader.exe "asdlakdfkfrefggfdgdfg-rgedfgd-adfsfdf3e" "d:\Backups" "/Backups" "password" + +## How to get Dropbox token +http://99rabbits.com/get-dropbox-access-token/ diff --git a/packages.config b/packages.config index 480820a..b0c7381 100644 --- a/packages.config +++ b/packages.config @@ -2,5 +2,5 @@ - + \ No newline at end of file