From 44bfab0043cf33d5ef31162f95c938cc1547a5bb Mon Sep 17 00:00:00 2001 From: mgabelmann Date: Sun, 29 Dec 2024 17:29:21 -0800 Subject: [PATCH 1/3] last modified didnt seem to be working correctly. added a change so that if last modified date is different, that a checksum can be calculated to see if the local/remote timestamps are out of sync. if the checksum is the same the file is not copied, otherwise the file is copied. --- .../mgabelmann/photo/workflow/io/Backup.java | 95 +++++++++++++------ 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/src/main/java/mgabelmann/photo/workflow/io/Backup.java b/src/main/java/mgabelmann/photo/workflow/io/Backup.java index 8d96bd3..cbab5b0 100644 --- a/src/main/java/mgabelmann/photo/workflow/io/Backup.java +++ b/src/main/java/mgabelmann/photo/workflow/io/Backup.java @@ -29,15 +29,28 @@ public final class Backup extends AbstractWorkflow { /** Service that ensures that the file checksums are threaded for optimum performance. */ private final ExecutorService service; + /** Take action, if false no changes are made by the application. */ + private boolean action = false; + + /** Use checksum to verify if last modified date is different. */ + private boolean useChecksum = false; + /** * Main method. * @param args arguments */ public static void main(final String[] args) { +// Backup backup = new Backup( +// new File("P:/Mike/catalog1/03_raw/01_working/2024/2024-01-26"), +// new File("Z:/catalog1/03_raw/01_working/2024/2024-01-26"), +// true +// ); + Backup backup = new Backup( - new File("P:/Mike/catalog1/03_raw/01_working/2024"), - new File("Z:/catalog1/03_raw/01_working/2024") + new File("P:/Mike/catalog1/03_raw/01_working/2024"), + new File("Z:/catalog1/03_raw/01_working/2024"), + true ); try { @@ -52,9 +65,10 @@ public static void main(final String[] args) { * Constructor. * @param workDir dirLocal local directory (original files) * @param dirRemote dirRemote remote directory (backup files) + * @param action */ - public Backup(final File workDir, final File dirRemote) { - this(workDir, dirRemote, DEFAULT_HASHTYPE, DEFAULT_VERIFY); + public Backup(final File workDir, final File dirRemote, final boolean action) { + this(workDir, dirRemote, DEFAULT_HASHTYPE, DEFAULT_VERIFY, action); } /** @@ -64,9 +78,10 @@ public Backup(final File workDir, final File dirRemote) { * @param type checksum type * @param verify verify copy */ - public Backup(final File dirLocal, final File dirRemote, final HashType type, final boolean verify) { + public Backup(final File dirLocal, final File dirRemote, final HashType type, final boolean verify, final boolean action) { super(dirLocal, dirRemote, type, verify); - + + this.action = action; this.service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 5); } @@ -81,9 +96,9 @@ public void process() throws WorkflowException { service.shutdown(); - boolean timeout = service.awaitTermination(15, TimeUnit.SECONDS); + boolean timeout = service.awaitTermination(5, TimeUnit.SECONDS); - if (timeout) { + if (!timeout) { LOG.warn("service timed out"); } @@ -158,46 +173,64 @@ private void backupDirectory( * @throws IOException error backing up file */ private void backupFile(final File localFile, final File remoteFile) throws IOException { - boolean copied = true; + boolean copied = false; //FIXME: switch to do this instead //BasicFileAttributes attributesLocal = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class); //BasicFileAttributes attributesRemote = Files.readAttributes(remoteFile.toPath(), BasicFileAttributes.class); - if (remoteFile.exists()) { - /*if (attributesLocal.size() != attributesRemote.size()) { + boolean equalLength = localFile.length() == remoteFile.length(); + boolean equalLastModified = localFile.lastModified() == remoteFile.lastModified(); + if (!equalLength) { + LOG.info("FILE: {} {} - different length", localFile.getAbsolutePath(), (action ? "replacing" : "")); - } else if (attributesLocal.lastModifiedTime() != attributesRemote.lastModifiedTime()) { + if (action) { + FileUtil.copyFile(localFile, remoteFile, true); + copied = true; + } + } else if (!equalLastModified) { + if (useChecksum) { + boolean equalChecksum = FileUtil.verifyCopy(localFile, remoteFile, type); - } else { + if (!equalChecksum) { + LOG.info("FILE: {} {} - different last modified and checksum", localFile.getAbsolutePath(), (action ? "replacing" : "")); - }*/ + if (action) { + FileUtil.copyFile(localFile, remoteFile, true); + copied = true; + } + + } else { + //last modified different, but files have same checksum + LOG.info("FILE: {} {} - different last modified, equal checksum", localFile.getAbsolutePath(), (action ? "skipping" : "")); + } + + } else { + //faster, but could be error-prone + LOG.info("FILE: {} {} - different last modified", localFile.getAbsolutePath(), (action ? "replacing" : "")); + + if (action) { + FileUtil.copyFile(localFile, remoteFile, true); + copied = true; + } + } - if (localFile.length() != remoteFile.length()) { - //different file lengths - LOG.info("FILE: {} replacing - different length", localFile.getAbsolutePath()); - - FileUtil.copyFile(localFile, remoteFile, true); - - } else if (localFile.lastModified() != remoteFile.lastModified()) { - //same file lengths and different timestamps - LOG.info("FILE: {} replacing - different timestamp", localFile.getAbsolutePath()); - - FileUtil.copyFile(localFile, remoteFile, true); - } else { //same file length and timestamps - LOG.debug("FILE: {} skipping - identical", localFile.getAbsolutePath()); - copied = false; + LOG.debug("FILE: {} {} - identical", localFile.getAbsolutePath(), (action ? "skipping" : "")); } } else { - //copy file - LOG.info("FILE: {} copying - new", localFile.getAbsolutePath()); - FileUtil.copyFile(localFile, remoteFile, true); + //copy file since it does not exist in remote location + LOG.info("FILE: {} {} - new", localFile.getAbsolutePath(), (action ? "copying" : "")); + + if (action) { + FileUtil.copyFile(localFile, remoteFile, true); + copied = true; + } } //if copied, verify it From 5466a9483b9deaea07bd734a41e09dcf150356d1 Mon Sep 17 00:00:00 2001 From: mgabelmann Date: Thu, 2 Jan 2025 13:06:47 -0800 Subject: [PATCH 2/3] Fixed an issue where the count was off by 1 when splitting titles into groups of 1950 characters or less. First group was +1 and last group was -1. Simply moved the increment after the length check --- src/main/java/mgabelmann/photo/copyright/Copyright.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/mgabelmann/photo/copyright/Copyright.java b/src/main/java/mgabelmann/photo/copyright/Copyright.java index 296ff80..652e5ba 100644 --- a/src/main/java/mgabelmann/photo/copyright/Copyright.java +++ b/src/main/java/mgabelmann/photo/copyright/Copyright.java @@ -473,7 +473,6 @@ List getAllTitles(final Map> titleRecords) { date = value.getDate().getMonth().toString(); } - ++counter; String name = value.getName(); if (sb.length() + name.length() > TITLES_GROUP_MAX_CHARACTERS) { @@ -485,6 +484,7 @@ List getAllTitles(final Map> titleRecords) { counter = 0; } + ++counter; sb.append(name).append(", "); } From 37a5d2df35f75bb9184618bb9abb17532428518e Mon Sep 17 00:00:00 2001 From: mgabelmann Date: Thu, 2 Jan 2025 13:12:08 -0800 Subject: [PATCH 3/3] fixed unit test failure --- .../photo/workflow/io/BackupTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/test/java/mgabelmann/photo/workflow/io/BackupTest.java b/src/test/java/mgabelmann/photo/workflow/io/BackupTest.java index 5b0d7a9..428b65d 100644 --- a/src/test/java/mgabelmann/photo/workflow/io/BackupTest.java +++ b/src/test/java/mgabelmann/photo/workflow/io/BackupTest.java @@ -50,7 +50,7 @@ void test1_process() { HashType type = HashType.SHA256; boolean verify = false; - IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(null, dstDir.toFile(), type, verify)); + IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(null, dstDir.toFile(), type, verify, true)); Assertions.assertEquals("dirLocal cannot be null", iae.getMessage()); } @@ -62,7 +62,7 @@ void test2_process() { HashType type = HashType.SHA256; boolean verify = false; - IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify)); + IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true)); Assertions.assertEquals("dirLocal does not exist", iae.getMessage()); } @@ -73,7 +73,7 @@ void test3_process() { HashType type = HashType.SHA256; boolean verify = false; - IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), null, type, verify)); + IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), null, type, verify, true)); Assertions.assertEquals("dirRemote cannot be null", iae.getMessage()); } @@ -85,7 +85,7 @@ void test4_process() { HashType type = HashType.SHA256; boolean verify = false; - IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify)); + IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true)); Assertions.assertEquals("dirRemote does not exist", iae.getMessage()); } @@ -97,7 +97,7 @@ void test5_process() { HashType type = null; boolean verify = false; - IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify)); + IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true)); Assertions.assertEquals("type cannot be null", iae.getMessage()); } @@ -120,7 +120,7 @@ void test7_process() throws Exception { Assertions.fail("could not set to read only"); } - IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify)); + IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true)); Assertions.assertEquals("dirRemote is not writable", iae.getMessage()); Assertions.assertTrue(dstDir.toFile().setWritable(true)); @@ -135,7 +135,7 @@ void test10_process() throws Exception { Path dstDir = this.createDirectory(tempDir, "dstDir"); - Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false); + Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true); b.process(); Path dstDir2 = Paths.get(dstDir + File.separator + "srcDir2"); @@ -152,7 +152,7 @@ void test11_process() throws Exception { Path srcFile1 = this.createFile(srcDir, "srcFile1.jpg"); - Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false); + Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true); b.process(); Path dstFile1 = Paths.get(dstDir + File.separator + "srcFile1.jpg"); @@ -169,7 +169,7 @@ void test12_process() throws Exception { Path srcFile1 = this.createFileWithData(srcDir, "srcFile1.jpg", "updated data"); Path dstFile1 = this.createFileWithData(dstDir, "srcFile1.jpg", "old data"); - Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false); + Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true); b.process(); Assertions.assertEquals(12, dstFile1.toFile().length()); @@ -186,7 +186,7 @@ void test13_process() throws Exception { Assertions.assertTrue(dstFile1.toFile().setLastModified(Instant.now().minus(1, ChronoUnit.DAYS).toEpochMilli())); - Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false); + Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true); b.process(); Assertions.assertEquals(srcFile1.toFile().lastModified(), dstFile1.toFile().lastModified()); @@ -203,7 +203,7 @@ void test14_process() throws Exception { Assertions.assertTrue(dstFile1.toFile().setLastModified(srcFile1.toFile().lastModified())); - Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false); + Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true); b.process(); Assertions.assertEquals(srcFile1.toFile().lastModified(), dstFile1.toFile().lastModified()); @@ -226,7 +226,7 @@ void test15_process() throws Exception { Path srcFile1 = this.createFileWithData(srcDir, "srcFile1.jpg", "updated data"); - Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false); + Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false,true); IOException ie = Assertions.assertThrows(IOException.class, b::process); Assertions.assertEquals("unable to create directory " + dstDir.toAbsolutePath(), ie.getMessage());