Skip to content

Feature/lastmodified #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mgabelmann merged 3 commits into from
Jan 2, 2025
Merged

Feature/lastmodified #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
44bfab0
last modified didnt seem to be working correctly. added a change so t…
mgabelmann Dec 30, 2024
5466a94
Fixed an issue where the count was off by 1 when splitting titles int…
mgabelmann Jan 2, 2025
37a5d2d
fixed unit test failure
mgabelmann Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/mgabelmann/photo/copyright/Copyright.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,6 @@ List<String> getAllTitles(final Map<String, List<FileInfo>> titleRecords) {
date = value.getDate().getMonth().toString();
}

++counter;
String name = value.getName();

if (sb.length() + name.length() > TITLES_GROUP_MAX_CHARACTERS) {
Expand All @@ -485,6 +484,7 @@ List<String> getAllTitles(final Map<String, List<FileInfo>> titleRecords) {
counter = 0;
}

++counter;
sb.append(name).append(", ");
}

Expand Down
95 changes: 64 additions & 31 deletions src/main/java/mgabelmann/photo/workflow/io/Backup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,28 @@ public final class Backup extends AbstractWorkflow {
/** Service that ensures that the file checksums are threaded for optimum performance. */
private final ExecutorService service;

/** Take action, if false no changes are made by the application. */
private boolean action = false;

/** Use checksum to verify if last modified date is different. */
private boolean useChecksum = false;


/**
* Main method.
* @param args arguments
*/
public static void main(final String[] args) {
// Backup backup = new Backup(
// new File("P:/Mike/catalog1/03_raw/01_working/2024/2024-01-26"),
// new File("Z:/catalog1/03_raw/01_working/2024/2024-01-26"),
// true
// );

Backup backup = new Backup(
new File("P:/Mike/catalog1/03_raw/01_working/2024"),
new File("Z:/catalog1/03_raw/01_working/2024")
new File("P:/Mike/catalog1/03_raw/01_working/2024"),
new File("Z:/catalog1/03_raw/01_working/2024"),
true
);

try {
Expand All @@ -52,9 +65,10 @@ public static void main(final String[] args) {
* Constructor.
* @param workDir dirLocal local directory (original files)
* @param dirRemote dirRemote remote directory (backup files)
* @param action
*/
public Backup(final File workDir, final File dirRemote) {
this(workDir, dirRemote, DEFAULT_HASHTYPE, DEFAULT_VERIFY);
public Backup(final File workDir, final File dirRemote, final boolean action) {
this(workDir, dirRemote, DEFAULT_HASHTYPE, DEFAULT_VERIFY, action);
}

/**
Expand All @@ -64,9 +78,10 @@ public Backup(final File workDir, final File dirRemote) {
* @param type checksum type
* @param verify verify copy
*/
public Backup(final File dirLocal, final File dirRemote, final HashType type, final boolean verify) {
public Backup(final File dirLocal, final File dirRemote, final HashType type, final boolean verify, final boolean action) {
super(dirLocal, dirRemote, type, verify);


this.action = action;
this.service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 5);
}

Expand All @@ -81,9 +96,9 @@ public void process() throws WorkflowException {

service.shutdown();

boolean timeout = service.awaitTermination(15, TimeUnit.SECONDS);
boolean timeout = service.awaitTermination(5, TimeUnit.SECONDS);

if (timeout) {
if (!timeout) {
LOG.warn("service timed out");
}

Expand Down Expand Up @@ -158,46 +173,64 @@ private void backupDirectory(
* @throws IOException error backing up file
*/
private void backupFile(final File localFile, final File remoteFile) throws IOException {
boolean copied = true;
boolean copied = false;

//FIXME: switch to do this instead
//BasicFileAttributes attributesLocal = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class);
//BasicFileAttributes attributesRemote = Files.readAttributes(remoteFile.toPath(), BasicFileAttributes.class);


if (remoteFile.exists()) {
/*if (attributesLocal.size() != attributesRemote.size()) {
boolean equalLength = localFile.length() == remoteFile.length();
boolean equalLastModified = localFile.lastModified() == remoteFile.lastModified();

if (!equalLength) {
LOG.info("FILE: {} {} - different length", localFile.getAbsolutePath(), (action ? "replacing" : ""));

} else if (attributesLocal.lastModifiedTime() != attributesRemote.lastModifiedTime()) {
if (action) {
FileUtil.copyFile(localFile, remoteFile, true);
copied = true;
}

} else if (!equalLastModified) {
if (useChecksum) {
boolean equalChecksum = FileUtil.verifyCopy(localFile, remoteFile, type);

} else {
if (!equalChecksum) {
LOG.info("FILE: {} {} - different last modified and checksum", localFile.getAbsolutePath(), (action ? "replacing" : ""));

}*/
if (action) {
FileUtil.copyFile(localFile, remoteFile, true);
copied = true;
}

} else {
//last modified different, but files have same checksum
LOG.info("FILE: {} {} - different last modified, equal checksum", localFile.getAbsolutePath(), (action ? "skipping" : ""));
}

} else {
//faster, but could be error-prone
LOG.info("FILE: {} {} - different last modified", localFile.getAbsolutePath(), (action ? "replacing" : ""));

if (action) {
FileUtil.copyFile(localFile, remoteFile, true);
copied = true;
}
}

if (localFile.length() != remoteFile.length()) {
//different file lengths
LOG.info("FILE: {} replacing - different length", localFile.getAbsolutePath());

FileUtil.copyFile(localFile, remoteFile, true);

} else if (localFile.lastModified() != remoteFile.lastModified()) {
//same file lengths and different timestamps
LOG.info("FILE: {} replacing - different timestamp", localFile.getAbsolutePath());

FileUtil.copyFile(localFile, remoteFile, true);

} else {
//same file length and timestamps
LOG.debug("FILE: {} skipping - identical", localFile.getAbsolutePath());
copied = false;
LOG.debug("FILE: {} {} - identical", localFile.getAbsolutePath(), (action ? "skipping" : ""));
}

} else {
//copy file
LOG.info("FILE: {} copying - new", localFile.getAbsolutePath());
FileUtil.copyFile(localFile, remoteFile, true);
//copy file since it does not exist in remote location
LOG.info("FILE: {} {} - new", localFile.getAbsolutePath(), (action ? "copying" : ""));

if (action) {
FileUtil.copyFile(localFile, remoteFile, true);
copied = true;
}
}

//if copied, verify it
Expand Down
24 changes: 12 additions & 12 deletions src/test/java/mgabelmann/photo/workflow/io/BackupTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ void test1_process() {
HashType type = HashType.SHA256;
boolean verify = false;

IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(null, dstDir.toFile(), type, verify));
IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(null, dstDir.toFile(), type, verify, true));
Assertions.assertEquals("dirLocal cannot be null", iae.getMessage());
}

Expand All @@ -62,7 +62,7 @@ void test2_process() {
HashType type = HashType.SHA256;
boolean verify = false;

IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify));
IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true));
Assertions.assertEquals("dirLocal does not exist", iae.getMessage());
}

Expand All @@ -73,7 +73,7 @@ void test3_process() {
HashType type = HashType.SHA256;
boolean verify = false;

IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), null, type, verify));
IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), null, type, verify, true));
Assertions.assertEquals("dirRemote cannot be null", iae.getMessage());
}

Expand All @@ -85,7 +85,7 @@ void test4_process() {
HashType type = HashType.SHA256;
boolean verify = false;

IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify));
IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true));
Assertions.assertEquals("dirRemote does not exist", iae.getMessage());
}

Expand All @@ -97,7 +97,7 @@ void test5_process() {
HashType type = null;
boolean verify = false;

IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify));
IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true));
Assertions.assertEquals("type cannot be null", iae.getMessage());
}

Expand All @@ -120,7 +120,7 @@ void test7_process() throws Exception {
Assertions.fail("could not set to read only");
}

IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify));
IllegalArgumentException iae = Assertions.assertThrows(IllegalArgumentException.class, () -> new Backup(srcDir.toFile(), dstDir.toFile(), type, verify, true));
Assertions.assertEquals("dirRemote is not writable", iae.getMessage());

Assertions.assertTrue(dstDir.toFile().setWritable(true));
Expand All @@ -135,7 +135,7 @@ void test10_process() throws Exception {

Path dstDir = this.createDirectory(tempDir, "dstDir");

Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false);
Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true);
b.process();

Path dstDir2 = Paths.get(dstDir + File.separator + "srcDir2");
Expand All @@ -152,7 +152,7 @@ void test11_process() throws Exception {

Path srcFile1 = this.createFile(srcDir, "srcFile1.jpg");

Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false);
Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true);
b.process();

Path dstFile1 = Paths.get(dstDir + File.separator + "srcFile1.jpg");
Expand All @@ -169,7 +169,7 @@ void test12_process() throws Exception {
Path srcFile1 = this.createFileWithData(srcDir, "srcFile1.jpg", "updated data");
Path dstFile1 = this.createFileWithData(dstDir, "srcFile1.jpg", "old data");

Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false);
Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true);
b.process();

Assertions.assertEquals(12, dstFile1.toFile().length());
Expand All @@ -186,7 +186,7 @@ void test13_process() throws Exception {

Assertions.assertTrue(dstFile1.toFile().setLastModified(Instant.now().minus(1, ChronoUnit.DAYS).toEpochMilli()));

Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false);
Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true);
b.process();

Assertions.assertEquals(srcFile1.toFile().lastModified(), dstFile1.toFile().lastModified());
Expand All @@ -203,7 +203,7 @@ void test14_process() throws Exception {

Assertions.assertTrue(dstFile1.toFile().setLastModified(srcFile1.toFile().lastModified()));

Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false);
Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false, true);
b.process();

Assertions.assertEquals(srcFile1.toFile().lastModified(), dstFile1.toFile().lastModified());
Expand All @@ -226,7 +226,7 @@ void test15_process() throws Exception {

Path srcFile1 = this.createFileWithData(srcDir, "srcFile1.jpg", "updated data");

Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false);
Backup b = new Backup(srcDir.toFile(), dstDir.toFile(), HashType.SHA256, false,true);

IOException ie = Assertions.assertThrows(IOException.class, b::process);
Assertions.assertEquals("unable to create directory " + dstDir.toAbsolutePath(), ie.getMessage());
Expand Down
Loading