From 05899323643aa55b2f0c60deecc91bfc4868d954 Mon Sep 17 00:00:00 2001 From: Cole-Greer Date: Fri, 2 May 2025 15:35:24 -0700 Subject: [PATCH 1/6] replace system file separator with when assembling S3 paths --- .../com/amazonaws/services/neptune/util/S3ObjectInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java b/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java index 7b86e7e0..7924ec82 100644 --- a/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java +++ b/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java @@ -73,7 +73,7 @@ public File createDownloadFile(String parent) { public S3ObjectInfo withNewKeySuffix(String suffix) { File file = StringUtils.isNotEmpty(key) ? new File(key, suffix) : new File(suffix); - return new S3ObjectInfo( String.format("s3://%s/%s", bucket, file.getPath())); + return new S3ObjectInfo( String.format("s3://%s/%s", bucket, file.getPath().replace(File.separatorChar, '/'))); } public S3ObjectInfo replaceOrAppendKey(String placeholder, String ifPresent, String ifAbsent) { @@ -82,7 +82,7 @@ public S3ObjectInfo replaceOrAppendKey(String placeholder, String ifPresent, Str new File(key.replace(placeholder, ifPresent)) : new File(key, ifAbsent); - return new S3ObjectInfo( String.format("s3://%s/%s", bucket, file.getPath())); + return new S3ObjectInfo( String.format("s3://%s/%s", bucket, file.getPath().replace(File.separatorChar, '/'))); } public S3ObjectInfo replaceOrAppendKey(String placeholder, String ifPresent) { From 785bd41cc44461e88e9c752d2ab796dfaf7dcdf1 Mon Sep 17 00:00:00 2001 From: Cole-Greer Date: Fri, 2 May 2025 16:09:39 -0700 Subject: [PATCH 2/6] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c498a53b..5d10be30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Add retries to Neptune clone creation - Fix missing csv headers from `export-pg-from-queries` - Disable DFE for all Gremlin queries +- Fix invalid S3 paths when using Windows ## Neptune Export v1.1.11 (Release Date: Mar 10, 2025): From 00b35583170860fe17a508c58ff13abda7a76d54 Mon Sep 17 00:00:00 2001 From: Cole-Greer Date: Mon, 12 May 2025 12:26:33 -0700 Subject: [PATCH 3/6] Add windows build to GHA --- .github/workflows/maven_build_and_verify.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven_build_and_verify.yml b/.github/workflows/maven_build_and_verify.yml index ef72068c..11a76002 100644 --- a/.github/workflows/maven_build_and_verify.yml +++ b/.github/workflows/maven_build_and_verify.yml @@ -27,8 +27,8 @@ env: SIGNING_ENABLED: ${{ github.event.inputs.signJarArtifacts }} jobs: - build: - + build-ubuntu: + name: Build on Ubuntu runs-on: ubuntu-latest steps: @@ -127,3 +127,18 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 + + build-windows: + name: Build on Windows + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'corretto' + cache: maven + - name: Build with Maven + run: mvn -B clean io.github.git-commit-id:git-commit-id-maven-plugin:revision io.github.git-commit-id:git-commit-id-maven-plugin:validateRevision install --file pom.xml From 31f6a4b9ced9cec9c00255604da41fc9854e25d8 Mon Sep 17 00:00:00 2001 From: Cole-Greer Date: Thu, 15 May 2025 13:48:51 -0700 Subject: [PATCH 4/6] Fix failing Windows tests --- .../services/neptune/io/DirectoriesTest.java | 2 +- .../propertygraph/ExportStatsTest.java | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java b/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java index 28084867..6a7b0271 100644 --- a/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java +++ b/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java @@ -41,7 +41,7 @@ public void createsDigestFilePathsForVeryLongFilenames() throws IOException { Directories directories = Directories.createFor(DirectoryStructure.PropertyGraph, new File("home"), "export-id", "", ""); Path filePath = directories.createFilePath(path, longName, PropertyGraphExportFormat.csv); - assertEquals("/export/8044f12c352773b7ff400ef524da6e90db419e4a.csv", filePath.toString()); + assertEquals(Paths.get("/export","8044f12c352773b7ff400ef524da6e90db419e4a.csv").toString(), filePath.toString()); } diff --git a/src/test/java/com/amazonaws/services/neptune/propertygraph/ExportStatsTest.java b/src/test/java/com/amazonaws/services/neptune/propertygraph/ExportStatsTest.java index 149b51c2..16d605c0 100644 --- a/src/test/java/com/amazonaws/services/neptune/propertygraph/ExportStatsTest.java +++ b/src/test/java/com/amazonaws/services/neptune/propertygraph/ExportStatsTest.java @@ -57,24 +57,24 @@ public void testExportStats() throws JsonProcessingException { String formattedStats = stats.formatStats(schema); String expectedStats = - "Source:\n" + - " Nodes: 0\n" + - " Edges: 0\n" + - "Export:\n" + - " Nodes: 2\n" + - " Edges: 2\n" + - " Properties: 0\n" + - "Details:\n" + - " Nodes: \n" + - " node1: 1\n" + - " |_ prop1 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}\n" + - " |_ prop2 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}\n" + - " node2: 1\n" + - " Edges: \n" + - " edge2: 1\n" + - " edge1: 1\n" + - " |_ prop1 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}\n" + - " |_ prop2 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}\n"; + "Source:" + System.lineSeparator() + + " Nodes: 0" + System.lineSeparator() + + " Edges: 0" + System.lineSeparator() + + "Export:" + System.lineSeparator() + + " Nodes: 2" + System.lineSeparator() + + " Edges: 2" + System.lineSeparator() + + " Properties: 0" + System.lineSeparator() + + "Details:" + System.lineSeparator() + + " Nodes: " + System.lineSeparator() + + " node1: 1" + System.lineSeparator() + + " |_ prop1 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}" + System.lineSeparator() + + " |_ prop2 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}" + System.lineSeparator() + + " node2: 1" + System.lineSeparator() + + " Edges: " + System.lineSeparator() + + " edge2: 1" + System.lineSeparator() + + " edge1: 1" + System.lineSeparator() + + " |_ prop1 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}" + System.lineSeparator() + + " |_ prop2 {propertyCount=0, minCardinality=-1, maxCardinality=-1, recordCount=0, dataTypeCounts=[]}" + System.lineSeparator(); assertEquals(expectedStats, formattedStats); } From c9d3ccba41fafb8066375bb0a09373fdde3bf25f Mon Sep 17 00:00:00 2001 From: Cole-Greer Date: Tue, 10 Jun 2025 14:31:49 -0700 Subject: [PATCH 5/6] fix windows incompabible tests --- .github/workflows/maven_build_and_verify.yml | 2 +- .../propertygraph/io/CsvPropertyGraphPrinterTest.java | 5 +++-- .../propertygraph/io/JsonPropertyGraphPrinterTest.java | 8 ++++---- .../io/VariableRowCsvPropertyGraphPrinterTest.java | 10 +++++----- .../neptune/propertygraph/schema/DataTypeTest.java | 8 ++++---- .../services/neptune/rdf/NeptuneSparqlClientTest.java | 4 +++- .../services/neptune/util/AWSCredentialsUtilTest.java | 4 ++-- 7 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/workflows/maven_build_and_verify.yml b/.github/workflows/maven_build_and_verify.yml index 11a76002..819131af 100644 --- a/.github/workflows/maven_build_and_verify.yml +++ b/.github/workflows/maven_build_and_verify.yml @@ -141,4 +141,4 @@ jobs: distribution: 'corretto' cache: maven - name: Build with Maven - run: mvn -B clean io.github.git-commit-id:git-commit-id-maven-plugin:revision io.github.git-commit-id:git-commit-id-maven-plugin:validateRevision install --file pom.xml + run: mvn -B clean io.github.git-commit-id:git-commit-id-maven-plugin:revision io.github.git-commit-id:git-commit-id-maven-plugin:validateRevision -Dtest=\!KinesisConfigTest* install --file pom.xml diff --git a/src/test/java/com/amazonaws/services/neptune/propertygraph/io/CsvPropertyGraphPrinterTest.java b/src/test/java/com/amazonaws/services/neptune/propertygraph/io/CsvPropertyGraphPrinterTest.java index 9f06d9d4..56a484dc 100644 --- a/src/test/java/com/amazonaws/services/neptune/propertygraph/io/CsvPropertyGraphPrinterTest.java +++ b/src/test/java/com/amazonaws/services/neptune/propertygraph/io/CsvPropertyGraphPrinterTest.java @@ -145,7 +145,7 @@ public void shouldNotEscapeNewlineCharAfterPrintPropertiesToCSVAndRewrite() thro @Test public void shouldNotEscapeNewlineAfterPrintPropertiesToCSVAndRewrite() throws Exception { - testEscapeCharacterAfterPrintPropertiesAndRewrite("A" + System.lineSeparator() + "B", "\"A\nB\"", + testEscapeCharacterAfterPrintPropertiesAndRewrite("A" + System.lineSeparator() + "B", "\"A"+System.lineSeparator()+"B\"", new PrinterOptions(CsvPrinterOptions.builder().build())); } @@ -158,8 +158,9 @@ public void shouldEscapeNewlineCharSetTrueAfterPrintPropertiesToCSVAndRewrite() @Test public void shouldEscapeNewlineSetTrueAfterPrintPropertiesToCSVAndRewrite() throws Exception { + final String escapedLineSeparator = System.lineSeparator().replace("\r", "\\r").replace("\n", "\\n"); testEscapeCharacterAfterPrintPropertiesAndRewrite("A" + System.lineSeparator() + "B", - "\"A\\nB\"", + "\"A"+escapedLineSeparator+"B\"", new PrinterOptions(CsvPrinterOptions.builder().setEscapeNewline(true).build())); } diff --git a/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java b/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java index 3f5acb2d..a5219575 100644 --- a/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java +++ b/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java @@ -197,10 +197,10 @@ public void appendsPreviouslyUnseenValuesToObjectWhenInferringSchema() throws IO map(entry("fname", "fname5"), entry("lname", "lname5"), entry("age", 50)) ); - String expectedOutput = "{\"fname\":\"fname1\"}\n" + - "{\"fname\":\"fname2\",\"lname\":\"lname2\"}\n" + - "{\"fname\":\"fname3\",\"age\":30}\n" + - "{\"lname\":\"lname4\",\"age\":40}\n" + + String expectedOutput = "{\"fname\":\"fname1\"}" + System.lineSeparator() + + "{\"fname\":\"fname2\",\"lname\":\"lname2\"}" + System.lineSeparator() + + "{\"fname\":\"fname3\",\"age\":30}" + System.lineSeparator() + + "{\"lname\":\"lname4\",\"age\":40}" + System.lineSeparator() + "{\"fname\":\"fname5\",\"lname\":\"lname5\",\"age\":50}"; assertEquals(expectedOutput, stringWriter.toString()); diff --git a/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java b/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java index fddb73bc..0e0fa1ee 100644 --- a/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java +++ b/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java @@ -50,11 +50,11 @@ public void appendsPreviouslyUnseenColumnsToEndOfRow() throws IOException { map(entry("fname", "fname5"), entry("lname", "lname5"), entry("age", 50)) ); - String expectedOutput = "\"fname1\"\n" + - "\"fname2\",\"lname2\"\n" + - "\"fname3\",,30\n" + - ",\"lname4\",40\n" + - "\"fname5\",\"lname5\",50\n"; + String expectedOutput = "\"fname1\"" + System.lineSeparator() + + "\"fname2\",\"lname2\"" + System.lineSeparator() + + "\"fname3\",,30" + System.lineSeparator() + + ",\"lname4\",40" + System.lineSeparator() + + "\"fname5\",\"lname5\",50" + System.lineSeparator(); assertEquals(expectedOutput, stringWriter.toString()); diff --git a/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java b/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java index d445bfff..08cdbf91 100644 --- a/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java +++ b/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java @@ -14,12 +14,10 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import org.junit.Test; import java.io.IOException; import java.io.StringWriter; -import java.io.Writer; import java.util.Date; import static org.junit.Assert.assertEquals; @@ -77,7 +75,7 @@ public void shouldNotEscapeNewlineChar(){ @Test public void shouldNotEscapeNewline(){ String result = DataType.String.format("A" + System.lineSeparator() + "B"); - assertEquals("\"A\nB\"", result); + assertEquals("\"A" + System.lineSeparator() + "B\"", result); } @Test @@ -88,8 +86,10 @@ public void shouldEscapeNewlineCharIfEscapeNewlineSetToTrue(){ @Test public void shouldEscapeNewlineIfEscapeNewlineSetToTrue(){ + final String escapedLineSeparator = System.lineSeparator().replace("\r", "\\r").replace("\n", "\\n"); + String result = DataType.String.format("A" + System.lineSeparator() + "B", true); - assertEquals("\"A\\nB\"", result); + assertEquals("\"A"+escapedLineSeparator+"B\"", result); } @Test diff --git a/src/test/java/com/amazonaws/services/neptune/rdf/NeptuneSparqlClientTest.java b/src/test/java/com/amazonaws/services/neptune/rdf/NeptuneSparqlClientTest.java index 718818ea..be34e0a4 100644 --- a/src/test/java/com/amazonaws/services/neptune/rdf/NeptuneSparqlClientTest.java +++ b/src/test/java/com/amazonaws/services/neptune/rdf/NeptuneSparqlClientTest.java @@ -49,12 +49,14 @@ public class NeptuneSparqlClientTest { + private static final String escapedLineSeparator = System.lineSeparator().replace("\r", "\\r").replace("\n", "\\n"); + private SPARQLRepository mockSPARQLRepository; private SailRepository sailRepository; private final String testDataNTriples = " .\n" + " \"version\" .\n" + " \"0.77\" .\n" + - " \"Version: 0.77 Generated: 2017-10-06 16:24:52 UTC\\nGraph created by Kelvin R. Lawrence\\nPlease let me know of any errors you find in the graph.\" .\n"; + " \"Version: 0.77 Generated: 2017-10-06 16:24:52 UTC"+escapedLineSeparator+"Graph created by Kelvin R. Lawrence"+escapedLineSeparator+"Please let me know of any errors you find in the graph.\" .\n"; private OutputWriter writer; diff --git a/src/test/java/com/amazonaws/services/neptune/util/AWSCredentialsUtilTest.java b/src/test/java/com/amazonaws/services/neptune/util/AWSCredentialsUtilTest.java index 2f77a807..b22a8032 100644 --- a/src/test/java/com/amazonaws/services/neptune/util/AWSCredentialsUtilTest.java +++ b/src/test/java/com/amazonaws/services/neptune/util/AWSCredentialsUtilTest.java @@ -54,9 +54,9 @@ public void shouldGetDefaultCredsIfConfigIsNull() { @Test public void shouldAttemptToUseProvidedPath() { Throwable t = assertThrows(IllegalStateException.class, () -> getProfileCredentialsProvider( - null, tempFolder.getRoot().getAbsolutePath()+"/non-existent-file").resolveCredentials()); + null, tempFolder.getRoot().toPath().resolve("non-existent-file").toString()).resolveCredentials()); assertEquals("Profile file '"+ - tempFolder.getRoot().getAbsolutePath()+"/non-existent-file' does not exist.", t.getMessage()); + tempFolder.getRoot().toPath().resolve("non-existent-file").toAbsolutePath()+"' does not exist.", t.getMessage()); } @Test From e62035b139920f1009d838d24b8a5cd94f8918af Mon Sep 17 00:00:00 2001 From: Cole-Greer Date: Tue, 10 Jun 2025 15:14:12 -0700 Subject: [PATCH 6/6] ignoring KinesisConfigTests due to windows incompatibility --- .github/workflows/maven_build_and_verify.yml | 2 +- .../amazonaws/services/neptune/io/KinesisConfigTest.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_build_and_verify.yml b/.github/workflows/maven_build_and_verify.yml index 819131af..11a76002 100644 --- a/.github/workflows/maven_build_and_verify.yml +++ b/.github/workflows/maven_build_and_verify.yml @@ -141,4 +141,4 @@ jobs: distribution: 'corretto' cache: maven - name: Build with Maven - run: mvn -B clean io.github.git-commit-id:git-commit-id-maven-plugin:revision io.github.git-commit-id:git-commit-id-maven-plugin:validateRevision -Dtest=\!KinesisConfigTest* install --file pom.xml + run: mvn -B clean io.github.git-commit-id:git-commit-id-maven-plugin:revision io.github.git-commit-id:git-commit-id-maven-plugin:validateRevision install --file pom.xml diff --git a/src/test/java/com/amazonaws/services/neptune/io/KinesisConfigTest.java b/src/test/java/com/amazonaws/services/neptune/io/KinesisConfigTest.java index f6c8c47f..29c05716 100644 --- a/src/test/java/com/amazonaws/services/neptune/io/KinesisConfigTest.java +++ b/src/test/java/com/amazonaws/services/neptune/io/KinesisConfigTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assume.assumeTrue; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -38,6 +39,10 @@ public void resetTargetModule() { @Test public void shouldCreateStreamIfNameAndRegionAreProvided() { + // Ignoring test for Windows due to incompatibility + String osName = System.getProperty("os.name"); + assumeTrue(!osName.startsWith("Windows")); + when(target.getStreamName()).thenReturn("test"); when(target.getRegion()).thenReturn("us-west-2"); KinesisConfig config = new KinesisConfig(target); @@ -67,6 +72,10 @@ public void shouldNotCreateStreamIfRegionNotProvided() { @Test public void shouldUseProvidedCredentialsProvider() throws InterruptedException { + // Ignoring test for Windows due to incompatibility + String osName = System.getProperty("os.name"); + assumeTrue(!osName.startsWith("Windows")); + when(target.getStreamName()).thenReturn("test"); when(target.getRegion()).thenReturn("us-west-2"); AwsCredentialsProvider credentialsProvider = spy(AnonymousCredentialsProvider.create());