Improve Integration Testing Around S3 Upload

docs/dev/IntegrationTests.md

@@ -2,16 +2,20 @@
 
 ## Overview
 
-The integration tests for Neptune Export run a set of full export jobs against a Neptune instance and validate that the output is correct. A user must provide their own Neptune instance with the correct sample data pre-loaded in order to run these tests.
+The integration tests for Neptune Export run a set of full export jobs against a Neptune instance and validate that the output is correct. A user must provide their own Neptune instance with the correct sample data pre-loaded in order to run these tests. The tests
+assume that IAM authentication is enabled on the provided Neptune Cluster.
 
 ## Setup
 
 - Setup a clean Neptune instance to run integration tests against.
 - Using the Neptune bulk loader, fill the database with Kelvin Lawrence's air-routes-small graph data [found here](https://github.com/krlawrence/graph/blob/master/sample-data/air-routes-small.graphml). GraphML files can be run through [graphml2csv](https://github.com/awslabs/amazon-neptune-tools/tree/master/graphml2csv) for property graph loading and csv files can go through [csv to rdf](https://github.com/aws/amazon-neptune-csv-to-rdf-converter) for RDF loading.
-- Ensure your Neptune instance is accessible to your testing environment (either run tests from within the
-same VPC as your instance or connect through a bastion host)
+- Ensure your Neptune instance is accessible to your testing environment (use a public endpoint,run tests from within the
+same VPC as your instance, or connect through a bastion host)
+- Create an S3 bucket which can be read and written to by `ExportServiceIntegrationTest`.
 
 ## Running Tests
 
 - Set a `NEPTUNE_ENDPOINT` environment variable to your Neptune cluster endpoint
+- Set a `SERVICE_REGION` environment variable for IAM auth
+- Set a `S3_PATH` environment variable for ExportServiceIntegrationTest
 - Run Junit integration tests through your IDE

src/main/java/com/amazonaws/services/neptune/export/NeptuneExportLambda.java

@@ -181,9 +181,7 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co
         }
 
         if (outputS3ObjectInfo != null) {
-            try (Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream, UTF_8))) {
-                writer.write(outputS3ObjectInfo.toString());
-            }
+            outputStream.write(outputS3ObjectInfo.toString().getBytes(UTF_8));
         } else {
             System.exit(-1);
         }

src/test/java/com/amazonaws/services/neptune/CreatePgConfigIntegrationTest.java

@@ -21,7 +21,7 @@ public class CreatePgConfigIntegrationTest extends AbstractExportIntegrationTest
 
     @Test
     public void testCreatePgConfig() {
-        final String[] command = {"create-pg-config", "-e", neptuneEndpoint, "-d", outputDir.getPath()};
+        final String[] command = {"create-pg-config", "-e", neptuneEndpoint, "-d", outputDir.getPath(), "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -33,7 +33,7 @@ public void testCreatePgConfig() {
     @Test
     public void testCreatePgConfigWithGremlinFilter() {
         final String[] command = {"create-pg-config", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--gremlin-filter", "has(\"runways\", 2)"};
+                "--gremlin-filter", "has(\"runways\", 2)", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -45,7 +45,7 @@ public void testCreatePgConfigWithGremlinFilter() {
     @Test
     public void testCreatePgConfigWithEdgeGremlinFilter() {
         final String[] command = {"create-pg-config", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--gremlin-filter", "hasLabel(\"route\")"};
+                "--gremlin-filter", "hasLabel(\"route\")", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -57,7 +57,7 @@ public void testCreatePgConfigWithEdgeGremlinFilter() {
     @Test
     public void testCreatePgConfigWithEdgeGremlinFilterAndEarlyGremlinFilter() {
         final String[] command = {"create-pg-config", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--gremlin-filter", "hasLabel(\"route\")", "--filter-edges-early"};
+                "--gremlin-filter", "hasLabel(\"route\")", "--filter-edges-early", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 

src/test/java/com/amazonaws/services/neptune/ExportPgFromConfigIntegrationTest.java

@@ -23,7 +23,7 @@ public class ExportPgFromConfigIntegrationTest extends AbstractExportIntegration
     public void testExportPgFromConfig() {
         final String[] command = {"export-pg-from-config", "-e", neptuneEndpoint,
                 "-c", "src/test/resources/IntegrationTest/ExportPgFromConfigIntegrationTest/input/config.json",
-                "-d", outputDir.getPath()};
+                "-d", outputDir.getPath(), "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -36,7 +36,7 @@ public void testExportPgFromConfig() {
     public void testExportPgFromConfigWithGremlinFilter() {
         final String[] command = {"export-pg-from-config", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
                 "-c", "src/test/resources/IntegrationTest/ExportPgFromConfigIntegrationTest/input/config.json",
-                "--gremlin-filter", "has(\"runways\", 2)"};
+                "--gremlin-filter", "has(\"runways\", 2)", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -49,7 +49,7 @@ public void testExportPgFromConfigWithGremlinFilter() {
     public void testExportEdgesFromConfigWithGremlinFilter() {
         final String[] command = {"export-pg-from-config", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
                 "-c", "src/test/resources/IntegrationTest/ExportPgFromConfigIntegrationTest/input/config.json",
-                "--gremlin-filter", "hasLabel(\"route\")"};
+                "--gremlin-filter", "hasLabel(\"route\")", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -62,7 +62,7 @@ public void testExportEdgesFromConfigWithGremlinFilter() {
     public void testExportEdgesFromConfigWithGremlinFilterWithEarlyGremlinFilter() {
         final String[] command = {"export-pg-from-config", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
                 "-c", "src/test/resources/IntegrationTest/ExportPgFromConfigIntegrationTest/input/config.json",
-                "--gremlin-filter", "hasLabel(\"route\")", "--filter-edges-early"};
+                "--gremlin-filter", "hasLabel(\"route\")", "--filter-edges-early", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 

src/test/java/com/amazonaws/services/neptune/ExportPgFromQueriesIntegrationTest.java

@@ -25,7 +25,8 @@ public class ExportPgFromQueriesIntegrationTest extends AbstractExportIntegratio
     public void testExportPgFromQueries() {
         final String[] command = {"export-pg-from-queries", "-e", neptuneEndpoint,
                 "-d", outputDir.getPath(),
-                "-q", "airport=g.V().hasLabel('airport').has('runways', gt(2)).project('code', 'runways', 'city', 'country').by('code').by('runways').by('city').by('country')"
+                "-q", "airport=g.V().hasLabel('airport').has('runways', gt(2)).project('code', 'runways', 'city', 'country').by('code').by('runways').by('city').by('country')",
+                "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -39,7 +40,8 @@ public void testExportPgFromQueries() {
     public void testExportPgFromQueriesNoHeaders() {
         final String[] command = {"export-pg-from-queries", "-e", neptuneEndpoint,
                 "-d", outputDir.getPath(), "--format", "csvNoHeaders",
-                "-q", "airport=g.V().hasLabel('airport').has('runways', gt(2)).project('code', 'runways', 'city', 'country').by('code').by('runways').by('city').by('country')"
+                "-q", "airport=g.V().hasLabel('airport').has('runways', gt(2)).project('code', 'runways', 'city', 'country').by('code').by('runways').by('city').by('country')",
+                "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -53,7 +55,8 @@ public void testExportPgFromQueriesNoHeaders() {
     public void testExportPgFromQueriesWithStaggeredResults() {
         final String[] command = {"export-pg-from-queries", "-e", neptuneEndpoint,
                 "-d", outputDir.getPath(),
-                "-q", "airport=g.inject(['code': 'YYC'], ['city': 'Vancouver', 'code': 'YVR'], ['code':'SEA', 'city':'Seattle', 'runways': 3])"
+                "-q", "airport=g.inject(['code': 'YYC'], ['city': 'Vancouver', 'code': 'YVR'], ['code':'SEA', 'city':'Seattle', 'runways': 3])",
+                "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -67,7 +70,8 @@ public void testExportPgFromQueriesWithStaggeredResults() {
     public void testExportPgFromQueriesWithStaggeredResultsNoHeaders() {
         final String[] command = {"export-pg-from-queries", "-e", neptuneEndpoint,
                 "-d", outputDir.getPath(), "--format", "csvNoHeaders",
-                "-q", "airport=g.inject(['code':'SEA', 'city':'Seattle', 'runways': 3], ['city': 'Vancouver', 'code': 'YVR'], ['code': 'YYC'])"
+                "-q", "airport=g.inject(['code':'SEA', 'city':'Seattle', 'runways': 3], ['city': 'Vancouver', 'code': 'YVR'], ['code': 'YYC'])",
+                "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -82,7 +86,7 @@ public void testExportPgFromQueriesSplitQueries() {
         final String[] command = {"export-pg-from-queries", "-e", neptuneEndpoint,
                 "-d", outputDir.getPath(),
                 "-q", "airport=g.V().hasLabel('airport').has('runways', gt(2)).project('code', 'runways', 'city', 'country').by('code').by('runways').by('city').by('country')",
-                "--split-queries"
+                "--split-queries", "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -97,7 +101,7 @@ public void testExportPgFromQueriesSplitQueriesAndRange() {
         final String[] command = {"export-pg-from-queries", "-e", neptuneEndpoint,
                 "-d", outputDir.getPath(),
                 "-q", "airport=g.V().hasLabel('airport').has('runways', gt(2)).project('code', 'runways', 'city', 'country').by('code').by('runways').by('city').by('country')",
-                "--split-queries", "--range", "25"
+                "--split-queries", "--range", "25", "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -113,7 +117,7 @@ public void testExportPgFromQueriesWithStructuredOutput() {
                 "-d", outputDir.getPath(),
                 "-q", "airport=g.V().union(hasLabel('airport'), outE()).elementMap()",
                 "--include-type-definitions",
-                "--structured-output"
+                "--structured-output", "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -130,7 +134,7 @@ public void testExportPgFromQueriesWithStructuredOutputSplitQueries() {
                 "-q", "airport=g.V().union(hasLabel('airport'), outE()).elementMap()",
                 "--include-type-definitions",
                 "--split-queries", "--range", "25",
-                "--structured-output"
+                "--structured-output", "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -145,7 +149,7 @@ public void testExportPgFromQueriesWithStructuredOutputWithEdgeAndVertexLabels()
         final String[] command = {"export-pg-from-queries", "-e", neptuneEndpoint,
                 "-d", outputDir.getPath(),
                 "-q", "all=g.V().union(elementMap(), outE().elementMap())",
-                "--edge-label-strategy", "edgeAndVertexLabels", "--structured-output"
+                "--edge-label-strategy", "edgeAndVertexLabels", "--structured-output", "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
@@ -161,7 +165,7 @@ public void testExportPgFromQueriesWithStructuredOutputWithEdgeAndVertexLabelsIn
                 "-d", outputDir.getPath(),
                 "-q", "airport=g.V().union(hasLabel('airport'), outE()).elementMap()",
                 "--include-type-definitions", "--edge-label-strategy", "edgeAndVertexLabels",
-                "--structured-output"
+                "--structured-output", "--use-iam-auth"
         };
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();

src/test/java/com/amazonaws/services/neptune/ExportPgIntegrationTest.java

@@ -22,7 +22,7 @@ public class ExportPgIntegrationTest extends AbstractExportIntegrationTest{
 
     @Test
     public void testExportPgToCsv() {
-        final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath()};
+        final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath(), "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -34,7 +34,7 @@ public void testExportPgToCsv() {
     @Test
     public void testExportPgWithEdgeAndVertexLabels() {
         final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--edge-label-strategy", "edgeAndVertexLabels"};
+                "--edge-label-strategy", "edgeAndVertexLabels", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -45,7 +45,7 @@ public void testExportPgWithEdgeAndVertexLabels() {
 
     @Test
     public void testExportPgToCsvWithJanus() {
-        final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath(), "--janus"};
+        final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath(), "--janus", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -57,7 +57,7 @@ public void testExportPgToCsvWithJanus() {
     @Test
     public void testExportPgToCsvWithGremlinFilter() {
         final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--gremlin-filter", "has(\"runways\", 2)"};
+                "--gremlin-filter", "has(\"runways\", 2)", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -69,7 +69,7 @@ public void testExportPgToCsvWithGremlinFilter() {
     @Test
     public void testExportEdgesToCsvWithGremlinFilter() {
         final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--gremlin-filter", "hasLabel(\"route\")"};
+                "--gremlin-filter", "hasLabel(\"route\")", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -81,7 +81,7 @@ public void testExportEdgesToCsvWithGremlinFilter() {
     @Test
     public void testExportEdgesToCsvWithGremlinFilterWithEarlyGremlinFilter() {
         final String[] command = {"export-pg", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--gremlin-filter", "hasLabel(\"route\")", "--filter-edges-early"};
+                "--gremlin-filter", "hasLabel(\"route\")", "--filter-edges-early", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 

src/test/java/com/amazonaws/services/neptune/ExportRdfIntegrationTest.java

@@ -29,7 +29,7 @@ public class ExportRdfIntegrationTest extends AbstractExportIntegrationTest{
 
     @Test
     public void testExportRdf() {
-        final String[] command = {"export-rdf", "-e", neptuneEndpoint, "-d", outputDir.getPath()};
+        final String[] command = {"export-rdf", "-e", neptuneEndpoint, "-d", outputDir.getPath(), "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -41,7 +41,7 @@ public void testExportRdf() {
     @Test
     public void testExportRdfSingleNamedGraphVersion() {
         final String[] command = {"export-rdf", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--named-graph", "http://aws.amazon.com/neptune/csv2rdf/graph/version"};
+                "--named-graph", "http://aws.amazon.com/neptune/csv2rdf/graph/version", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -53,7 +53,7 @@ public void testExportRdfSingleNamedGraphVersion() {
     @Test
     public void testExportRdfSingleNamedGraphDefault() {
         final String[] command = {"export-rdf", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--named-graph", "http://aws.amazon.com/neptune/vocab/v01/DefaultNamedGraph"};
+                "--named-graph", "http://aws.amazon.com/neptune/vocab/v01/DefaultNamedGraph", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -65,7 +65,7 @@ public void testExportRdfSingleNamedGraphDefault() {
     @Test
     public void testExportRdfNoGSP() {
         final String[] command = {"export-rdf", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
-                "--feature-toggle", "No_GSP"};
+                "--feature-toggle", "No_GSP", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -78,7 +78,7 @@ public void testExportRdfNoGSP() {
     public void testExportRdfSingleNamedGraphVersionNoGSP() {
         final String[] command = {"export-rdf", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
                 "--named-graph", "http://aws.amazon.com/neptune/csv2rdf/graph/version",
-                "--feature-toggle", "No_GSP"};
+                "--feature-toggle", "No_GSP", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();
 
@@ -91,7 +91,7 @@ public void testExportRdfSingleNamedGraphVersionNoGSP() {
     public void testExportRdfSingleNamedGraphDefaultNoGSP() {
         final String[] command = {"export-rdf", "-e", neptuneEndpoint, "-d", outputDir.getPath(),
                 "--named-graph", "http://aws.amazon.com/neptune/vocab/v01/DefaultNamedGraph",
-                "--feature-toggle", "No_GSP"};
+                "--feature-toggle", "No_GSP", "--use-iam-auth"};
         final NeptuneExportRunner runner = new NeptuneExportRunner(command);
         runner.run();