Fix: Windows-safe credential setup and OpenSSL subject preservation

README.md

@@ -10,6 +10,11 @@ In the following pages you will find information about [migrating existing](docs
 You can quickly find commonly used commands in our [cheat sheet](./docs/cheat_sheet.md). Config file formats were copied from [cloud-automation](https://github.com/uc-cdis/cloud-automation) and stored in the `Secrets` directory and modified for local use with Docker Compose. Setup scripts for some of the containers are kept in the `scripts` directory.
 
 
+
+# Prerequisites
+
+This repository is supported on Linux, MacOS, and Windows (using Git Bash or MSYS2 environments). Most scripts, including `creds_setup.sh`, are optimized for Linux/MacOS shells but now include compatibility fixes for Windows file-system constraints and OpenSSL subject formatting. On Windows, run scripts in a Unix-like shell (e.g., Git Bash). Native Windows Command Prompt or PowerShell is not supported for shell scripts.
+
 # Key Documentation
 
 * [Database Information](docs/database_information.md)

creds_setup.sh

@@ -53,8 +53,15 @@ rm "$tempFile"
 
 cd Secrets
 
-# make directories for temporary credentials
-timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+# OS detection for Windows (Git Bash/MSYS), Linux, or MacOS
+OS_TYPE="$(uname -s)"
+if [[ "$OS_TYPE" =~ (MINGW|MSYS|CYGWIN) ]]; then
+  # Windows: replace colons in timestamp with dashes
+  timestamp=$(date -u +"%Y-%m-%dT%H-%M-%SZ")
+else
+  timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+fi
 
 # generate private and public key for fence
 yearMonth="$(date +%Y-%m)"
@@ -95,9 +102,14 @@ authorityKeyIdentifier = keyid:always,issuer:always
       # erase old certs if they exist
       /bin/rm -rf service.key service.crt
       commonName=${1:-localhost}
-      SUBJ="/countryName=US/stateOrProvinceName=IL/localityName=Chicago/organizationName=CDIS/organizationalUnitName=PlanX/commonName=$commonName/emailAddress=cdis@uchicago.edu"
+      # Use double slashes and backslashes for Windows OpenSSL, single slashes for others
+      if [[ "$OS_TYPE" =~ (MINGW|MSYS|CYGWIN) ]]; then
+        SUBJ="//countryName=US\\stateOrProvinceName=IL\\localityName=Chicago\\organizationName=CDIS\\organizationalUnitName=PlanX\\commonName=$commonName\\emailAddress=cdis@uchicago.edu"
+      else
+        SUBJ="/countryName=US/stateOrProvinceName=IL/localityName=Chicago/organizationName=CDIS/organizationalUnitName=PlanX/commonName=$commonName/emailAddress=cdis@uchicago.edu"
+      fi
       openssl req -new -x509 -nodes -extensions v3_ca -keyout ca-key.pem \
-          -out ca.pem -days 365 -subj $SUBJ $OPTS
+          -out ca.pem -days 365 -subj "$SUBJ" $OPTS
       if [[ $? -eq 1 ]]; then
           echo "problem with creds_setup.sh script, refer to compose-services wiki"
           rm -rf temp*
@@ -151,8 +163,8 @@ EOM
     fi
     if [[ ! -f service.key || ! -f service.crt ]]; then
       openssl genrsa -out "service.key" 2048
-      openssl req -new -key "service.key" \
-          -out "service.csr" -subj $SUBJ
+        openssl req -new -key "service.key" \
+          -out "service.csr" -subj "$SUBJ"
       openssl ca -batch -in "service.csr" -config openssl.cnf \
           -extensions server_cert -days 365 -notext -out "service.crt"
     else

docker-compose.yml

@@ -142,23 +142,23 @@ services:
       retries: 5
     depends_on:
       - postgres
-  guppy-service:
-    image: "quay.io/cdis/guppy:2021.03"
-    container_name: guppy-service
-    networks:
-      - devnet
-    volumes:
-      - ./Secrets/guppy_config.json:/guppy/guppy_config.json
-      - ./scripts/wait_for_esproxy.sh:/usr/bin/wait_for_esproxy.sh:ro
-    entrypoint: /usr/bin/wait_for_esproxy.sh
-    command: node --max-http-header-size 16000 dist/server/server.js
-    environment:
-      - GUPPY_CONFIG_FILEPATH=/guppy/guppy_config.json
-      - GEN3_ARBORIST_ENDPOINT=http://arborist-service
-      - GEN3_ES_ENDPOINT=http://esproxy-service:9200
-    depends_on:
-      - arborist-service
-      - esproxy-service
+  # guppy-service:
+  #   image: "quay.io/cdis/guppy:2021.03"
+  #   container_name: guppy-service
+  #   networks:
+  #     - devnet
+  #     volumes:
+  #       - ./Secrets/guppy_config.json:/guppy/guppy_config.json
+  #       - ./scripts/wait_for_esproxy.sh:/usr/bin/wait_for_esproxy.sh:ro
+  #     entrypoint: /usr/bin/wait_for_esproxy.sh
+  #     command: node --max-http-header-size 16000 dist/server/server.js
+  #     environment:
+  #       - GUPPY_CONFIG_FILEPATH=/guppy/guppy_config.json
+  #       - GEN3_ARBORIST_ENDPOINT=http://arborist-service
+  #       - GEN3_ES_ENDPOINT=http://esproxy-service:9200
+  #     depends_on:
+  #       - arborist-service
+  #       - esproxy-service
   esproxy-service:
     image: quay.io/cdis/elasticsearch-oss:6.8.12
     container_name: esproxy-service
@@ -306,18 +306,18 @@ services:
     environment:
       - HADOOP_URL=hdfs://0.0.0.0:9000
       - HADOOP_HOST=0.0.0.0
-  kibana-service:
-    image: quay.io/cdis/kibana-oss:6.5.4
-    container_name: kibana-service
-    environment:
-      - SERVER_NAME=kibana-service
-      - ELASTICSEARCH_URL=http://esproxy-service:9200
-    ports:
-      - 5601:5601
-    networks:
-      - devnet
-    depends_on:
-      - esproxy-service
+  # kibana-service:
+  #   image: quay.io/cdis/kibana-oss:6.5.4
+  #   container_name: kibana-service
+  #   environment:
+  #     - SERVER_NAME=kibana-service
+  #     - ELASTICSEARCH_URL=http://esproxy-service:9200
+  #   ports:
+  #     - 5601:5601
+  #   networks:
+  #     - devnet
+  #   depends_on:
+  #     - esproxy-service
 networks:
   devnet:
 volumes:

docs/using_the_commons.md

@@ -22,9 +22,37 @@ Make sure to update user privileges:
 docker exec -it fence-service fence-create sync --arborist http://arborist-service --yaml user.yaml
 ```
 
-To create a program, visit the URL where your Gen3 Commons is hosted and append `/_root`. If you are running the Docker Compose setup locally, then this will be `localhost/_root`. Otherwise, this will be whatever you set the `hostname` field to in the creds files for the services with `/_root` added to the end. Here, you can choose to either use form submission or upload a file.  I will go through the process of using form submission here, as it will show you what your file would need to look like if you were using file upload. Choose form submission, search for "program" in the drop-down list and then fill in the "dbgap_accession_number" and "name" fields. As an example, you can use "123" as "dbgap accession number" and "Program1" as "name". Click 'Upload submission json from form' and then 'Submit'. If the message is green ("succeeded:200"), that indicates success, while a grey message indicates failure. More details can be viewed by clicking on the "DETAILS" button. If you don't see the green message, you can control the sheepdog logs for possible errors and check the Sheepdog database (`/datadictionary`), where programs and projects are stored. If you see your program in the data dictionary, neglect the fact that at this time the green message does not appear and continue to create a project.
 
-To create a project, visit the URL where your Gen3 Commons is hosted and append the name of the program you want to create the project under. For example, if you are running the Docker Compose setup locally and would like to create a project under the program "Program1", the URL you will visit will be `localhost/Program1`. You will see the same options to use form submission or upload a file. This time, search for "project" in the drop-down list and then fill in the fields. As an example, you can use "P1" as "code", "phs1" as "dbgap_accession_number", and "project1" as "name". If you use different entries, make a note of the dbgap_accession_number for later. Click 'Upload submission json from form' and then 'Submit'. Again, a green message indicates success while a grey message indicates failure, and more details can be viewed by clicking on the "DETAILS" button. You can control in the `/datadictionary` whether the program and project have been correctly stored.
+### Submitting Program or Project JSON via the UI
+
+You can submit a program or project using the Data Commons web UI. The submission form allows you to either:
+
+- **Upload a JSON file** (drag-and-drop or file picker)
+- **Paste JSON directly** into a text area
+
+The form validates your JSON and provides feedback before submission. Example for a program:
+
+```json
+{
+  "dbgap_accession_number": "123",
+  "name": "Program1"
+}
+```
+
+After uploading or pasting your JSON, click **Submit**. A green message indicates success; a grey message indicates failure. You can view more details by clicking the "DETAILS" button in the UI.
+
+
+To create a project, visit your Gen3 Commons UI and navigate to the appropriate program (e.g., `localhost/Program1`). Use the submission form as above, but select **project** in the drop-down. Example project JSON:
+
+```json
+{
+  "code": "P1",
+  "dbgap_accession_number": "phs1",
+  "name": "project1"
+}
+```
+
+After submitting, check for a green success message. You can verify the new program/project in the `/datadictionary` section.
 
 After that, you're ready to start submitting data for that project! Please note that Data Submission refers to metadata regarding the file(s) (Image, Sequencing files, etc.) that are to be uploaded. Please refer to the [Gen3 website](https://gen3.org/resources/user/submit-data/) for additional details.
 

portal/src/components/UploadSubmissionForm.jsx

@@ -0,0 +1,60 @@
+import React, { useState } from 'react';
+
+/**
+ * UploadSubmissionForm - allows users to upload a JSON file or submit JSON via a form for Gen3 submission.
+ * SRP: Handles only upload/submission logic, delegates validation and API to parent.
+ */
+export default function UploadSubmissionForm({ onSubmit }) {
+  const [jsonText, setJsonText] = useState('');
+  const [fileError, setFileError] = useState('');
+  const [status, setStatus] = useState('');
+
+  const handleFileChange = (e) => {
+    const file = e.target.files[0];
+    if (!file) return;
+    if (!file.name.endsWith('.json')) {
+      setFileError('Please select a JSON file.');
+      return;
+    }
+    const reader = new FileReader();
+    reader.onload = (event) => {
+      setJsonText(event.target.result);
+      setFileError('');
+    };
+    reader.readAsText(file);
+  };
+
+  const handleChange = (e) => {
+    setJsonText(e.target.value);
+  };
+
+  const handleSubmit = (e) => {
+    e.preventDefault();
+    try {
+      const json = JSON.parse(jsonText);
+      setStatus('Submitting...');
+      onSubmit(json)
+        .then(() => setStatus('Submission successful!'))
+        .catch((err) => setStatus('Submission failed: ' + err.message));
+    } catch (err) {
+      setStatus('Invalid JSON: ' + err.message);
+    }
+  };
+
+  return (
+    <form onSubmit={handleSubmit} style={{ maxWidth: 600, margin: '2em auto', padding: 20, border: '1px solid #ccc', borderRadius: 8 }}>
+      <h2>Upload Submission JSON</h2>
+      <div style={{ marginBottom: 12 }}>
+        <label>Upload JSON file: </label>
+        <input type="file" accept="application/json" onChange={handleFileChange} />
+        {fileError && <div style={{ color: 'red' }}>{fileError}</div>}
+      </div>
+      <div style={{ marginBottom: 12 }}>
+        <label>Or paste JSON here:</label>
+        <textarea value={jsonText} onChange={handleChange} rows={8} style={{ width: '100%' }} />
+      </div>
+      <button type="submit">Submit</button>
+      {status && <div style={{ marginTop: 10 }}>{status}</div>}
+    </form>
+  );
+}

portal/src/components/UploadSubmissionForm.test.jsx

@@ -0,0 +1,25 @@
+import React from 'react';
+import { render, fireEvent, screen } from '@testing-library/react';
+import UploadSubmissionForm from './UploadSubmissionForm';
+
+describe('UploadSubmissionForm', () => {
+  it('renders and submits valid JSON', async () => {
+    const mockSubmit = jest.fn(() => Promise.resolve());
+    render(<UploadSubmissionForm onSubmit={mockSubmit} />);
+    const textarea = screen.getByLabelText(/paste json/i);
+    fireEvent.change(textarea, { target: { value: '{"foo": "bar"}' } });
+    fireEvent.click(screen.getByText(/submit/i));
+    expect(await screen.findByText(/submission successful/i)).toBeInTheDocument();
+    expect(mockSubmit).toHaveBeenCalledWith({ foo: 'bar' });
+  });
+
+  it('shows error for invalid JSON', () => {
+    const mockSubmit = jest.fn();
+    render(<UploadSubmissionForm onSubmit={mockSubmit} />);
+    const textarea = screen.getByLabelText(/paste json/i);
+    fireEvent.change(textarea, { target: { value: '{foo: bar}' } });
+    fireEvent.click(screen.getByText(/submit/i));
+    expect(screen.getByText(/invalid json/i)).toBeInTheDocument();
+    expect(mockSubmit).not.toHaveBeenCalled();
+  });
+});

scripts/postgres_run.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env bash
 
 # Thing shim around the normal Docker postgres entrypoint that allows us to run
 # non-application migrations. Things like DB and user creations that would be

scripts/postgres_run_fixed.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Always run the postgres init script
+postgres_always.sh
+
+# Run postgres
+exec postgres "$@"