From 356e8a65d94f338d73cbe50a42de4eed2a7fe141 Mon Sep 17 00:00:00 2001 From: Anusha-janardhan Date: Fri, 31 Oct 2025 08:54:16 +0000 Subject: [PATCH 1/2] feat(catalog): remove deprecated catalog items and update Amazon Sagemaker name --- dump/standardcatalogitems.json | 373 +-------------------------------- 1 file changed, 1 insertion(+), 372 deletions(-) diff --git a/dump/standardcatalogitems.json b/dump/standardcatalogitems.json index 21ed41b..fda684c 100644 --- a/dump/standardcatalogitems.json +++ b/dump/standardcatalogitems.json @@ -53,66 +53,6 @@ ] } }, - { - "_id": { - "$oid": "60504955135663522b880ade" - }, - "name": "Docker on Amazon EC2 Linux", - "description": "Use Docker, a popular container orchestration software, on Amazon Elastic Compute Cloud (Amazon EC2).", - "fileName": "ec2-linux-docker.yml", - "tags": [ - { - "Key": "EstimatedTimeToProvision", - "Value": "10 Minutes" - }, - { - "Key": "DetailsLink", - "Value": "https://researchgateway.readthedocs.io/en/latest/DockerEC2linux.html" - }, - { - "Key": "Service", - "Value": "EC2" - }, - { - "Key": "TypeOfProduct", - "Value": "Research" - } - ], - "owner": "RL", - "portfolio": "RGPortfolio", - "availableRegions": [], - "assignedOU": [], - "metaData": { - "pre_provisioning": [ - { - "code": "CFT_PARAMS", - "params": [ - { - "name": "EnvironmentInstanceFiles", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "IamPolicyDocument", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "S3Mounts", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "Namespace", - "type": "RL::SC::PARAM::HD" - } - ] - } - ], - "post_provisioning": [], - "checks_before_assigning_product": [], - "checks_after_assigning_product": [], - "permission_required": {}, - "cost_resource": true - } - }, { "_id": { "$oid": "60504955135663522b880edb" @@ -238,7 +178,7 @@ "_id": { "$oid": "60504955135663522b880cff" }, - "name": "Amazon Sagemaker", + "name": "Amazon Sagemaker Notebook", "description": "Amazon SageMaker is a fully managed service that provides the ability to build, train, and deploy ML models", "fileName": "sagemaker-template-with-url.yml", "tags": [ @@ -390,318 +330,7 @@ "permission_required": {}, "cost_resource": true } - }, - { - "_id": { - "$oid": "6076865ad0881f46a105da23" - }, - "name": "Amazon EC2 Windows", - "description": "Amazon Elastic Compute Cloud (Amazon EC2) is a web service that provides secure, resizable compute capacity in the cloud.", - "fileName": "windows-ec2.yaml", - "tags": [ - { - "Key": "EstimatedTimeToProvision", - "Value": "10 Minutes" - }, - { - "Key": "DetailsLink", - "Value": "https://researchgateway.readthedocs.io/en/latest/windows.html" - }, - { - "Key": "Service", - "Value": "EC2" - }, - { - "Key": "TypeOfProduct", - "Value": "Research" - } - ], - "owner": "RL", - "portfolio": "RGPortfolio", - "availableRegions": [], - "assignedOU": [], - "metaData": { - "pre_provisioning": [], - "post_provisioning": [], - "checks_before_assigning_product": [], - "checks_after_assigning_product": [], - "permission_required": {}, - "cost_resource": true - } - }, - { - "_id": { - "$oid": "610166d6bbe50a00122a5139" - }, - "name" : "Ubuntu 20 04 on Amazon EC2", - "description" : "Use Ubuntu 20.04, on Amazon Elastic Compute Cloud (Amazon EC2).", - "fileName" : "ec2-ubuntu.yml", - "tags" : [ - { - "Key" : "EstimatedTimeToProvision", - "Value" : "10 Minutes" - }, - { - "Key" : "DetailsLink", - "Value" : "https://researchgateway.readthedocs.io/en/latest/ubuntu.html" - }, - { - "Key" : "Service", - "Value" : "EC2" - }, - { - "Key" : "TypeOfProduct", - "Value" : "Research" - } - ], - "owner" : "RL", - "portfolio" : "RGPortfolio", - "availableRegions" : [ ], - "assignedOU" : [ ], - "metaData" : { - "pre_provisioning" : [ - { - "code" : "CFT_PARAMS", - "params" : [ - { - "name" : "EnvironmentInstanceFiles", - "type" : "RL::SC::PARAM::HD" - }, - { - "name" : "IamPolicyDocument", - "type" : "RL::SC::PARAM::HD" - }, - { - "name" : "S3Mounts", - "type" : "RL::SC::PARAM::HD" - }, - { - "name" : "Namespace", - "type" : "RL::SC::PARAM::HD" - } - ] - } - ], - "post_provisioning" : [ ], - "checks_before_assigning_product" : [ ], - "checks_after_assigning_product" : [ ], - "permission_required" : { - - }, - "cost_resource": true - } }, - { - "_id": { - "$oid": "611b96eb4548787205d67866" - }, - "name": "Nextflow Advanced", - "description": "Use the scalability of AWS Batch to run Nextflow workflows. This is ideal for large workloads and larger data-sets.", - "fileName": "nextflow-advanced.yaml", - "tags": [ - { - "Key": "EstimatedTimeToProvision", - "Value": "10 Minutes" - }, - { - "Key": "DetailsLink", - "Value": "https://researchgateway.readthedocs.io/en/latest/nextflow.html" - }, - { - "Key": "TypeOfProduct", - "Value": "Research" - }, - { - "Key": "ALB", - "Value": "true" - }, - { - "Key": "ApplicationPort", - "Value": "443" - }, - { - "Key": "Service", - "Value": "Nextflow" - } - ], - "owner": "RL", - "portfolio": "RGPortfolio", - "availableRegions": [], - "assignedOU": [], - "metaData": { - "pre_provisioning": [ - { - "code": "CFT_PARAMS", - "params": [ - { - "name": "PipelineName", - "type": "RL::SC::PIPELINE" - }, - { - "name": "PipelineContainer", - "type": "RL::SC::PIPELINE_CONTAINER" - }, - { - "name": "InputDataLocation", - "type": "RL::SC::S3", - "productName": "Amazon S3" - }, - { - "name": "OutputDataLocation", - "type": "RL::SC::S3", - "productName": "Amazon S3" - }, - { - "name": "ResearcherName", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "ProjectId", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "EnvironmentInstanceFiles", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "IamPolicyDocument", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "S3Mounts", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "Namespace", - "type": "RL::SC::PARAM::HD" - } - ] - } - ], - "post_provisioning": [], - "checks_before_assigning_product": [ - { - "code": "AMI_ID_REQUIRED", - "is_ami_required": true, - "ami_id_list": [ - { - "us-east-1": "ami-02161943742b2f0e1" - }, - { - "us-east-2": "ami-06a28664a50da1083" - }, - { - "us-west-1": "ami-083064b2ce8cff688" - }, - { - "us-west-2": "ami-02a57220b3320a450" - }, - { - "ap-south-1": "ami-0922d7f6b2b855794" - }, - { - "ap-southeast-1": "ami-0ee126c404d54fc5b" - }, - { - "ap-southeast-2": "ami-0cdcce7966c66cd6b" - } - ], - "ami_path": "/RL/RG/StandardCatalog/nextflow-latest" - } - ], - "checks_after_assigning_product": [], - "permission_required": { - }, - "cost_resource" : true, - "aws_batch" : true - } - }, - { - "_id": { - "$oid": "62010969011b8d521ccce386" - }, - "name": "Cromwell Advanced", - "description": "Use the scalability of AWS Batch to run Cromwell workflows. This is ideal for large workloads and larger data-sets.", - "fileName": "cromwell-advanced.yaml", - "tags": [ - { - "Key": "EstimatedTimeToProvision", - "Value": "10 Minutes" - }, - { - "Key": "DetailsLink", - "Value": "https://researchgateway.readthedocs.io/en/latest/cromwell.html" - }, - { - "Key": "TypeOfProduct", - "Value": "Research" - }, - { - "Key": "ALB", - "Value": "true" - }, - { - "Key": "ApplicationPort", - "Value": "443" - }, - { - "Key": "Service", - "Value": "Cromwell" - } - ], - "owner": "RL", - "portfolio": "RGPortfolio", - "availableRegions": [], - "assignedOU": [], - "metaData": { - "pre_provisioning": [ - { - "code": "CFT_PARAMS", - "params": [ - { - "name": "PipelineName", - "type": "RL::SC::PIPELINE" - }, - { - "name": "InputDataLocation", - "type": "RL::SC::S3", - "productName": "Amazon S3" - }, - { - "name": "OutputDataLocation", - "type": "RL::SC::S3", - "productName": "Amazon S3" - }, - { - "name": "ResearcherName", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "ProjectId", - "type": "RL::SC::PARAM::HD" - }, - { - "name": "Namespace", - "type": "RL::SC::PARAM::HD" - } - ] - } - ], - "post_provisioning": [], - "checks_before_assigning_product": [ - { - "code": "AMI_ID_REQUIRED", - "is_ami_required": true, - "ami_id_list": [], - "ami_path": "/RL/RG/StandardCatalog/cromwell-latest" - } - ], - "checks_after_assigning_product": [], - "permission_required": {}, - "cost_resource": true, - "aws_batch": true - } - }, { "_id" : { "$oid":"62839f80e82248514eba03b6" From 77af9b1369874a5650d30ee1dbcb90d95280491c Mon Sep 17 00:00:00 2001 From: Anusha-janardhan Date: Tue, 4 Nov 2025 10:34:22 +0000 Subject: [PATCH 2/2] feat(upload-assets): add script for uploading assets to S3 with validation and error handling --- upload-assets.sh | 117 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 upload-assets.sh diff --git a/upload-assets.sh b/upload-assets.sh new file mode 100644 index 0000000..5a8f668 --- /dev/null +++ b/upload-assets.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Verifying required utilities +echo "Verifying utilities are installed" +apps=(jq aws) +for program in "${apps[@]}"; do + if ! command -v "$program" >/dev/null 2>&1; then + echo "$program not found. This Script needs jq and aws cli. Please install the application/s and restart deployment, Exiting." + exit + else + echo "$program found" + fi +done + +# Check required parameters +if [ $# -lt 2 ]; then + echo "Usage: $0 " + echo ' Param 1: The AMI from which the EC2 for Research Gateway should be created' + echo ' Param 2: The S3 bucket to create for holding the CFT templates' + exit 1 +fi + +amiid=$1 +bucketname=$2 +region=$(aws configure get region) +[ -z "$region" ] && echo "AWS region not configured. Run 'aws configure'" && exit 1 + +# Validate AMI ID +echo "Validating AMI ID: $amiid" +if ! aws ec2 describe-images --image-id "$amiid" >/dev/null 2>&1; then + echo "Error: AMI ID $amiid does not exist." + exit 1 +fi + +# Validate bucket name +echo "Validating bucket name: $bucketname" +if ! echo "$bucketname" | grep -q -P '(?=^.{3,63}$)(?!^xn--)(?!.*s3alias$)(?!^(\d+\.)+\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$)'; then + echo "Error: Invalid S3 bucket name." + exit 1 +fi + +localhome=$(pwd) +bucketstackname="RG-PortalStack-Bucket-$(date +%s)" + +# Create or check bucket +if aws s3api head-bucket --bucket "$bucketname" 2>/dev/null; then + echo "Bucket $bucketname exists. Proceeding to upload files..." +else + echo "Bucket $bucketname does not exist. Creating it using CloudFormation..." + aws cloudformation deploy --template-file rg_deploy_bucket.yml --stack-name "$bucketstackname" \ + --parameter-overrides S3NewBucketName="$bucketname" + aws cloudformation wait stack-create-complete --stack-name "$bucketstackname" + if [ $? -ne 0 ]; then + echo "Error: Failed to create S3 bucket via CloudFormation." + exit 1 + fi +fi + +echo "Uploading required files to s3://$bucketname..." + +# Upload individual config and script files +aws s3 cp "$localhome"/docker-compose.yml s3://"$bucketname" +aws s3 cp "$localhome"/nginx.conf s3://"$bucketname" +aws s3 cp "$localhome"/updatescripts.sh s3://"$bucketname" +aws s3 cp "$localhome"/makeconfigs.sh s3://"$bucketname" +aws s3 cp "$localhome"/mainonly.sh s3://"$bucketname" +aws s3 cp "$localhome"/makestudies.sh s3://"$bucketname" + +# Sync CFT templates +aws s3 sync "$localhome/cft-templates/" s3://"$bucketname/" + +# Upload config files +tar -czf config.tar.gz config/* +tar -tf config.tar.gz +aws s3 cp config.tar.gz s3://"$bucketname" +rm -f config.tar.gz + +# Upload scripts +tar -czf scripts.tar.gz scripts/* +tar -tf scripts.tar.gz +aws s3 cp scripts.tar.gz s3://"$bucketname" +rm -f scripts.tar.gz + +# Upload bootstrap scripts +aws s3 cp "$localhome"/scripts/bootstrap-scripts/ s3://"$bucketname/bootstrap-scripts" --recursive + +# Upload Lambda zip files +cd "$localhome"/lambdas || exit +zip -j pre_verification_custom_message.zip pre_verification_custom_message/index.js +zip -j post_verification_send_message.zip post_verification_send_message/index.js +aws s3 cp pre_verification_custom_message.zip s3://"$bucketname" +aws s3 cp post_verification_send_message.zip s3://"$bucketname" +rm -f pre_verification_custom_message.zip post_verification_send_message.zip + +# Upload Image Builder products +cd "$localhome"/products || exit + +tar -czf nicedcv.tar.gz Nicedcv/* +aws s3 cp nicedcv.tar.gz s3://"$bucketname/" +rm -f nicedcv.tar.gz + + +zip -r ec2-winsecure-image.zip ec2-secure-windows/* +aws s3 cp ec2-winsecure-image.zip s3://"$bucketname/" +rm -f ec2-winsecure-image.zip + +# Upload dump data +cd "$localhome" || exit +zip dump.zip dump/* +unzip -l dump.zip +aws s3 cp dump.zip s3://"$bucketname/" +rm -f dump.zip +echo "✅ All files uploaded successfully to s3://$bucketname" +exit 0 + + +# ./upload-assets.sh ami-0f78b782bj5ef10a6 single-cft-test-s3 \ No newline at end of file