openapi-crawler/example_all_databases.sh at main · psui3905/openapi-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
#!/bin/bash
#
# Example script: Download data from all 3 verified databases
#
# This script demonstrates how to use openapi_downloader.py with:
# - HuBMAP: Human tissue samples, donors, datasets
# - SenNet: Cellular senescence data
# - CFDE: Gene regulation data
#

set -e

echo "=========================================="
echo "Download Data from 3 Biomedical Databases"
echo "=========================================="
echo ""

# Create output directory
mkdir -p data
cd data

# ==================== HuBMAP ====================
echo "1/3: Downloading HuBMAP (Human Tissue Data)"
echo "--------------------------------------------"
echo "Downloading OpenAPI spec..."
curl -s "https://smart-api.info/api/metadata/7aaf02b838022d564da776b03f357158" > hubmap.json

echo "Downloading data (limited to 50 rows per entity for testing)..."
python3 ../openapi_downloader.py \
  --openapi hubmap.json \
  --out-dir ./hubmap \
  --max-rows-per-entity 50

echo "✓ HuBMAP complete"
echo ""

# ==================== SenNet ====================
echo "2/3: Downloading SenNet (Cellular Senescence Data)"
echo "---------------------------------------------------"
echo "Downloading OpenAPI spec..."
curl -s "https://smart-api.info/api/metadata/7d838c9dee0caa2f8fe57173282c5812" > sennet.json

echo "Downloading data (limited to 50 rows per entity for testing)..."
python3 ../openapi_downloader.py \
  --openapi sennet.json \
  --out-dir ./sennet \
  --max-rows-per-entity 50

echo "✓ SenNet complete"
echo ""

# ==================== CFDE ====================
echo "3/3: Downloading CFDE (Gene Regulation Data)"
echo "----------------------------------------------"
echo "Downloading OpenAPI spec..."
curl -s "https://smart-api.info/api/metadata/d1ac2227e079aa3cae4e1cd696431ff8" > cfde.json

echo "Downloading data (limited to 50 rows per entity for testing)..."
python3 ../openapi_downloader.py \
  --openapi cfde.json \
  --out-dir ./cfde \
  --max-rows-per-entity 50

echo "✓ CFDE complete"
echo ""

# ==================== Summary ====================
echo "=========================================="
echo "Summary"
echo "=========================================="
echo ""
echo "HuBMAP files:"
ls -lh hubmap/*.csv 2>/dev/null || echo "  No CSV files generated"
echo ""
echo "SenNet files:"
ls -lh sennet/*.csv 2>/dev/null || echo "  No CSV files generated"
echo ""
echo "CFDE files:"
ls -lh cfde/*.csv 2>/dev/null || echo "  No CSV files generated"
echo ""

echo "=========================================="
echo "✓ All downloads complete!"
echo "=========================================="
echo ""
echo "Data saved to: $(pwd)"
echo ""
echo "To download FULL datasets (not limited to 50 rows),"
echo "remove the --max-rows-per-entity flag from the commands."