Skip to content

Commit c379cbe

Browse files
jonathanpeterwujonathanpeterwu
committed
fix: Railway build cache issues and add frame sync scripts
- Remove node_modules cache before install to fix EBUSY error - Disable cache directories in nixpacks to prevent conflicts - Add sync script to pull frames from Railway PostgreSQL - Add recreate script for local SQLite database
1 parent 06ad44b commit c379cbe

File tree

4 files changed

+319
-5
lines changed

4 files changed

+319
-5
lines changed

‎nixpacks.toml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,17 @@ nixPkgs = ["nodejs_20", "npm-10_x"]
66

77
[phases.install]
88
cmds = [
9-
"rm -rf node_modules/.cache",
10-
"npm ci --omit=dev --ignore-scripts"
9+
"rm -rf node_modules/.cache node_modules",
10+
"npm ci"
1111
]
12-
cacheDirectories = ["/root/.npm"]
12+
cacheDirectories = []
1313

1414
[phases.build]
1515
dependsOn = ["install"]
1616
cmds = ["npm run build"]
1717

1818
[start]
19-
cmd = "node dist/servers/railway/index.js"
19+
cmd = "node dist/servers/railway/minimal.js"
2020

2121
[variables]
2222
NODE_ENV = "production"

‎railway.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[build]
22
builder = "nixpacks"
3-
buildCommand = "rm -rf node_modules/.cache && npm ci --omit=dev --ignore-scripts && npm run build"
3+
buildCommand = "rm -rf node_modules/.cache node_modules && npm ci && npm run build"
44

55
[deploy]
66
startCommand = "node dist/servers/railway/minimal.js"

‎scripts/recreate-frames-db.js‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/usr/bin/env node
2+
import 'dotenv/config';
3+
import Database from 'better-sqlite3';
4+
import { mkdirSync } from 'fs';
5+
import { dirname, join } from 'path';
6+
import { homedir } from 'os';
7+
8+
// Create the database directory if it doesn't exist
9+
const dbPath = join(homedir(), '.stackmemory', 'context.db');
10+
mkdirSync(dirname(dbPath), { recursive: true });
11+
12+
// Connect to the database
13+
const db = new Database(dbPath);
14+
15+
// Create frames table with proper schema
16+
db.exec(`
17+
CREATE TABLE IF NOT EXISTS frames (
18+
frame_id TEXT PRIMARY KEY,
19+
run_id TEXT NOT NULL,
20+
project_id TEXT NOT NULL,
21+
parent_frame_id TEXT REFERENCES frames(frame_id),
22+
depth INTEGER NOT NULL DEFAULT 0,
23+
type TEXT NOT NULL,
24+
name TEXT NOT NULL,
25+
state TEXT DEFAULT 'active',
26+
inputs TEXT DEFAULT '{}',
27+
outputs TEXT DEFAULT '{}',
28+
digest_text TEXT,
29+
digest_json TEXT DEFAULT '{}',
30+
created_at INTEGER DEFAULT (unixepoch()),
31+
closed_at INTEGER
32+
);
33+
34+
CREATE TABLE IF NOT EXISTS events (
35+
event_id TEXT PRIMARY KEY,
36+
run_id TEXT NOT NULL,
37+
frame_id TEXT NOT NULL,
38+
seq INTEGER NOT NULL,
39+
event_type TEXT NOT NULL,
40+
payload TEXT NOT NULL,
41+
ts INTEGER DEFAULT (unixepoch()),
42+
FOREIGN KEY(frame_id) REFERENCES frames(frame_id)
43+
);
44+
45+
CREATE TABLE IF NOT EXISTS anchors (
46+
anchor_id TEXT PRIMARY KEY,
47+
frame_id TEXT NOT NULL,
48+
project_id TEXT NOT NULL,
49+
type TEXT NOT NULL,
50+
text TEXT NOT NULL,
51+
priority INTEGER DEFAULT 0,
52+
created_at INTEGER DEFAULT (unixepoch()),
53+
metadata TEXT DEFAULT '{}',
54+
FOREIGN KEY(frame_id) REFERENCES frames(frame_id)
55+
);
56+
57+
CREATE TABLE IF NOT EXISTS handoff_requests (
58+
request_id TEXT PRIMARY KEY,
59+
source_stack_id TEXT NOT NULL,
60+
target_stack_id TEXT NOT NULL,
61+
frame_ids TEXT NOT NULL,
62+
status TEXT NOT NULL DEFAULT 'pending',
63+
created_at INTEGER DEFAULT (unixepoch()),
64+
expires_at INTEGER,
65+
target_user_id TEXT,
66+
message TEXT
67+
);
68+
69+
CREATE INDEX IF NOT EXISTS idx_frames_run ON frames(run_id);
70+
CREATE INDEX IF NOT EXISTS idx_frames_parent ON frames(parent_frame_id);
71+
CREATE INDEX IF NOT EXISTS idx_frames_state ON frames(state);
72+
CREATE INDEX IF NOT EXISTS idx_events_frame ON events(frame_id);
73+
CREATE INDEX IF NOT EXISTS idx_events_seq ON events(frame_id, seq);
74+
CREATE INDEX IF NOT EXISTS idx_anchors_frame ON anchors(frame_id);
75+
CREATE INDEX IF NOT EXISTS idx_handoff_requests_status ON handoff_requests(status);
76+
CREATE INDEX IF NOT EXISTS idx_handoff_requests_target ON handoff_requests(target_stack_id);
77+
`);
78+
79+
console.log('✅ Frames database recreated at:', dbPath);
80+
81+
// Verify tables exist
82+
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all();
83+
console.log('📊 Tables created:', tables.map(t => t.name).join(', '));
84+
85+
// Check if we have any existing frames
86+
const frameCount = db.prepare('SELECT COUNT(*) as count FROM frames').get();
87+
console.log('📈 Existing frames:', frameCount.count);
88+
89+
db.close();

‎scripts/sync-frames-from-railway.js‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
#!/usr/bin/env node
2+
import 'dotenv/config';
3+
import pg from 'pg';
4+
import Database from 'better-sqlite3';
5+
import { mkdirSync } from 'fs';
6+
import { dirname, join } from 'path';
7+
import { homedir } from 'os';
8+
9+
const { Client } = pg;
10+
11+
// Railway PostgreSQL connection
12+
const RAILWAY_DATABASE_URL = 'postgresql://postgres:YTSFXqPzFhghOcefgwPvJyWOBTYHbYxd@postgres.railway.internal:5432/railway';
13+
14+
// Local SQLite database path
15+
const dbPath = join(homedir(), '.stackmemory', 'context.db');
16+
mkdirSync(dirname(dbPath), { recursive: true });
17+
18+
async function syncFramesFromRailway() {
19+
console.log('🔄 Starting sync from Railway database...\n');
20+
21+
// Connect to Railway PostgreSQL
22+
const pgClient = new Client({
23+
connectionString: RAILWAY_DATABASE_URL,
24+
});
25+
26+
try {
27+
await pgClient.connect();
28+
console.log('✅ Connected to Railway PostgreSQL database');
29+
30+
// Connect to local SQLite
31+
const sqliteDb = new Database(dbPath);
32+
console.log('✅ Connected to local SQLite database\n');
33+
34+
// Check if frames table exists in PostgreSQL
35+
const tableCheckQuery = `
36+
SELECT EXISTS (
37+
SELECT FROM information_schema.tables
38+
WHERE table_name = 'frames'
39+
);
40+
`;
41+
42+
const tableExists = await pgClient.query(tableCheckQuery);
43+
44+
if (!tableExists.rows[0].exists) {
45+
console.log('⚠️ No frames table found in Railway database');
46+
console.log(' The Railway deployment may not have created frames yet.\n');
47+
48+
// Check for other relevant tables
49+
const tablesQuery = `
50+
SELECT table_name
51+
FROM information_schema.tables
52+
WHERE table_schema = 'public'
53+
ORDER BY table_name;
54+
`;
55+
56+
const tables = await pgClient.query(tablesQuery);
57+
console.log('📊 Available tables in Railway database:');
58+
tables.rows.forEach(row => {
59+
console.log(` - ${row.table_name}`);
60+
});
61+
62+
return;
63+
}
64+
65+
// Fetch frames from Railway
66+
const framesQuery = `
67+
SELECT
68+
frame_id,
69+
run_id,
70+
project_id,
71+
parent_frame_id,
72+
depth,
73+
type,
74+
name,
75+
state,
76+
inputs,
77+
outputs,
78+
digest_text,
79+
digest_json,
80+
created_at,
81+
closed_at
82+
FROM frames
83+
ORDER BY created_at DESC
84+
LIMIT 1000;
85+
`;
86+
87+
const framesResult = await pgClient.query(framesQuery);
88+
console.log(`📥 Found ${framesResult.rows.length} frames in Railway database\n`);
89+
90+
if (framesResult.rows.length === 0) {
91+
console.log('ℹ️ No frames to sync. The Railway database is empty.');
92+
return;
93+
}
94+
95+
// Prepare SQLite insert statement
96+
const insertStmt = sqliteDb.prepare(`
97+
INSERT OR REPLACE INTO frames (
98+
frame_id, run_id, project_id, parent_frame_id, depth,
99+
type, name, state, inputs, outputs, digest_text, digest_json,
100+
created_at, closed_at
101+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
102+
`);
103+
104+
// Begin transaction for bulk insert
105+
const insertMany = sqliteDb.transaction((frames) => {
106+
for (const frame of frames) {
107+
insertStmt.run(
108+
frame.frame_id,
109+
frame.run_id,
110+
frame.project_id,
111+
frame.parent_frame_id,
112+
frame.depth,
113+
frame.type,
114+
frame.name,
115+
frame.state,
116+
typeof frame.inputs === 'object' ? JSON.stringify(frame.inputs) : frame.inputs,
117+
typeof frame.outputs === 'object' ? JSON.stringify(frame.outputs) : frame.outputs,
118+
frame.digest_text,
119+
typeof frame.digest_json === 'object' ? JSON.stringify(frame.digest_json) : frame.digest_json,
120+
frame.created_at ? new Date(frame.created_at).getTime() : Date.now(),
121+
frame.closed_at ? new Date(frame.closed_at).getTime() : null
122+
);
123+
}
124+
});
125+
126+
// Execute bulk insert
127+
insertMany(framesResult.rows);
128+
console.log(`✅ Synced ${framesResult.rows.length} frames to local database\n`);
129+
130+
// Also sync events if they exist
131+
const eventsCheckQuery = `
132+
SELECT EXISTS (
133+
SELECT FROM information_schema.tables
134+
WHERE table_name = 'events'
135+
);
136+
`;
137+
138+
const eventsExist = await pgClient.query(eventsCheckQuery);
139+
140+
if (eventsExist.rows[0].exists) {
141+
const eventsQuery = `
142+
SELECT
143+
event_id,
144+
run_id,
145+
frame_id,
146+
seq,
147+
event_type,
148+
payload,
149+
ts
150+
FROM events
151+
ORDER BY ts DESC
152+
LIMIT 5000;
153+
`;
154+
155+
const eventsResult = await pgClient.query(eventsQuery);
156+
157+
if (eventsResult.rows.length > 0) {
158+
console.log(`📥 Found ${eventsResult.rows.length} events in Railway database`);
159+
160+
const eventInsertStmt = sqliteDb.prepare(`
161+
INSERT OR REPLACE INTO events (
162+
event_id, run_id, frame_id, seq, event_type, payload, ts
163+
) VALUES (?, ?, ?, ?, ?, ?, ?)
164+
`);
165+
166+
const insertEvents = sqliteDb.transaction((events) => {
167+
for (const event of events) {
168+
eventInsertStmt.run(
169+
event.event_id,
170+
event.run_id,
171+
event.frame_id,
172+
event.seq,
173+
event.event_type,
174+
typeof event.payload === 'object' ? JSON.stringify(event.payload) : event.payload,
175+
event.ts ? new Date(event.ts).getTime() : Date.now()
176+
);
177+
}
178+
});
179+
180+
insertEvents(eventsResult.rows);
181+
console.log(`✅ Synced ${eventsResult.rows.length} events to local database\n`);
182+
}
183+
}
184+
185+
// Verify the sync
186+
const frameCount = sqliteDb.prepare('SELECT COUNT(*) as count FROM frames').get();
187+
const eventCount = sqliteDb.prepare('SELECT COUNT(*) as count FROM events').get();
188+
189+
console.log('📊 Local database statistics:');
190+
console.log(` - Frames: ${frameCount.count}`);
191+
console.log(` - Events: ${eventCount.count}`);
192+
193+
// Show recent frames
194+
const recentFrames = sqliteDb.prepare(`
195+
SELECT frame_id, name, type, state, datetime(created_at/1000, 'unixepoch') as created
196+
FROM frames
197+
ORDER BY created_at DESC
198+
LIMIT 5
199+
`).all();
200+
201+
if (recentFrames.length > 0) {
202+
console.log('\n🕐 Recent frames:');
203+
recentFrames.forEach(frame => {
204+
console.log(` - ${frame.name} (${frame.type}) - ${frame.state} - ${frame.created}`);
205+
});
206+
}
207+
208+
sqliteDb.close();
209+
210+
} catch (error) {
211+
console.error('❌ Error syncing frames:', error.message);
212+
213+
// If connection failed due to internal network, try external URL
214+
if (error.message.includes('ENOTFOUND') || error.message.includes('postgres.railway.internal')) {
215+
console.log('\n🔄 Retrying with external Railway database URL...');
216+
console.log(' Note: You may need to get the external DATABASE_URL from Railway dashboard.');
217+
console.log(' Run: railway variables --json | jq -r .DATABASE_URL');
218+
}
219+
} finally {
220+
await pgClient.end();
221+
}
222+
}
223+
224+
// Run the sync
225+
syncFramesFromRailway().catch(console.error);

0 commit comments

Comments
 (0)