Skip to content

Commit caa8cfa

Browse files
author
StackMemory Bot (CLI)
committed
feat(database): add generational context compression to GC
- Add compressFrame() and compressFrames() methods with digest_only and anchors_only strategies - Wire generational GC into runGC() as Phase 1 before deletion Phase 2 - Mature frames (1-7d) compress to digest_only: strip inputs/outputs, delete events - Old frames (7-30d) compress to anchors_only: also clear digest_json - Add getDatabaseSize() via PRAGMA page_count * page_size - Skip already-compressed frames (inputs='{}'), keep_forever, active, and protected runs - Add 16 tests covering all compression strategies, edge cases, and protections
1 parent 3ee724f commit caa8cfa

File tree

2 files changed

+541
-5
lines changed

2 files changed

+541
-5
lines changed

src/core/database/__tests__/gc.test.ts

Lines changed: 326 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,332 @@ describe('Garbage Collection', () => {
491491
expect(score).toBeLessThanOrEqual(1.0);
492492
});
493493

494+
it('should return framesCompressed: 0 when no generational GC', async () => {
495+
insertFrame({ frameId: 'old-1', createdAt: daysAgo(100) });
496+
const result = await adapter.runGC({ retentionDays: 90 });
497+
expect(result.framesCompressed).toBe(0);
498+
});
499+
});
500+
501+
// --- Generational compression ---
502+
503+
describe('Generational compression', () => {
504+
it('should compress mature frames with digest_only strategy', async () => {
505+
// 3 days old = mature (between young=1d and old=30d cutoffs)
506+
insertFrame({ frameId: 'mature-1', createdAt: daysAgo(3) });
507+
// Set non-empty inputs so it qualifies for compression
508+
const db = adapter.getRawDatabase()!;
509+
db.prepare(
510+
'UPDATE frames SET inputs = \'{"key":"value"}\', outputs = \'{"out":"data"}\' WHERE frame_id = \'mature-1\''
511+
).run();
512+
insertEvent('mature-1', 'evt-1');
513+
insertEvent('mature-1', 'evt-2');
514+
515+
const result = await adapter.runGC({
516+
retentionDays: 90,
517+
generationalGc: {
518+
mature_strategy: 'digest_only',
519+
youngCutoffDays: 1,
520+
matureCutoffDays: 7,
521+
oldCutoffDays: 30,
522+
},
523+
});
524+
525+
expect(result.framesCompressed).toBe(1);
526+
expect(result.framesDeleted).toBe(0);
527+
528+
// Frame still exists but inputs/outputs stripped
529+
const frame = await adapter.getFrame('mature-1');
530+
expect(frame).not.toBeNull();
531+
expect(frame!.inputs).toEqual({});
532+
expect(frame!.outputs).toEqual({});
533+
534+
// Events deleted
535+
expect(countRows('events')).toBe(0);
536+
});
537+
538+
it('should compress old frames with anchors_only strategy', async () => {
539+
// 15 days old = old (between mature=7d and deletion=30d)
540+
insertFrame({
541+
frameId: 'old-1',
542+
createdAt: daysAgo(15),
543+
digestText: 'important decision',
544+
});
545+
const db = adapter.getRawDatabase()!;
546+
db.prepare(
547+
'UPDATE frames SET inputs = \'{"key":"value"}\', outputs = \'{"out":"data"}\', digest_json = \'{"summary":"test"}\' WHERE frame_id = \'old-1\''
548+
).run();
549+
insertEvent('old-1', 'evt-1');
550+
insertAnchor('old-1', 'anc-1', 'DECISION');
551+
552+
const result = await adapter.runGC({
553+
retentionDays: 90,
554+
generationalGc: {
555+
old_strategy: 'anchors_only',
556+
youngCutoffDays: 1,
557+
matureCutoffDays: 7,
558+
oldCutoffDays: 30,
559+
},
560+
});
561+
562+
expect(result.framesCompressed).toBe(1);
563+
564+
// Frame exists: inputs/outputs/digest_json stripped
565+
const frame = await adapter.getFrame('old-1');
566+
expect(frame).not.toBeNull();
567+
expect(frame!.inputs).toEqual({});
568+
expect(frame!.outputs).toEqual({});
569+
expect(frame!.digest_json).toEqual({});
570+
571+
// digest_text preserved (for search)
572+
expect(frame!.digest_text).toBe('important decision');
573+
574+
// Anchors preserved
575+
expect(countRows('anchors')).toBe(1);
576+
577+
// Events deleted
578+
expect(countRows('events')).toBe(0);
579+
});
580+
581+
it('should skip already-compressed frames (inputs already empty)', async () => {
582+
// Already compressed (inputs = '{}')
583+
insertFrame({ frameId: 'already-compressed', createdAt: daysAgo(3) });
584+
585+
const result = await adapter.runGC({
586+
retentionDays: 90,
587+
generationalGc: {
588+
mature_strategy: 'digest_only',
589+
youngCutoffDays: 1,
590+
matureCutoffDays: 7,
591+
},
592+
});
593+
594+
expect(result.framesCompressed).toBe(0);
595+
});
596+
597+
it('should not compress young frames', async () => {
598+
// 12 hours old = young
599+
insertFrame({
600+
frameId: 'young-1',
601+
createdAt: nowSec - 43200,
602+
});
603+
const db = adapter.getRawDatabase()!;
604+
db.prepare(
605+
'UPDATE frames SET inputs = \'{"key":"value"}\' WHERE frame_id = \'young-1\''
606+
).run();
607+
insertEvent('young-1', 'evt-1');
608+
609+
const result = await adapter.runGC({
610+
retentionDays: 90,
611+
generationalGc: {
612+
mature_strategy: 'digest_only',
613+
youngCutoffDays: 1,
614+
matureCutoffDays: 7,
615+
},
616+
});
617+
618+
expect(result.framesCompressed).toBe(0);
619+
// Events still there
620+
expect(countRows('events')).toBe(1);
621+
});
622+
623+
it('should not compress keep_forever frames', async () => {
624+
insertFrame({
625+
frameId: 'forever-1',
626+
createdAt: daysAgo(15),
627+
retentionPolicy: 'keep_forever',
628+
});
629+
const db = adapter.getRawDatabase()!;
630+
db.prepare(
631+
'UPDATE frames SET inputs = \'{"key":"value"}\' WHERE frame_id = \'forever-1\''
632+
).run();
633+
634+
const result = await adapter.runGC({
635+
retentionDays: 90,
636+
generationalGc: {
637+
old_strategy: 'anchors_only',
638+
youngCutoffDays: 1,
639+
matureCutoffDays: 7,
640+
oldCutoffDays: 30,
641+
},
642+
});
643+
644+
expect(result.framesCompressed).toBe(0);
645+
});
646+
647+
it('should not compress active frames', async () => {
648+
insertFrame({
649+
frameId: 'active-1',
650+
createdAt: daysAgo(5),
651+
state: 'active',
652+
});
653+
const db = adapter.getRawDatabase()!;
654+
db.prepare(
655+
'UPDATE frames SET inputs = \'{"key":"value"}\' WHERE frame_id = \'active-1\''
656+
).run();
657+
658+
const result = await adapter.runGC({
659+
retentionDays: 90,
660+
generationalGc: {
661+
mature_strategy: 'digest_only',
662+
youngCutoffDays: 1,
663+
matureCutoffDays: 7,
664+
},
665+
});
666+
667+
expect(result.framesCompressed).toBe(0);
668+
});
669+
670+
it('should not compress protected run_id frames', async () => {
671+
insertFrame({
672+
frameId: 'protected-1',
673+
createdAt: daysAgo(5),
674+
runId: 'active-session',
675+
});
676+
const db = adapter.getRawDatabase()!;
677+
db.prepare(
678+
'UPDATE frames SET inputs = \'{"key":"value"}\' WHERE frame_id = \'protected-1\''
679+
).run();
680+
681+
const result = await adapter.runGC({
682+
retentionDays: 90,
683+
protectedRunIds: ['active-session'],
684+
generationalGc: {
685+
mature_strategy: 'digest_only',
686+
youngCutoffDays: 1,
687+
matureCutoffDays: 7,
688+
},
689+
});
690+
691+
expect(result.framesCompressed).toBe(0);
692+
});
693+
694+
it('should skip compression in dryRun mode', async () => {
695+
insertFrame({ frameId: 'mature-1', createdAt: daysAgo(3) });
696+
const db = adapter.getRawDatabase()!;
697+
db.prepare(
698+
'UPDATE frames SET inputs = \'{"key":"value"}\' WHERE frame_id = \'mature-1\''
699+
).run();
700+
701+
const result = await adapter.runGC({
702+
retentionDays: 90,
703+
dryRun: true,
704+
generationalGc: {
705+
mature_strategy: 'digest_only',
706+
youngCutoffDays: 1,
707+
matureCutoffDays: 7,
708+
},
709+
});
710+
711+
expect(result.framesCompressed).toBe(0);
712+
// Inputs still intact
713+
const frame = await adapter.getFrame('mature-1');
714+
expect(frame!.inputs).toEqual({ key: 'value' });
715+
});
716+
717+
it('should compress both mature and old tiers in one run', async () => {
718+
// Mature frame (3 days old)
719+
insertFrame({ frameId: 'mature-1', createdAt: daysAgo(3) });
720+
// Old frame (15 days old)
721+
insertFrame({ frameId: 'old-1', createdAt: daysAgo(15) });
722+
723+
const db = adapter.getRawDatabase()!;
724+
db.prepare(
725+
"UPDATE frames SET inputs = '{\"data\":\"yes\"}' WHERE frame_id IN ('mature-1', 'old-1')"
726+
).run();
727+
728+
const result = await adapter.runGC({
729+
retentionDays: 90,
730+
generationalGc: {
731+
mature_strategy: 'digest_only',
732+
old_strategy: 'anchors_only',
733+
youngCutoffDays: 1,
734+
matureCutoffDays: 7,
735+
oldCutoffDays: 30,
736+
},
737+
});
738+
739+
expect(result.framesCompressed).toBe(2);
740+
});
741+
742+
it('should keep_all when strategy is keep_all', async () => {
743+
insertFrame({ frameId: 'mature-1', createdAt: daysAgo(3) });
744+
const db = adapter.getRawDatabase()!;
745+
db.prepare(
746+
'UPDATE frames SET inputs = \'{"data":"yes"}\' WHERE frame_id = \'mature-1\''
747+
).run();
748+
insertEvent('mature-1', 'evt-1');
749+
750+
const result = await adapter.runGC({
751+
retentionDays: 90,
752+
generationalGc: {
753+
mature_strategy: 'keep_all',
754+
youngCutoffDays: 1,
755+
matureCutoffDays: 7,
756+
},
757+
});
758+
759+
expect(result.framesCompressed).toBe(0);
760+
expect(countRows('events')).toBe(1);
761+
const frame = await adapter.getFrame('mature-1');
762+
expect(frame!.inputs).toEqual({ data: 'yes' });
763+
});
764+
});
765+
766+
describe('compressFrame', () => {
767+
it('should return false for non-existent frame', () => {
768+
const result = adapter.compressFrame('nonexistent', 'digest_only');
769+
expect(result).toBe(false);
770+
});
771+
772+
it('digest_only should strip inputs/outputs and delete events', async () => {
773+
insertFrame({ frameId: 'f1', createdAt: daysAgo(5), digestText: 'kept' });
774+
const db = adapter.getRawDatabase()!;
775+
db.prepare(
776+
"UPDATE frames SET inputs = '{\"a\":1}', outputs = '{\"b\":2}' WHERE frame_id = 'f1'"
777+
).run();
778+
insertEvent('f1', 'evt-1');
779+
780+
const result = adapter.compressFrame('f1', 'digest_only');
781+
expect(result).toBe(true);
782+
783+
const frame = await adapter.getFrame('f1');
784+
expect(frame!.inputs).toEqual({});
785+
expect(frame!.outputs).toEqual({});
786+
expect(frame!.digest_text).toBe('kept');
787+
expect(countRows('events')).toBe(0);
788+
});
789+
790+
it('anchors_only should also clear digest_json', async () => {
791+
insertFrame({ frameId: 'f1', createdAt: daysAgo(5), digestText: 'kept' });
792+
const db = adapter.getRawDatabase()!;
793+
db.prepare(
794+
'UPDATE frames SET inputs = \'{"a":1}\', digest_json = \'{"s":"t"}\' WHERE frame_id = \'f1\''
795+
).run();
796+
insertAnchor('f1', 'anc-1', 'DECISION');
797+
798+
const result = adapter.compressFrame('f1', 'anchors_only');
799+
expect(result).toBe(true);
800+
801+
const frame = await adapter.getFrame('f1');
802+
expect(frame!.inputs).toEqual({});
803+
expect(frame!.digest_json).toEqual({});
804+
expect(frame!.digest_text).toBe('kept');
805+
// Anchors preserved
806+
expect(countRows('anchors')).toBe(1);
807+
});
808+
});
809+
810+
describe('getDatabaseSize', () => {
811+
it('should return a positive number', () => {
812+
const size = adapter.getDatabaseSize();
813+
expect(size).toBeGreaterThan(0);
814+
});
815+
});
816+
817+
// --- Importance scoring ---
818+
819+
describe('Importance scoring', () => {
494820
it('should recompute scores in batches', async () => {
495821
// Insert frames with default score of 0.5
496822
insertFrame({

0 commit comments

Comments
 (0)