diff --git a/documentation/getting-started.md b/documentation/getting-started.md index 308fae45..b0f31b98 100644 --- a/documentation/getting-started.md +++ b/documentation/getting-started.md @@ -26,9 +26,13 @@ so some information might change depending on which version and branch you're us ## Index - [Getting started](#getting-started) + * [Index](#index) * [Starting the server](#starting-the-server) - * [Authenticating the Resource Server](#authenticating-the-resource-server) - * [Locating the Authorization Server](#locating-the-authorization-server) + * [Authenticating as Resource Owner](#authenticating-as-resource-owner) + * [Authenticating as Resource Server](#authenticating-as-resource-server) + + [Requesting client credentials](#requesting-client-credentials) + + [Sending the credentials to the RS (CSS specific)](#sending-the-credentials-to-the-rs--css-specific-) + + [Requesting a PAT as RS](#requesting-a-pat-as-rs) * [Resource registration](#resource-registration) + [About identifiers](#about-identifiers) * [Resource access](#resource-access) @@ -41,7 +45,9 @@ so some information might change depending on which version and branch you're us + [Generate token](#generate-token) + [Use token](#use-token) * [Policies](#policies) + + [Client application identification](#client-application-identification) * [Adding or changing policies](#adding-or-changing-policies) + * [Policy backups](#policy-backups) Table of contents generated with markdown-toc @@ -439,3 +445,18 @@ ex:constraint odrl:leftOperand odrl:purpose ; ## Adding or changing policies For more details, see the [policy management API documentation](policy-management.md). + +## Policy backups + +Policies are stored in memory, meaning these will be lost when the AS is restarted. +To prevent this from happening, +there is a backup system which copies all policy data to a file every 5 minutes, +and reads it in again on server start. + +By default, this is disabled. +To enable this, you have to edit the Components.js variables +which get passed along in [`packages/uma/bin/main.js`](../packages/uma/bin/main.js). +You want to change the line that defines the `urn:uma:variables:backupFilePath` variable, +and set the string value to the path where you want the backup file to be stored, +e.g., `backup.ttl`. +When restarting the server, the contents of that file will be read to initialize policies on the server. diff --git a/packages/uma/bin/demo.js b/packages/uma/bin/demo.js index 341c7039..8c5072d0 100644 --- a/packages/uma/bin/demo.js +++ b/packages/uma/bin/demo.js @@ -18,6 +18,7 @@ const launch = async () => { // variables['urn:uma:variables:policyDir'] = path.join(rootDir, './config/rules/policy'); variables['urn:uma:variables:policyContainer'] = 'http://localhost:3000/settings/policies/'; variables['urn:uma:variables:eyePath'] = 'eye'; + variables['urn:uma:variables:backupFilePath'] = ''; const configPath = path.join(rootDir, './config/demo.json'); diff --git a/packages/uma/bin/main.js b/packages/uma/bin/main.js index 29ac9954..503552dc 100644 --- a/packages/uma/bin/main.js +++ b/packages/uma/bin/main.js @@ -18,6 +18,7 @@ const launch = async () => { variables['urn:uma:variables:policyBaseIRI'] = 'http://localhost:3000/'; variables['urn:uma:variables:policyDir'] = path.join(rootDir, './config/rules/policy'); variables['urn:uma:variables:eyePath'] = 'eye'; + variables['urn:uma:variables:backupFilePath'] = 'backup.ttl'; const configPath = path.join(rootDir, './config/default.json'); diff --git a/packages/uma/bin/odrl.js b/packages/uma/bin/odrl.js index 608452d8..91f2517f 100644 --- a/packages/uma/bin/odrl.js +++ b/packages/uma/bin/odrl.js @@ -18,6 +18,7 @@ const launch = async () => { variables['urn:uma:variables:policyBaseIRI'] = 'http://localhost:3000/'; variables['urn:uma:variables:policyDir'] = path.join(rootDir, './config/rules/odrl'); variables['urn:uma:variables:eyePath'] = 'eye'; + variables['urn:uma:variables:backupFilePath'] = ''; const configPath = path.join(rootDir, './config/odrl.json'); diff --git a/packages/uma/config/default.json b/packages/uma/config/default.json index 1dbdf4a9..88618c11 100644 --- a/packages/uma/config/default.json +++ b/packages/uma/config/default.json @@ -34,7 +34,18 @@ "comment": "This is the entry point to the application. It can be used to both start and stop the server.", "@id": "urn:uma:default:App", "@type": "App", - "initializer": { "@id": "urn:uma:default:NodeHttpServer" }, + "initializer": { + "@id": "urn:uma:default:Initializer", + "@type": "SequenceHandler", + "handlers": [ + { + "@id": "urn:uma:default:FlexibleInitializer", + "@type": "SequenceHandler", + "handlers": [] + }, + { "@id": "urn:uma:default:NodeHttpServer" } + ] + }, "finalizer": { "@id": "urn:uma:default:Finalizer", "@type": "SequenceHandler", diff --git a/packages/uma/config/demo.json b/packages/uma/config/demo.json index d74fab6e..6f84b71b 100644 --- a/packages/uma/config/demo.json +++ b/packages/uma/config/demo.json @@ -1,6 +1,7 @@ { "@context": [ - "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld" + "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/asynchronous-handlers/^1.0.0/components/context.jsonld" ], "import": [ "sai-uma:config/default.json" @@ -48,6 +49,15 @@ "@id": "urn:uma:variables:policyContainer" } } + }, + { + "comment": "Replace the InitializableHandler as the storage is no longer an Initializer", + "@id": "urn:solid-server:override:RulesStorageInitializableHandler", + "@type": "Override", + "overrideInstance": { "@id": "urn:uma:default:RulesStorageInitializableHandler" }, + "overrideParameters": { + "@type": "StaticHandler" + } } ] } diff --git a/packages/uma/config/policies/authorizers/default.json b/packages/uma/config/policies/authorizers/default.json index f5f71972..19775a38 100644 --- a/packages/uma/config/policies/authorizers/default.json +++ b/packages/uma/config/policies/authorizers/default.json @@ -1,6 +1,8 @@ { "@context": [ - "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld" + "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^8.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/asynchronous-handlers/^1.0.0/components/context.jsonld" ], "@graph": [ { @@ -53,10 +55,23 @@ "eyePath": { "@id": "urn:uma:variables:eyePath" }, "policies": { "@id": "urn:uma:default:RulesStorage", - "@type": "MemoryUCRulesStorage" + "@type": "FileBackupUCRulesStorage", + "filePath": { "@id": "urn:uma:variables:backupFilePath" } } }, "registrationStore": { "@id": "urn:uma:default:ResourceRegistrationStore" } + }, + { + "@id": "urn:uma:default:FlexibleInitializer", + "@type": "SequenceHandler", + "handlers": [ + { + "comment": "Reads in the policies stored in the backup file when starting the server", + "@id": "urn:uma:default:RulesStorageInitializableHandler", + "@type": "InitializableHandler", + "initializable": { "@id": "urn:uma:default:RulesStorage" } + } + ] } ] } diff --git a/packages/uma/config/variables/default.json b/packages/uma/config/variables/default.json index ed9164dd..c355cdb4 100644 --- a/packages/uma/config/variables/default.json +++ b/packages/uma/config/variables/default.json @@ -31,6 +31,11 @@ "comment": "URL of container where policies are stored.", "@id": "urn:uma:variables:policyContainer", "@type": "Variable" + }, + { + "comment": "File path pointing to where policies should be backed up. Set to empty string to disable backups.", + "@id": "urn:uma:variables:backupFilePath", + "@type": "Variable" } ] } diff --git a/packages/uma/src/index.ts b/packages/uma/src/index.ts index 17443d33..46367eeb 100644 --- a/packages/uma/src/index.ts +++ b/packages/uma/src/index.ts @@ -84,6 +84,7 @@ export * from './ucp/policy/ODRL'; export * from './ucp/policy/UsageControlPolicy'; export * from './ucp/storage/ContainerUCRulesStorage'; export * from './ucp/storage/DirectoryUCRulesStorage'; +export * from './ucp/storage/FileBackupUCRulesStorage'; export * from './ucp/storage/MemoryUCRulesStorage'; export * from './ucp/storage/UCRulesStorage'; export * from './ucp/util/Util'; diff --git a/packages/uma/src/ucp/storage/FileBackupUCRulesStorage.ts b/packages/uma/src/ucp/storage/FileBackupUCRulesStorage.ts new file mode 100644 index 00000000..edcf359f --- /dev/null +++ b/packages/uma/src/ucp/storage/FileBackupUCRulesStorage.ts @@ -0,0 +1,108 @@ +import { createErrorMessage, Initializable, isSystemError, setSafeInterval } from '@solid/community-server'; +import { getLoggerFor } from 'global-logger-factory'; +import { Parser, Store, Writer } from 'n3'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { rename, unlink } from 'node:fs/promises'; +import { MemoryUCRulesStorage } from './MemoryUCRulesStorage'; + +/** + * Backs up all the stored policies to a backup file every 5 minutes (default). + * Reads the policies from this file on startup. + * If no file path is defined, this is just a MemoryUCRulesStorage. + */ +export class FileBackupUCRulesStorage extends MemoryUCRulesStorage implements Initializable { + protected readonly logger = getLoggerFor(this); + + protected doingBackup = false; + protected dataChanged = false; + + public constructor(protected readonly filePath?: string, interval = 5 * 60) { + super(); + this.logger.info(`STARTING ${filePath}`); + if (filePath) { + const timer = setSafeInterval( + this.logger, + 'Failed to backup policies', + this.backup.bind(this), + interval * 1000); + timer.unref(); + } + } + + public async initialize(): Promise { + this.logger.info('CALLING INITIALIZE'); + if (!this.filePath) { + return; + } + this.logger.info(`Reading policy backup from ${this.filePath}`); + const parser = new Parser(); + const stream = createReadStream(this.filePath, 'utf8'); + parser.parse(stream, (err, quad) => { + if (err) { + if (isSystemError(err) && err.code === 'ENOENT') { + this.logger.info(`No backup file found at ${this.filePath}`); + return; + } + this.logger.error(`Problem parsing backup policies file: ${createErrorMessage(err)}`); + } + if (quad) { + this.store.add(quad); + } + }); + } + + protected async backup(): Promise { + if (!this.filePath || this.doingBackup || !this.dataChanged) { + return; + } + this.doingBackup = true; + this.logger.info(`Creating policy backup in ${this.filePath}`); + try { + // Move the previous backup just in case of a crash during backup + const oldPath = this.filePath + '.old'; + let removeOld = false; + try { + await rename(this.filePath, oldPath); + removeOld = true; + } catch (error: unknown) { + if (!isSystemError(error) || error.code !== 'ENOENT') { + throw error; + } + } + + // Backup the triples + const stream = createWriteStream(this.filePath, 'utf8'); + const writer = new Writer(stream); + writer.addQuads(this.store.getQuads(null, null, null, null)); + writer.end(); + this.dataChanged = false; + + // Remove the previous backup + if (removeOld) { + await unlink(oldPath); + } + } finally { + this.doingBackup = false; + } + } + + public async addRule(rule: Store): Promise { + this.dataChanged = true; + return super.addRule(rule); + } + + public async deleteRule(identifier: string): Promise { + this.dataChanged = true; + return super.deleteRule(identifier); + } + + public async deleteRuleFromPolicy(ruleID: string, PolicyID: string): Promise { + this.dataChanged = true; + return super.deleteRuleFromPolicy(ruleID, PolicyID); + } + + public async removeData(data: Store): Promise { + this.dataChanged = true; + return super.removeData(data); + } +} diff --git a/packages/uma/test/unit/ucp/storage/FileBackupUCRulesStorage.test.ts b/packages/uma/test/unit/ucp/storage/FileBackupUCRulesStorage.test.ts new file mode 100644 index 00000000..d1fb3deb --- /dev/null +++ b/packages/uma/test/unit/ucp/storage/FileBackupUCRulesStorage.test.ts @@ -0,0 +1,124 @@ +import { DataFactory as DF, Parser, Store } from 'n3'; +import { Readable } from 'node:stream'; +import { flushPromises } from '../../../../../../test/util/Util'; +import { FileBackupUCRulesStorage } from '../../../../src/ucp/storage/FileBackupUCRulesStorage'; +import * as fs from 'node:fs'; +import * as fsPromises from 'node:fs/promises'; + +vi.useFakeTimers(); + +vi.mock('node:fs', () => ({ + createReadStream: vi.fn().mockReturnValue(Readable.from(' .')), + createWriteStream: vi.fn(), +})); + +vi.mock('node:fs/promises', () => ({ + rename: vi.fn(), + unlink: vi.fn(), +})); + +describe('FileBackupUCRulesStorage', (): void => { + const policyString = ` + . + . + . + . + . + . + .` + const ruleIRI = 'http://example.org/1705937573496#permission'; + const filePath = 'backup.ttl'; + let policy: Store; + let storage: FileBackupUCRulesStorage; + + beforeEach(async(): Promise => { + vi.clearAllMocks(); + + const parser = new Parser(); + const quads = parser.parse(policyString); + policy = new Store(quads); + + storage = new FileBackupUCRulesStorage(filePath); + }); + + it('loads in the stored policies when initializing.', async(): Promise => { + await expect(storage.initialize()).resolves.toBeUndefined(); + + // Give parser time to parse + await flushPromises(); + + const store = await storage.getStore(); + expect(store.size).toBe(1); + expect(store.has(DF.quad(DF.namedNode('urn:a'), DF.namedNode('urn:b'), DF.namedNode('urn:c')))).toBe(true); + }); + + it('does not initialize any data if no file path is provided.', async(): Promise => { + storage = new FileBackupUCRulesStorage(); + await expect(storage.initialize()).resolves.toBeUndefined(); + + await flushPromises(); + + const store = await storage.getStore(); + expect(store.size).toBe(0); + expect(fs.createReadStream).toHaveBeenCalledTimes(0); + }); + + it('backs up data every 5 minutes.', async(): Promise => { + // Need to modify data so it will be backed up + await storage.addRule(policy); + + expect(fs.createWriteStream).not.toHaveBeenCalledOnce(); + + vi.advanceTimersByTime(5 * 60 * 1000 + 1); + await flushPromises(); + + // Something weird with the mock, `toHaveBeenCalledTimes`, seems to not work correctly + expect(fs.createWriteStream).toHaveBeenCalledExactlyOnceWith('backup.ttl', 'utf8'); + expect(fsPromises.rename).toHaveBeenCalledExactlyOnceWith('backup.ttl', 'backup.ttl.old'); + expect(fsPromises.unlink).toHaveBeenCalledExactlyOnceWith('backup.ttl.old'); + }); + + it('does not remove the old file if it does not exist.', async(): Promise => { + await storage.addRule(policy); + const error = new Error(); + (error as any).code = 'ENOENT'; + (error as any).syscall = 'call'; + vi.mocked(fsPromises.rename).mockRejectedValueOnce(error) + + vi.advanceTimersByTime(5 * 60 * 1000 + 1); + await flushPromises(); + + expect(fs.createWriteStream).toHaveBeenCalledExactlyOnceWith('backup.ttl', 'utf8'); + expect(fsPromises.unlink).not.toHaveBeenCalledExactlyOnceWith('backup.ttl.old'); + }); + + it('does not backup data if no file path is provided.', async(): Promise => { + storage = new FileBackupUCRulesStorage(); + + vi.advanceTimersByTime(5 * 60 * 1000 + 1); + await flushPromises(); + + expect(fsPromises.rename).not.toHaveBeenCalledOnce(); + }); + + it('does not start a backup if there is one already in progress.', async(): Promise => { + await storage.addRule(policy); + // Promise that does not end + vi.mocked(fsPromises.unlink).mockResolvedValueOnce(new Promise(() => {}) as any); + + vi.advanceTimersByTime(5 * 60 * 1000 + 1); + await flushPromises(); + + vi.advanceTimersByTime(5 * 60 * 1000 + 1); + await flushPromises(); + + expect(fs.createWriteStream).toHaveBeenCalledExactlyOnceWith('backup.ttl', 'utf8'); + }); + + it('does not backup data if no data was changed.', async(): Promise => { + vi.advanceTimersByTime(5 * 60 * 1000 + 1); + await flushPromises(); + + expect(fs.createWriteStream).not.toHaveBeenCalledExactlyOnceWith('backup.ttl', 'utf8'); + }); +}); diff --git a/test/integration/Base.test.ts b/test/integration/Base.test.ts index 042c8cd5..639d9fe9 100644 --- a/test/integration/Base.test.ts +++ b/test/integration/Base.test.ts @@ -23,6 +23,7 @@ describe('A server setup', (): void => { 'urn:uma:variables:port': umaPort, 'urn:uma:variables:baseUrl': `http://localhost:${umaPort}/uma`, 'urn:uma:variables:eyePath': 'eye', + 'urn:uma:variables:backupFilePath': '', } ) as App; diff --git a/test/integration/Demo.test.ts b/test/integration/Demo.test.ts index f6250424..551ed14e 100644 --- a/test/integration/Demo.test.ts +++ b/test/integration/Demo.test.ts @@ -36,6 +36,7 @@ describe('A demo server setup', (): void => { 'urn:uma:variables:baseUrl': `http://localhost:${umaPort}/uma`, 'urn:uma:variables:eyePath': 'eye', 'urn:uma:variables:policyContainer': policyContainer, + 'urn:uma:variables:backupFilePath': '', } ) as App; diff --git a/test/integration/Odrl.test.ts b/test/integration/Odrl.test.ts index 4a02633b..466b98df 100644 --- a/test/integration/Odrl.test.ts +++ b/test/integration/Odrl.test.ts @@ -24,6 +24,7 @@ describe('An ODRL server setup', (): void => { 'urn:uma:variables:port': umaPort, 'urn:uma:variables:baseUrl': `http://localhost:${umaPort}/uma`, 'urn:uma:variables:eyePath': 'eye', + 'urn:uma:variables:backupFilePath': '', } ) as App; diff --git a/test/integration/Oidc.test.ts b/test/integration/Oidc.test.ts index 863f8508..0b3032b1 100644 --- a/test/integration/Oidc.test.ts +++ b/test/integration/Oidc.test.ts @@ -30,6 +30,7 @@ describe('A server supporting OIDC tokens', (): void => { 'urn:uma:variables:port': umaPort, 'urn:uma:variables:baseUrl': `http://localhost:${umaPort}/uma`, 'urn:uma:variables:eyePath': 'eye', + 'urn:uma:variables:backupFilePath': '', } ) as App; diff --git a/test/integration/Policies.test.ts b/test/integration/Policies.test.ts index 98c51123..3a608b27 100644 --- a/test/integration/Policies.test.ts +++ b/test/integration/Policies.test.ts @@ -89,6 +89,7 @@ describe('A policy server setup', (): void => { 'urn:uma:variables:port': umaPort, 'urn:uma:variables:baseUrl': `http://localhost:${umaPort}/uma`, 'urn:uma:variables:eyePath': 'eye', + 'urn:uma:variables:backupFilePath': '', } ) as App;