feat: Backup memory-based policies to file

documentation/getting-started.md

@@ -26,9 +26,13 @@ so some information might change depending on which version and branch you're us
 ## Index
 
 - [Getting started](#getting-started)
+  * [Index](#index)
   * [Starting the server](#starting-the-server)
-  * [Authenticating the Resource Server](#authenticating-the-resource-server)
-  * [Locating the Authorization Server](#locating-the-authorization-server)
+  * [Authenticating as Resource Owner](#authenticating-as-resource-owner)
+  * [Authenticating as Resource Server](#authenticating-as-resource-server)
+    + [Requesting client credentials](#requesting-client-credentials)
+    + [Sending the credentials to the RS (CSS specific)](#sending-the-credentials-to-the-rs--css-specific-)
+    + [Requesting a PAT as RS](#requesting-a-pat-as-rs)
   * [Resource registration](#resource-registration)
     + [About identifiers](#about-identifiers)
   * [Resource access](#resource-access)
@@ -41,7 +45,9 @@ so some information might change depending on which version and branch you're us
     + [Generate token](#generate-token)
     + [Use token](#use-token)
   * [Policies](#policies)
+    + [Client application identification](#client-application-identification)
   * [Adding or changing policies](#adding-or-changing-policies)
+  * [Policy backups](#policy-backups)
 
 <small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small>
 
@@ -439,3 +445,18 @@ ex:constraint odrl:leftOperand odrl:purpose ;
 ## Adding or changing policies
 
 For more details, see the [policy management API documentation](policy-management.md).
+
+## Policy backups
+
+Policies are stored in memory, meaning these will be lost when the AS is restarted.
+To prevent this from happening,
+there is a backup system which copies all policy data to a file every 5 minutes,
+and reads it in again on server start.
+
+By default, this is disabled.
+To enable this, you have to edit the Components.js variables
+which get passed along in [`packages/uma/bin/main.js`](../packages/uma/bin/main.js).
+You want to change the line that defines the `urn:uma:variables:backupFilePath` variable,
+and set the string value to the path where you want the backup file to be stored,
+e.g., `backup.ttl`.
+When restarting the server, the contents of that file will be read to initialize policies on the server.

packages/uma/bin/demo.js

@@ -18,6 +18,7 @@ const launch = async () => {
   // variables['urn:uma:variables:policyDir'] = path.join(rootDir, './config/rules/policy');
   variables['urn:uma:variables:policyContainer'] = 'http://localhost:3000/settings/policies/';
   variables['urn:uma:variables:eyePath'] = 'eye';
+  variables['urn:uma:variables:backupFilePath'] = '';
 
   const configPath = path.join(rootDir, './config/demo.json');
 

packages/uma/bin/main.js

@@ -18,6 +18,7 @@ const launch = async () => {
   variables['urn:uma:variables:policyBaseIRI'] = 'http://localhost:3000/';
   variables['urn:uma:variables:policyDir'] = path.join(rootDir, './config/rules/policy');
   variables['urn:uma:variables:eyePath'] = 'eye';
+  variables['urn:uma:variables:backupFilePath'] = 'backup.ttl';
 
   const configPath = path.join(rootDir, './config/default.json');
 

packages/uma/bin/odrl.js

@@ -18,6 +18,7 @@ const launch = async () => {
     variables['urn:uma:variables:policyBaseIRI'] = 'http://localhost:3000/';
     variables['urn:uma:variables:policyDir'] = path.join(rootDir, './config/rules/odrl');
     variables['urn:uma:variables:eyePath'] = 'eye';
+    variables['urn:uma:variables:backupFilePath'] = '';
 
     const configPath = path.join(rootDir, './config/odrl.json');
 

packages/uma/config/default.json

@@ -34,7 +34,18 @@
       "comment": "This is the entry point to the application. It can be used to both start and stop the server.",
       "@id": "urn:uma:default:App",
       "@type": "App",
-      "initializer": { "@id": "urn:uma:default:NodeHttpServer" },
+      "initializer": {
+        "@id": "urn:uma:default:Initializer",
+        "@type": "SequenceHandler",
+        "handlers": [
+          {
+            "@id": "urn:uma:default:FlexibleInitializer",
+            "@type": "SequenceHandler",
+            "handlers": []
+          },
+          { "@id": "urn:uma:default:NodeHttpServer" }
+        ]
+      },
       "finalizer": {
         "@id": "urn:uma:default:Finalizer",
         "@type": "SequenceHandler",

packages/uma/config/demo.json

@@ -1,6 +1,7 @@
 {
   "@context": [
-    "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld"
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/asynchronous-handlers/^1.0.0/components/context.jsonld"
   ],
   "import": [
     "sai-uma:config/default.json"
@@ -48,6 +49,15 @@
           "@id": "urn:uma:variables:policyContainer"
         }
       }
+    },
+    {
+      "comment": "Replace the InitializableHandler as the storage is no longer an Initializer",
+      "@id": "urn:solid-server:override:RulesStorageInitializableHandler",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:uma:default:RulesStorageInitializableHandler" },
+      "overrideParameters": {
+        "@type": "StaticHandler"
+      }
     }
   ]
 }

packages/uma/config/policies/authorizers/default.json

@@ -1,6 +1,8 @@
 {
   "@context": [
-    "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld"
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^8.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solidlab/uma/^0.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/asynchronous-handlers/^1.0.0/components/context.jsonld"
   ],
   "@graph": [
     {
@@ -53,10 +55,23 @@
         "eyePath": { "@id": "urn:uma:variables:eyePath" },
         "policies": {
           "@id": "urn:uma:default:RulesStorage",
-          "@type": "MemoryUCRulesStorage"
+          "@type": "FileBackupUCRulesStorage",
+          "filePath": { "@id": "urn:uma:variables:backupFilePath" }
         }
       },
       "registrationStore": { "@id": "urn:uma:default:ResourceRegistrationStore" }
+    },
+    {
+      "@id": "urn:uma:default:FlexibleInitializer",
+      "@type": "SequenceHandler",
+      "handlers": [
+        {
+          "comment": "Reads in the policies stored in the backup file when starting the server",
+          "@id": "urn:uma:default:RulesStorageInitializableHandler",
+          "@type": "InitializableHandler",
+          "initializable": { "@id": "urn:uma:default:RulesStorage" }
+        }
+      ]
     }
   ]
 }

packages/uma/config/variables/default.json

@@ -31,6 +31,11 @@
         "comment": "URL of container where policies are stored.",
         "@id": "urn:uma:variables:policyContainer",
         "@type": "Variable"
+      },
+      {
+        "comment": "File path pointing to where policies should be backed up. Set to empty string to disable backups.",
+        "@id": "urn:uma:variables:backupFilePath",
+        "@type": "Variable"
       }
     ]
   }

packages/uma/src/index.ts

@@ -84,6 +84,7 @@ export * from './ucp/policy/ODRL';
 export * from './ucp/policy/UsageControlPolicy';
 export * from './ucp/storage/ContainerUCRulesStorage';
 export * from './ucp/storage/DirectoryUCRulesStorage';
+export * from './ucp/storage/FileBackupUCRulesStorage';
 export * from './ucp/storage/MemoryUCRulesStorage';
 export * from './ucp/storage/UCRulesStorage';
 export * from './ucp/util/Util';

packages/uma/src/ucp/storage/FileBackupUCRulesStorage.ts

@@ -0,0 +1,108 @@
+import { createErrorMessage, Initializable, isSystemError, setSafeInterval } from '@solid/community-server';
+import { getLoggerFor } from 'global-logger-factory';
+import { Parser, Store, Writer } from 'n3';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { rename, unlink } from 'node:fs/promises';
+import { MemoryUCRulesStorage } from './MemoryUCRulesStorage';
+
+/**
+ * Backs up all the stored policies to a backup file every 5 minutes (default).
+ * Reads the policies from this file on startup.
+ * If no file path is defined, this is just a MemoryUCRulesStorage.
+ */
+export class FileBackupUCRulesStorage extends MemoryUCRulesStorage implements Initializable {
+  protected readonly logger = getLoggerFor(this);
+
+  protected doingBackup = false;
+  protected dataChanged = false;
+
+  public constructor(protected readonly filePath?: string, interval = 5 * 60) {
+    super();
+    this.logger.info(`STARTING ${filePath}`);
+    if (filePath) {
+      const timer = setSafeInterval(
+        this.logger,
+        'Failed to backup policies',
+        this.backup.bind(this),
+        interval * 1000);
+      timer.unref();
+    }
+  }
+
+  public async initialize(): Promise<void> {
+    this.logger.info('CALLING INITIALIZE');
+    if (!this.filePath) {
+      return;
+    }
+    this.logger.info(`Reading policy backup from ${this.filePath}`);
+    const parser = new Parser();
+    const stream = createReadStream(this.filePath, 'utf8');
+    parser.parse(stream, (err, quad) => {
+      if (err) {
+        if (isSystemError(err) && err.code === 'ENOENT') {
+          this.logger.info(`No backup file found at ${this.filePath}`);
+          return;
+        }
+        this.logger.error(`Problem parsing backup policies file: ${createErrorMessage(err)}`);
+      }
+      if (quad) {
+        this.store.add(quad);
+      }
+    });
+  }
+
+  protected async backup(): Promise<void> {
+    if (!this.filePath || this.doingBackup || !this.dataChanged) {
+      return;
+    }
+    this.doingBackup = true;
+    this.logger.info(`Creating policy backup in ${this.filePath}`);
+    try {
+      // Move the previous backup just in case of a crash during backup
+      const oldPath = this.filePath + '.old';
+      let removeOld = false;
+      try {
+        await rename(this.filePath, oldPath);
+        removeOld = true;
+      } catch (error: unknown) {
+        if (!isSystemError(error) || error.code !== 'ENOENT') {
+          throw error;
+        }
+      }
+
+      // Backup the triples
+      const stream = createWriteStream(this.filePath, 'utf8');
+      const writer = new Writer(stream);
+      writer.addQuads(this.store.getQuads(null, null, null, null));
+      writer.end();
+      this.dataChanged = false;
+
+      // Remove the previous backup
+      if (removeOld) {
+        await unlink(oldPath);
+      }
+    } finally {
+      this.doingBackup = false;
+    }
+  }
+
+  public async addRule(rule: Store): Promise<void> {
+    this.dataChanged = true;
+    return super.addRule(rule);
+  }
+
+  public async deleteRule(identifier: string): Promise<void> {
+    this.dataChanged = true;
+    return super.deleteRule(identifier);
+  }
+
+  public async deleteRuleFromPolicy(ruleID: string, PolicyID: string): Promise<void> {
+    this.dataChanged = true;
+    return super.deleteRuleFromPolicy(ruleID, PolicyID);
+  }
+
+  public async removeData(data: Store): Promise<void> {
+    this.dataChanged = true;
+    return super.removeData(data);
+  }
+}