Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/lib/Subscription.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { ChannelOptions, SubscriptionSettings } from "./types";
import { Attachment } from "./Attachment";
import { settings } from "./helpers/index";
import { Video } from "./Video";
import { compilePatterns, matchesPatterns, CompiledPatterns } from "./helpers/patternMatcher";

const removeRepeatedSentences = (postTitle: string, attachmentTitle: string) => {
const separators = /(?:\s+|^)((?:[^.,;:!?-]+[\s]*[.,;:!?-]+)+)(?:\s+|$)/g;
Expand All @@ -28,11 +29,21 @@ export default class Subscription {
public readonly creatorId: string;
public readonly channels: SubscriptionSettings["channels"];
public readonly plan: string;
private compiledPatternsCache: Map<string, CompiledPatterns> = new Map();

constructor(subscription: SubscriptionSettings) {
this.creatorId = subscription.creatorId;
this.channels = subscription.channels;
this.plan = subscription.plan;

// Precompile patterns to cache
for (const channel of this.channels) {
const cacheKey = `${channel.title}`;
this.compiledPatternsCache.set(
cacheKey,
compilePatterns(channel.includePatterns, channel.excludePatterns)
)
}
}

public deleteOldVideos = async () => {
Expand Down Expand Up @@ -84,6 +95,10 @@ export default class Subscription {
if (channel.skip) break;
if (channel.daysToKeepVideos !== undefined && new Date(post.releaseDate).getTime() < Subscription.getIgnoreBeforeTimestamp(channel)) return;

// Pattern match the video title if patterns are defined for the channel
const compiledPatterns = this.compiledPatternsCache.get(channel.title);
if (compiledPatterns !== undefined && !matchesPatterns(video.title, compiledPatterns)) continue;

// Remove the identifier from the video title if to give a nicer title
if (settings.extras.stripSubchannelPrefix === true) {
const replacers = [
Expand Down
59 changes: 59 additions & 0 deletions src/lib/helpers/patternMatcher.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
type PatternMatcher = (title: string) => boolean;

export interface CompiledPatterns {
includeMatchers: PatternMatcher[];
excludeMatchers: PatternMatcher[];
}

/**
* Compiles and validates include and exclude patterns.
* @param includePatterns Optional array of patterns to include when matching titles.
* @param excludePatterns Optional array of patterns to exclude when matching titles.
* @returns An object containing precompiled include and exclude pattern matchers.
*/
export const compilePatterns = (includePatterns?: string[], excludePatterns?: string[]): CompiledPatterns => {
const compilePattern = (pattern: string): PatternMatcher => {
const normalized = pattern.trim().toLowerCase();

// Literal match
if (!normalized.includes("*")) {
return (title) => title.includes(normalized);
}

// Wildcard pattern conversion to regex
const regexPattern = normalized.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");

const regex = new RegExp(`^${regexPattern}$`);
return (title) => regex.test(title);
};

return {
includeMatchers: (includePatterns || []).map(compilePattern),
excludeMatchers: (excludePatterns || []).map(compilePattern),
};
};

/**
* Checks if a title matches the compiled include and exclude patterns.
* @param title The video title to check against the patterns.
* @param compiled An object containing precompiled include and exclude pattern matchers.
* @returns true if the video title is valid, false if it should be excluded.
*/
export const matchesPatterns = (title: string, compiled: CompiledPatterns): boolean => {
const lowerTitle = title.toLowerCase();

if (compiled.excludeMatchers.length > 0 && compiled.excludeMatchers.some((matcher) => matcher(lowerTitle))) {
console.log(`Excluding video "${title}".`);
return false;
}

// If no include patterns, include anything
if (compiled.includeMatchers.length === 0) {
return true;
}

// Log included videos for validation
const match = compiled.includeMatchers.some((matcher) => matcher(lowerTitle));
if (match) console.log(`Including video "${title}".`);
return match;
};
2 changes: 2 additions & 0 deletions src/lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ export type ChannelOptions = {
skip: boolean;
isChannel: string;
daysToKeepVideos?: number;
includePatterns?: string[];
excludePatterns?: string[];
};

export type Channels = ChannelOptions[];
Expand Down
34 changes: 32 additions & 2 deletions wiki/settings.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,10 +224,12 @@ You can add custom channels to a creator if you want.
First come first served, the first channel a video matches to is what it goes into, channels are checked top to bottom in the config. Videos cannot be sorted into multiple channels.
<br>

A **channel** is made up of a `title`, `skip`, `isChannel` and optionally `daysToKeepVideos`.
A **channel** is made up of a `title`, `skip`, `isChannel`, `includePatterns`, `excludePatterns` and optionally `daysToKeepVideos`.
`title` is the nice name used for the channel.
`skip` can be set to true to skip downloading videos matched on the given channel.
`isChannel` function that returns true or false if the video should be sorted into this channel (more on this further down).
`isChannel` function that returns true or false if the video should be sorted into this channel (more on this further down).
`includePatterns` is an optional array of plain text patterns for matching videos (more on this [here](#pattern-matching)).
`excludePatterns` is an optional array of plain text patterns to exclude from matching videos (more on this [here](#pattern-matching)).
`daysToKeepVideos` is the optional number of days to keep videos for this channel. **2** would mean only videos released within the **last two days** are downloaded and any older will be **automatically deleted** if previously downloaded.

<br>
Expand All @@ -251,6 +253,34 @@ For example:

<br>

### Pattern Matching

**includePatterns** and **excludePatterns** are arrays of plain text patterns to include or exclude videos from being downloaded. It uses a simple wildcard system where `*` can be used to match any characters. For example `*beat saber*` would match any video with "beat saber" in the title, but `beat saber*` would only match videos that start with "beat saber".

The patterns are applied in a Exclude-Then-Include order:

* If the patterns are empty then all videos are included by default.
* If `includePatterns` is provided but `excludePatterns` is empty then only videos that match the `includePatterns` are included.
* If `excludePatterns` is provided but `includePatterns` is empty then all videos that do not match the `excludePatterns` are included.
* If both `includePatterns` and `excludePatterns` are provided then videos that match the `excludePatterns` are excluded first, then from the remaining videos only those that match the `includePatterns` are included.

Example:

```json
{
"title": "FP Exclusives",
"skip": false,
"excludePatterns": [
"*exec*",
],
"includePatterns": [
"*week*",
]
}
```

The example shown above will include all videos with "week" in the title except those that also have "exec" in the title.

## Metrics:

**metrics.prometheusExporterPort**:
Expand Down