diff --git a/app/client/src/components/SettingModal/ArticleShortcutSetting.tsx b/app/client/src/components/SettingModal/ArticleShortcutSetting.tsx index d141afd1..3412ddc8 100644 --- a/app/client/src/components/SettingModal/ArticleShortcutSetting.tsx +++ b/app/client/src/components/SettingModal/ArticleShortcutSetting.tsx @@ -20,6 +20,7 @@ import { Typography, } from "@mui/material"; import React, { useState } from "react"; +import { createPortal } from "react-dom"; import DeleteIcon from '@mui/icons-material/Delete'; import AddIcon from '@mui/icons-material/Add'; import EditIcon from '@mui/icons-material/Edit'; @@ -387,130 +388,134 @@ const ArticleShortcutSetting = () => { draggableId={shortcut.id?.toString() || `new-${index}`} index={index} > - {(provided, snapshot) => ( - - {/* Drag Handle */} + {(provided, snapshot) => { + const draggable = ( - - - - {/* Toggle Switch */} - - handleToggleShortcut(shortcut)} + {/* Drag Handle */} + - - - {/* Content */} - - - - {shortcut.name} - + > + - {shortcut.description && ( - - {shortcut.description} - - )} - - {/* Actions */} - - - + handleEditShortcut(shortcut)} + checked={shortcut.enabled} + onChange={() => handleToggleShortcut(shortcut)} sx={{ - color: '#94a3b8', - '&:hover': { + '& .MuiSwitch-switchBase.Mui-checked': { color: '#3b82f6', - bgcolor: alpha('#3b82f6', 0.08) + '&:hover': { bgcolor: alpha('#3b82f6', 0.08) } + }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { + bgcolor: '#3b82f6' } }} - > - - - - - shortcut.id && handleDeleteShortcut(shortcut.id)} - sx={{ - color: '#94a3b8', - '&:hover': { - color: '#ef4444', - bgcolor: alpha('#ef4444', 0.08) - } - }} - > - - + /> + + {/* Content */} + + + + {shortcut.name} + + + {shortcut.description && ( + + {shortcut.description} + + )} + + + {/* Actions */} + + + handleEditShortcut(shortcut)} + sx={{ + color: '#94a3b8', + '&:hover': { + color: '#3b82f6', + bgcolor: alpha('#3b82f6', 0.08) + } + }} + > + + + + + shortcut.id && handleDeleteShortcut(shortcut.id)} + sx={{ + color: '#94a3b8', + '&:hover': { + color: '#ef4444', + bgcolor: alpha('#ef4444', 0.08) + } + }} + > + + + + - - )} + ); + + return snapshot.isDragging ? createPortal(draggable, document.body) : draggable; + }} ))} {provided.placeholder} @@ -865,4 +870,4 @@ const ArticleShortcutSetting = () => { ); }; -export default ArticleShortcutSetting; \ No newline at end of file +export default ArticleShortcutSetting; diff --git a/app/client/src/components/SettingModal/FeedsSetting.tsx b/app/client/src/components/SettingModal/FeedsSetting.tsx index e6269eca..3a4f1178 100644 --- a/app/client/src/components/SettingModal/FeedsSetting.tsx +++ b/app/client/src/components/SettingModal/FeedsSetting.tsx @@ -18,6 +18,7 @@ import { } from "@mui/material"; import SettingSectionTitle from "./SettingSectionTitle"; import React, { useState } from "react"; +import { createPortal } from "react-dom"; import { PreviewFeedsInfo, SettingControllerApiFactory } from "../../api"; import { useSnackbar } from "notistack"; import { useFormik } from "formik"; @@ -411,54 +412,58 @@ function FoldersTabContent() { { - (dragProvided, snapshot) => ( - + (dragProvided, snapshot) => { + const draggable = ( { - setFolderId(folder.id || 0) + p: 1, + ...(snapshot.isDragging && { + background: 'white', + boxShadow: '0 2px 8px rgba(0,0,0,0.15)', + }), }} > - - - - - - + { + setFolderId(folder.id || 0) + }} + > + + + + + + + + { + setEditFolderId(folder.id) + }}> + + - { - setEditFolderId(folder.id) - }}> - - - - ) + ); + + return snapshot.isDragging ? createPortal(draggable, document.body) : draggable; + } } ) } @@ -486,44 +491,48 @@ function FoldersTabContent() { connectors.map((conn, index) => { - (dragProvided, snapshot) => ( - - - - { - conn.iconUrl && - {conn.name} - } - { - !conn.iconUrl && - } - - - - { - setEditFeedsId(conn.id) - }}> - - - - ) + (dragProvided, snapshot) => { + const draggable = ( + + + + { + conn.iconUrl && + {conn.name} + } + { + !conn.iconUrl && + } + + + + { + setEditFeedsId(conn.id) + }}> + + + + ); + + return snapshot.isDragging ? createPortal(draggable, document.body) : draggable; + } } ) } @@ -551,4 +560,4 @@ function FoldersTabContent() { } ); -} \ No newline at end of file +} diff --git a/app/extension/src/background.ts b/app/extension/src/background.ts index 0e9a2d5d..aaabc0af 100644 --- a/app/extension/src/background.ts +++ b/app/extension/src/background.ts @@ -116,6 +116,8 @@ chrome.runtime.onMessage.addListener(function (msg: Message, sender, sendRespons } else if (msg.type === 'auto_save_tweets') { readSyncStorageSettings().then((settings) => { if (settings.autoSaveTweet) { + // Add minLikes to payload + msg.payload.minLikes = settings.autoSaveTweetMinLikes; sendData("tweet/saveTweets", msg.payload); } }); diff --git a/app/extension/src/settings.tsx b/app/extension/src/settings.tsx index 156338d3..7064c3ee 100644 --- a/app/extension/src/settings.tsx +++ b/app/extension/src/settings.tsx @@ -1,6 +1,6 @@ import React, {useEffect, useState} from "react"; import './options.css'; -import {Alert, Button, Divider, FormControlLabel, IconButton, Slider, Snackbar, Switch, TextField} from "@mui/material"; +import {Alert, Button, Divider, FormControlLabel, IconButton, Snackbar, Switch, TextField} from "@mui/material"; import * as yup from 'yup'; import {FieldArray, Form, Formik, getIn} from "formik"; import DeleteIcon from "@mui/icons-material/Delete"; @@ -16,16 +16,14 @@ export const Settings = ({onOptionsChange}: SettingsProps) => { const [enabledServerIndex, setEnabledServerIndex] = useState(0); const [autoSaveEnabled, setAutoSaveEnabled] = useState(true); const [autoSaveTweet, setAutoSaveTweet] = useState(false); - const [autoSaveMinScore, setAutoSaveMinScore] = useState(20); - const [autoSaveMinContentLength, setAutoSaveMinContentLength] = useState(40); + const [autoSaveTweetMinLikes, setAutoSaveTweetMinLikes] = useState(0); const [showSavedTip, setShowSavedTip] = useState(false); useEffect(() => { readSyncStorageSettings().then((settings) => { setAutoSaveEnabled(settings.autoSaveEnabled); - setAutoSaveMinScore(settings.autoSaveMinScore); - setAutoSaveMinContentLength(settings.autoSaveMinContentLength); setAutoSaveTweet(settings.autoSaveTweet); + setAutoSaveTweetMinLikes(settings.autoSaveTweetMinLikes); if (settings.serverUrlList.length > 0) { setServerUrlList(settings.serverUrlList); settings.serverUrlList.forEach((item, index) => { @@ -44,10 +42,9 @@ export const Settings = ({onOptionsChange}: SettingsProps) => { 'Enter correct url!' ).required('Url is required.') })), - autoSaveMinScore: yup.number().min(0).max(200).required('Min score is required.'), - autoSaveMinContentLength: yup.number().min(10).max(200).required('Min content length is required.'), autoSaveEnabled: yup.boolean().required('Auto save enabled is required.'), - autoSaveTweet: yup.boolean().required('Auto save tweet is required.') + autoSaveTweet: yup.boolean().required('Auto save tweet is required.'), + autoSaveTweetMinLikes: yup.number().min(0).max(10000).required('Min likes is required.') }); return ( @@ -69,20 +66,27 @@ export const Settings = ({onOptionsChange}: SettingsProps) => { initialValues={{ settings: serverUrlList, autoSaveEnabled: autoSaveEnabled, - autoSaveMinScore: autoSaveMinScore, - autoSaveMinContentLength: autoSaveMinContentLength, - autoSaveTweet: autoSaveTweet + autoSaveTweet: autoSaveTweet, + autoSaveTweetMinLikes: autoSaveTweetMinLikes }} validationSchema={urlValidation} - onSubmit={(values, formikHelpers) => { + onSubmit={(values) => { const serverUrl = values.settings[enabledServerIndex].url; const storageSettings: StorageSettings = { "serverUrl": serverUrl, "serverUrlList": values.settings, - ...values + "autoSaveEnabled": values.autoSaveEnabled, + "autoSaveTweet": values.autoSaveTweet, + "autoSaveTweetMinLikes": values.autoSaveTweetMinLikes }; chrome.storage.sync.set( - storageSettings, + { + "serverUrl": serverUrl, + "serverUrlList": values.settings, + "autoSaveEnabled": values.autoSaveEnabled, + "autoSaveTweet": values.autoSaveTweet, + "autoSaveTweetMinLikes": values.autoSaveTweetMinLikes + }, () => { setShowSavedTip(true); if (onOptionsChange) { @@ -170,17 +174,6 @@ export const Settings = ({onOptionsChange}: SettingsProps) => { } label="Enabled"/> -
-
Probably readerable min score (the default is 20)
- -
-
-
The minimum length of a paragraph (the default is 40)
- -
@@ -190,6 +183,18 @@ export const Settings = ({onOptionsChange}: SettingsProps) => { } label="Enabled"/> +
+ +
diff --git a/app/extension/src/storage.ts b/app/extension/src/storage.ts index 0133c997..feac715b 100644 --- a/app/extension/src/storage.ts +++ b/app/extension/src/storage.ts @@ -1,9 +1,8 @@ export const STORAGE_SERVER_URL = "serverUrl"; export const STORAGE_SERVER_URL_LIST = "serverUrlList"; export const STORAGE_AUTO_SAVE_ENABLED = "autoSaveEnabled"; -export const STORAGE_AUTO_SAVE_MIN_SCORE = "autoSaveMinScore"; -export const STORAGE_AUTO_SAVE_MIN_CONTENT_LENGTH = "autoSaveMinContentLength"; export const STORAGE_AUTO_SAVE_TWEET = "autoSaveTweet"; +export const STORAGE_AUTO_SAVE_TWEET_MIN_LIKES = "autoSaveTweetMinLikes"; export type ServerUrlItem = { url: string, @@ -13,18 +12,16 @@ export type StorageSettings = { serverUrl: string; serverUrlList: ServerUrlItem[]; autoSaveEnabled: boolean; - autoSaveMinScore: number; - autoSaveMinContentLength: number; autoSaveTweet: boolean; + autoSaveTweetMinLikes: number; } export const DefaultStorageSettings: StorageSettings = { serverUrl: "", serverUrlList: [], - autoSaveEnabled: true, - autoSaveMinScore: 20, - autoSaveMinContentLength: 40, - autoSaveTweet: false + autoSaveEnabled: false, + autoSaveTweet: false, + autoSaveTweetMinLikes: 0 } export async function readSyncStorageSettings(): Promise { @@ -32,9 +29,8 @@ export async function readSyncStorageSettings(): Promise { return { serverUrl: items[STORAGE_SERVER_URL] || DefaultStorageSettings.serverUrl, serverUrlList: items[STORAGE_SERVER_URL_LIST] || DefaultStorageSettings.serverUrlList, - autoSaveEnabled: items[STORAGE_AUTO_SAVE_ENABLED], - autoSaveMinScore: items[STORAGE_AUTO_SAVE_MIN_SCORE] || DefaultStorageSettings.autoSaveMinScore, - autoSaveMinContentLength: items[STORAGE_AUTO_SAVE_MIN_CONTENT_LENGTH] || DefaultStorageSettings.autoSaveMinContentLength, - autoSaveTweet: items[STORAGE_AUTO_SAVE_TWEET] + autoSaveEnabled: items[STORAGE_AUTO_SAVE_ENABLED] ?? DefaultStorageSettings.autoSaveEnabled, + autoSaveTweet: items[STORAGE_AUTO_SAVE_TWEET] ?? DefaultStorageSettings.autoSaveTweet, + autoSaveTweetMinLikes: items[STORAGE_AUTO_SAVE_TWEET_MIN_LIKES] ?? DefaultStorageSettings.autoSaveTweetMinLikes }; } \ No newline at end of file diff --git a/app/extension/src/web_clipper.tsx b/app/extension/src/web_clipper.tsx index ccc28d05..ca5f8c3f 100644 --- a/app/extension/src/web_clipper.tsx +++ b/app/extension/src/web_clipper.tsx @@ -117,27 +117,22 @@ chrome.runtime.onMessage.addListener(function (msg: Message, sender, sendRespons } readSyncStorageSettings().then((settings) => { if (settings.autoSaveEnabled) { - timeoutSavePureRead({minScore: settings.autoSaveMinScore, minContentLength: settings.autoSaveMinContentLength}); + timeoutSavePureRead(); } }); }); -type AutoSaveSetting = { - minScore: number, - minContentLength: number -} - -function timeoutSavePureRead(saveSetting: AutoSaveSetting) { +function timeoutSavePureRead() { setTimeout(() => { const webClipper = new WebClipper() - webClipper.autoSavePureRead(saveSetting); + webClipper.autoSavePureRead(); }, 2000); } export class WebClipper { - autoSavePureRead(saveSetting: AutoSaveSetting) { - if (!this.isMaybeReadable(saveSetting)) { + autoSavePureRead() { + if (!this.isMaybeReadable()) { return; } @@ -149,10 +144,10 @@ export class WebClipper { return doc.querySelector("meta[data-huntly='1']"); } - isMaybeReadable(saveSetting: AutoSaveSetting) { + isMaybeReadable() { return !this.hasHuntlyMeta(document) && isProbablyReaderable(document, { - minScore: saveSetting.minScore, - minContentLength: saveSetting.minContentLength + minScore: 20, + minContentLength: 40 }); } diff --git a/app/server/huntly-interfaces/src/main/java/com/huntly/interfaces/external/model/InterceptTweets.java b/app/server/huntly-interfaces/src/main/java/com/huntly/interfaces/external/model/InterceptTweets.java index 078e4a0e..8906269c 100644 --- a/app/server/huntly-interfaces/src/main/java/com/huntly/interfaces/external/model/InterceptTweets.java +++ b/app/server/huntly-interfaces/src/main/java/com/huntly/interfaces/external/model/InterceptTweets.java @@ -9,12 +9,19 @@ @Getter @Setter public class InterceptTweets { - + private String category; - + private String jsonData; - + private String loginScreenName; - - private String browserScreenName; + + private String browserScreenName; + + /** + * Minimum likes count for auto-save filtering. + * Only tweets with favoriteCount >= minLikes will be saved. + * If null or 0, all tweets will be saved. + */ + private Integer minLikes; } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/twitter/ParsedTweetPage.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/twitter/ParsedTweetPage.java new file mode 100644 index 00000000..de1a7005 --- /dev/null +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/twitter/ParsedTweetPage.java @@ -0,0 +1,42 @@ +package com.huntly.server.connector.twitter; + +import com.huntly.interfaces.external.model.TweetProperties; +import com.huntly.server.domain.entity.Page; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +/** + * Container for a parsed tweet page with its properties. + * This allows passing the TweetProperties along with the Page + * to avoid re-parsing the JSON later. + * + * @author lcomplete + */ +@Getter +@Setter +@AllArgsConstructor +public class ParsedTweetPage { + private Page page; + private TweetProperties tweetProperties; + + /** + * Get the favorite count from the tweet properties. + * For retweets, returns the count from the retweeted tweet. + * + * @return the favorite count, or 0 if not available + */ + public int getFavoriteCount() { + if (tweetProperties == null) { + return 0; + } + // For retweets, get the count from the retweeted tweet + TweetProperties effectiveProperties = tweetProperties.getRetweetedTweet() != null + ? tweetProperties.getRetweetedTweet() + : tweetProperties; + return effectiveProperties.getFavoriteCount() != null + ? effectiveProperties.getFavoriteCount() + : 0; + } +} + diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/twitter/TweetParser.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/twitter/TweetParser.java index 4a01357a..952d3d21 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/connector/twitter/TweetParser.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/twitter/TweetParser.java @@ -29,14 +29,15 @@ public class TweetParser { Pattern textMatchPattern = Pattern.compile("\\P{M}\\p{M}*+"); - public List tweetsToPages(InterceptTweets tweets) { + public List tweetsToPages(InterceptTweets tweets) { ObjectMapper mapper = new ObjectMapper(); - List pages = Lists.newArrayList(); + List parsedPages = Lists.newArrayList(); + try { TweetsRoot root = mapper.readValue(tweets.getJsonData(), TweetsRoot.class); TweetsRoot.Timeline timeline = getTimeline(root); if (timeline == null || timeline.instructions == null) { - return pages; + return parsedPages; } timeline.instructions.forEach(ins -> { @@ -53,8 +54,8 @@ public List tweetsToPages(InterceptTweets tweets) { contents.add(entry.content.itemContent); } contents.forEach(content -> { - var itemPages = itemContentToPages(tweets.getCategory(), content); - pages.addAll(itemPages); + var itemParsedPages = itemContentToParsedPages(tweets.getCategory(), content); + parsedPages.addAll(itemParsedPages); }); }); } @@ -64,20 +65,20 @@ public List tweetsToPages(InterceptTweets tweets) { } // reverse pages to make sure the latest tweet is the first one - Collections.reverse(pages); - return pages; + Collections.reverse(parsedPages); + return parsedPages; } - private List itemContentToPages(String category, TweetsRoot.ItemContent itemContent) { - List pages = Lists.newArrayList(); + private List itemContentToParsedPages(String category, TweetsRoot.ItemContent itemContent) { + List parsedPages = Lists.newArrayList(); if (itemContent == null || itemContent.tweet_results == null || itemContent.tweet_results.result == null) { - return pages; + return parsedPages; } var tweet = ObjectUtils.firstNonNull(itemContent.tweet_results.result.tweet, itemContent.tweet_results.result); var user = tweet.core.user_results.result.legacy; if (user == null) { - return pages; + return parsedPages; } var views = tweet.views; @@ -90,16 +91,16 @@ private List itemContentToPages(String category, TweetsRoot.ItemContent it quotedTweet = null; } - Page page = getPageFromTweetDetail(category, tweet, user, views, quotedTweet, false); - pages.add(page); + ParsedTweetPage parsedPage = getParsedTweetPageFromDetail(category, tweet, user, views, quotedTweet, false); + parsedPages.add(parsedPage); if (quotedTweet != null) { - Page quotedPage = getPageFromTweetDetail(category, quotedTweet.result, quotedTweet.result.core.user_results.result.legacy, null, null, true); - pages.add(quotedPage); + ParsedTweetPage quotedParsedPage = getParsedTweetPageFromDetail(category, quotedTweet.result, quotedTweet.result.core.user_results.result.legacy, null, null, true); + parsedPages.add(quotedParsedPage); } - return pages; + return parsedPages; } - private Page getPageFromTweetDetail(String category, TweetsRoot.Result tweetResult, TweetsRoot.Legacy user, TweetsRoot.Views views, TweetsRoot.QuotedStatusResult quotedTweet, boolean isFromQuote) { + private ParsedTweetPage getParsedTweetPageFromDetail(String category, TweetsRoot.Result tweetResult, TweetsRoot.Legacy user, TweetsRoot.Views views, TweetsRoot.QuotedStatusResult quotedTweet, boolean isFromQuote) { TweetProperties tweetProperties = getTweetProperties(user, tweetResult, views, quotedTweet); var tweet = tweetResult.legacy; @@ -119,7 +120,7 @@ private Page getPageFromTweetDetail(String category, TweetsRoot.Result tweetResu page.setPageJsonProperties(JSONUtils.toJson(tweetProperties)); - return page; + return new ParsedTweetPage(page, tweetProperties); } private long calcVoteScore(TweetProperties tweetProperties) { @@ -174,7 +175,7 @@ private TweetProperties getTweetProperties(TweetsRoot.Legacy user, TweetsRoot.Re //tweetProperties.setFullText(String.join("", textList)); // FE handle full text tweetProperties.setFullText(tweet.full_text); - tweetProperties.setUrl("https://twitter.com/" + user.screen_name + "/status/" + tweet.id_str); + tweetProperties.setUrl("https://twitter.com/" + tweetProperties.getUserScreeName() + "/status/" + tweet.id_str); // convert twitter datetime to instant String pattern = "EEE MMM dd HH:mm:ss ZZZZZ yyyy"; SimpleDateFormat simpleDateFormat = new SimpleDateFormat(pattern, Locale.ENGLISH); diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/controller/TweetController.java b/app/server/huntly-server/src/main/java/com/huntly/server/controller/TweetController.java index 5f5b53ef..daa1f446 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/controller/TweetController.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/controller/TweetController.java @@ -44,11 +44,12 @@ public TweetController(CapturePageService capturePageService, TweetParser tweetP @PostMapping("/saveTweets") public ApiResult saveTweets(@RequestBody InterceptTweets tweets) { - var pages = tweetParser.tweetsToPages(tweets); + var parsedPages = tweetParser.tweetsToPages(tweets); AtomicInteger count = new AtomicInteger(); - pages.forEach(page -> { - // SQLite only supports one connection. To avoid other threads from being unable to obtain the SQLite connection, asynchronous events are used - eventPublisher.publishTweetPageCaptureEvent(new TweetPageCaptureEvent(page, tweets.getLoginScreenName(), tweets.getBrowserScreenName())); + Integer minLikes = tweets.getMinLikes(); + parsedPages.forEach(parsedPage -> { + // SQLite only supports one connection. To avoid other threads from being unable to obtain the SQLite connection, asynchronous events are used + eventPublisher.publishTweetPageCaptureEvent(new TweetPageCaptureEvent(parsedPage, tweets.getLoginScreenName(), tweets.getBrowserScreenName(), minLikes)); count.getAndIncrement(); }); return ApiResult.ok(count.get()); diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureEvent.java b/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureEvent.java index b8be54de..06ba7336 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureEvent.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureEvent.java @@ -1,6 +1,6 @@ package com.huntly.server.event; -import com.huntly.server.domain.entity.Page; +import com.huntly.server.connector.twitter.ParsedTweetPage; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; @@ -12,9 +12,15 @@ @Setter @AllArgsConstructor public class TweetPageCaptureEvent { - private Page page; - + private ParsedTweetPage parsedTweetPage; + private String loginScreenName; - + private String browserScreenName; + + /** + * Minimum likes count for auto-save filtering. + * If null or 0, no filtering is applied. + */ + private Integer minLikes; } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureListener.java b/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureListener.java index fcff5ae6..d36604f0 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureListener.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/event/TweetPageCaptureListener.java @@ -19,6 +19,13 @@ public TweetPageCaptureListener(CapturePageService capturePageService) { @EventListener @Async public void tweetPageCaptureEvent(TweetPageCaptureEvent event) { - capturePageService.saveTweetPage(event.getPage(), event.getLoginScreenName(), event.getBrowserScreenName()); + var parsedPage = event.getParsedTweetPage(); + capturePageService.saveTweetPage( + parsedPage.getPage(), + event.getLoginScreenName(), + event.getBrowserScreenName(), + event.getMinLikes(), + parsedPage.getFavoriteCount() + ); } } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/service/CapturePageService.java b/app/server/huntly-server/src/main/java/com/huntly/server/service/CapturePageService.java index 65f976e6..0cb0aafc 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/service/CapturePageService.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/service/CapturePageService.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -34,9 +35,11 @@ public class CapturePageService extends BasePageService { private final SourceRepository sourceRepository; private final ConnectorRepository connectorRepository; - private final TwitterUserSettingRepository twitterUserSettingRepository; + // Lock map for preventing concurrent saves of the same tweet + private final ConcurrentHashMap tweetSaveLocks = new ConcurrentHashMap<>(); + public CapturePageService(PageRepository pageRepository, LuceneService luceneService, SourceRepository sourceRepository, ConnectorRepository connectorRepository, TwitterUserSettingRepository twitterUserSettingRepository) { super(pageRepository, luceneService); @@ -114,8 +117,30 @@ public Page save(CapturePage capturePage) { return save(page); } - public Page saveTweetPage(Page page, String loginScreenName, String browserScreenName) { - var existPage = pageRepository.findTop1ByUrl(page.getUrl()); + public Page saveTweetPage(Page page, String loginScreenName, String browserScreenName, Integer minLikes, int favoriteCount) { + // Use pageUniqueId (tweet id) as lock key for preventing concurrent saves of the same tweet + String lockKey = StringUtils.isNotBlank(page.getPageUniqueId()) ? page.getPageUniqueId() : page.getUrl(); + Object lock = tweetSaveLocks.computeIfAbsent(lockKey, k -> new Object()); + + try { + synchronized (lock) { + return doSaveTweetPage(page, loginScreenName, browserScreenName, minLikes, favoriteCount); + } + } finally { + tweetSaveLocks.remove(lockKey); + } + } + + private Page doSaveTweetPage(Page page, String loginScreenName, String browserScreenName, Integer minLikes, int favoriteCount) { + // Use pageUniqueId (tweet id) for duplicate detection, more reliable than URL + Optional existPage = Optional.empty(); + if (StringUtils.isNotBlank(page.getPageUniqueId())) { + existPage = pageRepository.findTop1ByPageUniqueId(page.getPageUniqueId()); + } + // Fallback to URL if pageUniqueId not found + if (existPage.isEmpty()) { + existPage = pageRepository.findTop1ByUrl(page.getUrl()); + } if (existPage.isPresent()) { var currentPage = existPage.get(); currentPage.setContent(page.getContent()); @@ -125,11 +150,19 @@ public Page saveTweetPage(Page page, String loginScreenName, String browserScree currentPage.setPageJsonProperties(page.getPageJsonProperties()); currentPage.setCategory(page.getCategory()); currentPage.setVoteScore(page.getVoteScore()); + // Update URL if it was previously null-based + if (StringUtils.isNotBlank(page.getUrl()) && !page.getUrl().contains("/null/")) { + currentPage.setUrl(page.getUrl()); + } + // Update pageUniqueId if it was previously empty + if (StringUtils.isNotBlank(page.getPageUniqueId()) && StringUtils.isBlank(currentPage.getPageUniqueId())) { + currentPage.setPageUniqueId(page.getPageUniqueId()); + } page = currentPage; } else { page.setCreatedAt(Instant.now()); } - // tweet auto save + // tweet auto save rules String toUseScreenName = page.getAuthorScreenName(); if (Objects.equals(page.getCategory(), "like") && StringUtils.isNotBlank(browserScreenName)) { toUseScreenName = browserScreenName; @@ -137,10 +170,14 @@ public Page saveTweetPage(Page page, String loginScreenName, String browserScree toUseScreenName = loginScreenName; } var twitterUserSetting = twitterUserSettingRepository.findByScreenName(toUseScreenName); + + // Check if this tweet matches a TwitterUserSetting rule with actual configuration + boolean matchesSaveRule = false; + Integer libraryType = null; + Long collectionId = null; + if (twitterUserSetting.isPresent()) { var setting = twitterUserSetting.get(); - Integer libraryType = null; - Long collectionId = null; // Priority: tweet (user's own) > bookmark > like // Only use if the setting has libraryType or collectionId configured @@ -152,75 +189,88 @@ public Page saveTweetPage(Page page, String loginScreenName, String browserScree if (isOwnTweet && hasSetting(setting.getTweetToLibraryType(), setting.getTweetToCollectionId())) { libraryType = setting.getTweetToLibraryType(); collectionId = setting.getTweetToCollectionId(); + matchesSaveRule = true; } // 2. Second priority: bookmark settings else if (isBookmark && hasSetting(setting.getBookmarkToLibraryType(), setting.getBookmarkToCollectionId())) { libraryType = setting.getBookmarkToLibraryType(); collectionId = setting.getBookmarkToCollectionId(); + matchesSaveRule = true; } // 3. Third priority: like settings else if (isLike && hasSetting(setting.getLikeToLibraryType(), setting.getLikeToCollectionId())) { libraryType = setting.getLikeToLibraryType(); collectionId = setting.getLikeToCollectionId(); + matchesSaveRule = true; } + } - // If only collectionId is set (no libraryType), default to MY_LIST (code = 1) - if (collectionId != null && (libraryType == null || libraryType == 0)) { - libraryType = 1; // MY_LIST + // Apply minLikes filter only if the tweet does NOT match a TwitterUserSetting rule + // Tweets that match rules should always be saved regardless of likes count + if (!matchesSaveRule && minLikes != null && minLikes > 0) { + if (favoriteCount < minLikes) { + // Skip saving this tweet - it doesn't meet the minimum likes requirement + return null; } + } - // Apply library type settings - LibrarySaveType librarySaveType = LibrarySaveType.fromCode(libraryType); - if (librarySaveType != null) { - switch (librarySaveType) { - case STARRED: - page.setStarred(true); - page.setLibrarySaveStatus(LibrarySaveStatus.SAVED.getCode()); - if (page.getSavedAt() == null) { - page.setSavedAt(Instant.now()); - } - if (page.getStarredAt() == null) { - page.setStarredAt(Instant.now()); - } - break; - case READ_LATER: - page.setReadLater(true); - page.setLibrarySaveStatus(LibrarySaveStatus.SAVED.getCode()); - if (page.getSavedAt() == null) { - page.setSavedAt(Instant.now()); - } - if (page.getReadLaterAt() == null) { - page.setReadLaterAt(Instant.now()); - } - break; - case MY_LIST: - page.setLibrarySaveStatus(LibrarySaveStatus.SAVED.getCode()); - if (page.getSavedAt() == null) { - page.setSavedAt(Instant.now()); - } - break; - case ARCHIVE: - page.setLibrarySaveStatus(LibrarySaveStatus.ARCHIVED.getCode()); - if (page.getArchivedAt() == null) { - page.setArchivedAt(Instant.now()); - } - break; - default: - break; - } - if(page.getCollectedAt() == null) { - page.setCollectedAt(Instant.now()); - } + // If only collectionId is set (no libraryType), default to MY_LIST (code = 1) + if (collectionId != null && (libraryType == null || libraryType == 0)) { + libraryType = 1; // MY_LIST + } + + // Apply library type settings + LibrarySaveType librarySaveType = LibrarySaveType.fromCode(libraryType); + if (librarySaveType != null) { + switch (librarySaveType) { + case STARRED: + page.setStarred(true); + page.setLibrarySaveStatus(LibrarySaveStatus.SAVED.getCode()); + if (page.getSavedAt() == null) { + page.setSavedAt(Instant.now()); + } + if (page.getStarredAt() == null) { + page.setStarredAt(Instant.now()); + } + break; + case READ_LATER: + page.setReadLater(true); + page.setLibrarySaveStatus(LibrarySaveStatus.SAVED.getCode()); + if (page.getSavedAt() == null) { + page.setSavedAt(Instant.now()); + } + if (page.getReadLaterAt() == null) { + page.setReadLaterAt(Instant.now()); + } + break; + case MY_LIST: + page.setLibrarySaveStatus(LibrarySaveStatus.SAVED.getCode()); + if (page.getSavedAt() == null) { + page.setSavedAt(Instant.now()); + } + break; + case ARCHIVE: + page.setLibrarySaveStatus(LibrarySaveStatus.ARCHIVED.getCode()); + if (page.getArchivedAt() == null) { + page.setArchivedAt(Instant.now()); + } + break; + default: + break; + } + if(page.getCollectedAt() == null) { + page.setCollectedAt(Instant.now()); } + } - if (collectionId != null) { - page.setCollectionId(collectionId); - // Ensure collectedAt is set when collectionId is set - if (page.getCollectedAt() == null) { - page.setCollectedAt(page.getCreatedAt() != null ? page.getCreatedAt() : Instant.now()); - } + if (collectionId != null) { + page.setCollectionId(collectionId); + // Ensure collectedAt is set when collectionId is set + if (page.getCollectedAt() == null) { + page.setCollectedAt(page.getCreatedAt() != null ? page.getCreatedAt() : Instant.now()); } } + return save(page); }