diff --git a/.github/workflows/self-hosted-docker-build.yml b/.github/workflows/self-hosted-docker-build.yml index bce8703a..330ab982 100644 --- a/.github/workflows/self-hosted-docker-build.yml +++ b/.github/workflows/self-hosted-docker-build.yml @@ -27,7 +27,7 @@ jobs: run: | # 保留 latest 和 previous，删除其他 huntly 镜像 docker images lcomplete/huntly --format "table {{.Repository}}:{{.Tag}}" | grep -v "latest\|previous\|REPOSITORY" | xargs -r docker rmi || true - continue-on-error: true + continue-on-error: false - name: Ensure data directory exists run: | diff --git a/app/client/src/components/SettingModal/BatchOrganizeSetting.tsx b/app/client/src/components/SettingModal/BatchOrganizeSetting.tsx index ad730f78..851d39a5 100644 --- a/app/client/src/components/SettingModal/BatchOrganizeSetting.tsx +++ b/app/client/src/components/SettingModal/BatchOrganizeSetting.tsx @@ -262,11 +262,11 @@ export default function BatchOrganizeSetting() { {filterResult.items.length > 0 && ( <> - {filterResult.totalCount > 5 && ( - - )} + )} diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/ConnectorProperties.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/ConnectorProperties.java index 57941d73..ea7b479d 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/connector/ConnectorProperties.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/ConnectorProperties.java @@ -13,12 +13,16 @@ @Setter public class ConnectorProperties { private Instant lastFetchAt; - + private String subscribeUrl; - + private String apiToken; private Boolean crawlFullContent; - + private ProxySetting proxySetting; + + private String httpEtag; + + private String httpLastModified; } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/FetchPagesResult.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/FetchPagesResult.java new file mode 100644 index 00000000..4ddda2f7 --- /dev/null +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/FetchPagesResult.java @@ -0,0 +1,49 @@ +package com.huntly.server.connector; + +import com.huntly.interfaces.external.model.CapturePage; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * Result of fetching pages from a connector, including HTTP cache headers. + */ +@Getter +@Setter +public class FetchPagesResult { + /** + * The fetched pages, empty if the feed was not modified. + */ + private List pages; + + /** + * Whether the feed was not modified (HTTP 304). + */ + private boolean notModified; + + /** + * ETag header from the response. + */ + private String httpEtag; + + /** + * Last-Modified header from the response. + */ + private String httpLastModified; + + public static FetchPagesResult notModified() { + FetchPagesResult result = new FetchPagesResult(); + result.setNotModified(true); + return result; + } + + public static FetchPagesResult of(List pages, String httpEtag, String httpLastModified) { + FetchPagesResult result = new FetchPagesResult(); + result.setPages(pages); + result.setNotModified(false); + result.setHttpEtag(httpEtag); + result.setHttpLastModified(httpLastModified); + return result; + } +} diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/InfoConnector.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/InfoConnector.java index e5da9b73..48a516fe 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/connector/InfoConnector.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/InfoConnector.java @@ -24,4 +24,15 @@ protected HttpClient buildHttpClient(ConnectorProperties properties) { public abstract List fetchAllPages(); public abstract CapturePage fetchPageContent(CapturePage capturePage); + + /** + * Fetch newest pages with HTTP 304 cache support. + * Default implementation delegates to fetchNewestPages() without cache support. + * + * @return FetchPagesResult containing pages and cache headers + */ + public FetchPagesResult fetchNewestPagesWithCache() { + List pages = fetchNewestPages(); + return FetchPagesResult.of(pages, null, null); + } } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedFetchResult.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedFetchResult.java new file mode 100644 index 00000000..6e20c10b --- /dev/null +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedFetchResult.java @@ -0,0 +1,47 @@ +package com.huntly.server.connector.rss; + +import com.rometools.rome.feed.synd.SyndFeed; +import lombok.Getter; +import lombok.Setter; + +/** + * Result of fetching a feed, containing the parsed feed and HTTP cache headers. + */ +@Getter +@Setter +public class FeedFetchResult { + /** + * The parsed feed, null if the response was 304 Not Modified. + */ + private SyndFeed feed; + + /** + * Whether the feed was not modified (HTTP 304). + */ + private boolean notModified; + + /** + * ETag header from the response. + */ + private String etag; + + /** + * Last-Modified header from the response. + */ + private String lastModified; + + public static FeedFetchResult notModified() { + FeedFetchResult result = new FeedFetchResult(); + result.setNotModified(true); + return result; + } + + public static FeedFetchResult of(SyndFeed feed, String etag, String lastModified) { + FeedFetchResult result = new FeedFetchResult(); + result.setFeed(feed); + result.setNotModified(false); + result.setEtag(etag); + result.setLastModified(lastModified); + return result; + } +} diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedUtils.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedUtils.java index 31704db8..fea28c79 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedUtils.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedUtils.java @@ -23,23 +23,55 @@ */ @UtilityClass public class FeedUtils { - public static SyndFeed parseFeedUrl(String feedUrl, OkHttpClient client) { - Request request = new Request.Builder() - .url(feedUrl) - .build(); - try(Response response = client.newCall(request).execute()) { - byte[] xmlBytes = null; + + private static final int HTTP_NOT_MODIFIED = 304; + + /** + * Fetch feed with conditional request support (HTTP 304). + * + * @param feedUrl The feed URL to fetch + * @param client The OkHttp client + * @param etag The ETag from previous request (can be null) + * @param lastModified The Last-Modified from previous request (can be null) + * @return FeedFetchResult containing the feed or notModified flag + */ + public static FeedFetchResult fetchFeed(String feedUrl, OkHttpClient client, String etag, String lastModified) { + Request.Builder requestBuilder = new Request.Builder().url(feedUrl); + + // Add conditional request headers if available + if (StringUtils.isNotBlank(etag)) { + requestBuilder.header("If-None-Match", etag); + } + if (StringUtils.isNotBlank(lastModified)) { + requestBuilder.header("If-Modified-Since", lastModified); + } + + Request request = requestBuilder.build(); + + try (Response response = client.newCall(request).execute()) { + // Check for 304 Not Modified + if (response.code() == HTTP_NOT_MODIFIED) { + return FeedFetchResult.notModified(); + } + if (response.body() == null) { throw new ConnectorFetchException("xml response null for url: " + feedUrl); } - xmlBytes = response.body().bytes(); + byte[] xmlBytes = response.body().bytes(); Charset encoding = FeedUtils.guessEncoding(xmlBytes); String xmlString = XmlUtils.removeInvalidXmlCharacters(new String(xmlBytes, encoding)); if (xmlString == null) { throw new ConnectorFetchException("xml fetch failed for url: " + feedUrl); } - return new SyndFeedInput().build(new StringReader(xmlString)); + + SyndFeed feed = new SyndFeedInput().build(new StringReader(xmlString)); + + // Extract cache headers from response + String responseEtag = response.header("ETag"); + String responseLastModified = response.header("Last-Modified"); + + return FeedFetchResult.of(feed, responseEtag, responseLastModified); } catch (IOException e) { throw new RuntimeException(e); } catch (FeedException e) { @@ -47,37 +79,48 @@ public static SyndFeed parseFeedUrl(String feedUrl, OkHttpClient client) { } } -// public static SyndFeed parseFeedUrl(String feedUrl, HttpClient client) { -// HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(feedUrl)) -// .build(); -// HttpResponse response = null; -// try { -// response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); -// } catch (IOException e) { -// throw new RuntimeException(e); -// } catch (InterruptedException e) { -// throw new RuntimeException(e); -// } -// var xmlBytes = response.body(); -// Charset encoding = FeedUtils.guessEncoding(xmlBytes); -// String xmlString = XmlUtils.removeInvalidXmlCharacters(new String(xmlBytes, encoding)); -// if (xmlString == null) { -// throw new ConnectorFetchException("xml fetch failed for url: " + feedUrl); -// } -// -// try { -// SyndFeed feed = new SyndFeedInput().build(new StringReader(xmlString)); -// return feed; -// } catch (FeedException e) { -// throw new RuntimeException(e); -// } -// } - -// public static SyndFeed parseFeedUrl(String feedUrl) { -// var client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60)) -// .followRedirects(HttpClient.Redirect.ALWAYS).build(); -// return parseFeedUrl(feedUrl, client); -// } + /** + * @deprecated Use {@link #fetchFeed(String, OkHttpClient, String, String)} for + * HTTP 304 support + */ + @Deprecated + public static SyndFeed parseFeedUrl(String feedUrl, OkHttpClient client) { + FeedFetchResult result = fetchFeed(feedUrl, client, null, null); + return result.getFeed(); + } + + // public static SyndFeed parseFeedUrl(String feedUrl, HttpClient client) { + // HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(feedUrl)) + // .build(); + // HttpResponse response = null; + // try { + // response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + // } catch (IOException e) { + // throw new RuntimeException(e); + // } catch (InterruptedException e) { + // throw new RuntimeException(e); + // } + // var xmlBytes = response.body(); + // Charset encoding = FeedUtils.guessEncoding(xmlBytes); + // String xmlString = XmlUtils.removeInvalidXmlCharacters(new String(xmlBytes, + // encoding)); + // if (xmlString == null) { + // throw new ConnectorFetchException("xml fetch failed for url: " + feedUrl); + // } + // + // try { + // SyndFeed feed = new SyndFeedInput().build(new StringReader(xmlString)); + // return feed; + // } catch (FeedException e) { + // throw new RuntimeException(e); + // } + // } + + // public static SyndFeed parseFeedUrl(String feedUrl) { + // var client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60)) + // .followRedirects(HttpClient.Redirect.ALWAYS).build(); + // return parseFeedUrl(feedUrl, client); + // } public static Charset guessEncoding(byte[] bytes) { String extracted = extractDeclaredEncoding(bytes); diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/RSSConnector.java b/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/RSSConnector.java index 03ca2610..b878b5ad 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/RSSConnector.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/RSSConnector.java @@ -3,6 +3,7 @@ import com.huntly.common.util.UrlUtils; import com.huntly.interfaces.external.model.CapturePage; import com.huntly.server.connector.ConnectorProperties; +import com.huntly.server.connector.FetchPagesResult; import com.huntly.server.connector.InfoConnector; import com.huntly.server.domain.exceptions.ConnectorFetchException; import com.huntly.server.util.HttpUtils; @@ -47,31 +48,54 @@ public List fetchAllPages() { @Override public List fetchNewestPages() { + FetchPagesResult result = fetchNewestPagesWithCache(); + return result.getPages() != null ? result.getPages() : new ArrayList<>(); + } + + @Override + public FetchPagesResult fetchNewestPagesWithCache() { if (StringUtils.isBlank(connectorProperties.getSubscribeUrl())) { - return new ArrayList<>(); + return FetchPagesResult.of(new ArrayList<>(), null, null); } try { - SyndFeed feed = FeedUtils.parseFeedUrl(connectorProperties.getSubscribeUrl(), okClient); + // Use conditional request with cached ETag and Last-Modified + FeedFetchResult feedResult = FeedUtils.fetchFeed( + connectorProperties.getSubscribeUrl(), + okClient, + connectorProperties.getHttpEtag(), + connectorProperties.getHttpLastModified()); + + // If feed was not modified, return early with notModified flag + if (feedResult.isNotModified()) { + log.debug("Feed not modified (HTTP 304): {}", connectorProperties.getSubscribeUrl()); + return FetchPagesResult.notModified(); + } + + SyndFeed feed = feedResult.getFeed(); var entries = feed.getEntries(); List pages = new ArrayList<>(); for (var entry : entries) { CapturePage capturePage = new CapturePage(); String content = getContent(entry); - String description = StringUtils.trimToEmpty(entry.getDescription() == null ? null : entry.getDescription().getValue()); + String description = StringUtils + .trimToEmpty(entry.getDescription() == null ? null : entry.getDescription().getValue()); capturePage.setUrl(entry.getLink()); capturePage.setDomain(UrlUtils.getDomainName(entry.getLink())); capturePage.setContent(content); capturePage.setDescription(description); capturePage.setTitle(getTitle(entry)); - capturePage.setConnectedAt(ObjectUtils.firstNonNull(entry.getPublishedDate(), entry.getUpdatedDate(), feed.getPublishedDate(), new Date()).toInstant()); + capturePage.setConnectedAt(ObjectUtils.firstNonNull(entry.getPublishedDate(), entry.getUpdatedDate(), + feed.getPublishedDate(), new Date()).toInstant()); capturePage.setAuthor(StringUtils.trimToEmpty(entry.getAuthor())); - capturePage.setCategory(entry.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", "))); + capturePage.setCategory( + entry.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", "))); capturePage.setNeedFindThumbUrl(true); pages.add(capturePage); } - return pages; + // Return pages with cache headers from response + return FetchPagesResult.of(pages, feedResult.getEtag(), feedResult.getLastModified()); } catch (Exception e) { throw new ConnectorFetchException(e); } @@ -93,7 +117,8 @@ private String getTitle(SyndEntry item) { private String getContent(SyndEntry entry) { String content = null; if (!entry.getContents().isEmpty()) { - content = entry.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator())); + content = entry.getContents().stream().map(SyndContent::getValue) + .collect(Collectors.joining(System.lineSeparator())); } return StringUtils.trimToEmpty(content); } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/domain/entity/Connector.java b/app/server/huntly-server/src/main/java/com/huntly/server/domain/entity/Connector.java index 5fa69286..b916a9ed 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/domain/entity/Connector.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/domain/entity/Connector.java @@ -22,7 +22,7 @@ public class Connector implements Serializable { @Column(name = "id") @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; - + @Column(name = "type") private Integer type; @@ -31,7 +31,7 @@ public class Connector implements Serializable { @Column(name = "subscribe_url") private String subscribeUrl; - + @Column(name = "api_token") private String apiToken; @@ -64,10 +64,16 @@ public class Connector implements Serializable { @Column(name = "crawl_full_content") private Boolean crawlFullContent; - + @Column(name = "is_enabled") private Boolean enabled; @Column(name = "created_at") private Instant createdAt; + + @Column(name = "http_etag") + private String httpEtag; + + @Column(name = "http_last_modified") + private String httpLastModified; } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorFetchService.java b/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorFetchService.java index c1f4a2c7..156cc2ce 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorFetchService.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorFetchService.java @@ -47,7 +47,9 @@ public class ConnectorFetchService { ThreadPoolExecutor fetchExecutor; - public ConnectorFetchService(HuntlyProperties huntlyProperties, ConnectorService connectorService, CapturePageService capturePageService, PageArticleContentService pageArticleContentService, EventPublisher eventPublisher, GlobalSettingService globalSettingService, PageService pageService) { + public ConnectorFetchService(HuntlyProperties huntlyProperties, ConnectorService connectorService, + CapturePageService capturePageService, PageArticleContentService pageArticleContentService, + EventPublisher eventPublisher, GlobalSettingService globalSettingService, PageService pageService) { this.huntlyProperties = huntlyProperties; this.connectorService = connectorService; this.capturePageService = capturePageService; @@ -57,12 +59,13 @@ public ConnectorFetchService(HuntlyProperties huntlyProperties, ConnectorService inProcessConnectorIds = Collections.synchronizedSet(new HashSet<>()); fetchExecutor = new ThreadPoolExecutor( - ObjectUtils.defaultIfNull(huntlyProperties.getConnectorFetchCorePoolSize(), AppConstants.DEFAULT_CONNECTOR_FETCH_CORE_POOL_SIZE), - ObjectUtils.defaultIfNull(huntlyProperties.getConnectorFetchMaxPoolSize(), AppConstants.DEFAULT_CONNECTOR_FETCH_MAX_POOL_SIZE), + ObjectUtils.defaultIfNull(huntlyProperties.getConnectorFetchCorePoolSize(), + AppConstants.DEFAULT_CONNECTOR_FETCH_CORE_POOL_SIZE), + ObjectUtils.defaultIfNull(huntlyProperties.getConnectorFetchMaxPoolSize(), + AppConstants.DEFAULT_CONNECTOR_FETCH_MAX_POOL_SIZE), 300, TimeUnit.SECONDS, new LinkedBlockingQueue<>(10000), - r -> new Thread(r, "connector_fetch_thread") - ); + r -> new Thread(r, "connector_fetch_thread")); } public void fetchAllConnectPages() { @@ -118,15 +121,45 @@ public void fetchPagesImmediately(Integer connectorId) { private void fetchPages(Connector connector) { var connectorProperties = connectorService.getConnectorProperties(connector.getId()); - InfoConnector infoConnector = InfoConnectorFactory.createInfoConnector(connector.getType(), connectorProperties); + InfoConnector infoConnector = InfoConnectorFactory.createInfoConnector(connector.getType(), + connectorProperties); if (infoConnector == null) { return; } - var pages = connector.getLastFetchBeginAt() == null ? infoConnector.fetchAllPages() : infoConnector.fetchNewestPages(); - boolean inboxChangedTriggered = false; + boolean isRssFetch = Objects.equals(connector.getType(), ConnectorType.RSS.getCode()); boolean isGithubFetch = Objects.equals(connector.getType(), ConnectorType.GITHUB.getCode()); + List pages; + + // For RSS feeds, use cache-aware fetching (HTTP 304 support) + if (isRssFetch && connector.getLastFetchBeginAt() != null) { + var fetchResult = infoConnector.fetchNewestPagesWithCache(); + + // If feed was not modified (HTTP 304), we can skip processing + if (fetchResult.isNotModified()) { + log.info("Feed not modified (HTTP 304), skipping: {}", connector.getName()); + return; + } + + pages = fetchResult.getPages() != null ? fetchResult.getPages() : new ArrayList<>(); + + // Update HTTP cache headers for future conditional requests + if (StringUtils.isNotBlank(fetchResult.getHttpEtag()) + || StringUtils.isNotBlank(fetchResult.getHttpLastModified())) { + connectorService.updateHttpCacheHeaders( + connector.getId(), + fetchResult.getHttpEtag(), + fetchResult.getHttpLastModified()); + } + } else { + // For first fetch or non-RSS connectors, use regular fetch + pages = connector.getLastFetchBeginAt() == null ? infoConnector.fetchAllPages() + : infoConnector.fetchNewestPages(); + } + + boolean inboxChangedTriggered = false; + for (CapturePage page : pages) { page.setConnectorId(connector.getId()); String rawContent = page.getContent(); @@ -152,15 +185,18 @@ private void fetchPages(Connector connector) { } Page savedPage = null; - //Avoid frequent updates of RSS articles. - if (isRssFetch && existPage != null && Objects.equals(existPage.getConnectorId(), page.getConnectorId()) && Objects.equals(existPage.getTitle(), page.getTitle()) && Objects.equals(existPage.getConnectedAt(), page.getConnectedAt())) { + // Avoid frequent updates of RSS articles. + if (isRssFetch && existPage != null && Objects.equals(existPage.getConnectorId(), page.getConnectorId()) + && Objects.equals(existPage.getTitle(), page.getTitle()) + && Objects.equals(existPage.getConnectedAt(), page.getConnectedAt())) { savedPage = existPage; } else { savedPage = capturePageService.save(page); } if (isRssFetch && isExecuteFetch) { - pageArticleContentService.saveContent(savedPage.getId(), rawContent, ArticleContentCategory.RAW_CONTENT); + pageArticleContentService.saveContent(savedPage.getId(), rawContent, + ArticleContentCategory.RAW_CONTENT); } if (savedPage.getMarkRead() == null || Objects.equals(savedPage.getMarkRead(), false)) { @@ -177,7 +213,8 @@ private void fetchPages(Connector connector) { // update rss connector site icon if (isRssFetch) { if (StringUtils.isBlank(connector.getIconUrl())) { - var icon = SiteUtils.getFaviconFromHome(connector.getSubscribeUrl(), HttpUtils.buildHttpClient(globalSettingService.getProxySetting(), 10)); + var icon = SiteUtils.getFaviconFromHome(connector.getSubscribeUrl(), + HttpUtils.buildHttpClient(globalSettingService.getProxySetting(), 10)); if (icon != null) { connectorService.updateIconUrl(connector.getId(), icon.getIconUrl()); } @@ -199,7 +236,9 @@ private boolean isAtFetchTime(Connector connector) { if (connector == null) { return false; } - Integer fetchIntervalSeconds = ObjectUtils.defaultIfNull(connector.getFetchIntervalSeconds(), huntlyProperties.getDefaultFeedFetchIntervalSeconds()); - return connector.getLastFetchBeginAt() == null || connector.getLastFetchBeginAt().plusSeconds(fetchIntervalSeconds).isBefore(Instant.now()); + Integer fetchIntervalSeconds = ObjectUtils.defaultIfNull(connector.getFetchIntervalSeconds(), + huntlyProperties.getDefaultFeedFetchIntervalSeconds()); + return connector.getLastFetchBeginAt() == null + || connector.getLastFetchBeginAt().plusSeconds(fetchIntervalSeconds).isBefore(Instant.now()); } } diff --git a/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorService.java b/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorService.java index 99428300..e3fbab8c 100644 --- a/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorService.java +++ b/app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorService.java @@ -42,8 +42,10 @@ public class ConnectorService { private GlobalSettingService globalSettingService; - public ConnectorService(HuntlyProperties huntlyProperties, FolderRepository folderRepository, ConnectorSettingRepository connectorSettingRepository, - ConnectorRepository connectorRepository, PageRepository pageRepository, GlobalSettingService globalSettingService) { + public ConnectorService(HuntlyProperties huntlyProperties, FolderRepository folderRepository, + ConnectorSettingRepository connectorSettingRepository, + ConnectorRepository connectorRepository, PageRepository pageRepository, + GlobalSettingService globalSettingService) { this.huntlyProperties = huntlyProperties; this.folderRepository = folderRepository; this.connectorSettingRepository = connectorSettingRepository; @@ -69,6 +71,8 @@ public ConnectorProperties getConnectorProperties(Integer connectorId) { properties.setSubscribeUrl(con.getSubscribeUrl()); properties.setLastFetchAt(con.getLastFetchBeginAt()); properties.setProxySetting(globalSettingService.getProxySetting()); + properties.setHttpEtag(con.getHttpEtag()); + properties.setHttpLastModified(con.getHttpLastModified()); }); return properties; } @@ -95,8 +99,22 @@ public void updateLastFetchEndAt(Integer connectorId, Instant endAt, boolean suc } } + public void updateHttpCacheHeaders(Integer connectorId, String httpEtag, String httpLastModified) { + var connector = connectorRepository.findById(connectorId).orElse(null); + if (connector != null) { + if (httpEtag != null) { + connector.setHttpEtag(httpEtag); + } + if (httpLastModified != null) { + connector.setHttpLastModified(httpLastModified); + } + connectorRepository.save(connector); + } + } + public Connector saveWhenNotExist(Connector connector) { - var existsConnector = connectorRepository.findBySubscribeUrlAndType(connector.getSubscribeUrl(), connector.getType()); + var existsConnector = connectorRepository.findBySubscribeUrlAndType(connector.getSubscribeUrl(), + connector.getType()); if (existsConnector.isEmpty()) { connectorRepository.save(connector); } @@ -118,7 +136,8 @@ public FolderConnectorView getFolderConnectorView(boolean onlyEnabled) { } private List getFolderFeedConnectors(List folders, List connectors) { - connectors = connectors.stream().filter(t -> ConnectorType.RSS.getCode().equals(t.getType())).collect(Collectors.toList()); + connectors = connectors.stream().filter(t -> ConnectorType.RSS.getCode().equals(t.getType())) + .collect(Collectors.toList()); List folderConnectorsList = new ArrayList<>(); // add connectors without folder first @@ -136,7 +155,8 @@ private List getFolderFeedConnectors(List folders, Lis return folderConnectorsList; } - private void fillFolderConnectors(List folderConnectorsList, Folder folder, List childConnectors) { + private void fillFolderConnectors(List folderConnectorsList, Folder folder, + List childConnectors) { if (!CollectionUtils.isEmpty(childConnectors)) { FolderConnectors folderConnectors = new FolderConnectors(); folderConnectors.setId(folder.getId()); @@ -275,8 +295,9 @@ public Connector saveGitHubSetting(GitHubSetting gitHubSetting) { return null; } - var connector = gitHubSetting.getConnectorId() > 0 ? - connectorRepository.findById(gitHubSetting.getConnectorId()).orElse(null) : null; + var connector = gitHubSetting.getConnectorId() > 0 + ? connectorRepository.findById(gitHubSetting.getConnectorId()).orElse(null) + : null; if (connector == null) { connector = new Connector(); connector.setCreatedAt(Instant.now());