diff --git a/README.md b/README.md index 8feec9f..8df8b69 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # JavaDownloadLibrary -A java library to download files and process the download speed,progress and other things +A Java library to download files and process the download speed, progress and other things Features: @@ -8,37 +8,61 @@ Features: - Check download progress - Directly cast download objects - Download Text -- Convert file sizes (MB;GB;KB...) +- Use (Netscape format) cookies to download files that require login, optionally filter which domain name to load (to check if the cookie file contains the domain name that the programme expects) +- Able to cancel during download +- Get JSON (e.g. from a website API) as String objects +- Convert file sizes from bytes to KB, MB, GB dynamically # Download: -https://github.com/MrMarnic/JDL/releases/download/1.0/JDL.jar - - -# Getting Started: - -# Download File: +https://github.com/MrMarnic/JavaDownloadLibrary/releases/ +# Example Usage: +**Simple download, with auto filename retrieval (without cookies/progress/speed/custom filename/custom user agent string)** +Note that a '\\' is expected at the end of the file path ``` -Downloader downloader = new Downloader(false); -downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtract/releases/download/1.0/JIconExtract.jar","C:\\Downloads\\download.zip"); +Downloader downloader = new Downloader(true); +downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtractReloaded/releases/download/v1.0/JIconExtractReloaded.jar", "C:\\demo\\test downloads\\"); ``` -# Add Handler (Check Speed,progress...): +**Download with file size preview, progress and speed display** ``` Downloader downloader = new Downloader(false); downloader.setDownloadHandler(new CombinedSpeedProgressDownloadHandler(downloader) { + @Override + public void onDownloadStart() { + super.onDownloadStart(); + // define custom actions to do before download starts + System.out.println("Download starting..."); + } @Override public void onDownloadSpeedProgress(int downloaded, int maxDownload, int percent, int bytesPerSec) { - System.out.println(SizeUtil.toMBFB(bytesPerSec)+"/s " + percent + "%"); + // define actions to do on each progress update + // (by default updates once every 250ms as defined in CombinedSpeedProgressDownloadHandler's onDownloadStart()) + System.out.println(SizeUtil.toHumanReadableFromBytes(bytesPerSec) + "/s " + percent + "%"); } - @Override public void onDownloadFinish() { super.onDownloadFinish(); + // define custom actions to do after download finishes System.out.println("Download finished"); } }); -downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtract/releases/download/1.0/JIconExtract.jar","C:\\Downloads\\download.zi"); +System.out.println(SizeUtil.toHumanReadableFromBytes(downloader.getDownloadLength("https://github.com/MrMarnic/JIconExtractReloaded/releases/download/v1.0/JIconExtractReloaded.jar"))); +downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtractReloaded/releases/download/v1.0/JIconExtractReloaded.jar", "C:\\demo\\test downloads\\"); +``` + +**Download files that require login using cookies, with custom file name** +``` +Downloader downloader = new Downloader(true); +downloader.setCookies(new File("C:\\demo\\cookies-github-com.txt")); +downloader.downloadFileToLocation("https://github.com/settings/profile", "C:\\demo\\test downloads\\", "GitHub Settings page.html"); +``` + +**Use custom agent string to pass through sites that block Java** +``` +Downloader downloader = new Downloader(true); +downloader.setCustomUserAgentString("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); +downloader.downloadFileToLocation("https://www.whatismybrowser.com/detect/what-is-my-user-agent/", "C:\\demo\\test downloads\\", "UserAgentTest.html"); ``` # Handler @@ -76,12 +100,14 @@ public class ExampleDownloadHandler extends DownloadHandler{ # Convert File sizes -Syntax: SizeUtil.toMBFB() = toMegaBytesFromBytes - SizeUtil.toGBFB() = toGigiaBytesFromBytes - - +**Explaination:** +SizeUtil.toMBFB() = toMegaBytesFromBytes +SizeUtil.toGBFB() = toGigiaBytesFromBytes +SizeUtil.toHumanReadableFromBytes() = convert bytes to KB, MB, GB (e.g. for displaying download size / download speed); +**Usage** ``` double mb = SizeUtil.toMBFB(2000000000); double kb = SizeUtil.toKBFB(1000000); +String size = SizeUtil.toHumanReadableFromBytes(4096); ``` diff --git a/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java b/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java index cd93bd8..3148544 100644 --- a/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java +++ b/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java @@ -33,7 +33,7 @@ public void run() { onDownloadProgress(downloader.downloadedBytes,downloader.downloadLength,percent); onDownloadSpeedProgress(downloader.downloadedBytes,downloader.downloadLength,percent,deltaDownload); } - },0,1000); + },0,250); } public void onDownloadTickPerSec(int bytesPerSec) {} diff --git a/src/me/marnic/jdl/DownloadProgressDownloadHandler.java b/src/me/marnic/jdl/DownloadProgressDownloadHandler.java index 27907be..eb666c8 100644 --- a/src/me/marnic/jdl/DownloadProgressDownloadHandler.java +++ b/src/me/marnic/jdl/DownloadProgressDownloadHandler.java @@ -25,7 +25,7 @@ public void onDownloadStart() { public void run() { onDownloadProgress(downloader.downloadedBytes,downloader.downloadLength,(int)(((double)downloader.downloadedBytes/downloader.downloadLength)*100)); } - },0,1000); + },0,250); } public abstract void onDownloadProgress(int downloaded,int maxDownload,int percent); diff --git a/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java b/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java index 4a95005..95c184d 100644 --- a/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java +++ b/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java @@ -29,7 +29,7 @@ public void run() { onDownloadTickPerSec(deltaDownload); lastDownloadSize = downloader.downloadedBytes; } - },0,1000); + },0,250); } public abstract void onDownloadTickPerSec(int bytesPerSec); diff --git a/src/me/marnic/jdl/Downloader.java b/src/me/marnic/jdl/Downloader.java index 9961ca4..e47a517 100644 --- a/src/me/marnic/jdl/Downloader.java +++ b/src/me/marnic/jdl/Downloader.java @@ -1,10 +1,11 @@ package me.marnic.jdl; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.net.URL; -import java.net.URLConnection; +import java.io.*; +import java.net.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; /** * Copyright (c) 12.01.2019 @@ -12,14 +13,19 @@ * GitHub: https://github.com/MrMarnic */ public class Downloader { - private DownloadHandler downloadHandler; - int downloadedBytes; int downloadLength; + private HashMap cookiesMap; + private boolean bUseCookies = false; + private boolean bUseDomainFilter = false; + private String domainFilter; + private boolean bIsCancelled = false; + private boolean bUseCustomUserAgentString = false; + private String customUserAgentString; public Downloader(boolean createDefaultHandler) { - if(createDefaultHandler) { + if (createDefaultHandler) { downloadHandler = new DownloadHandler(this) { @Override public void onDownloadStart() { @@ -39,36 +45,252 @@ public void onDownloadError() { } } + public void setCookies(File cookiesFile) { + this.bUseCookies = true; + this.cookiesMap = readCookies(cookiesFile); + } + + public void setDomainFilter(String domainFilter) { + this.bUseDomainFilter = true; + this.domainFilter = domainFilter; + } + + public void setCustomUserAgentString(String userAgentString) { + this.bUseCustomUserAgentString = true; + this.customUserAgentString = userAgentString; + } + + // get HashMap formatted cookies that have finished reading from a txt file + public HashMap getCookiesMap() { + return cookiesMap; + } + + // read HashMap formatted cookies from txt file for use with libraries like jsoup + public HashMap readCookies(File cookiesFile) { + try { + Scanner myReader = new Scanner(cookiesFile); + String data; + List partList; + HashMap result = new HashMap<>(); + while (myReader.hasNextLine()) { + data = myReader.nextLine(); + // skip comment and empty lines + if (!data.contains("#") && !data.trim().isEmpty()) { + partList = Arrays.asList(data.split("\\s+")); // split line as array by space + if (bUseDomainFilter) { + if (partList.get(0).contains(this.domainFilter)) { + result.put(partList.get(5), partList.get(6)); // name and value fields in Netscape format cookies + } + } else { + result.put(partList.get(5), partList.get(6)); + } + } + } + myReader.close(); + return result; + } catch (FileNotFoundException e) { + System.err.println("Cookies not found"); + e.printStackTrace(); + } + return null; + } + + private URLConnection setupConnectionWithCookies(HashMap cookiesMap, String urlStr) { + URLConnection connection = null; + URL url = null; + try { + url = new URL(urlStr); + } catch (MalformedURLException e) { + System.err.println("An error occurred."); + e.printStackTrace(); + } + try { + connection = url.openConnection(); + } catch (IOException e) { + System.err.println("An error occurred."); + e.printStackTrace(); + } + String cookies = ""; + + for (Map.Entry entry : cookiesMap.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + cookies += key + "=" + value + "; "; + connection.addRequestProperty("Cookie", cookies); + } + return connection; + } + + public String getServerFileName(String urlStr) { + HttpURLConnection con = null; + try { + URL url = new URL(urlStr); + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } + con.connect(); + + if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { + System.err.println("Server returned HTTP " + con.getResponseCode() + " " + con.getResponseMessage()); + return "error"; + } + } catch (Exception e) { + System.err.println("An error occurred."); + return "error"; + } finally { + if (con != null) con.disconnect(); + } + + String filename; + try { + if (con.getHeaderField("Content-Disposition") != null) { + filename = con.getHeaderField("Content-Disposition").replaceFirst("(?i)^.*filename=\"?([^\"]+)\"?.*$", "$1"); + } else { + filename = Paths.get(new URI(con.getURL().toString()).getPath()).getFileName().toString(); + } + return filename; + } + catch (Exception e) { + System.err.println("An error occurred."); + e.printStackTrace(); + } + return null; + } + + public void setbIsCancelled(boolean bIsCancelled) { + this.bIsCancelled = bIsCancelled; + } + + public void cancelDownload() { + bIsCancelled = true; + } + + private void resetValues() { + downloadedBytes = 0; + downloadLength = 0; + } + + public Integer getDownloadLength(String urlStr) { + try { + URL url = new URL(urlStr); + HttpURLConnection con; + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } + con.connect(); + downloadLength = con.getContentLength(); + return downloadLength; + } catch (Exception e) { + System.err.println("An error occurred."); + e.printStackTrace(); + } + return null; + } + public void downloadFileToLocation(String urlStr, String pathToDownload) { + handleDownloadFileToLocation(urlStr, pathToDownload, null); + } + + public void downloadFileToLocation(String urlStr, String pathToDownload, String customFileName) { + handleDownloadFileToLocation(urlStr, pathToDownload, customFileName); + } + private void handleDownloadFileToLocation(String urlStr, String pathToDownload, String customFileName) { try { URL url = new URL(urlStr); - URLConnection con = url.openConnection(); + HttpURLConnection con; + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } + con.connect(); downloadLength = con.getContentLength(); InputStream in = con.getInputStream(); - FileOutputStream out = new FileOutputStream(pathToDownload); + String fileName; + if (customFileName != null) { + fileName = customFileName; + } else { + fileName = getServerFileName(urlStr); + } + Files.createDirectories(Paths.get(pathToDownload)); + FileOutputStream out = new FileOutputStream(pathToDownload + fileName); byte[] data = new byte[1024]; int length; downloadHandler.onDownloadStart(); - while ((length = in.read(data,0,1024))!=-1) { - out.write(data,0,length); - downloadedBytes+=length; + while (!bIsCancelled && (length = in.read(data, 0, 1024)) != -1) { + out.write(data, 0, length); + downloadedBytes += length; } + out.flush(); in.close(); out.close(); + resetValues(); // for repeated use of the same downloader object, in order to get correct download speed values downloadHandler.onDownloadFinish(); - }catch (Exception e) { + } catch (Exception e) { + System.err.println("An error occurred."); e.printStackTrace(); downloadHandler.onDownloadError(); } } + public String downloadJSONString(String urlStr) { + try { + URL url = new URL(urlStr); + HttpURLConnection con; + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } + con.connect(); + downloadLength = con.getContentLength(); + + InputStream in = con.getInputStream(); + BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + sb.append(line).append("\n"); + } + br.close(); + return sb.toString(); + } catch (UnknownHostException e) { + System.err.println("No Internet available"); + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + downloadHandler.onDownloadError(); + } + return null; + } + + // This method never worked anyway public Object downloadObject(String urlStr) { try { @@ -83,15 +305,16 @@ public Object downloadObject(String urlStr) { downloadHandler.onDownloadStart(); - while ((length = in.read(data,0,1024))!=-1) { - downloadedBytes+=length; + while ((length = in.read(data, 0, 1024)) != -1) { + downloadedBytes += length; } in.close(); downloadHandler.onDownloadFinish(); return in.readObject(); - }catch (Exception e) { + } catch (Exception e) { + System.err.println("An error occurred."); e.printStackTrace(); downloadHandler.onDownloadError(); } diff --git a/src/me/marnic/jdl/SizeUtil.java b/src/me/marnic/jdl/SizeUtil.java index 9ef07b6..8e20d68 100644 --- a/src/me/marnic/jdl/SizeUtil.java +++ b/src/me/marnic/jdl/SizeUtil.java @@ -7,30 +7,46 @@ */ public class SizeUtil { public static double toKBFB(int bytes) { - return ((double)bytes/1000); + return ((double) bytes / 1024); } public static double toMBFB(int bytes) { - return ((double)bytes/1000000); + return ((double) bytes / (1024 * 1024)); } public static double toGBFB(int bytes) { - return ((double)bytes/1000000000); + return ((double) bytes / 1024 * 1024 * 1024); } - public static double toMBFKB(int kb) { - return ((double)kb/1000); + return ((double) kb / 1024); } public static double toGBFKB(int kb) { - return ((double)kb/1000000); + return ((double) kb / (1024 * 1024)); } - public static double toGBFMB(int mb) { - return ((double)mb/1000); + return ((double) mb / 1024); + } + + public static String toHumanReadableFromBytes(int bytes) { + if (bytes < 0) { + return "unknown size"; + } + + String[] units = {"B", "KB", "MB", "GB"}; + int unitIndex = 0; + double value = bytes; + + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024.0; + unitIndex++; + } + + String formattedValue = String.format("%.2f", value); + return formattedValue + " " + units[unitIndex]; } }