diff --git a/.gitignore b/.gitignore index 8b4d536..3ca4d0a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,5 @@ *.rar *.txt -.idea - +.idea/ +.gradle/ \ No newline at end of file diff --git a/.gradle/4.8/fileHashes/fileHashes.bin b/.gradle/4.8/fileHashes/fileHashes.bin index e8a286e..df5a168 100644 Binary files a/.gradle/4.8/fileHashes/fileHashes.bin and b/.gradle/4.8/fileHashes/fileHashes.bin differ diff --git a/.gradle/4.8/fileHashes/fileHashes.lock b/.gradle/4.8/fileHashes/fileHashes.lock index 69ce6a2..dd8f4c6 100644 Binary files a/.gradle/4.8/fileHashes/fileHashes.lock and b/.gradle/4.8/fileHashes/fileHashes.lock differ diff --git a/.idea/gradle.xml b/.idea/gradle.xml deleted file mode 100644 index ba1ec5c..0000000 --- a/.idea/gradle.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index bc8d0a3..b2526f7 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,7 +1,13 @@ - - - + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 664c567..94a25f7 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -2,6 +2,5 @@ - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 7a33c51..a1b69e2 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,695 +1,290 @@ - - + + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - + - - + + - - + + - - + + - + - - + + - + - - + + - + - + - - + + - - + + - - + + - + - - - + + - + - - - - - - - - - - - - + @@ -699,36 +294,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - 1539109262430 + + 1544020048059 - - 1542664417856 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - - - - - - + + + + + + + + + + + + + - - - + + + - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - totalLinksFound - JAVA - EXPRESSION - - - shorterStudyGuideLink - JAVA - EXPRESSION - - - badLinks - JAVA - EXPRESSION - - - link - JAVA - EXPRESSION - - - response - JAVA - EXPRESSION - - - respone - JAVA - EXPRESSION - + - studyGuideLinks + new File("application.properties").exists() JAVA + java.io.File EXPRESSION - - - matcher + new File("src/main/resources/application.properties").exists() JAVA + java.io.File EXPRESSION - matcher.group(1); + new File("main/resources/application.properties").exists() JAVA + java.io.File EXPRESSION - matcher.group(1) + new File("resources/application.properties").exists() JAVA + java.io.File EXPRESSION - startIndex + new File("../application.properties").exists() JAVA + java.io.File EXPRESSION - endIndex + new File("./application.properties").exists() JAVA + java.io.File EXPRESSION - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - + - - + + - + - - + + - + - - + + - - - - + - - + + - + - - + + - - - - + - - - - - - - - - - - - - - - - - - + - - - - - + + - + - - - - - + + - + - - - - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - No facets are configured - - - - - - - - - - - - - - - 1.8 - - - - - - - - TestingBase64 - - - - - - - - Gradle: junit:junit:4.12 - - - - - - \ No newline at end of file diff --git a/TestingBase64.zip b/TestingBase64.zip deleted file mode 100644 index 4b2a635..0000000 Binary files a/TestingBase64.zip and /dev/null differ diff --git a/TestingBase64_2.zip b/TestingBase64_2.zip deleted file mode 100644 index 692d5ca..0000000 Binary files a/TestingBase64_2.zip and /dev/null differ diff --git a/build.gradle b/build.gradle.backup similarity index 100% rename from build.gradle rename to build.gradle.backup diff --git a/gradlew b/gradlew deleted file mode 100644 index cccdd3d..0000000 --- a/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat deleted file mode 100644 index f955316..0000000 --- a/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/out/production/classes/com/base64/d/AllStudyGuideLinks.class b/out/production/classes/com/base64/d/AllStudyGuideLinks.class deleted file mode 100644 index 62ca9e4..0000000 Binary files a/out/production/classes/com/base64/d/AllStudyGuideLinks.class and /dev/null differ diff --git a/out/production/classes/com/base64/d/AllStudyGuideLinks2.class b/out/production/classes/com/base64/d/AllStudyGuideLinks2.class deleted file mode 100644 index 6597641..0000000 Binary files a/out/production/classes/com/base64/d/AllStudyGuideLinks2.class and /dev/null differ diff --git a/out/production/classes/com/base64/d/KnownFalsePositives.class b/out/production/classes/com/base64/d/KnownFalsePositives.class deleted file mode 100644 index 06fbc08..0000000 Binary files a/out/production/classes/com/base64/d/KnownFalsePositives.class and /dev/null differ diff --git a/out/production/classes/com/base64/d/StudyGuideChecker.class b/out/production/classes/com/base64/d/StudyGuideChecker.class deleted file mode 100644 index 89c258a..0000000 Binary files a/out/production/classes/com/base64/d/StudyGuideChecker.class and /dev/null differ diff --git a/out/production/classes/com/base64/d/WriteToFile.class b/out/production/classes/com/base64/d/WriteToFile.class deleted file mode 100644 index 3635944..0000000 Binary files a/out/production/classes/com/base64/d/WriteToFile.class and /dev/null differ diff --git a/out/production/resources/studyguidelink.html b/out/production/resources/studyguidelink.html deleted file mode 100644 index bb5f379..0000000 --- a/out/production/resources/studyguidelink.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - Title - - - -
- URL: - -
- - - - - - - - - - - - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..4e61671 --- /dev/null +++ b/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + com.infinitecampus + study-guide-checker + 1.0-SNAPSHOT + + + + + org.jsoup + jsoup + 1.11.3 + + + + + org.apache.logging.log4j + log4j-api + 2.7 + + + org.apache.logging.log4j + log4j-core + 2.7 + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.7 + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + 1.8 + 1.8 + + + + + + \ No newline at end of file diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index e706738..0000000 --- a/settings.gradle +++ /dev/null @@ -1,2 +0,0 @@ -rootProject.name = 'TestingBase64' - diff --git a/src/main/java/com/base64/d/AllStudyGuideLinks.java b/src/main/java/com/base64/d/AllStudyGuideLinks.java deleted file mode 100644 index b4729a6..0000000 --- a/src/main/java/com/base64/d/AllStudyGuideLinks.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.base64.d; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class AllStudyGuideLinks { - - public static final String STUDY_GUIDE_PATH = "https://content.infinitecampus.com/sis/latest/study-guide/"; - - //System.out.println(getTotalLinksFound) - - public static final Set ALL_STUDY_GUIDES = Stream.of( - "academic-planner-system-set-up", - "academic-planner-use-and-management-", - "ad-hoc-filters-letters-and-data-viewer", - "ad-hoc-functions-and-logical-expressions", - "attendance", - "behavior-admin-set-up", - "behavior-data-management-and-reporting", - "behavior-messages-and-letters", - "calendar-rights-user-groups", - "campus-instruction-part-1-the-fundamentals", - "campus-instruction-part-2-grade-book-basics", - "campus-instruction-part-3-advanced-grade-book-and-posting-grades", - "campus-instruction-part-4-campus-learning", - "census---new-personfamily-set-up", - "census---personhousehold-maintenance", - "census-reports", - "flags-and-programs", - "grade-submission-process", - "grading-setup", - "health-module-system-setup", - "health-module-view-and-manage-student-health-information", - "messenger-setup-contacts-and-reports", - "messenger-for-the-end-user", - "online-payments-integrated-card-swipe-for-the-end-user", - "schedule-wizard-mass-schedule-students", - "tool-rights-user-groups", - "user-account-creation-maintenance-and-reporting", - "walk-in-scheduler-complete-or-change-a-students-schedule" - //combining first and second part of the url. - ).map(studyGuideName -> STUDY_GUIDE_PATH + studyGuideName) - .collect(Collectors.toSet()); - - - - - - -} diff --git a/src/main/java/com/base64/d/AllStudyGuideLinks2.java b/src/main/java/com/base64/d/AllStudyGuideLinks2.java deleted file mode 100644 index 7eb4203..0000000 --- a/src/main/java/com/base64/d/AllStudyGuideLinks2.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.base64.d; - -public class AllStudyGuideLinks2 { - - -} diff --git a/src/main/java/com/base64/d/Application.java b/src/main/java/com/base64/d/Application.java deleted file mode 100644 index 0c4405a..0000000 --- a/src/main/java/com/base64/d/Application.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.base64.d; - -import org.jsoup.Connection; -import org.jsoup.Jsoup; -import org.jsoup.helper.HttpConnection; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.io.IOException; - -public class Application { - public static void main(String[] args) { - - StudyGuideChecker checker = new StudyGuideChecker(); - checker.checkStudyGuides(); -// String communityURL = "https://www.cnn.com/"; -//// -////// Connection linkConnection = Jsoup.connect(communityURL); -//// Document communityLogin = Jsoup.parse(communityURL); -//// System.out.println(communityLogin); -//// Element link = communityLogin.select("a").first(); -//// System.out.println(link); - - -// Elements content = communityLogin.getElementsByClass("breaking-news__background"); -// System.out.println(content); - -// Connection.Response loginForm = Jsoup.connect(String.valueOf(communityLogin)) -// .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0") -// .method(Connection.Method.POST) -// .get -// .timeout(3000) -// .data("dave.hemmesch", signin-form_login) -// .data("password", password) -// .execute(); -// -// System.out.println(loginForm.cookies()); - - - - - - - -// communityLogin.body() = (Element) Jsoup.connect("login Site URL") -// .method(Connection.Method.GET) -// .timeout(10000) -// .execute(); - } - -// String sessionID = res.cookie("SESSION ID for site"); - -} - - - diff --git a/src/main/java/com/base64/d/KnownFalsePositives.java b/src/main/java/com/base64/d/KnownFalsePositives.java deleted file mode 100644 index 95deccd..0000000 --- a/src/main/java/com/base64/d/KnownFalsePositives.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.base64.d; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - - -public class KnownFalsePositives { - - public static final String BLACK_LIST_PATH = "https://content.infinitecampus.com/sis/latest/"; - - public static final Set BLACK_LIST = Stream.of( - "simulation/send-a-behavior-message-sc-01-28-01", - "simulation/schedule-attendance-message-sc-01-24-02", - "simulation/schedule-a-behavior-message-sc-01-27-02", - "simulation/create-a-behavior-message-template-sc-01-27-01", - "simulation/view-individual-student-attendance", - "simulation/cancel-a-user-created-future-dated-message-sc-03-08-04" - ).map(blackList -> BLACK_LIST_PATH + blackList) - .collect(Collectors.toSet()); - - - - -// public KnownFalsePositives(Set knownFalsePositives) { -// this.knownFalsePositives = knownFalsePositives; -// } -// -// public Set getKnownFalsePositives() { -// return knownFalsePositives; -// } -// -// public void setKnownFalsePositives(Set knownFalsePositives) { -// this.knownFalsePositives = knownFalsePositives; -// } -} diff --git a/src/main/java/com/base64/d/StudyGuideChecker.java b/src/main/java/com/base64/d/StudyGuideChecker.java deleted file mode 100644 index 4d0ae63..0000000 --- a/src/main/java/com/base64/d/StudyGuideChecker.java +++ /dev/null @@ -1,161 +0,0 @@ -package com.base64.d; - -import org.jsoup.Connection; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.io.IOException; -import java.util.Base64; -import java.util.Date; -import java.util.HashSet; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class StudyGuideChecker { - - private int totalLinksFound = 0; - - public int getTotalLinksFound() { - return totalLinksFound; - } - - public void checkStudyGuides() { - int count = 0; - - - String decodedText = ""; - String iframeSrc; - //why null here? - String titleOfStudyGuide = null; - - Set studyGuideLinks = new HashSet<>(); - Set badLinks = new HashSet<>(); - - WriteToFile writeToFile = new WriteToFile(); - - for (String studyGuideLink : AllStudyGuideLinks.ALL_STUDY_GUIDES) { - try { - - //Connect to the study guide url - Document wrapperDocument = Jsoup.connect(studyGuideLink).get(); - - //Grab the iframe - Element iframe = wrapperDocument.select("iframe").first(); - - //Get iframe scr url - iframeSrc = iframe.attr("src"); - - //Connect to the iframe source url - Document studyGuideDocument = Jsoup.connect(iframeSrc).get(); - - titleOfStudyGuide = wrapperDocument.title(); - System.out.println(titleOfStudyGuide + " " + iframeSrc); - - //Get the script tag info - Elements scriptTag = studyGuideDocument.getElementsByTag("script"); - String jsScripts = scriptTag.toString(); - - //Match any string of characters over 100 in length. - Pattern courseDataPattern = Pattern.compile("(\\w{100,}.+)"); - - Matcher courseDataMatcher = courseDataPattern.matcher(jsScripts); - - String base64String = ""; - while (courseDataMatcher.find()) { - System.out.println("--------"); - - base64String = courseDataMatcher.group(1); - } - //Reduced the giant string by 2 because it included the " and the ; - String trimFoundBase64String = base64String.substring(0, (base64String.length() - 2)); - - //Put in this try block because one study guide had different script tag layout and threw error. - try { - // Decoding a base64 string and putting into a byte array - byte[] decodedArr = Base64.getDecoder().decode(trimFoundBase64String); - decodedText = new String(decodedArr, "UTF-8"); - //System.out.println("decoded base64: " + decodedText); - } catch - (IllegalArgumentException exc) { - exc.printStackTrace(); - totalLinksFound -= studyGuideLinks.size(); - System.out.println(titleOfStudyGuide + " not checked because of decoding error"); - } - - // Regex meanings: - // . => means any character - // + => means 1 or more - // * => means 0 or more - // ? +> means 0 or 1 - // Everything in the () is courseDataPattern/courseDataMatcher "group" syntax. then \" is looking for the next quote - Pattern contentLinkPattern = Pattern.compile("(https://content.infinitecampus.com/sis.*?)\""); - - Matcher contentLinkMatcher = contentLinkPattern.matcher(decodedText); - //System.out.println("This is the courseDataMatcher " + contentLinkMatcher); - - - while (contentLinkMatcher.find()) { - count++; - //Some study guide links had '\' added for unknown reason. this code checks for it and removes it before adding it to the studyGuide. - String shorterStudyGuideLink = contentLinkMatcher.group(1).replaceAll("\\\\+$", ""); - //Links are good because regex replace all removes "\" if there. - studyGuideLinks.add(shorterStudyGuideLink); - - } - - System.out.println(studyGuideLinks.size() + " Unique Campus Community links found"); - //Remove all the false positives that are in JSON of study guide but not visible to front end user. - studyGuideLinks.removeAll(KnownFalsePositives.BLACK_LIST); - System.out.println(studyGuideLinks); - totalLinksFound += studyGuideLinks.size(); - - } catch (IOException e) { - e.printStackTrace(); - } - - - int response; - for (String link : studyGuideLinks) { - - try { - Connection linkConnection = Jsoup.connect(link).ignoreHttpErrors(true); - - // Although the `linkContents` is never utilized, it is a - // required call. This is because Jsoup expects that `get()` - // is called before the `response()` object is available. - Document linkContents = linkConnection.get(); - response = linkConnection.response() - .statusCode(); - // Report a bad link if anything other than a "200 OK" response is returned. - if (response != 200) { - badLinks.add(link); - writeToFile.createTxtFile(titleOfStudyGuide, link, studyGuideLink); - } - } catch (IOException ie) { - ie.printStackTrace(); - } - } - - System.out.println(badLinks.size() + " Broken links: " + badLinks); - System.out.println(getTotalLinksFound() + " total links checked"); - - - System.out.println("----------------------------------------------------"); - System.out.println("----------------------------------------------------"); - - studyGuideLinks.clear(); - } - System.out.println(AllStudyGuideLinks.ALL_STUDY_GUIDES.size() + " Total Study Guides"); - - String totalLinksFoundAsString = Integer.toString(totalLinksFound); -// totalLinksFoundAsString = totalLinksFound.to; - - writeToFile.writeTotalCount(totalLinksFoundAsString); - } - - -} - diff --git a/src/main/java/com/base64/d/WriteToFile.java b/src/main/java/com/base64/d/WriteToFile.java deleted file mode 100644 index a921b63..0000000 --- a/src/main/java/com/base64/d/WriteToFile.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.base64.d; - -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.text.SimpleDateFormat; -import java.util.Date; - - - - -public class WriteToFile { - - private SimpleDateFormat dateFormat = new SimpleDateFormat("MM-dd-yyyy HHmm"); - private String date = dateFormat.format(new Date()); - - public void createTxtFile(String titleOfStudyGuide, String link, String link2) { - try { - FileWriter fileWriter = new FileWriter("DeadLinks " + date + ".txt", true); - PrintWriter printWriter = new PrintWriter(fileWriter); - - printWriter.println("\n"); - printWriter.println("---------------------------------------"); - - printWriter.println(titleOfStudyGuide); - printWriter.println("Link to Study Guide: " + link2); - printWriter.println("This link is broken: " + link); - printWriter.println(date); - printWriter.println("---------------------------------------"); - - - printWriter.close(); - } catch ( - IOException exception) { - System.out.println("ERROR"); - exception.printStackTrace(); - - } - } - - public void writeTotalCount(String totalLinksFound){ - try { - Files.write(Paths.get("DeadLinks " + date + ".txt"), (String.format("%s total links checked\n", totalLinksFound)).getBytes(), StandardOpenOption.APPEND); - }catch (IOException e) { - e.printStackTrace(); - //exception handling left as an exercise for the reader - } - } - -} diff --git a/src/main/java/com/infinitecampus/StudyGuideChecker.java b/src/main/java/com/infinitecampus/StudyGuideChecker.java new file mode 100644 index 0000000..fa74fde --- /dev/null +++ b/src/main/java/com/infinitecampus/StudyGuideChecker.java @@ -0,0 +1,188 @@ +package com.infinitecampus; + +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class StudyGuideChecker { + + private static Logger log = LoggerFactory.getLogger(StudyGuideChecker.class); + + public static void main(String[] args) { + + Properties properties = loadProperties(); + + List studyGuideUrls = Arrays.asList( + properties.getProperty("study-guide-urls").split(",")); + List knownBadLinks = Arrays.asList( + properties.getProperty("known-bad-links").split(",")); + + // Retrieve all the content links from all the Study Guides + Map> allLinksToCheck = new HashMap<>(); + for (String studyGuideUrl : studyGuideUrls) { + List links = getLinksInStudyGuide(studyGuideUrl); + // In order to have a single list of all links, and also keep track + // of the study guide(s) using the link, invert these collection, + // and have each link point to a List of Study Guide links. + for (String link : links) { + // Only process links that are not known to be bad. + if (!knownBadLinks.contains(link)) { + // Check to see if the link is already in the collection, and + // if so, simply add the studyGuideUrl to the List + if (allLinksToCheck.containsKey(link)) { + List studyGuidesUsingThisLink = allLinksToCheck.get(link); + studyGuidesUsingThisLink.add(studyGuideUrl); + } + else { + List studyGuidesUsingThisLink = new ArrayList<>(); + studyGuidesUsingThisLink.add(studyGuideUrl); + allLinksToCheck.put(link, studyGuidesUsingThisLink); + } + } + } + } + + // Validate each link + Map> foundBadLinks = new HashMap<>(); + for (Map.Entry> linkToCheck : allLinksToCheck.entrySet()) { + String link = linkToCheck.getKey(); + Connection linkConnection = Jsoup.connect(link).ignoreHttpErrors(true); + + // Although the `linkContents` is never utilized, it is a + // required call. This is because Jsoup expects that `get()` + // is called before the `response()` object is available. + Document linkContents = null; + try { + linkContents = linkConnection.get(); + } catch (IOException e) { + e.printStackTrace(); + } + int response = linkConnection.response().statusCode(); + // Report a bad link if anything other than a "200 OK" response is returned. + if (response != 200) { + foundBadLinks.put(link, linkToCheck.getValue()); + } + } + + // Report, indicating each bad link, and the associated study guide(s) + for (Map.Entry> badLink : foundBadLinks.entrySet()) { + log.info("Bad Link: " + badLink.getKey()); + for (String studyGuideUrl : badLink.getValue()) { + log.info(" Found in Study Guide: " + studyGuideUrl); + } + } + } + + /** + * Retrieve the Properties file for this application. + * @return + */ + private static Properties loadProperties() { + Properties properties = new Properties(); + try { + InputStream inputStream = new FileInputStream("src/main/resources/application.properties"); + properties.load(inputStream); + } catch (IOException e) { + e.printStackTrace(); + } + return properties; + } + + /** + * Given a URL to a Study Guide, find all of the links to content, and + * return them as a List. + * @param studyGuideUrl + * @return + */ + private static List getLinksInStudyGuide(String studyGuideUrl) { + List linksInStudyGuide = new ArrayList<>(); + + // Connect to the study guide url + Document wrapperDocument = null; + try { + wrapperDocument = Jsoup.connect(studyGuideUrl).get(); + } catch (IOException e) { + e.printStackTrace(); + } + + // Grab the iframe + Element iframe = wrapperDocument.select("iframe").first(); + + // Get iframe scr url + String iframeSrc = iframe.attr("src"); + + // Connect to the iframe source url + Document studyGuideDocument = null; + try { + studyGuideDocument = Jsoup.connect(iframeSrc).get(); + } catch (IOException e) { + e.printStackTrace(); + } + + String titleOfStudyGuide = wrapperDocument.title(); + + //Get the script tag info + Elements scriptTag = studyGuideDocument.getElementsByTag("script"); + String jsScripts = scriptTag.toString(); + + //Match any string of characters over 100 in length. + Pattern courseDataPattern = Pattern.compile("(\\w{100,}.+)"); + + Matcher courseDataMatcher = courseDataPattern.matcher(jsScripts); + + String base64String = null; + while (courseDataMatcher.find()) { + base64String = courseDataMatcher.group(1); + } + //Reduced the giant string by 2 because it included the " and the ; + String trimFoundBase64String = base64String.substring(0, (base64String.length() - 2)); + + // Put in this try block because one study guide had different script tag layout and threw error. + String decodedText = null; + try { + // Decoding a base64 string and putting into a byte array + byte[] decodedArr = Base64.getDecoder().decode(trimFoundBase64String); + decodedText = new String(decodedArr, "UTF-8"); + //System.out.println("decoded base64: " + decodedText); + } catch (IllegalArgumentException | UnsupportedEncodingException e) { + e.printStackTrace(); + } + + // Regex meanings: + // . => means any character + // + => means 1 or more + // * => means 0 or more + // ? +> means 0 or 1 + // Everything in the () is courseDataPattern/courseDataMatcher "group" syntax. then \" is looking for the next quote + Pattern contentLinkPattern = Pattern.compile("(https://content.infinitecampus.com/sis.*?)\""); + + Matcher contentLinkMatcher = contentLinkPattern.matcher(decodedText); + + while (contentLinkMatcher.find()) { + // Some study guide links had '\' added for unknown reason. this code checks for it and removes it before adding it to the studyGuide. + String shorterStudyGuideLink = contentLinkMatcher.group(1).replaceAll("\\\\+$", ""); + // Links are good because regex replace all removes "\" if there. + linksInStudyGuide.add(shorterStudyGuideLink); + } + + return linksInStudyGuide; + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..d484de9 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,32 @@ +study-guide-urls=https://content.infinitecampus.com/sis/latest/study-guide/academic-planner-system-set-up,\ +https://content.infinitecampus.com/sis/latest/study-guide/academic-planner-use-and-management-,\ +https://content.infinitecampus.com/sis/latest/study-guide/ad-hoc-filters-letters-and-data-viewer,\ +https://content.infinitecampus.com/sis/latest/study-guide/ad-hoc-functions-and-logical-expressions,\ +https://content.infinitecampus.com/sis/latest/study-guide/attendance,\ +https://content.infinitecampus.com/sis/latest/study-guide/behavior-admin-set-up,\ +https://content.infinitecampus.com/sis/latest/study-guide/behavior-data-management-and-reporting,\ +https://content.infinitecampus.com/sis/latest/study-guide/behavior-messages-and-letters,\ +https://content.infinitecampus.com/sis/latest/study-guide/calendar-rights-user-groups,\ +https://content.infinitecampus.com/sis/latest/study-guide/campus-instruction-part-1-the-fundamentals,\ +https://content.infinitecampus.com/sis/latest/study-guide/campus-instruction-part-2-grade-book-basics,\ +https://content.infinitecampus.com/sis/latest/study-guide/campus-instruction-part-3-advanced-grade-book-and-posting-grades,\ +https://content.infinitecampus.com/sis/latest/study-guide/campus-instruction-part-4-campus-learning,\ +https://content.infinitecampus.com/sis/latest/study-guide/census---new-personfamily-set-up,\ +https://content.infinitecampus.com/sis/latest/study-guide/census---personhousehold-maintenance,\ +https://content.infinitecampus.com/sis/latest/study-guide/census-reports,\ +https://content.infinitecampus.com/sis/latest/study-guide/flags-and-programs,\ +https://content.infinitecampus.com/sis/latest/study-guide/grade-submission-process,\ +https://content.infinitecampus.com/sis/latest/study-guide/grading-setup,\ +https://content.infinitecampus.com/sis/latest/study-guide/health-module-system-setup,\ +https://content.infinitecampus.com/sis/latest/study-guide/health-module-view-and-manage-student-health-information,\ +https://content.infinitecampus.com/sis/latest/study-guide/messenger-for-the-end-user,\ +https://content.infinitecampus.com/sis/latest/study-guide/tool-rights-user-groups,\ +https://content.infinitecampus.com/sis/latest/study-guide/user-account-creation-maintenance-and-reporting,\ +https://content.infinitecampus.com/sis/latest/study-guide/walk-in-scheduler-complete-or-change-a-students-schedule + +known-bad-links=https://content.infinitecampus.com/sis/latest/simulation/send-a-behavior-message-sc-01-28-01,\ +https://content.infinitecampus.com/sis/latest/simulation/schedule-attendance-message-sc-01-24-02,\ +https://content.infinitecampus.com/sis/latest/simulation/schedule-a-behavior-message-sc-01-27-02,\ +https://content.infinitecampus.com/sis/latest/simulation/create-a-behavior-message-template-sc-01-27-01,\ +https://content.infinitecampus.com/sis/latest/simulation/view-individual-student-attendance,\ +https://content.infinitecampus.com/sis/latest/simulation/cancel-a-user-created-future-dated-message-sc-03-08-04 diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 0000000..91eb083 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/studyguidelink.html b/src/main/resources/studyguidelink.html deleted file mode 100644 index bb5f379..0000000 --- a/src/main/resources/studyguidelink.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - Title - - - -
- URL: - -
- - - - - - - - - - - - - - \ No newline at end of file