Added method to fetch project versions using GitHub API, removed hardcoded versions in tests

l2tdevtools/download_helpers/github.py

@@ -3,6 +3,7 @@
 
 from __future__ import unicode_literals
 
+import json
 import re
 
 from l2tdevtools.download_helpers import project
@@ -76,6 +77,51 @@ def _GetAvailableVersions(self, version_strings):
 
     return available_versions
 
+  def GetLatestVersionWithAPI(self, version_definition):
+    """Uses the GitHub API to retrieve the latest version number for a project.
+
+    This method is intended for use in tests only, due to the API rate-limit.
+
+    Args:
+      version_definition (ProjectVersionDefinition): project version definition
+          or None if not set.
+
+    Returns:
+      str: latest version number or None if not available.
+    """
+    earliest_version = None
+    latest_version = None
+
+    if version_definition:
+      earliest_version = version_definition.GetEarliestVersion()
+      if earliest_version and earliest_version[0] == '==':
+        return '.'.join(earliest_version[1:])
+
+      latest_version = version_definition.GetLatestVersion()
+
+    github_url = 'https://api.github.com/repos/{0:s}/{1:s}/releases'.format(
+        self._organization, self._repository)
+    page_content = self.DownloadPageContent(github_url)
+
+
+    api_response = json.loads(page_content)
+    release_names = [release['name'] for release in api_response]
+
+    version_expressions = [
+        '({0:s})$'.format(version_expression)
+        for version_expression
+        in self._VERSION_EXPRESSIONS]
+
+    versions = []
+    for release in release_names:
+      for version_expression in version_expressions:
+        version_strings = re.findall(version_expression, release)
+        versions.extend(version_strings)
+
+    available_versions = self._GetAvailableVersions(versions)
+    return self._GetLatestVersion(
+        earliest_version, latest_version, available_versions)
+
   def GetLatestVersion(self, project_name, version_definition):
     """Retrieves the latest version number for a given project name.
 

l2tdevtools/download_helpers/interface.py

@@ -103,3 +103,30 @@ def DownloadPageContent(self, download_url, encoding='utf-8'):
       self._cached_url = download_url
 
     return self._cached_page_content
+
+  def DownloadAPIPageContent(self, download_url, encoding='utf-8'):
+    """Download content from a github API url"""
+    if not download_url:
+      return None
+
+    if self._cached_url != download_url:
+      try:
+        url_object = urllib_request.urlopen(download_url)
+      except urllib_error.URLError as exception:
+        logging.warning(
+            'Unable to download URL: {0:s} with error: {1!s}'.format(
+                download_url, exception))
+        return None
+
+      if url_object.code != 403:
+        return None
+
+      page_content = url_object.read()
+
+      if encoding and isinstance(page_content, py2to3.BYTES_TYPE):
+        page_content = page_content.decode(encoding)
+
+      self._cached_page_content = page_content
+      self._cached_url = download_url
+
+    return self._cached_page_content

tests/download_helpers/github.py

@@ -22,6 +22,8 @@ class DocoptGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase):
 
   _PROJECT_ORGANIZATION = 'docopt'
   _PROJECT_NAME = 'docopt'
+  # Hardcoded version to check parsing of the GitHub page, as the GitHub API
+  # does not return release information for this project.
   _PROJECT_VERSION = '0.6.2'
 
   def testGetLatestVersion(self):
@@ -76,21 +78,24 @@ def testGetLatestVersion(self):
     download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL)
 
     latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None)
+    latest_version_api = download_helper.GetLatestVersionWithAPI(None)
 
-    self.assertEqual(latest_version, self._PROJECT_VERSION)
+    self.assertEqual(latest_version, latest_version_api)
 
   def testGetDownloadURL(self):
     """Tests the GetDownloadURL functions."""
     download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL)
 
+    project_version = download_helper.GetLatestVersionWithAPI(None)
+
     download_url = download_helper.GetDownloadURL(
-        self._PROJECT_NAME, self._PROJECT_VERSION)
+        self._PROJECT_NAME, project_version)
 
     expected_download_url = (
         'https://github.com/{0:s}/{1:s}/releases/download/{3:s}/'
         '{1:s}-{2:s}-{3:s}.tar.gz').format(
             self._PROJECT_ORGANIZATION, self._PROJECT_NAME,
-            self._PROJECT_STATUS, self._PROJECT_VERSION)
+            self._PROJECT_STATUS, project_version)
 
     self.assertEqual(download_url, expected_download_url)
 
@@ -116,29 +121,31 @@ class Log2TimelineGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase):
 
   _PROJECT_ORGANIZATION = 'log2timeline'
   _PROJECT_NAME = 'dfvfs'
-  # Hard-coded version to check parsing of GitHub page.
-  _PROJECT_VERSION = '20190128'
 
   def testGetLatestVersion(self):
     """Tests the GetLatestVersion functions."""
     download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL)
 
     latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None)
+    latest_version_api = download_helper.GetLatestVersionWithAPI(None)
 
-    self.assertEqual(latest_version, self._PROJECT_VERSION)
+
+    self.assertEqual(latest_version, latest_version_api)
 
   def testGetDownloadURL(self):
     """Tests the GetDownloadURL functions."""
     download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL)
 
+    project_version = download_helper.GetLatestVersionWithAPI(None)
+
     download_url = download_helper.GetDownloadURL(
-        self._PROJECT_NAME, self._PROJECT_VERSION)
+        self._PROJECT_NAME, project_version)
 
     expected_download_url = (
         'https://github.com/{0:s}/{1:s}/releases/download/{2:s}/'
         '{1:s}-{2:s}.tar.gz').format(
             self._PROJECT_ORGANIZATION, self._PROJECT_NAME,
-            self._PROJECT_VERSION)
+            project_version)
 
     self.assertEqual(download_url, expected_download_url)