Add the ability to access remote resources, more cleanup

.github/workflows/python_package.yaml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.8 ]
+        python-version: [ 3.9 ]
 
     steps:
       - uses: actions/checkout@v3
@@ -19,8 +19,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest pytest-cov
+          pip install flake8 pytest pytest-cov setuptools
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f setup.py ]; then python setup.py install; fi
           sudo apt-get install graphviz
       - name: Lint with flake8
         run: |

onto_tool/__init__.py

@@ -1 +1 @@
-VERSION = '1.8.0'
+VERSION = '1.9.0'

onto_tool/onto_tool.py

@@ -12,7 +12,7 @@
 
 import onto_tool
 
-from .bundle import bundle_ontology
+from .bundle import BundleFileException, bundle_ontology
 from .command_line import configure_arg_parser
 from .ontograph import OntoGraf
 from .utils import isfile, expand_file_ref, perform_export, find_single_ontology, \
@@ -292,7 +292,10 @@ def main(arguments):
         __suppress_ssl_certificate_check()
 
     if args.command == 'bundle':
-        bundle_ontology(args.variables, args.bundle)
+        try:
+            bundle_ontology(args.variables, args.bundle)
+        except BundleFileException as bfe:
+            logging.error('Encountered %s', str(bfe))
         return
 
     if args.command == 'graphic':

onto_tool/utils.py

@@ -3,8 +3,11 @@
 import re
 from glob import glob
 from os.path import isdir, isfile, join
+import tempfile
 import sys
+from urllib.parse import urlparse
 
+import requests
 from rdflib import ConjunctiveGraph, Graph, Literal, URIRef
 from rdflib.namespace import OWL, RDF, RDFS, SKOS, XSD
 from rdflib.plugins.parsers.notation3 import BadSyntax
@@ -156,6 +159,27 @@ def clean_merge_artifacts(g, iri, version):
         g.add((iri, OWL.imports, i))
 
 
+def is_remote(path: str) -> bool:
+    """True if resource is remote (downloable)"""
+    as_url = urlparse(path)
+    return as_url.scheme in ('http', 'https')
+
+
+def download_if_needed(path: str) -> str:
+    """If the path is a HTTP(S) URL, download to temporary file"""
+    if is_remote(path):
+        # Use instead of urlretrieve to get content after redirection
+        r = requests.get(path, allow_redirects=True, timeout=300)
+        if not r.ok:
+            return None
+        # TODO Won't be cleaned up for now, create context manager
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        with tmp_file:
+            tmp_file.write(r.content)
+        return tmp_file.name
+    return path
+
+
 def perform_export(output, output_format, paths, context=None,
                    remove_dependency_versions=False,
                    merge=None,

setup.py

@@ -21,6 +21,7 @@
         'pydot',
         'jinja2',
         'markdown2',
+        'requests==2.22.0',
         'pyyaml==5.2',
         'jsonschema==3.2.0',
         'SPARQLWrapper==1.8.4',
@@ -46,5 +47,5 @@
             "onto_tool = onto_tool.onto_tool:run_tool"
         ]
     },
-    python_requires='>=3.8',
+    python_requires='>=3.9',
 )

tests/bundle/download/invalid_wildcard.yaml

@@ -0,0 +1,10 @@
+bundle: test
+variables:
+  input: "https://w3id.org/semanticarts/ontology/"
+  output: "tests/bundle/download"
+actions:
+  - action: 'move'
+    source: '{input}'
+    target: "{output}/"
+    excludes:
+      - allThe*.ttl

tests/bundle/download/missing_include.yaml

@@ -0,0 +1,10 @@
+bundle: test
+variables:
+  input: "https://w3id.org/semanticarts/ontology/"
+  output: "tests/bundle/download"
+actions:
+  - action: 'move'
+    source: '{input}'
+    target: "{output}/"
+    excludes:
+      - disregard.ttl

tests/bundle/download/missing_reference.yaml

@@ -0,0 +1,13 @@
+bundle: test
+variables:
+  input: "https://w3id.org/semanticarts/ontology/"
+  output: "tests/bundle/download"
+actions:
+  - action: 'copy'
+    source: '{input}'
+    includes:
+      - 'non-existent.ttl'
+    target: "{output}/"
+  - action: 'copy'
+    source: '{input}/non-existent.ttl'
+    target: "{output}/"

tests/bundle/download/remote_direct.yaml

@@ -0,0 +1,16 @@
+bundle: test
+variables:
+  input: " https://ontologies.semanticarts.com/ontology/"
+  output: "tests/bundle/download"
+actions:
+  - action: 'sparql'
+    source: '{input}'
+    includes:
+      - gistCore.ttl
+    target: '{output}/sparql.csv'
+    query: >
+      prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+      SELECT DISTINCT ?def_by
+      WHERE {{
+        [ rdfs:isDefinedBy ?def_by ] .
+      }} ORDER BY ?def_by

tests/bundle/download/remote_rename.yaml

@@ -0,0 +1,24 @@
+bundle: test
+variables:
+  input: "https://w3id.org/semanticarts/ontology/"
+  output: "tests/bundle/download"
+actions:
+  - action: 'copy'
+    source: '{input}'
+    includes:
+      - versioning
+    target: "{output}"
+    rename:
+        from: "^(.*)$"
+        to: "\\g<1>.ttl"
+  - action: 'sparql'
+    source: '{output}'
+    includes:
+      - '*.ttl'
+    target: '{output}/sparql.csv'
+    query: >
+      prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+      SELECT DISTINCT ?def_by
+      WHERE {{
+        [ rdfs:isDefinedBy ?def_by ] .
+      }}

tests/bundle/download/test_remote_download.py

@@ -0,0 +1,62 @@
+import csv
+import logging
+from tempfile import TemporaryDirectory
+
+from onto_tool import onto_tool
+
+
+def test_download():
+    with TemporaryDirectory() as tempdir:
+        onto_tool.main([
+            'bundle', '-v', 'output', tempdir, 'tests/bundle/download/remote_rename.yaml'
+        ])
+        with open(f'{tempdir}/sparql.csv') as csvfile:
+            actual = list(row for row in csv.DictReader(csvfile))
+        expected = [
+            {'def_by': 'https://w3id.org/semanticarts/ontology/versioning'}
+        ]
+        assert actual == expected
+
+
+def test_remote_query():
+    with TemporaryDirectory() as tempdir:
+        onto_tool.main([
+            'bundle', '-v', 'output', tempdir, 'tests/bundle/download/remote_direct.yaml'
+        ])
+        with open(f'{tempdir}/sparql.csv') as csvfile:
+            actual = list(row for row in csv.DictReader(csvfile))
+        expected = [
+            {'def_by': 'https://w3id.org/semanticarts/ontology/gistCore'}
+        ]
+        assert actual == expected
+
+
+def test_missing_download_reported(caplog):
+    with TemporaryDirectory() as tempdir:
+        onto_tool.main([
+            'bundle', '-v', 'output', tempdir, 'tests/bundle/download/missing_reference.yaml'
+        ])
+
+    logs = caplog.text
+    assert 'non-existent.ttl' in logs
+
+
+def test_missing_include_reported(caplog):
+    caplog.set_level(logging.INFO)
+    with TemporaryDirectory() as tempdir:
+        onto_tool.main([
+            'bundle', '-v', 'output', tempdir, 'tests/bundle/download/missing_include.yaml'
+        ])
+
+    logs = caplog.text
+    assert 'must have an include' in logs
+
+
+def test_wildcard_reported(caplog):
+    with TemporaryDirectory() as tempdir:
+        onto_tool.main([
+            'bundle', '-v', 'output', tempdir, 'tests/bundle/download/invalid_wildcard.yaml'
+        ])
+
+    logs = caplog.text
+    assert 'wildcard' in logs

tests/bundle/test_markdown.py

@@ -27,4 +27,3 @@ def test_markdown_bulk():
 
     inc_and_exc = [basename(f) for f in glob('tests-output/bundle/bulk_md/inc_and_exc/*')]
     assert sorted(inc_and_exc) == ['b2.html', 'c3.html']
-