Merge pull request #1 from 1kbgz/tkp/init

Add initial support for loading from remote

Author: timkpaine

.github/workflows/build.yaml

@@ -53,6 +53,8 @@ jobs:
 
     - name: Test
       run: make coverage
+      env:
+        FSSPEC_S3_ENDPOINT_URL: ${{ secrets.FSSPEC_S3_ENDPOINT_URL }}
 
     - name: Upload test results (Python)
       uses: actions/upload-artifact@v4

fsspec_python/__init__.py

@@ -1 +1,4 @@
+from .importer import *
+from .open import *
+
 __version__ = "0.1.0"

fsspec_python/importer.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import sys
+from importlib.abc import MetaPathFinder, SourceLoader
+from importlib.machinery import SOURCE_SUFFIXES, ModuleSpec
+from os.path import join
+from types import ModuleType
+from typing import TYPE_CHECKING
+
+from fsspec import url_to_fs
+from fsspec.implementations.local import AbstractFileSystem
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+__all__ = (
+    "FSSpecImportFinder",
+    "FSSpecImportLoader",
+    "install_importer",
+    "uninstall_importer",
+)
+
+
+class FSSpecImportFinder(MetaPathFinder):
+    def __init__(self, fsspec: str, **fsspec_args: str) -> None:
+        self.fsspec_fs: AbstractFileSystem
+        self.root: str
+        self.fsspec_fs, self.root = url_to_fs(fsspec, **fsspec_args)
+        self.remote_modules: dict[str, str] = {}
+
+    def find_spec(self, fullname: str, path: Sequence[str | bytes] | None, target: ModuleType | None = None) -> ModuleSpec | None:
+        for suffix in SOURCE_SUFFIXES:
+            filename = join(self.root, fullname.split(".")[-1] + suffix)
+            if not self.fsspec_fs.exists(filename):
+                continue
+            self.remote_modules[fullname] = ModuleSpec(
+                name=fullname, loader=FSSpecImportLoader(fullname, filename, self.fsspec_fs), origin=filename, is_package=False
+            )
+            return self.remote_modules[fullname]
+        return None
+
+    def unload(self) -> None:
+        # unimport all remote modules from sys.modules
+        for mod in self.remote_modules:
+            if mod in sys.modules:
+                del sys.modules[mod]
+        self.remote_modules = {}
+
+
+# Singleton for use elsewhere
+_finder: FSSpecImportFinder = None
+
+
+class FSSpecImportLoader(SourceLoader):
+    def __init__(self, fullname: str, path: str, fsspec_fs: AbstractFileSystem):
+        self.fullname = fullname
+        self.path = path
+        self.fsspec_fs = fsspec_fs
+
+    def get_filename(self, fullname: str) -> str:  # noqa: ARG002
+        return self.path
+
+    def get_data(self, path: str | bytes) -> bytes:
+        with self.fsspec_fs.open(path, "rb") as f:
+            return f.read()
+
+    # def exec_module(self, module: ModuleType) -> None:
+    #     source_bytes = self.get_data(self.get_filename(self.fullname))
+    #     source = source_bytes.decode("utf-8")
+
+
+def install_importer(fsspec: str, **fsspec_args: str) -> FSSpecImportFinder:
+    """Install the fsspec importer.
+
+    Args:
+        fsspec: fsspec filesystem string
+    Returns: The finder instance that was installed.
+    """
+    global _finder
+    if _finder is None:
+        _finder = FSSpecImportFinder(fsspec, **fsspec_args)
+
+    sys.meta_path.insert(0, _finder)
+    return _finder
+
+
+def uninstall_importer() -> None:
+    """Uninstall the fsspec importer."""
+    global _finder
+    if _finder is not None and _finder in sys.meta_path:
+        _finder.unload()
+        sys.meta_path.remove(_finder)
+        _finder = None

fsspec_python/open.py

@@ -0,0 +1,15 @@
+import builtins
+
+_original_open = builtins.open
+
+__all__ = ("open_from_fsspec", "install_open_hook")
+
+
+def open_from_fsspec(file, mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None):
+    print(f"Intercepted open call for: '{file}' with mode: '{mode}'")
+    return _original_open(file, mode, buffering, encoding, errors, newline, closefd, opener)
+
+
+def install_open_hook():
+    builtins.open = open_from_fsspec
+    globals()["open"] = open_from_fsspec

fsspec_python/tests/conftest.py

@@ -0,0 +1,22 @@
+import os
+from pathlib import Path
+
+import pytest
+
+from fsspec_python import install_importer, uninstall_importer
+
+
+@pytest.fixture()
+def s3_importer():
+    if not os.environ.get("FSSPEC_S3_ENDPOINT_URL"):
+        pytest.skip("S3 not configured")
+    install_importer("s3://timkpaine-public/projects/fsspec-python")
+    yield
+    uninstall_importer()
+
+
+@pytest.fixture()
+def local_importer():
+    install_importer(f"file://{Path(__file__).parent}/local")
+    yield
+    uninstall_importer()

fsspec_python/tests/local/my_local_file.py

@@ -0,0 +1,2 @@
+def bar():
+    return "This is a local file."

fsspec_python/tests/test_remote_import.py

@@ -0,0 +1,16 @@
+import os
+
+import pytest
+
+
+class TestRemoteImport:
+    @pytest.mark.skipif(not os.environ.get("FSSPEC_S3_ENDPOINT_URL"), reason="S3 not configured")
+    def test_importer_s3(self, s3_importer):
+        import my_remote_file
+
+        assert my_remote_file.foo() == "This is a remote file."
+
+    def test_importer_local(self, local_importer):
+        import my_local_file
+
+        assert my_local_file.bar() == "This is a local file."

pyproject.toml

@@ -45,6 +45,8 @@ develop = [
     "twine",
     "uv",
     "wheel",
+    # Tests
+    "s3fs",
 ]
 
 [project.scripts]