Win port vtk

etc/spack/defaults/windows/config.yaml

@@ -3,3 +3,4 @@ config:
   concretizer: clingo
   build_stage::
     - '$spack/.staging'
+  cmake_ext_build: true

etc/spack/defaults/windows/packages.yaml

@@ -19,3 +19,4 @@ packages:
     - msvc
     providers:
       mpi: [msmpi]
+      gl: [wgl]

lib/spack/spack/build_systems/cmake.py

@@ -375,7 +375,12 @@ def build_dirname(self):
     @property
     def build_directory(self):
         """Full-path to the directory to use when building the package."""
-        return os.path.join(self.pkg.stage.path, self.build_dirname)
+        stage_path = (
+            os.path.join(self.pkg.stage.path, self.build_dirname)
+            if not self.pkg.spec.dag_hash(7) in spack.stage.CMakeBuildStage.dispatch
+            else spack.stage.CMakeBuildStage.dispatch[self.pkg.spec.dag_hash(7)]
+        )
+        return stage_path
 
     def cmake_args(self):
         """List of all the arguments that must be passed to cmake, except:

lib/spack/spack/cmd/install.py

@@ -51,6 +51,7 @@ def install_kwargs_from_args(args):
         "fail_fast": args.fail_fast,
         "keep_prefix": args.keep_prefix,
         "keep_stage": args.keep_stage,
+        "cmake_stage": args.cmake_external_stage,
         "restage": not args.dont_restage,
         "install_source": args.install_source,
         "verbose": args.verbose or args.install_verbose,
@@ -114,7 +115,13 @@ def setup_parser(subparser):
         action="store_true",
         help="if a partial install is detected, don't delete prior state",
     )
-
+    subparser.add_argument(
+        "--cmake-external-stage",
+        action="store",
+        dest="cmake_external_stage",
+        help="""Path to root directory where CMake should build relevant
+                projects external to Spack's stage (no impact on non Windows systems)""",
+    )
     cache_group = subparser.add_mutually_exclusive_group()
     cache_group.add_argument(
         "--use-cache",

lib/spack/spack/compilers/msvc.py

@@ -122,7 +122,11 @@ def platform_toolset_ver(self):
     @property
     def cl_version(self):
         """Cl toolset version"""
-        return spack.compiler.get_compiler_version_output(self.cc)
+        return Version(re.search(Msvc.version_regex, spack.compiler.get_compiler_version_output(self.cc, "")).group(1))
+
+    @property
+    def vs_root(self):
+        return os.path.abspath(os.path.join(self.cc, "../../../../../../../.."))
 
     def setup_custom_environment(self, pkg, env):
         """Set environment variables for MSVC using the

lib/spack/spack/installer.py

@@ -45,6 +45,7 @@
 from llnl.util.tty.log import log_output
 
 import spack.binary_distribution as binary_distribution
+import spack.build_systems.cmake
 import spack.compilers
 import spack.error
 import spack.hooks
@@ -588,7 +589,10 @@ def log(pkg):
             # Check that we are trying to copy things that are
             # in the stage tree (not arbitrary files)
             abs_expr = os.path.realpath(glob_expr)
-            if os.path.realpath(pkg.stage.path) not in abs_expr:
+            if os.path.realpath(pkg.stage.path) not in abs_expr and not (
+                isinstance(pkg.builder, spack.build_systems.cmake.CMakeBuilder)
+                and spack.config.get("config:cmake_ext_build")
+            ):
                 errors.write("[OUTSIDE SOURCE PATH]: {0}\n".format(glob_expr))
                 continue
             # Now that we are sure that the path is within the correct
@@ -1880,6 +1884,8 @@ def __init__(self, pkg, install_args):
         # whether to keep the build stage after installation
         self.keep_stage = install_args.get("keep_stage", False)
 
+        self.cmake_build_stage = install_args.get("cmake_stage", "")
+
         # whether to skip the patch phase
         self.skip_patch = install_args.get("skip_patch", False)
 
@@ -1909,6 +1915,9 @@ def run(self):
 
         self.timer.start("stage")
 
+        if self.cmake_build_stage:
+            self.pkg.cmake_stage_dir = self.cmake_build_stage
+
         if not self.fake:
             if not self.skip_patch:
                 self.pkg.do_patch()
@@ -2398,6 +2407,7 @@ def _add_default_args(self):
             ("package_use_cache", True),
             ("keep_prefix", False),
             ("keep_stage", False),
+            ("cmake_stage", ""),
             ("restage", False),
             ("skip_patch", False),
             ("tests", False),

lib/spack/spack/package_base.py

@@ -57,7 +57,13 @@
 from spack.filesystem_view import YamlFilesystemView
 from spack.install_test import TestFailure, TestSuite
 from spack.installer import InstallError, PackageInstaller
-from spack.stage import ResourceStage, Stage, StageComposite, stage_prefix
+from spack.stage import (
+    CMakeBuildStage,
+    ResourceStage,
+    Stage,
+    StageComposite,
+    stage_prefix,
+)
 from spack.util.executable import ProcessError, which
 from spack.util.package_hash import package_hash
 from spack.util.prefix import Prefix
@@ -1038,6 +1044,13 @@ def _make_root_stage(self, fetcher):
         )
         return stage
 
+    def _make_cmake_build_stage(self):
+        root_stage_name = "{0}{1}-{2}-{3}".format(
+            stage_prefix, self.spec.name, self.spec.version, self.spec.dag_hash()
+        )
+        stage = CMakeBuildStage(self.spec.dag_hash(7), root_stage_name, root=self.cmake_stage_dir)
+        return stage
+
     def _make_stage(self):
         # If it's a dev package (not transitively), use a DIY stage object
         dev_path_var = self.spec.variants.get("dev_path", None)
@@ -1059,6 +1072,17 @@ def _make_stage(self):
             # Append the item to the composite
             composite_stage.append(stage)
 
+        # if we're building a CMake package on Windows
+        # and the user set the requsite config option, setup a
+        # custom CMake build stage to relocate the cmake build dir to
+        # add here to take advantage of stage cleanup
+        if (
+            is_windows
+            and self.spec.variants["build_system"].value == "cmake"
+            and spack.config.get("config:cmake_ext_build", False)
+        ):
+            composite_stage.append(self._make_cmake_build_stage())
+
         return composite_stage
 
     @property
@@ -1660,6 +1684,16 @@ def content_hash(self, content=None):
 
         return b32_hash
 
+    @property
+    def cmake_stage_dir(self):
+        if not getattr(self, "_cmake_build_stage", False):
+            return ""
+        return self._cmake_build_stage
+
+    @cmake_stage_dir.setter
+    def cmake_stage_dir(self, val):
+        self._cmake_build_stage = val
+
     @property
     def cmake_prefix_paths(self):
         return [self.prefix]

lib/spack/spack/stage.py

@@ -14,6 +14,8 @@
 import stat
 import sys
 import tempfile
+import time
+from pathlib import Path
 from typing import Dict
 
 import llnl.util.lang
@@ -28,6 +30,7 @@
     partition_path,
     remove_linked_tree,
 )
+from llnl.util.symlink import symlink
 
 import spack.caches
 import spack.config
@@ -725,6 +728,191 @@ def _add_to_root_stage(self):
                     install(src, destination_path)
 
 
+class CMakeBuildStage:
+    """Interface abstracting a CMake build tree at a location outside
+    of a Spack stage directory but still managed by Spack.
+
+    CMake's build tree can be located arbitrarily on a filesystem independent
+    from a source directory. This class relocates that build directory out of the
+    stage and to a path under a Users home directory or to a location of a users chosing set
+    either by config or the command line via the --cmake-build-dir cl argument
+    to the install command.
+    Interactions with the stage expecting an in stage build tree will work as normal
+    as this class serves to obfucscate the external stage and allow for all stage behavior
+    to perform as normal.
+    The external build tree is given the same lifespan as its stage dir, is spun up when the stage
+    spins up, and destroyed when the stage is destroyed. After the build and installation are done,
+    the build tree is first relocated to what would be its proper place in the stage.
+    and a symlink is placed in place of the build tree pointing at the external directory
+
+    Note: This is not, nor should it be, used on *nix platforms and is intended as a solution to
+    reduce file path lengths on Windows during compilation/linking. This class should be removed
+    when MSVC fully supports the LongPath feature on Windows.
+    """
+
+    dispatch: Dict[str, str] = {}
+
+    def __init__(self, hash, name, root=None, keep=False):
+        # Users can override external cmake build dir, default is %USERPROFILE%
+        # overrides can come from command line or config, command line will override all
+        self._hash = hash
+        self._path = Path(get_stage_root(), name)
+        self._remote_stage = None
+        self.keep = keep
+        if not root:
+            fallback_path = Path(os.environ["USERPROFILE"], ".sp-stage")
+            self._root = Path(spack.config.get("config:cmake_ext_build_stage_dir", fallback_path))
+
+    def __enter__(self):
+        self.create()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.destroy()
+
+    def _establish_context_link(self):
+        symlink(str(self._remote_stage), str(self._path / ("spack-build-%s" % self._hash)))
+
+    def _remove_context_link(self):
+        Path(self._path, "spack-build-%s" % self._hash).unlink()
+
+    def _rebuild_remote_stage(self):
+        full_subdir = Path(CMakeBuildStage.dispatch[self._hash])
+        if not full_subdir.exists():
+            try:
+                full_subdir.mkdir()
+            except FileExistsError:
+                full_subdir = self._setup_remote_build_stage()
+        return full_subdir
+
+    def _setup_remote_build_stage(self):
+        # try to create root if it doesn't exist
+        self._root.mkdir(parents=True, exist_ok=True)
+        sub_dir = self._compute_next_open_subdir()
+        full_subdir = self._root / sub_dir
+        try:
+            full_subdir.mkdir()
+        except FileExistsError:
+            # another process must have created the same directory as us
+            # try again
+            return self._setup_remote_build_stage()
+        CMakeBuildStage.dispatch[self._hash] = str(full_subdir)
+        return full_subdir
+
+    def _compute_next_open_subdir(self, last=None):
+        def inc(c):
+            curr = ord(c[-1])
+            over = curr // 122
+            if over:
+                return (inc(c[:-1]) if len(c[:-1]) else "a") + "a"
+            return c[:-1] + chr(curr + 1)
+
+        sort_key = lambda x: (len(x.name), x.name)
+        current_ext_stages = list(Path(self._root).iterdir())
+        if not current_ext_stages:
+            # no currently extant external stages, start enumerating with 'a'
+            return "a"
+        last = sorted(current_ext_stages, key=sort_key)[-1]
+        return inc(last.name)
+
+    def _return_to_stage(self):
+        """Copy external build tree back to stage in normal CMake build dir location"""
+        dest = Path(self._path, "spack-build-%s" % self._hash)
+        install_tree(str(self._remote_stage), str(dest))
+
+    def _teardown_remote_stage(self):
+        """Destroy external build tree if not keep-stage
+        Otherwise this is kept as usual"""
+
+        def teardown(pth: Path):
+            for sub_item in pth.iterdir():
+                if sub_item.is_dir():
+                    teardown(sub_item)
+                else:
+                    sub_item.unlink()
+            pth.rmdir()
+
+        if self._remote_stage:
+            teardown(self._remote_stage)
+
+    def _reclaim_remote_stage(self):
+        # another Spack process or build has taken this directory
+        # the cmake build will not work from a different dir
+        # so wait until we can take it - try five times
+        # waiting a little longer each time.
+        # This will cause a hang but this should only be called if we're trying
+        # to rebuild a pre-existing stage, so we need to get the previous
+        # build dir or CMake will error
+        ii = 0
+        while self._remote_stage.exists() and ii < 5:
+            time.sleep(0.5)
+            ii += 1
+        if ii == 5:
+            raise StageError(
+                "Could not re-create external CMake stage, one exists for this package already"
+            )
+        self._remote_stage.mkdir()
+
+    def destroy(self):
+        # copy back to stage may fail in event of error, make sure we clean up the
+        # associated external build dir in that event unless we're keeping the
+        # parent stage on cleanup
+        # If remote stage is not set, we never created one, package is already
+        # installed and we should do nothing here
+        if self._remote_stage:
+            try:
+                self._remove_context_link()
+                self._return_to_stage()
+            finally:
+                self._teardown_remote_stage()
+
+    def restage(self):
+        if self._hash in CMakeBuildStage.dispatch:
+            self._remote_stage = Path(CMakeBuildStage.dispatch[self._hash])
+            if self._remote_stage.exists():
+                self._reclaim_remote_stage()
+        else:
+            self.create()
+
+    def create(self):
+        if not self.created:
+            try:
+                self._remote_stage = self._setup_remote_build_stage()
+                self._establish_context_link()
+            except Exception:
+                self._teardown_remote_stage()
+                raise
+
+    def steal_source(self, dest):
+        if not self._remote_stage:
+            self.create()
+
+        self._path = Path(dest)
+
+    @property
+    def created(self):
+        return bool(self._remote_stage) and self._remote_stage.exists()
+
+    @property
+    def managed_by_spack(self):
+        return True
+
+    def fetch(self, mirror_only=False, err_msg=None):
+        pass
+
+    def cache_local(self):
+        pass
+
+    def cache_mirror(self):
+        pass
+
+    def check(self):
+        pass
+
+    def expand_archive(self):
+        pass
+
+
 class StageComposite(pattern.Composite):
     """Composite for Stage type objects. The first item in this composite is
     considered to be the root package, and operations that return a value are