From fff36034ba5dc121146b5a98941704c66f7d8c42 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Mon, 29 Jul 2024 16:42:15 -0700 Subject: [PATCH 01/10] add extract_features_xfeat --- opensfm/features.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/opensfm/features.py b/opensfm/features.py index 26eb0c05a..5057ae70a 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -7,6 +7,7 @@ import cv2 import numpy as np +import torch from opensfm import context, pyfeatures @@ -548,6 +549,19 @@ def extract_features_orb( return points, desc +def extract_features_xfeat( + image: np.ndarray, config: Dict[str, Any], features_count: int, xfeat: Any +) -> Tuple[np.ndarray, np,.ndarray]: + logger.debug("Computing XFeats") + t = time.time() + + output = xfeat.detectAndCompute(im, top_k = 4096) + points = output['keypoints'].numpy() + desc = output['descriptors'].numpy() + + logger.debug("Found {0} points in {1}s".format(len(keypoints), time.time() - t)) + return points, desc + def extract_features( image: np.ndarray, config: Dict[str, Any], is_panorama: bool ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: @@ -606,6 +620,13 @@ def extract_features( points, desc = extract_features_hahog(image_gray, config, features_count) elif feature_type == "ORB": points, desc = extract_features_orb(image_gray, config, features_count) + elif feature_type == 'XFEAT': + xfeat = torch.hub.load( + 'verlab/accelerated_features', + 'XFeat', + pretrained = True, + top_k = 4096) + points, desc = extract_features_xfeat(image, config, features_count, xfeat) else: raise ValueError("Unknown feature type " + "(must be SURF, SIFT, AKAZE, HAHOG or ORB)") From 82b123a59a831bb1a252d8add33b56952f5c451c Mon Sep 17 00:00:00 2001 From: evanmoss Date: Mon, 29 Jul 2024 16:48:52 -0700 Subject: [PATCH 02/10] fix missing index --- opensfm/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/features.py b/opensfm/features.py index 5057ae70a..3e38a7ee0 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -555,7 +555,7 @@ def extract_features_xfeat( logger.debug("Computing XFeats") t = time.time() - output = xfeat.detectAndCompute(im, top_k = 4096) + output = xfeat.detectAndCompute(im, top_k = 4096)[0] points = output['keypoints'].numpy() desc = output['descriptors'].numpy() From 1ceb21a4c4b90c6757e02e9c9166de8fd7fce945 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Mon, 29 Jul 2024 17:15:03 -0700 Subject: [PATCH 03/10] add xfeat lighterglue support --- opensfm/matching.py | 102 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/opensfm/matching.py b/opensfm/matching.py index a96eee3f0..158d8e5e0 100644 --- a/opensfm/matching.py +++ b/opensfm/matching.py @@ -5,9 +5,11 @@ import cv2 import numpy as np +import torch from opensfm import ( context, feature_loader, + io, log, multiview, pairs_selection, @@ -434,6 +436,23 @@ def _match_descriptors_impl( matches = match_brute_force_symmetric(d1, d2, overriden_config) else: matches = match_brute_force(d1, d2, overriden_config) + elif matcher_type == "LIGHTGLUE": + assert not symmetric_matching + # assume im1.shape == im2.shape + im = io.imread(im1) + xfeat = torch.hub.load( + 'verlab/accelerated_features', + 'XFeat', + pretrained = True, + top_k = 4096) + match_lightglue( + features_data1.points, + d1, + features_data2.points, + d2, + overriden_config, + shape = shape, + xfeat = xfeat) else: raise ValueError("Invalid matcher_type: {}".format(matcher_type)) @@ -748,6 +767,89 @@ def match_brute_force( return _convert_matches_to_vector(good_matches) +def match_lightglue( + p1: np.ndarray, + d1: np.ndarray, + p2: np.ndarray, + d2: np.ndarray, + config: Dict[str, Any], + maskij: Optional[np.ndarray] = None, + xfeat: Any = None, + shape: np.ndarray, +) -> List[Tuple[int, int]]: + """LighterGlue feature matching from https://github.com/verlab/accelerated_features + + Args: + p1: feature keypoints of the first image + d1: feature descriptors of the first image + p2: feature keypoints of the second image + d2: feature descriptors of the second image + config: config parameters + maskij: optional boolean mask of len(i descriptors) x len(j descriptors) + xfeat: XFeat model + shape: shape of original image + """ + assert(xfeat is not None) + + def _kpt_idxs(output, mkpts): + m = {} + + for i, p in enumerate(mkpts): + x = np.floor(float(p[0])) + y = np.floor(float(p[1])) + m.setdefault(x, {}) + m[x][y] = -1 + + c = 0 + + for i, p in enumerate(output['keypoints']): + x = np.floor(float(p[0])) + y = np.floor(float(p[1])) + if x not in m: continue + if y not in m[x]: continue + m[x][y] = i + c += 1 + + assert(c == len(mkpts)) + + idxs = [] + for p in mkpts: + x = np.floor(float(p[0])) + y = np.floor(float(p[1])) + idxs.append(m[x][y]) + + return idxs + + extraction_size = ( + config["feature_process_size_panorama"] + if is_panorama + else config["feature_process_size"] + ) + + h, w = shape[:2] + size = max(w, h) + final_size = (h, w) + if 0 < extraction_size < size: + final_size = h * max_size // size, w * max_size // size + + output0 = { + 'keypoints': p1, + 'descriptors': d1, + 'image_size': final_size, + } + output1 = { + 'keypoints': p2, + 'descriptors': d2, + 'image_size': final_size, + } + mkpts_0, mkpts_1 = xfeat.match_lighterglue(output0, output1) + idxs_0 = _kpt_idxs(output0, mkpts_0) + idxs_1 = _kpt_idxs(output1, mkpts_1) + + # TODO can we justreturn the iterator? + return list(zip(idxs_0, idxs_1)) + + def _convert_matches_to_vector(matches: List[Any]) -> List[Tuple[int, int]]: """Convert Dmatch object to matrix form.""" return [(mm.queryIdx, mm.trainIdx) for mm in matches] From 9d5659fce949c14a4c4fbd12851a8c49431f5bc7 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Mon, 29 Jul 2024 17:16:23 -0700 Subject: [PATCH 04/10] move shape --- opensfm/matching.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opensfm/matching.py b/opensfm/matching.py index 158d8e5e0..e5c9d1192 100644 --- a/opensfm/matching.py +++ b/opensfm/matching.py @@ -450,8 +450,8 @@ def _match_descriptors_impl( d1, features_data2.points, d2, + shape, overriden_config, - shape = shape, xfeat = xfeat) else: raise ValueError("Invalid matcher_type: {}".format(matcher_type)) @@ -772,10 +772,10 @@ def match_lightglue( d1: np.ndarray, p2: np.ndarray, d2: np.ndarray, + shape: np.ndarray, config: Dict[str, Any], maskij: Optional[np.ndarray] = None, xfeat: Any = None, - shape: np.ndarray, ) -> List[Tuple[int, int]]: """LighterGlue feature matching from https://github.com/verlab/accelerated_features @@ -784,10 +784,10 @@ def match_lightglue( d1: feature descriptors of the first image p2: feature keypoints of the second image d2: feature descriptors of the second image + shape: shape of original image config: config parameters maskij: optional boolean mask of len(i descriptors) x len(j descriptors) xfeat: XFeat model - shape: shape of original image """ assert(xfeat is not None) From 0839d75d40edbf94c14be1aa199123747d2b11a7 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Tue, 30 Jul 2024 00:44:17 -0700 Subject: [PATCH 05/10] fix typo --- opensfm/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/features.py b/opensfm/features.py index 3e38a7ee0..b7853219d 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -551,7 +551,7 @@ def extract_features_orb( def extract_features_xfeat( image: np.ndarray, config: Dict[str, Any], features_count: int, xfeat: Any -) -> Tuple[np.ndarray, np,.ndarray]: +) -> Tuple[np.ndarray, np,ndarray]: logger.debug("Computing XFeats") t = time.time() From eaa94b2d9beb4faa44ea0569f9e2bdc545dbb0ba Mon Sep 17 00:00:00 2001 From: evanmoss Date: Tue, 30 Jul 2024 00:47:37 -0700 Subject: [PATCH 06/10] fix typo --- opensfm/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/features.py b/opensfm/features.py index b7853219d..af78dce0c 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -551,7 +551,7 @@ def extract_features_orb( def extract_features_xfeat( image: np.ndarray, config: Dict[str, Any], features_count: int, xfeat: Any -) -> Tuple[np.ndarray, np,ndarray]: +) -> Tuple[np.ndarray, np.ndarray]: logger.debug("Computing XFeats") t = time.time() From cfef83485643bfaa57c2827a436b51cb8599a640 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Tue, 30 Jul 2024 01:03:28 -0700 Subject: [PATCH 07/10] fix typo --- opensfm/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/features.py b/opensfm/features.py index af78dce0c..f81426e0b 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -555,7 +555,7 @@ def extract_features_xfeat( logger.debug("Computing XFeats") t = time.time() - output = xfeat.detectAndCompute(im, top_k = 4096)[0] + output = xfeat.detectAndCompute(image, top_k = 4096)[0] points = output['keypoints'].numpy() desc = output['descriptors'].numpy() From e1c9dd713bb31b1cfbe631f8d775d0135e38f3b6 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Tue, 30 Jul 2024 01:05:37 -0700 Subject: [PATCH 08/10] fix typo --- opensfm/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/features.py b/opensfm/features.py index f81426e0b..c25dd4546 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -559,7 +559,7 @@ def extract_features_xfeat( points = output['keypoints'].numpy() desc = output['descriptors'].numpy() - logger.debug("Found {0} points in {1}s".format(len(keypoints), time.time() - t)) + logger.debug("Found {0} points in {1}s".format(len(points), time.time() - t)) return points, desc def extract_features( From 0a94ef55eb77f4c8c45a57e82407cf1c47c1a7e9 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Tue, 30 Jul 2024 13:38:45 -0700 Subject: [PATCH 09/10] hstack reshaped score --- opensfm/features.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opensfm/features.py b/opensfm/features.py index c25dd4546..410551a3b 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -557,6 +557,8 @@ def extract_features_xfeat( output = xfeat.detectAndCompute(image, top_k = 4096)[0] points = output['keypoints'].numpy() + scores = output['scores'].numpy().reshape((4096, 1)) + points = np.hstack((points, scores)) desc = output['descriptors'].numpy() logger.debug("Found {0} points in {1}s".format(len(points), time.time() - t)) From 0b4e49d041f936fb6bc0c10fb740946744251dc8 Mon Sep 17 00:00:00 2001 From: evanmoss Date: Tue, 30 Jul 2024 15:49:00 -0700 Subject: [PATCH 10/10] fix typo --- opensfm/matching.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensfm/matching.py b/opensfm/matching.py index e5c9d1192..3b23d173a 100644 --- a/opensfm/matching.py +++ b/opensfm/matching.py @@ -439,7 +439,7 @@ def _match_descriptors_impl( elif matcher_type == "LIGHTGLUE": assert not symmetric_matching # assume im1.shape == im2.shape - im = io.imread(im1) + im = data.load_image(im1) xfeat = torch.hub.load( 'verlab/accelerated_features', 'XFeat', @@ -450,7 +450,7 @@ def _match_descriptors_impl( d1, features_data2.points, d2, - shape, + im.shape, overriden_config, xfeat = xfeat) else: