From 184495fcde7971539a1fb78f52247240bfb79c8e Mon Sep 17 00:00:00 2001 From: George Grigorev Date: Wed, 23 Aug 2023 11:48:31 +0100 Subject: [PATCH 1/4] update zoedepth to be compatible with torch 2.1 --- zoedepth/models/base_models/midas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zoedepth/models/base_models/midas.py b/zoedepth/models/base_models/midas.py index e26f85895..36f6c3794 100644 --- a/zoedepth/models/base_models/midas.py +++ b/zoedepth/models/base_models/midas.py @@ -170,7 +170,7 @@ def get_size(self, width, height): def __call__(self, x): width, height = self.get_size(*x.shape[-2:][::-1]) - return nn.functional.interpolate(x, (height, width), mode='bilinear', align_corners=True) + return nn.functional.interpolate(x, (int(height), int(width)), mode='bilinear', align_corners=True) class PrepForMidas(object): def __init__(self, resize_mode="minimal", keep_aspect_ratio=True, img_size=384, do_resize=True): From fda3a97d744629a8e71120c1306415860244fd49 Mon Sep 17 00:00:00 2001 From: utku-helvaci Date: Thu, 9 Nov 2023 20:59:47 +0200 Subject: [PATCH 2/4] Hacky way to run ZoeDepth with modern timm https://github.com/isl-org/ZoeDepth/issues/82 I implemented the fixes told in this issue --- README.md | 19 ++++++++++++++++--- ui/app.py | 14 ++++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 446dae32b..a471f92cc 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,26 @@ import torch repo = "isl-org/ZoeDepth" # Zoe_N -model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=True) +model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=False) +pretrained_dict = torch.hub.load_state_dict_from_url('https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_N.pt', map_location=DEVICE) +model_zoe_n.load_state_dict(pretrained_dict['model'], strict=False) +for b in model_zoe_n.core.core.pretrained.model.blocks: + b.drop_path = torch.nn.Identity() # Zoe_K -model_zoe_k = torch.hub.load(repo, "ZoeD_K", pretrained=True) +model_zoe_k = torch.hub.load(repo, "ZoeD_K", pretrained=False) +pretrained_dict = torch.hub.load_state_dict_from_url('https://github.com/isl-org/ZoeDepth/releases/#download/v1.0/ZoeD_M12_K.pt', map_location=DEVICE) +model_zoe_k.load_state_dict(pretrained_dict['model'], strict=False) +for b in model_zoe_k.core.core.pretrained.model.blocks: + b.drop_path = torch.nn.Identity() # Zoe_NK -model_zoe_nk = torch.hub.load(repo, "ZoeD_NK", pretrained=True) +model_zoe_nk = torch.hub.load(repo, "ZoeD_NK", pretrained=False) +pretrained_dict = torch.hub.load_state_dict_from_url('https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_NK.pt', map_location=DEVICE) +model_zoe_nk.load_state_dict(pretrained_dict['model'], strict=False) +for b in model_zoe_nk.core.core.pretrained.model.blocks: + b.drop_path = torch.nn.Identity() + ``` ### Using local copy Clone this repo: diff --git a/ui/app.py b/ui/app.py index e32721854..4d4e3e829 100644 --- a/ui/app.py +++ b/ui/app.py @@ -43,7 +43,17 @@ """ DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' -model = torch.hub.load('isl-org/ZoeDepth', "ZoeD_N", pretrained=True).to(DEVICE).eval() + +# https://github.com/isl-org/ZoeDepth/issues/82#issuecomment-1779799540 +repo = "isl-org/ZoeDepth" + +model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=False) +pretrained_dict = torch.hub.load_state_dict_from_url('https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_N.pt', map_location=DEVICE) +model_zoe_n.load_state_dict(pretrained_dict['model'], strict=False) +for b in model_zoe_n.core.core.pretrained.model.blocks: + b.drop_path = torch.nn.Identity() + +model = model_zoe_n title = "# ZoeDepth" description = """Official demo for **ZoeDepth: Zero-shot Transfer by Combining Relative and Metric Depth**. @@ -63,4 +73,4 @@ create_pano_to_3d_demo(model) if __name__ == '__main__': - demo.queue().launch() \ No newline at end of file + demo.queue().launch() From c8dd97e52b6ed4f0fec13e769f5fbd56215de6ee Mon Sep 17 00:00:00 2001 From: utku-helvaci Date: Thu, 9 Nov 2023 21:06:05 +0200 Subject: [PATCH 3/4] normally correct device wasn't being used when launched as gui, zoedepth would only use cpu --- README.md | 12 ++++++++++-- ui/app.py | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a471f92cc..5fa1cb173 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ torch.hub.help("intel-isl/MiDaS", "DPT_BEiT_L_384", force_reload=True) # Trigge ```python import torch +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + repo = "isl-org/ZoeDepth" # Zoe_N model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=False) @@ -50,6 +52,8 @@ model_zoe_n.load_state_dict(pretrained_dict['model'], strict=False) for b in model_zoe_n.core.core.pretrained.model.blocks: b.drop_path = torch.nn.Identity() +zoe = model_zoe_n.to(DEVICE) + # Zoe_K model_zoe_k = torch.hub.load(repo, "ZoeD_K", pretrained=False) pretrained_dict = torch.hub.load_state_dict_from_url('https://github.com/isl-org/ZoeDepth/releases/#download/v1.0/ZoeD_M12_K.pt', map_location=DEVICE) @@ -57,6 +61,8 @@ model_zoe_k.load_state_dict(pretrained_dict['model'], strict=False) for b in model_zoe_k.core.core.pretrained.model.blocks: b.drop_path = torch.nn.Identity() +zoe = model_zoe_k.to(DEVICE) + # Zoe_NK model_zoe_nk = torch.hub.load(repo, "ZoeD_NK", pretrained=False) pretrained_dict = torch.hub.load_state_dict_from_url('https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_NK.pt', map_location=DEVICE) @@ -64,6 +70,8 @@ model_zoe_nk.load_state_dict(pretrained_dict['model'], strict=False) for b in model_zoe_nk.core.core.pretrained.model.blocks: b.drop_path = torch.nn.Identity() +zoe = model_zoe_nk.to(DEVICE) + ``` ### Using local copy Clone this repo: @@ -100,8 +108,8 @@ model_zoe_nk = build_model(conf) ### Using ZoeD models to predict depth ```python ##### sample prediction -DEVICE = "cuda" if torch.cuda.is_available() else "cpu" -zoe = model_zoe_n.to(DEVICE) + + # Local file diff --git a/ui/app.py b/ui/app.py index 4d4e3e829..45951b72d 100644 --- a/ui/app.py +++ b/ui/app.py @@ -54,6 +54,7 @@ b.drop_path = torch.nn.Identity() model = model_zoe_n +zoe = model_zoe_n.to(DEVICE) title = "# ZoeDepth" description = """Official demo for **ZoeDepth: Zero-shot Transfer by Combining Relative and Metric Depth**. From 5c94142c2343e46025e9018b7071a9d029393796 Mon Sep 17 00:00:00 2001 From: utku-helvaci Date: Thu, 9 Nov 2023 21:21:47 +0200 Subject: [PATCH 4/4] hacky way to make it work on modern gradio --- ui/gradio_depth_pred.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/gradio_depth_pred.py b/ui/gradio_depth_pred.py index fb875451e..2b2226f49 100644 --- a/ui/gradio_depth_pred.py +++ b/ui/gradio_depth_pred.py @@ -34,7 +34,7 @@ def predict_depth(model, image): def create_demo(model): gr.Markdown("### Depth Prediction demo") with gr.Row(): - input_image = gr.Image(label="Input Image", type='pil', elem_id='img-display-input').style(height="auto") + input_image = gr.Image(label="Input Image", type='pil', elem_id='img-display-input') depth_image = gr.Image(label="Depth Map", elem_id='img-display-output') raw_file = gr.File(label="16-bit raw depth, multiplier:256") submit = gr.Button("Submit") @@ -49,4 +49,4 @@ def on_submit(image): submit.click(on_submit, inputs=[input_image], outputs=[depth_image, raw_file]) # examples = gr.Examples(examples=["examples/person_1.jpeg", "examples/person_2.jpeg", "examples/person-leaves.png", "examples/living-room.jpeg"], - # inputs=[input_image]) \ No newline at end of file + # inputs=[input_image])