From 431f7dcbb969934fc152a281fe9ee56ea5e495c9 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Mon, 1 Mar 2021 14:17:51 -0500 Subject: [PATCH 01/12] first try... --- opencv/README.md | 55 +++++++ opencv/detect.py | 175 +++++++++++++++++++++++ opencv/install_requirements.sh | 67 +++++++++ opencv/requirements_for_sort_tracker.txt | 2 + opencv/tracker.py | 42 ++++++ 5 files changed, 341 insertions(+) create mode 100644 opencv/README.md create mode 100644 opencv/detect.py create mode 100755 opencv/install_requirements.sh create mode 100644 opencv/requirements_for_sort_tracker.txt create mode 100755 opencv/tracker.py diff --git a/opencv/README.md b/opencv/README.md new file mode 100644 index 0000000..6a899e0 --- /dev/null +++ b/opencv/README.md @@ -0,0 +1,55 @@ +# OpenCV camera examples with Coral + +This folder contains example code using [OpenCV](https://github.com/opencv/opencv) to obtain +camera images and perform object detection on the Edge TPU. + +This code works on Linux/macOS/Windows using a webcam, Raspberry Pi with the Pi Camera, and on the Coral Dev +Board using the Coral Camera or a webcam. For all settings other than the Coral Dev Board, you also need a Coral +USB/PCIe/M.2 Accelerator. + + +## Set up your device + +1. First, be sure you have completed the [setup instructions for your Coral + device](https://coral.ai/docs/setup/). If it's been a while, repeat to be sure + you have the latest software. + + Importantly, you should have the latest TensorFlow Lite runtime installed + (as per the [Python quickstart]( + https://www.tensorflow.org/lite/guide/python)). You can check which version is installed + using the ```pip3 show tflite_runtime``` command. + +2. Clone this Git repo onto your computer or Dev Board: + + ``` + mkdir google-coral && cd google-coral + + git clone https://github.com/google-coral/examples-camera --depth 1 + ``` + +3. Download the models: + + ``` + cd examples-camera + + sh download_models.sh + ``` + +4. Install the OpenCV libraries: + + ``` + cd opencv + + bash install_requirements.sh + ``` + + +## Run the detection demo (SSD models) + +``` +python3 detect.py +``` + +By default, this uses the ```mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite``` model. + +You can change the model and the labels file using flags ```--model``` and ```--labels```. diff --git a/opencv/detect.py b/opencv/detect.py new file mode 100644 index 0000000..f4546c3 --- /dev/null +++ b/opencv/detect.py @@ -0,0 +1,175 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A demo that runs object detection on camera frames using OpenCV. + +TEST_DATA=../all_models + +Run face detection model: +python3 detect.py \ + --model ${TEST_DATA}/mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite + +Run coco model: +python3 detect.py \ + --model ${TEST_DATA}/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite \ + --labels ${TEST_DATA}/coco_labels.txt + +""" +import argparse +import cv2 +import os + +from pycoral.adapters.common import input_size +from pycoral.adapters.detect import get_objects +from pycoral.utils.dataset import read_label_file +from pycoral.utils.edgetpu import make_interpreter +from pycoral.utils.edgetpu import run_inference +from tracker import ObjectTracker + +mot_tracker = None + + +def detectCoralDevBoard(): + try: + if 'MX8MQ' in open('/sys/firmware/devicetree/base/model').read(): + print('Detected Edge TPU dev board.') + return True + except: pass + return False + + +def main(): + global mot_tracker + default_model_dir = '../all_models' + default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite' + default_labels = 'coco_labels.txt' + parser = argparse.ArgumentParser() + parser.add_argument('--model', help='.tflite model path', + default=os.path.join(default_model_dir,default_model)) + parser.add_argument('--labels', help='label file path', + default=os.path.join(default_model_dir, default_labels)) + parser.add_argument('--top_k', type=int, default=3, + help='number of categories with highest score to display') + parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default = 0) + parser.add_argument('--threshold', type=float, default=0.1, + help='classifier score threshold') + parser.add_argument('--tracker', help='Name of the Object Tracker To be used.', + default=None, + choices=[None, 'sort']) + parser.add_argument('--videosrc', help='Directly connected (dev) or Networked (net) video source. ', choices=['dev','net','file'], + default='dev') + parser.add_argument('--displayBool', help='Is a display attached', + default='False', + choices=['True', 'False']) + parser.add_argument('--netsrc', help="Networked video source, example format: rtsp://192.168.1.43/mpeg4/media.amp",) + parser.add_argument('--filesrc', help="Video file source. The videos subdirectory gets mapped into the Docker container, so place your files there.",) + + args = parser.parse_args() + + trackerName=args.tracker + ''' Check for the object tracker.''' + if trackerName != None: + if trackerName == 'mediapipe': + if detectCoralDevBoard(): + objectOfTracker = ObjectTracker('mediapipe') + else: + print("Tracker MediaPipe is only available on the Dev Board. Keeping the tracker as None") + trackerName = None + else: + objectOfTracker = ObjectTracker(trackerName) + else: + pass + + if objectOfTracker: + mot_tracker = objectOfTracker.trackerObject.mot_tracker + else: + mot_tracker = None + print('Loading {} with {} labels.'.format(args.model, args.labels)) + interpreter = make_interpreter(args.model) + interpreter.allocate_tensors() + labels = read_label_file(args.labels) + inference_size = input_size(interpreter) + + if args.videosrc=='dev': + cap = cv2.VideoCapture(args.camera_idx) + elif args.videosrc=='file': + cap = cv2.VideoCapture(args.filesrc) + else: + if args.netsrc==None: + print("--videosrc was set to net but --netsrc was not specified") + sys.exit() + cap = cv2.VideoCapture(args.netsrc) + + while cap.isOpened(): + ret, frame = cap.read() + if not ret: + if args.videosrc=='file': + cap = cv2.VideoCapture(args.filesrc) + continue + else: + break + cv2_im = frame + + cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB) + cv2_im_rgb = cv2.resize(cv2_im_rgb, inference_size) + run_inference(interpreter, cv2_im_rgb.tobytes()) + objs = get_objects(interpreter, args.threshold)[:args.top_k] + + detections = [] # np.array([]) + for n in range(0, len(objs)): + element = [] # np.array([]) + element.append(objs[n].bbox.xmin) + element.append(objs[n].bbox.ymin) + element.append(objs[n].bbox.xmax) + element.append(objs[n].bbox.ymax) + element.append(objs[n].score) # print('element= ',element) + detections.append(element) # print('dets: ',dets) + # convert to numpy array # print('npdets: ',dets) + detections = np.array(detections) + trdata = [] + trackerFlag = False + if detections.any(): + if mot_tracker != None: + trdata = mot_tracker.update(detections) + trackerFlag = True + + cv2_im = append_objs_to_img(cv2_im, inference_size, objs, labels) + + if args.displayBool == 'True': + cv2.imshow('frame', cv2_im) + + if cv2.waitKey(1) & 0xFF == ord('q'): + break + + cap.release() + cv2.destroyAllWindows() + +def append_objs_to_img(cv2_im, inference_size, objs, labels): + height, width, channels = cv2_im.shape + scale_x, scale_y = width / inference_size[0], height / inference_size[1] + for obj in objs: + bbox = obj.bbox.scale(scale_x, scale_y) + x0, y0 = int(bbox.xmin), int(bbox.ymin) + x1, y1 = int(bbox.xmax), int(bbox.ymax) + + percent = int(100 * obj.score) + label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) + + cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) + cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), + cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) + return cv2_im + +if __name__ == '__main__': + main() diff --git a/opencv/install_requirements.sh b/opencv/install_requirements.sh new file mode 100755 index 0000000..23ab3ce --- /dev/null +++ b/opencv/install_requirements.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if grep -s -q "Mendel" /etc/os-release; then + MENDEL_VER="$(cat /etc/mendel_version)" + if [[ "$MENDEL_VER" == "1.0" || "$MENDEL_VER" == "2.0" || "$MENDEL_VER" == "3.0" ]]; then + echo "Your version of Mendel is not compatible with OpenCV." + echo "You must upgrade to Mendel 4.0 or higher." + exit 1 + fi + sudo apt install python3-opencv +elif grep -s -q "Raspberry Pi" /sys/firmware/devicetree/base/model; then + RASPBIAN=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID="\([0-9]\+\)"/\1/') + echo "Raspbian Version: $RASPBIAN" + if [[ "$RASPBIAN" -ge "10" ]]; then + # Lock to version due to bug: https://github.com/piwheels/packages/issues/59 + sudo pip3 install opencv-contrib-python==4.1.0.25 + sudo apt-get -y install libjasper1 libhdf5-1* libqtgui4 libatlas-base-dev libqt4-test + else + echo "For Raspbian versions older than Buster (10) you have to build OpenCV yourself" + echo "or install the unofficial opencv-contrib-python package." + exit 1 + fi +else + sudo apt install python3-opencv +fi + +# Verify models are downloaded +if [ ! -d "../models" ] +then + cd .. + echo "Downloading models." + bash download_models.sh + cd - +fi + +# Install Tracker Dependencies +echo +echo "Installing tracker dependencies." +echo +echo "Note that the trackers have their own licensing, many of which +are not Apache. Care should be taken if using a tracker with restrictive +licenses for end applications." + +read -p "Install SORT (GPLv3)? " -n 1 -r +if [[ $REPLY =~ ^[Yy]$ ]] +then + wget https://github.com/abewley/sort/archive/master.zip -O sort.zip + unzip sort.zip -d ../third_party + rm sort.zip + sudo apt install python3-skimage + python3 -m pip install -r requirements_for_sort_tracker.txt +fi +echo diff --git a/opencv/requirements_for_sort_tracker.txt b/opencv/requirements_for_sort_tracker.txt new file mode 100644 index 0000000..5b3bd5a --- /dev/null +++ b/opencv/requirements_for_sort_tracker.txt @@ -0,0 +1,2 @@ +filterpy==1.1.0 +lap==0.4.0 diff --git a/opencv/tracker.py b/opencv/tracker.py new file mode 100755 index 0000000..a68e448 --- /dev/null +++ b/opencv/tracker.py @@ -0,0 +1,42 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module provides the support for Tracker Object. +This creates object for the specific tracker based on the name of the tracker provided +in the command line of the demo. + +To add more trackers here, simply replicate the SortTracker() code and replace it with +the new tracker as required. + +Developer simply needs to instantiate the object of ObjectTracker(trackerObjectName) with a valid +trackerObjectName. + +""" +import os,sys + +class ObjectTracker(object): + def __init__(self, trackerObjectName): + if trackerObjectName == 'sort': # Add more trackers in elif whenever needed + self.trackerObject = SortTracker() + else: + print("Invalid Tracker Name") + self.trackerObject = None + + +class SortTracker(ObjectTracker): + def __init__(self): + sys.path.append(os.path.join(os.path.dirname(__file__), '../third_party', 'sort-master')) + from sort import Sort + self.mot_tracker = Sort() From b27a93bbc1fb59762a25f7e38e2f8292719887f9 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 09:24:12 -0500 Subject: [PATCH 02/12] overlaying tracking --- opencv/README.md | 3 +++ opencv/detect.py | 59 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/opencv/README.md b/opencv/README.md index 6a899e0..a3cd06d 100644 --- a/opencv/README.md +++ b/opencv/README.md @@ -19,6 +19,9 @@ USB/PCIe/M.2 Accelerator. https://www.tensorflow.org/lite/guide/python)). You can check which version is installed using the ```pip3 show tflite_runtime``` command. +1.5 Install PyCoral: https://coral.ai/software/#pycoral-api + + 2. Clone this Git repo onto your computer or Dev Board: ``` diff --git a/opencv/detect.py b/opencv/detect.py index f4546c3..796b433 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -14,7 +14,7 @@ """A demo that runs object detection on camera frames using OpenCV. -TEST_DATA=../all_models +TEST_DATA=../models Run face detection model: python3 detect.py \ @@ -27,6 +27,7 @@ """ import argparse +import numpy as np import cv2 import os @@ -51,7 +52,7 @@ def detectCoralDevBoard(): def main(): global mot_tracker - default_model_dir = '../all_models' + default_model_dir = '../models' default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite' default_labels = 'coco_labels.txt' parser = argparse.ArgumentParser() @@ -144,7 +145,7 @@ def main(): trdata = mot_tracker.update(detections) trackerFlag = True - cv2_im = append_objs_to_img(cv2_im, inference_size, objs, labels) + cv2_im = append_objs_to_img(cv2_im, inference_size, objs, labels, trdata, trackerFlag) if args.displayBool == 'True': cv2.imshow('frame', cv2_im) @@ -155,20 +156,52 @@ def main(): cap.release() cv2.destroyAllWindows() -def append_objs_to_img(cv2_im, inference_size, objs, labels): +def append_objs_to_img(cv2_im, inference_size, objs, labels, trdata, trackerFlag): height, width, channels = cv2_im.shape + src_w, src_h = height, width + inf_w, inf_h = inference_size scale_x, scale_y = width / inference_size[0], height / inference_size[1] - for obj in objs: - bbox = obj.bbox.scale(scale_x, scale_y) - x0, y0 = int(bbox.xmin), int(bbox.ymin) - x1, y1 = int(bbox.xmax), int(bbox.ymax) + if trackerFlag and (np.array(trdata)).size: + for td in trdata: + x0, y0, x1, y1, trackID = td[0].item(), td[1].item(), td[2].item(), td[3].item(), td[4].item() + overlap = 0 + for ob in objs: + dx0, dy0, dx1, dy1 = ob.bbox.xmin.item(), ob.bbox.ymin.item(), ob.bbox.xmax.item(), ob.bbox.ymax.item() + area = (min(dx1, x1)-max(dx0, x0))*(min(dy1, y1)-max(dy0, y0)) + if (area > overlap): + overlap = area + obj = ob + + # Relative coordinates. + x, y, w, h = x0, y0, x1 - x0, y1 - y0 + # Absolute coordinates, input tensor space. + x, y, w, h = int(x * inf_w), int(y * inf_h), int(w * inf_w), int(h * inf_h) + # Subtract boxing offset. + x, y = x - box_x, y - box_y + # Scale to source coordinate space. + x, y, w, h = x * scale_x, y * scale_y, w * scale_x, h * scale_y + percent = int(100 * obj.score) + label = '{}% {} ID:{}'.format( + percent, labels.get(obj.id, obj.id), int(trackID)) + shadow_text(dwg, x, y - 5, label) + dwg.add(dwg.rect(insert=(x, y), size=(w, h), + fill='none', stroke='red', stroke_width='2')) + cv2_im = cv2.rectangle(cv2_im, (x, y), (x+w, y+h), (0, 255, 0), 2) + cv2_im = cv2.putText(cv2_im, label, (x, y+30), + cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) - percent = int(100 * obj.score) - label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) + else: + for obj in objs: + bbox = obj.bbox.scale(scale_x, scale_y) + x0, y0 = int(bbox.xmin), int(bbox.ymin) + x1, y1 = int(bbox.xmax), int(bbox.ymax) + + percent = int(100 * obj.score) + label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) - cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) - cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), - cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) + cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) + cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), + cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) return cv2_im if __name__ == '__main__': From 4467ce917e63825cece6abcd1522edb51cb5e1ab Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 10:14:58 -0500 Subject: [PATCH 03/12] Update detect.py --- opencv/detect.py | 52 +++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/opencv/detect.py b/opencv/detect.py index 796b433..d1ae1c7 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -92,7 +92,7 @@ def main(): else: pass - if objectOfTracker: + if trackerName != None and objectOfTracker: mot_tracker = objectOfTracker.trackerObject.mot_tracker else: mot_tracker = None @@ -126,15 +126,18 @@ def main(): cv2_im_rgb = cv2.resize(cv2_im_rgb, inference_size) run_inference(interpreter, cv2_im_rgb.tobytes()) objs = get_objects(interpreter, args.threshold)[:args.top_k] - + height, width, channels = cv2_im.shape + scale_x, scale_y = width / inference_size[0], height / inference_size[1] detections = [] # np.array([]) - for n in range(0, len(objs)): + for obj in objs: + bbox = obj.bbox.scale(scale_x, scale_y) + obj.bbox = bbox element = [] # np.array([]) - element.append(objs[n].bbox.xmin) - element.append(objs[n].bbox.ymin) - element.append(objs[n].bbox.xmax) - element.append(objs[n].bbox.ymax) - element.append(objs[n].score) # print('element= ',element) + element.append(bbox.xmin) + element.append(bbox.ymin) + element.append(bbox.xmax) + element.append(bbox.ymax) + element.append(obj.score) # print('element= ',element) detections.append(element) # print('dets: ',dets) # convert to numpy array # print('npdets: ',dets) detections = np.array(detections) @@ -145,7 +148,7 @@ def main(): trdata = mot_tracker.update(detections) trackerFlag = True - cv2_im = append_objs_to_img(cv2_im, inference_size, objs, labels, trdata, trackerFlag) + cv2_im = append_objs_to_img(cv2_im, objs, labels, trdata, trackerFlag) if args.displayBool == 'True': cv2.imshow('frame', cv2_im) @@ -156,45 +159,30 @@ def main(): cap.release() cv2.destroyAllWindows() -def append_objs_to_img(cv2_im, inference_size, objs, labels, trdata, trackerFlag): - height, width, channels = cv2_im.shape - src_w, src_h = height, width - inf_w, inf_h = inference_size - scale_x, scale_y = width / inference_size[0], height / inference_size[1] +def append_objs_to_img(cv2_im, objs, labels, trdata, trackerFlag): + if trackerFlag and (np.array(trdata)).size: for td in trdata: - x0, y0, x1, y1, trackID = td[0].item(), td[1].item(), td[2].item(), td[3].item(), td[4].item() + x0, y0, x1, y1, trackID = int(td[0].item()), int(td[1].item()), int(td[2].item()), int(td[3].item()), td[4].item() overlap = 0 for ob in objs: - dx0, dy0, dx1, dy1 = ob.bbox.xmin.item(), ob.bbox.ymin.item(), ob.bbox.xmax.item(), ob.bbox.ymax.item() + dx0, dy0, dx1, dy1 = int(ob.bbox.xmin), int(ob.bbox.ymin), int(ob.bbox.xmax), int(ob.bbox.ymax) area = (min(dx1, x1)-max(dx0, x0))*(min(dy1, y1)-max(dy0, y0)) if (area > overlap): overlap = area obj = ob - # Relative coordinates. - x, y, w, h = x0, y0, x1 - x0, y1 - y0 - # Absolute coordinates, input tensor space. - x, y, w, h = int(x * inf_w), int(y * inf_h), int(w * inf_w), int(h * inf_h) - # Subtract boxing offset. - x, y = x - box_x, y - box_y - # Scale to source coordinate space. - x, y, w, h = x * scale_x, y * scale_y, w * scale_x, h * scale_y percent = int(100 * obj.score) label = '{}% {} ID:{}'.format( percent, labels.get(obj.id, obj.id), int(trackID)) - shadow_text(dwg, x, y - 5, label) - dwg.add(dwg.rect(insert=(x, y), size=(w, h), - fill='none', stroke='red', stroke_width='2')) - cv2_im = cv2.rectangle(cv2_im, (x, y), (x+w, y+h), (0, 255, 0), 2) - cv2_im = cv2.putText(cv2_im, label, (x, y+30), + cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) + cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) else: for obj in objs: - bbox = obj.bbox.scale(scale_x, scale_y) - x0, y0 = int(bbox.xmin), int(bbox.ymin) - x1, y1 = int(bbox.xmax), int(bbox.ymax) + x0, y0 = int(obj.bbox.xmin), int(obj.bbox.ymin) + x1, y1 = int(obj.bbox.xmax), int(obj.bbox.ymax) percent = int(100 * obj.score) label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) From 184dd17473748bd2e44e1454b03ea105a230bd50 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 10:23:49 -0500 Subject: [PATCH 04/12] Update detect.py --- opencv/detect.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/opencv/detect.py b/opencv/detect.py index d1ae1c7..50857c9 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -70,7 +70,7 @@ def main(): choices=[None, 'sort']) parser.add_argument('--videosrc', help='Directly connected (dev) or Networked (net) video source. ', choices=['dev','net','file'], default='dev') - parser.add_argument('--displayBool', help='Is a display attached', + parser.add_argument('--display', help='Is a display attached', default='False', choices=['True', 'False']) parser.add_argument('--netsrc', help="Networked video source, example format: rtsp://192.168.1.43/mpeg4/media.amp",) @@ -131,13 +131,13 @@ def main(): detections = [] # np.array([]) for obj in objs: bbox = obj.bbox.scale(scale_x, scale_y) - obj.bbox = bbox element = [] # np.array([]) element.append(bbox.xmin) element.append(bbox.ymin) element.append(bbox.xmax) element.append(bbox.ymax) element.append(obj.score) # print('element= ',element) + element.append(obj.id) detections.append(element) # print('dets: ',dets) # convert to numpy array # print('npdets: ',dets) detections = np.array(detections) @@ -148,7 +148,7 @@ def main(): trdata = mot_tracker.update(detections) trackerFlag = True - cv2_im = append_objs_to_img(cv2_im, objs, labels, trdata, trackerFlag) + cv2_im = append_objs_to_img(cv2_im, detections, labels, trdata, trackerFlag) if args.displayBool == 'True': cv2.imshow('frame', cv2_im) @@ -166,26 +166,29 @@ def append_objs_to_img(cv2_im, objs, labels, trdata, trackerFlag): x0, y0, x1, y1, trackID = int(td[0].item()), int(td[1].item()), int(td[2].item()), int(td[3].item()), td[4].item() overlap = 0 for ob in objs: - dx0, dy0, dx1, dy1 = int(ob.bbox.xmin), int(ob.bbox.ymin), int(ob.bbox.xmax), int(ob.bbox.ymax) + dx0, dy0, dx1, dy1 = int(ob[0].item()), int(ob[1].item()), int(ob[2].item()), int(ob[3].item()) area = (min(dx1, x1)-max(dx0, x0))*(min(dy1, y1)-max(dy0, y0)) if (area > overlap): overlap = area obj = ob - percent = int(100 * obj.score) + obj_score = obj[4].item() + obj_id = int(obj[5].item) + percent = int(100 * obj_score) label = '{}% {} ID:{}'.format( - percent, labels.get(obj.id, obj.id), int(trackID)) + percent, labels.get(obj_id, obj_id), int(trackID)) cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) else: for obj in objs: - x0, y0 = int(obj.bbox.xmin), int(obj.bbox.ymin) - x1, y1 = int(obj.bbox.xmax), int(obj.bbox.ymax) + x0, y0, x1, y1 = int(ob[0].item()), int(ob[1].item()), int(ob[2].item()), int(ob[3].item()) + obj_score = obj[4].item() + obj_id = int(obj[5].item) - percent = int(100 * obj.score) - label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) + percent = int(100 * obj_score) + label = '{}% {}'.format(percent, labels.get(obj_id, obj_id)) cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), From 2242b61b146f7a574ba315f3f9484565a68257d2 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 10:25:01 -0500 Subject: [PATCH 05/12] Update detect.py --- opencv/detect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencv/detect.py b/opencv/detect.py index 50857c9..f03ef9a 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -173,7 +173,7 @@ def append_objs_to_img(cv2_im, objs, labels, trdata, trackerFlag): obj = ob obj_score = obj[4].item() - obj_id = int(obj[5].item) + obj_id = int(obj[5].item()) percent = int(100 * obj_score) label = '{}% {} ID:{}'.format( percent, labels.get(obj_id, obj_id), int(trackID)) @@ -185,7 +185,7 @@ def append_objs_to_img(cv2_im, objs, labels, trdata, trackerFlag): for obj in objs: x0, y0, x1, y1 = int(ob[0].item()), int(ob[1].item()), int(ob[2].item()), int(ob[3].item()) obj_score = obj[4].item() - obj_id = int(obj[5].item) + obj_id = int(obj[5].item()) percent = int(100 * obj_score) label = '{}% {}'.format(percent, labels.get(obj_id, obj_id)) From 2a4f991d31de63b9049c0cbf45eef9bc62a8802d Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 10:25:30 -0500 Subject: [PATCH 06/12] Update detect.py --- opencv/detect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencv/detect.py b/opencv/detect.py index f03ef9a..f40c0f1 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -150,7 +150,7 @@ def main(): cv2_im = append_objs_to_img(cv2_im, detections, labels, trdata, trackerFlag) - if args.displayBool == 'True': + if args.display == 'True': cv2.imshow('frame', cv2_im) if cv2.waitKey(1) & 0xFF == ord('q'): From 00f55402e972ca4c8092d723cd26327c91d861f1 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 10:27:29 -0500 Subject: [PATCH 07/12] Update detect.py --- opencv/detect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencv/detect.py b/opencv/detect.py index f40c0f1..f4b6925 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -183,7 +183,7 @@ def append_objs_to_img(cv2_im, objs, labels, trdata, trackerFlag): else: for obj in objs: - x0, y0, x1, y1 = int(ob[0].item()), int(ob[1].item()), int(ob[2].item()), int(ob[3].item()) + x0, y0, x1, y1 = int(obj[0].item()), int(obj[1].item()), int(obj[2].item()), int(obj[3].item()) obj_score = obj[4].item() obj_id = int(obj[5].item()) From b7cac8e8612640771abe541d32b5ae4df692df89 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 11:57:00 -0500 Subject: [PATCH 08/12] Int8 Model support --- opencv/detect.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/opencv/detect.py b/opencv/detect.py index f4b6925..dba97ae 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -75,6 +75,7 @@ def main(): choices=['True', 'False']) parser.add_argument('--netsrc', help="Networked video source, example format: rtsp://192.168.1.43/mpeg4/media.amp",) parser.add_argument('--filesrc', help="Video file source. The videos subdirectory gets mapped into the Docker container, so place your files there.",) + parser.add_argument('--modelInt8', help="Model expects input tensors to be Int8, not UInt8", default='False', choices=['True', 'False']) args = parser.parse_args() @@ -101,6 +102,10 @@ def main(): interpreter.allocate_tensors() labels = read_label_file(args.labels) inference_size = input_size(interpreter) + if args.modelInt8=='True': + model_int8 = True + else: + model_int8 = False if args.videosrc=='dev': cap = cv2.VideoCapture(args.camera_idx) @@ -124,7 +129,16 @@ def main(): cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB) cv2_im_rgb = cv2.resize(cv2_im_rgb, inference_size) - run_inference(interpreter, cv2_im_rgb.tobytes()) + + if model_int8: + im_pil = Image.fromarray(cv2_im_rgb) + input_type = common.input_details(interpreter, 'dtype') + img = (input_type(cv2_im_rgb)- 127.5) / 128.0 + + run_inference(interpreter, img.flatten()) + else: + run_inference(interpreter, cv2_im_rgb.tobytes()) + objs = get_objects(interpreter, args.threshold)[:args.top_k] height, width, channels = cv2_im.shape scale_x, scale_y = width / inference_size[0], height / inference_size[1] From a66024a19ae328fa9cec44ab16ad7ff8b16c61b4 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 12:02:13 -0500 Subject: [PATCH 09/12] Update detect.py --- opencv/detect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencv/detect.py b/opencv/detect.py index dba97ae..c7d1e5d 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -30,6 +30,7 @@ import numpy as np import cv2 import os +from PIL import Image from pycoral.adapters.common import input_size from pycoral.adapters.detect import get_objects @@ -138,7 +139,7 @@ def main(): run_inference(interpreter, img.flatten()) else: run_inference(interpreter, cv2_im_rgb.tobytes()) - + objs = get_objects(interpreter, args.threshold)[:args.top_k] height, width, channels = cv2_im.shape scale_x, scale_y = width / inference_size[0], height / inference_size[1] From 999131e6ea8889637de6be758107d5a125a056bb Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 12:03:15 -0500 Subject: [PATCH 10/12] Update detect.py --- opencv/detect.py | 1 + 1 file changed, 1 insertion(+) diff --git a/opencv/detect.py b/opencv/detect.py index c7d1e5d..86655ac 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -32,6 +32,7 @@ import os from PIL import Image +from pycoral.adapters import common from pycoral.adapters.common import input_size from pycoral.adapters.detect import get_objects from pycoral.utils.dataset import read_label_file From 1d60841ab24d029b3e5ad0d90d8ae4f43ea4c444 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 14:10:31 -0500 Subject: [PATCH 11/12] Update detect.py --- opencv/detect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencv/detect.py b/opencv/detect.py index 86655ac..eec991e 100644 --- a/opencv/detect.py +++ b/opencv/detect.py @@ -118,7 +118,8 @@ def main(): print("--videosrc was set to net but --netsrc was not specified") sys.exit() cap = cv2.VideoCapture(args.netsrc) - + + cap.set(cv2.CAP_PROP_BUFFERSIZE, 0) while cap.isOpened(): ret, frame = cap.read() if not ret: From a75b29c508aa00da30cfe4e9c67598a65ad3b779 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 2 Mar 2021 16:43:26 -0500 Subject: [PATCH 12/12] Update README.md --- opencv/README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/opencv/README.md b/opencv/README.md index a3cd06d..0ef508b 100644 --- a/opencv/README.md +++ b/opencv/README.md @@ -47,12 +47,36 @@ USB/PCIe/M.2 Accelerator. ``` -## Run the detection demo (SSD models) +## Run the detection model with Sort tracker +``` +python3 detect.py --tracker sort +``` + +## Run the detection demo without any tracker (SSD models) ``` python3 detect.py ``` +## Arguments + +*All of the arguments are optional and provide increasing control over the configuration* + + - **model** path to the model you want to use, defaults to COCO + - **labels** labels for the model you are using, default to COCO labels + - **top_k** number of categories with highest score to display, defaults to 3 + - **threshold** classifier score threshold + - **videosrc** what video source you want to use. Choices are `net` or `dev`. Default is `dev`: + - **dev** a directly connected (dev) camera, can be Coral cam or USB cam or Networked + - **net** network video source, using RTSP. The --netsrc argument must be specified. + - **file** a video file can be used as a source + - **camera_idx** Index of which video source to use. I am not sure how OpenCV enumerates them. Defaults to 0. + - **filesrc** the path to the video file. In the Docker container should be at /app/videos + - **netsrc** If the `videosrc` is `net` then specify the URL. Example: `rtsp://192.168.1.43/mpeg4/media.amp` + - **tracker** Name of the Object Tracker To be used. Choices are `None` or `sort`. + - +You can change the model and the labels file using ```--model``` and ```--labels```. + By default, this uses the ```mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite``` model. You can change the model and the labels file using flags ```--model``` and ```--labels```.