Module narya.linker.multitracker

Expand source code
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

"""

Cloned from https://github.com/Zhongdao/Towards-Realtime-MOT,
with some modifications to :
    * the tracking of the ball
    * the dual tracking - embedding models 

"""

import mxnet as mx
from collections import deque

import numpy as np
import torch
import torch.nn.functional as F
import tensorflow as tf

from .matching import embedding_distance, fuse_motion, linear_assignment, iou_distance
from .kalman_filter import KalmanFilter
from .basetrack import BaseTrack, TrackState
from ..utils.utils import to_torch, to_numpy
from ..utils.linker import ctdet_post_process
from ..models.gluon_models import TrackerModel
from ..models.torch_models import ReIdModel

GLUON_PATH = (
    "https://storage.googleapis.com/narya-bucket-1/models/player_tracker.params"
)
GLUON_NAME = "player_tracker.params"
GLUON_TOTAR = False

TORCH_PATH = "https://storage.googleapis.com/narya-bucket-1/models/player_reid.pth"
TORCH_NAME = "player_reid.pth"
TORCH_TOTAR = False


class STrack(BaseTrack):
    shared_kalman = KalmanFilter()

    def __init__(self, tlwh, score, temp_feat, buffer_size=30):

        # wait activate
        self._tlwh = np.asarray(tlwh, dtype=np.float)
        self.kalman_filter = None
        self.mean, self.covariance = None, None
        self.is_activated = False

        self.score = score
        self.tracklet_len = 0

        self.smooth_feat = None
        self.update_features(temp_feat)
        self.features = deque([], maxlen=buffer_size)
        self.alpha = 0.9

    def update_features(self, feat):
        feat /= np.linalg.norm(feat)
        self.curr_feat = feat
        if self.smooth_feat is None:
            self.smooth_feat = feat
        else:
            self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat
        self.features.append(feat)
        self.smooth_feat /= np.linalg.norm(self.smooth_feat)

    def predict(self):
        mean_state = self.mean.copy()
        if self.state != TrackState.Tracked:
            mean_state[7] = 0
        self.mean, self.covariance = self.kalman_filter.predict(
            mean_state, self.covariance
        )

    @staticmethod
    def multi_predict(stracks):
        if len(stracks) > 0:
            multi_mean = np.asarray([st.mean.copy() for st in stracks])
            multi_covariance = np.asarray([st.covariance for st in stracks])
            for i, st in enumerate(stracks):
                if st.state != TrackState.Tracked:
                    multi_mean[i][7] = 0
            multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(
                multi_mean, multi_covariance
            )
            for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
                stracks[i].mean = mean
                stracks[i].covariance = cov

    def activate(self, kalman_filter, frame_id):
        """Start a new tracklet"""
        self.kalman_filter = kalman_filter
        self.track_id = self.next_id()
        self.mean, self.covariance = self.kalman_filter.initiate(
            self.tlwh_to_xyah(self._tlwh)
        )

        self.tracklet_len = 0
        self.state = TrackState.Tracked
        if frame_id == 1:
            self.is_activated = True
        # self.is_activated = True
        self.frame_id = frame_id
        self.start_frame = frame_id

    def re_activate(self, new_track, frame_id, new_id=False):
        self.mean, self.covariance = self.kalman_filter.update(
            self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)
        )

        self.update_features(new_track.curr_feat)
        self.tracklet_len = 0
        self.state = TrackState.Tracked
        self.is_activated = True
        self.frame_id = frame_id
        if new_id:
            self.track_id = self.next_id()

    def update(self, new_track, frame_id, update_feature=True):
        """
        Update a matched track
        :type new_track: STrack
        :type frame_id: int
        :type update_feature: bool
        :return:
        """
        self.frame_id = frame_id
        self.tracklet_len += 1

        new_tlwh = new_track.tlwh
        self.mean, self.covariance = self.kalman_filter.update(
            self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)
        )
        self.state = TrackState.Tracked
        self.is_activated = True

        self.score = new_track.score
        if update_feature:
            self.update_features(new_track.curr_feat)

    @property
    # @jit(nopython=True)
    def tlwh(self):
        """Get current position in bounding box format `(top left x, top left y,
                width, height)`.
        """
        if self.mean is None:
            return self._tlwh.copy()
        ret = self.mean[:4].copy()
        ret[2] *= ret[3]
        ret[:2] -= ret[2:] / 2
        return ret

    @property
    # @jit(nopython=True)
    def tlbr(self):
        """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
        `(top left, bottom right)`.
        """
        ret = self.tlwh.copy()
        ret[2:] += ret[:2]
        return ret

    @staticmethod
    # @jit(nopython=True)
    def tlwh_to_xyah(tlwh):
        """Convert bounding box to format `(center x, center y, aspect ratio,
        height)`, where the aspect ratio is `width / height`.
        """
        ret = np.asarray(tlwh).copy()
        ret[:2] += ret[2:] / 2
        ret[2] /= ret[3]
        return ret

    def to_xyah(self):
        return self.tlwh_to_xyah(self.tlwh)

    @staticmethod
    # @jit(nopython=True)
    def tlbr_to_tlwh(tlbr):
        ret = np.asarray(tlbr).copy()
        ret[2:] -= ret[:2]
        return ret

    @staticmethod
    # @jit(nopython=True)
    def tlwh_to_tlbr(tlwh):
        ret = np.asarray(tlwh).copy()
        ret[2:] += ret[:2]
        return ret

    def __repr__(self):
        return "OT_{}_({}-{})".format(self.track_id, self.start_frame, self.end_frame)


class JDETracker(object):
    def __init__(
        self, conf_thres=0.5, track_buffer=30, K=100, frame_rate=30,
    ):

        print("Creating model...")

        self.tracker_model = TrackerModel(
            pretrained=True, backbone="ssd_512_resnet50_v1_coco"
        )
        self.reid_model = ReIdModel()

        gluon_tracker_params = tf.keras.utils.get_file(
            GLUON_NAME, GLUON_PATH, GLUON_TOTAR,
        )

        torch_reid_pth = tf.keras.utils.get_file(TORCH_NAME, TORCH_PATH, TORCH_TOTAR,)

        self.tracker_model.load_weights(gluon_tracker_params)
        self.reid_model.load_weights(torch_reid_pth)

        self.tracked_stracks = []  # type: list[STrack]
        self.lost_stracks = []  # type: list[STrack]
        self.removed_stracks = []  # type: list[STrack]
        self.last_ball_bbox = None

        self.frame_id = 0
        self.det_thresh = conf_thres
        self.buffer_size = int((frame_rate / 30.0) * track_buffer)
        self.max_time_lost = self.buffer_size
        self.max_per_image = K
        self.kalman_filter = KalmanFilter()

    def update(self, im_blob, img0, split_size=None, verbose=True):
        self.frame_id += 1
        activated_starcks = []
        refind_stracks = []
        lost_stracks = []
        removed_stracks = []

        """ Step 1: Network forward, get detections & embeddings"""

        cid, score, bbox = self.tracker_model(img0, split_size=split_size)
        if split_size is None:
            bbox = bbox.asnumpy()
            cid = cid.asnumpy()
            score = score.asnumpy()
        id_feature = self.reid_model._get_embeddings(
            img0, bbox, score, cid, self.det_thresh, (512, 512)
        )
        cid = to_torch(cid)
        score = to_torch(score)
        bbox = to_torch(bbox)
        id_feature = to_torch(id_feature)
        detections = torch.cat([bbox, score, cid], dim=2)
        detections = detections[0]
        ball_detections = torch.cat([bbox, score, cid], dim=2)[0]
        id_feature = id_feature[0]
        for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
            remain_inds = detections[:, indx] > tresh
            detections = detections[remain_inds]
            id_feature = id_feature[remain_inds]
        dets = detections[:, :5]

        cnt = 0
        for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
            remain_inds = (
                ball_detections[:, indx] > tresh
                if cnt == 0
                else ball_detections[:, indx] < tresh
            )
            ball_detections = ball_detections[remain_inds]
            cnt += 1
        dets_ball = ball_detections[:, :5]
        if len(dets_ball) > 0:
            best_candidate_ball = np.argmax(dets_ball[:, 4], axis=0)
            best_candidate_ball_bbox = STrack.tlbr_to_tlwh(
                dets_ball[best_candidate_ball, :4]
            )
            self.last_ball_bbox = best_candidate_ball_bbox
        else:
            best_candidate_ball_bbox = None

        # vis
        """
        for i in range(0, dets.shape[0]):
            bbox = dets[i][0:4]
            cv2.rectangle(img0, (bbox[0], bbox[1]),
                          (bbox[2], bbox[3]),
                          (0, 255, 0), 2)
        cv2.imshow('dets', img0)
        cv2.waitKey(0)
        id0 = id0-1
        """

        if len(dets) > 0:
            """Detections"""
            detections = [
                STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30)
                for (tlbrs, f) in zip(dets[:, :5], id_feature)
            ]
        else:
            detections = []

        """ Add newly detected tracklets to tracked_stracks"""
        unconfirmed = []
        tracked_stracks = []  # type: list[STrack]
        for track in self.tracked_stracks:
            if not track.is_activated:
                unconfirmed.append(track)
            else:
                tracked_stracks.append(track)

        """ Step 2: First association, with embedding"""
        strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)
        # Predict the current location with KF
        # for strack in strack_pool:
        # strack.predict()
        STrack.multi_predict(strack_pool)
        dists = embedding_distance(strack_pool, detections)
        dists = fuse_motion(self.kalman_filter, dists, strack_pool, detections)
        matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)

        for itracked, idet in matches:
            track = strack_pool[itracked]
            det = detections[idet]
            if track.state == TrackState.Tracked:
                track.update(detections[idet], self.frame_id)
                activated_starcks.append(track)
            else:
                track.re_activate(det, self.frame_id, new_id=False)
                refind_stracks.append(track)

        """ Step 3: Second association, with IOU"""
        detections = [detections[i] for i in u_detection]
        r_tracked_stracks = [
            strack_pool[i]
            for i in u_track
            if strack_pool[i].state == TrackState.Tracked
        ]
        dists = iou_distance(r_tracked_stracks, detections)
        matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)

        for itracked, idet in matches:
            track = r_tracked_stracks[itracked]
            det = detections[idet]
            if track.state == TrackState.Tracked:
                track.update(det, self.frame_id)
                activated_starcks.append(track)
            else:
                track.re_activate(det, self.frame_id, new_id=False)
                refind_stracks.append(track)

        for it in u_track:
            track = r_tracked_stracks[it]
            if not track.state == TrackState.Lost:
                track.mark_lost()
                lost_stracks.append(track)

        """Deal with unconfirmed tracks, usually tracks with only one beginning frame"""
        detections = [detections[i] for i in u_detection]
        dists = iou_distance(unconfirmed, detections)
        matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.9)
        for itracked, idet in matches:
            unconfirmed[itracked].update(detections[idet], self.frame_id)
            activated_starcks.append(unconfirmed[itracked])

        for it in u_unconfirmed:
            track = unconfirmed[it]
            track.mark_removed()
            removed_stracks.append(track)

        """ Step 4: Init new stracks"""
        for inew in u_detection:
            track = detections[inew]
            if track.score < self.det_thresh:
                continue
            track.activate(self.kalman_filter, self.frame_id)
            activated_starcks.append(track)
        """ Step 5: Update state"""

        for track in self.lost_stracks:
            if self.frame_id - track.end_frame > self.max_time_lost:
                track.mark_removed()
                removed_stracks.append(track)

        self.tracked_stracks = [
            t for t in self.tracked_stracks if t.state == TrackState.Tracked
        ]
        self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)
        self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)
        self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)
        self.lost_stracks.extend(lost_stracks)
        self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)
        self.removed_stracks.extend(removed_stracks)
        self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(
            self.tracked_stracks, self.lost_stracks
        )
        # get scores of lost tracks
        output_stracks = [track for track in self.tracked_stracks if track.is_activated]

        if verbose:
            print("===========Frame {}==========".format(self.frame_id))
            print(
                "Activated: {}".format([track.track_id for track in activated_starcks])
            )
            print("Refind: {}".format([track.track_id for track in refind_stracks]))
            print("Lost: {}".format([track.track_id for track in lost_stracks]))
            print("Removed: {}".format([track.track_id for track in removed_stracks]))

        return output_stracks, best_candidate_ball_bbox


def joint_stracks(tlista, tlistb):
    exists = {}
    res = []
    for t in tlista:
        exists[t.track_id] = 1
        res.append(t)
    for t in tlistb:
        tid = t.track_id
        if not exists.get(tid, 0):
            exists[tid] = 1
            res.append(t)
    return res


def sub_stracks(tlista, tlistb):
    stracks = {}
    for t in tlista:
        stracks[t.track_id] = t
    for t in tlistb:
        tid = t.track_id
        if stracks.get(tid, 0):
            del stracks[tid]
    return list(stracks.values())


def remove_duplicate_stracks(stracksa, stracksb):
    pdist = iou_distance(stracksa, stracksb)
    pairs = np.where(pdist < 0.15)
    dupa, dupb = list(), list()
    for p, q in zip(*pairs):
        timep = stracksa[p].frame_id - stracksa[p].start_frame
        timeq = stracksb[q].frame_id - stracksb[q].start_frame
        if timep > timeq:
            dupb.append(q)
        else:
            dupa.append(p)
    resa = [t for i, t in enumerate(stracksa) if not i in dupa]
    resb = [t for i, t in enumerate(stracksb) if not i in dupb]
    return resa, resb

Functions

def joint_stracks(tlista, tlistb)
Expand source code
def joint_stracks(tlista, tlistb):
    exists = {}
    res = []
    for t in tlista:
        exists[t.track_id] = 1
        res.append(t)
    for t in tlistb:
        tid = t.track_id
        if not exists.get(tid, 0):
            exists[tid] = 1
            res.append(t)
    return res
def remove_duplicate_stracks(stracksa, stracksb)
Expand source code
def remove_duplicate_stracks(stracksa, stracksb):
    pdist = iou_distance(stracksa, stracksb)
    pairs = np.where(pdist < 0.15)
    dupa, dupb = list(), list()
    for p, q in zip(*pairs):
        timep = stracksa[p].frame_id - stracksa[p].start_frame
        timeq = stracksb[q].frame_id - stracksb[q].start_frame
        if timep > timeq:
            dupb.append(q)
        else:
            dupa.append(p)
    resa = [t for i, t in enumerate(stracksa) if not i in dupa]
    resb = [t for i, t in enumerate(stracksb) if not i in dupb]
    return resa, resb
def sub_stracks(tlista, tlistb)
Expand source code
def sub_stracks(tlista, tlistb):
    stracks = {}
    for t in tlista:
        stracks[t.track_id] = t
    for t in tlistb:
        tid = t.track_id
        if stracks.get(tid, 0):
            del stracks[tid]
    return list(stracks.values())

Classes

class JDETracker (conf_thres=0.5, track_buffer=30, K=100, frame_rate=30)
Expand source code
class JDETracker(object):
    def __init__(
        self, conf_thres=0.5, track_buffer=30, K=100, frame_rate=30,
    ):

        print("Creating model...")

        self.tracker_model = TrackerModel(
            pretrained=True, backbone="ssd_512_resnet50_v1_coco"
        )
        self.reid_model = ReIdModel()

        gluon_tracker_params = tf.keras.utils.get_file(
            GLUON_NAME, GLUON_PATH, GLUON_TOTAR,
        )

        torch_reid_pth = tf.keras.utils.get_file(TORCH_NAME, TORCH_PATH, TORCH_TOTAR,)

        self.tracker_model.load_weights(gluon_tracker_params)
        self.reid_model.load_weights(torch_reid_pth)

        self.tracked_stracks = []  # type: list[STrack]
        self.lost_stracks = []  # type: list[STrack]
        self.removed_stracks = []  # type: list[STrack]
        self.last_ball_bbox = None

        self.frame_id = 0
        self.det_thresh = conf_thres
        self.buffer_size = int((frame_rate / 30.0) * track_buffer)
        self.max_time_lost = self.buffer_size
        self.max_per_image = K
        self.kalman_filter = KalmanFilter()

    def update(self, im_blob, img0, split_size=None, verbose=True):
        self.frame_id += 1
        activated_starcks = []
        refind_stracks = []
        lost_stracks = []
        removed_stracks = []

        """ Step 1: Network forward, get detections & embeddings"""

        cid, score, bbox = self.tracker_model(img0, split_size=split_size)
        if split_size is None:
            bbox = bbox.asnumpy()
            cid = cid.asnumpy()
            score = score.asnumpy()
        id_feature = self.reid_model._get_embeddings(
            img0, bbox, score, cid, self.det_thresh, (512, 512)
        )
        cid = to_torch(cid)
        score = to_torch(score)
        bbox = to_torch(bbox)
        id_feature = to_torch(id_feature)
        detections = torch.cat([bbox, score, cid], dim=2)
        detections = detections[0]
        ball_detections = torch.cat([bbox, score, cid], dim=2)[0]
        id_feature = id_feature[0]
        for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
            remain_inds = detections[:, indx] > tresh
            detections = detections[remain_inds]
            id_feature = id_feature[remain_inds]
        dets = detections[:, :5]

        cnt = 0
        for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
            remain_inds = (
                ball_detections[:, indx] > tresh
                if cnt == 0
                else ball_detections[:, indx] < tresh
            )
            ball_detections = ball_detections[remain_inds]
            cnt += 1
        dets_ball = ball_detections[:, :5]
        if len(dets_ball) > 0:
            best_candidate_ball = np.argmax(dets_ball[:, 4], axis=0)
            best_candidate_ball_bbox = STrack.tlbr_to_tlwh(
                dets_ball[best_candidate_ball, :4]
            )
            self.last_ball_bbox = best_candidate_ball_bbox
        else:
            best_candidate_ball_bbox = None

        # vis
        """
        for i in range(0, dets.shape[0]):
            bbox = dets[i][0:4]
            cv2.rectangle(img0, (bbox[0], bbox[1]),
                          (bbox[2], bbox[3]),
                          (0, 255, 0), 2)
        cv2.imshow('dets', img0)
        cv2.waitKey(0)
        id0 = id0-1
        """

        if len(dets) > 0:
            """Detections"""
            detections = [
                STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30)
                for (tlbrs, f) in zip(dets[:, :5], id_feature)
            ]
        else:
            detections = []

        """ Add newly detected tracklets to tracked_stracks"""
        unconfirmed = []
        tracked_stracks = []  # type: list[STrack]
        for track in self.tracked_stracks:
            if not track.is_activated:
                unconfirmed.append(track)
            else:
                tracked_stracks.append(track)

        """ Step 2: First association, with embedding"""
        strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)
        # Predict the current location with KF
        # for strack in strack_pool:
        # strack.predict()
        STrack.multi_predict(strack_pool)
        dists = embedding_distance(strack_pool, detections)
        dists = fuse_motion(self.kalman_filter, dists, strack_pool, detections)
        matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)

        for itracked, idet in matches:
            track = strack_pool[itracked]
            det = detections[idet]
            if track.state == TrackState.Tracked:
                track.update(detections[idet], self.frame_id)
                activated_starcks.append(track)
            else:
                track.re_activate(det, self.frame_id, new_id=False)
                refind_stracks.append(track)

        """ Step 3: Second association, with IOU"""
        detections = [detections[i] for i in u_detection]
        r_tracked_stracks = [
            strack_pool[i]
            for i in u_track
            if strack_pool[i].state == TrackState.Tracked
        ]
        dists = iou_distance(r_tracked_stracks, detections)
        matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)

        for itracked, idet in matches:
            track = r_tracked_stracks[itracked]
            det = detections[idet]
            if track.state == TrackState.Tracked:
                track.update(det, self.frame_id)
                activated_starcks.append(track)
            else:
                track.re_activate(det, self.frame_id, new_id=False)
                refind_stracks.append(track)

        for it in u_track:
            track = r_tracked_stracks[it]
            if not track.state == TrackState.Lost:
                track.mark_lost()
                lost_stracks.append(track)

        """Deal with unconfirmed tracks, usually tracks with only one beginning frame"""
        detections = [detections[i] for i in u_detection]
        dists = iou_distance(unconfirmed, detections)
        matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.9)
        for itracked, idet in matches:
            unconfirmed[itracked].update(detections[idet], self.frame_id)
            activated_starcks.append(unconfirmed[itracked])

        for it in u_unconfirmed:
            track = unconfirmed[it]
            track.mark_removed()
            removed_stracks.append(track)

        """ Step 4: Init new stracks"""
        for inew in u_detection:
            track = detections[inew]
            if track.score < self.det_thresh:
                continue
            track.activate(self.kalman_filter, self.frame_id)
            activated_starcks.append(track)
        """ Step 5: Update state"""

        for track in self.lost_stracks:
            if self.frame_id - track.end_frame > self.max_time_lost:
                track.mark_removed()
                removed_stracks.append(track)

        self.tracked_stracks = [
            t for t in self.tracked_stracks if t.state == TrackState.Tracked
        ]
        self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)
        self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)
        self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)
        self.lost_stracks.extend(lost_stracks)
        self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)
        self.removed_stracks.extend(removed_stracks)
        self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(
            self.tracked_stracks, self.lost_stracks
        )
        # get scores of lost tracks
        output_stracks = [track for track in self.tracked_stracks if track.is_activated]

        if verbose:
            print("===========Frame {}==========".format(self.frame_id))
            print(
                "Activated: {}".format([track.track_id for track in activated_starcks])
            )
            print("Refind: {}".format([track.track_id for track in refind_stracks]))
            print("Lost: {}".format([track.track_id for track in lost_stracks]))
            print("Removed: {}".format([track.track_id for track in removed_stracks]))

        return output_stracks, best_candidate_ball_bbox

Methods

def update(self, im_blob, img0, split_size=None, verbose=True)
Expand source code
def update(self, im_blob, img0, split_size=None, verbose=True):
    self.frame_id += 1
    activated_starcks = []
    refind_stracks = []
    lost_stracks = []
    removed_stracks = []

    """ Step 1: Network forward, get detections & embeddings"""

    cid, score, bbox = self.tracker_model(img0, split_size=split_size)
    if split_size is None:
        bbox = bbox.asnumpy()
        cid = cid.asnumpy()
        score = score.asnumpy()
    id_feature = self.reid_model._get_embeddings(
        img0, bbox, score, cid, self.det_thresh, (512, 512)
    )
    cid = to_torch(cid)
    score = to_torch(score)
    bbox = to_torch(bbox)
    id_feature = to_torch(id_feature)
    detections = torch.cat([bbox, score, cid], dim=2)
    detections = detections[0]
    ball_detections = torch.cat([bbox, score, cid], dim=2)[0]
    id_feature = id_feature[0]
    for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
        remain_inds = detections[:, indx] > tresh
        detections = detections[remain_inds]
        id_feature = id_feature[remain_inds]
    dets = detections[:, :5]

    cnt = 0
    for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
        remain_inds = (
            ball_detections[:, indx] > tresh
            if cnt == 0
            else ball_detections[:, indx] < tresh
        )
        ball_detections = ball_detections[remain_inds]
        cnt += 1
    dets_ball = ball_detections[:, :5]
    if len(dets_ball) > 0:
        best_candidate_ball = np.argmax(dets_ball[:, 4], axis=0)
        best_candidate_ball_bbox = STrack.tlbr_to_tlwh(
            dets_ball[best_candidate_ball, :4]
        )
        self.last_ball_bbox = best_candidate_ball_bbox
    else:
        best_candidate_ball_bbox = None

    # vis
    """
    for i in range(0, dets.shape[0]):
        bbox = dets[i][0:4]
        cv2.rectangle(img0, (bbox[0], bbox[1]),
                      (bbox[2], bbox[3]),
                      (0, 255, 0), 2)
    cv2.imshow('dets', img0)
    cv2.waitKey(0)
    id0 = id0-1
    """

    if len(dets) > 0:
        """Detections"""
        detections = [
            STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30)
            for (tlbrs, f) in zip(dets[:, :5], id_feature)
        ]
    else:
        detections = []

    """ Add newly detected tracklets to tracked_stracks"""
    unconfirmed = []
    tracked_stracks = []  # type: list[STrack]
    for track in self.tracked_stracks:
        if not track.is_activated:
            unconfirmed.append(track)
        else:
            tracked_stracks.append(track)

    """ Step 2: First association, with embedding"""
    strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)
    # Predict the current location with KF
    # for strack in strack_pool:
    # strack.predict()
    STrack.multi_predict(strack_pool)
    dists = embedding_distance(strack_pool, detections)
    dists = fuse_motion(self.kalman_filter, dists, strack_pool, detections)
    matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)

    for itracked, idet in matches:
        track = strack_pool[itracked]
        det = detections[idet]
        if track.state == TrackState.Tracked:
            track.update(detections[idet], self.frame_id)
            activated_starcks.append(track)
        else:
            track.re_activate(det, self.frame_id, new_id=False)
            refind_stracks.append(track)

    """ Step 3: Second association, with IOU"""
    detections = [detections[i] for i in u_detection]
    r_tracked_stracks = [
        strack_pool[i]
        for i in u_track
        if strack_pool[i].state == TrackState.Tracked
    ]
    dists = iou_distance(r_tracked_stracks, detections)
    matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)

    for itracked, idet in matches:
        track = r_tracked_stracks[itracked]
        det = detections[idet]
        if track.state == TrackState.Tracked:
            track.update(det, self.frame_id)
            activated_starcks.append(track)
        else:
            track.re_activate(det, self.frame_id, new_id=False)
            refind_stracks.append(track)

    for it in u_track:
        track = r_tracked_stracks[it]
        if not track.state == TrackState.Lost:
            track.mark_lost()
            lost_stracks.append(track)

    """Deal with unconfirmed tracks, usually tracks with only one beginning frame"""
    detections = [detections[i] for i in u_detection]
    dists = iou_distance(unconfirmed, detections)
    matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.9)
    for itracked, idet in matches:
        unconfirmed[itracked].update(detections[idet], self.frame_id)
        activated_starcks.append(unconfirmed[itracked])

    for it in u_unconfirmed:
        track = unconfirmed[it]
        track.mark_removed()
        removed_stracks.append(track)

    """ Step 4: Init new stracks"""
    for inew in u_detection:
        track = detections[inew]
        if track.score < self.det_thresh:
            continue
        track.activate(self.kalman_filter, self.frame_id)
        activated_starcks.append(track)
    """ Step 5: Update state"""

    for track in self.lost_stracks:
        if self.frame_id - track.end_frame > self.max_time_lost:
            track.mark_removed()
            removed_stracks.append(track)

    self.tracked_stracks = [
        t for t in self.tracked_stracks if t.state == TrackState.Tracked
    ]
    self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)
    self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)
    self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)
    self.lost_stracks.extend(lost_stracks)
    self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)
    self.removed_stracks.extend(removed_stracks)
    self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(
        self.tracked_stracks, self.lost_stracks
    )
    # get scores of lost tracks
    output_stracks = [track for track in self.tracked_stracks if track.is_activated]

    if verbose:
        print("===========Frame {}==========".format(self.frame_id))
        print(
            "Activated: {}".format([track.track_id for track in activated_starcks])
        )
        print("Refind: {}".format([track.track_id for track in refind_stracks]))
        print("Lost: {}".format([track.track_id for track in lost_stracks]))
        print("Removed: {}".format([track.track_id for track in removed_stracks]))

    return output_stracks, best_candidate_ball_bbox
class STrack (tlwh, score, temp_feat, buffer_size=30)
Expand source code
class STrack(BaseTrack):
    shared_kalman = KalmanFilter()

    def __init__(self, tlwh, score, temp_feat, buffer_size=30):

        # wait activate
        self._tlwh = np.asarray(tlwh, dtype=np.float)
        self.kalman_filter = None
        self.mean, self.covariance = None, None
        self.is_activated = False

        self.score = score
        self.tracklet_len = 0

        self.smooth_feat = None
        self.update_features(temp_feat)
        self.features = deque([], maxlen=buffer_size)
        self.alpha = 0.9

    def update_features(self, feat):
        feat /= np.linalg.norm(feat)
        self.curr_feat = feat
        if self.smooth_feat is None:
            self.smooth_feat = feat
        else:
            self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat
        self.features.append(feat)
        self.smooth_feat /= np.linalg.norm(self.smooth_feat)

    def predict(self):
        mean_state = self.mean.copy()
        if self.state != TrackState.Tracked:
            mean_state[7] = 0
        self.mean, self.covariance = self.kalman_filter.predict(
            mean_state, self.covariance
        )

    @staticmethod
    def multi_predict(stracks):
        if len(stracks) > 0:
            multi_mean = np.asarray([st.mean.copy() for st in stracks])
            multi_covariance = np.asarray([st.covariance for st in stracks])
            for i, st in enumerate(stracks):
                if st.state != TrackState.Tracked:
                    multi_mean[i][7] = 0
            multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(
                multi_mean, multi_covariance
            )
            for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
                stracks[i].mean = mean
                stracks[i].covariance = cov

    def activate(self, kalman_filter, frame_id):
        """Start a new tracklet"""
        self.kalman_filter = kalman_filter
        self.track_id = self.next_id()
        self.mean, self.covariance = self.kalman_filter.initiate(
            self.tlwh_to_xyah(self._tlwh)
        )

        self.tracklet_len = 0
        self.state = TrackState.Tracked
        if frame_id == 1:
            self.is_activated = True
        # self.is_activated = True
        self.frame_id = frame_id
        self.start_frame = frame_id

    def re_activate(self, new_track, frame_id, new_id=False):
        self.mean, self.covariance = self.kalman_filter.update(
            self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)
        )

        self.update_features(new_track.curr_feat)
        self.tracklet_len = 0
        self.state = TrackState.Tracked
        self.is_activated = True
        self.frame_id = frame_id
        if new_id:
            self.track_id = self.next_id()

    def update(self, new_track, frame_id, update_feature=True):
        """
        Update a matched track
        :type new_track: STrack
        :type frame_id: int
        :type update_feature: bool
        :return:
        """
        self.frame_id = frame_id
        self.tracklet_len += 1

        new_tlwh = new_track.tlwh
        self.mean, self.covariance = self.kalman_filter.update(
            self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)
        )
        self.state = TrackState.Tracked
        self.is_activated = True

        self.score = new_track.score
        if update_feature:
            self.update_features(new_track.curr_feat)

    @property
    # @jit(nopython=True)
    def tlwh(self):
        """Get current position in bounding box format `(top left x, top left y,
                width, height)`.
        """
        if self.mean is None:
            return self._tlwh.copy()
        ret = self.mean[:4].copy()
        ret[2] *= ret[3]
        ret[:2] -= ret[2:] / 2
        return ret

    @property
    # @jit(nopython=True)
    def tlbr(self):
        """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
        `(top left, bottom right)`.
        """
        ret = self.tlwh.copy()
        ret[2:] += ret[:2]
        return ret

    @staticmethod
    # @jit(nopython=True)
    def tlwh_to_xyah(tlwh):
        """Convert bounding box to format `(center x, center y, aspect ratio,
        height)`, where the aspect ratio is `width / height`.
        """
        ret = np.asarray(tlwh).copy()
        ret[:2] += ret[2:] / 2
        ret[2] /= ret[3]
        return ret

    def to_xyah(self):
        return self.tlwh_to_xyah(self.tlwh)

    @staticmethod
    # @jit(nopython=True)
    def tlbr_to_tlwh(tlbr):
        ret = np.asarray(tlbr).copy()
        ret[2:] -= ret[:2]
        return ret

    @staticmethod
    # @jit(nopython=True)
    def tlwh_to_tlbr(tlwh):
        ret = np.asarray(tlwh).copy()
        ret[2:] += ret[:2]
        return ret

    def __repr__(self):
        return "OT_{}_({}-{})".format(self.track_id, self.start_frame, self.end_frame)

Ancestors

Class variables

var shared_kalman

Static methods

def multi_predict(stracks)
Expand source code
@staticmethod
def multi_predict(stracks):
    if len(stracks) > 0:
        multi_mean = np.asarray([st.mean.copy() for st in stracks])
        multi_covariance = np.asarray([st.covariance for st in stracks])
        for i, st in enumerate(stracks):
            if st.state != TrackState.Tracked:
                multi_mean[i][7] = 0
        multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(
            multi_mean, multi_covariance
        )
        for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
            stracks[i].mean = mean
            stracks[i].covariance = cov
def tlbr_to_tlwh(tlbr)
Expand source code
@staticmethod
# @jit(nopython=True)
def tlbr_to_tlwh(tlbr):
    ret = np.asarray(tlbr).copy()
    ret[2:] -= ret[:2]
    return ret
def tlwh_to_tlbr(tlwh)
Expand source code
@staticmethod
# @jit(nopython=True)
def tlwh_to_tlbr(tlwh):
    ret = np.asarray(tlwh).copy()
    ret[2:] += ret[:2]
    return ret
def tlwh_to_xyah(tlwh)

Convert bounding box to format (center x, center y, aspect ratio, height)<code>, where the aspect ratio is </code>width / height.

Expand source code
@staticmethod
# @jit(nopython=True)
def tlwh_to_xyah(tlwh):
    """Convert bounding box to format `(center x, center y, aspect ratio,
    height)`, where the aspect ratio is `width / height`.
    """
    ret = np.asarray(tlwh).copy()
    ret[:2] += ret[2:] / 2
    ret[2] /= ret[3]
    return ret

Instance variables

var tlbr

Convert bounding box to format (min x, min y, max x, max y), i.e., (top left, bottom right).

Expand source code
@property
# @jit(nopython=True)
def tlbr(self):
    """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
    `(top left, bottom right)`.
    """
    ret = self.tlwh.copy()
    ret[2:] += ret[:2]
    return ret
var tlwh

Get current position in bounding box format (top left x, top left y, width, height).

Expand source code
@property
# @jit(nopython=True)
def tlwh(self):
    """Get current position in bounding box format `(top left x, top left y,
            width, height)`.
    """
    if self.mean is None:
        return self._tlwh.copy()
    ret = self.mean[:4].copy()
    ret[2] *= ret[3]
    ret[:2] -= ret[2:] / 2
    return ret

Methods

def activate(self, kalman_filter, frame_id)

Start a new tracklet

Expand source code
def activate(self, kalman_filter, frame_id):
    """Start a new tracklet"""
    self.kalman_filter = kalman_filter
    self.track_id = self.next_id()
    self.mean, self.covariance = self.kalman_filter.initiate(
        self.tlwh_to_xyah(self._tlwh)
    )

    self.tracklet_len = 0
    self.state = TrackState.Tracked
    if frame_id == 1:
        self.is_activated = True
    # self.is_activated = True
    self.frame_id = frame_id
    self.start_frame = frame_id
def predict(self)
Expand source code
def predict(self):
    mean_state = self.mean.copy()
    if self.state != TrackState.Tracked:
        mean_state[7] = 0
    self.mean, self.covariance = self.kalman_filter.predict(
        mean_state, self.covariance
    )
def re_activate(self, new_track, frame_id, new_id=False)
Expand source code
def re_activate(self, new_track, frame_id, new_id=False):
    self.mean, self.covariance = self.kalman_filter.update(
        self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)
    )

    self.update_features(new_track.curr_feat)
    self.tracklet_len = 0
    self.state = TrackState.Tracked
    self.is_activated = True
    self.frame_id = frame_id
    if new_id:
        self.track_id = self.next_id()
def to_xyah(self)
Expand source code
def to_xyah(self):
    return self.tlwh_to_xyah(self.tlwh)
def update(self, new_track, frame_id, update_feature=True)

Update a matched track :type new_track: STrack :type frame_id: int :type update_feature: bool :return:

Expand source code
def update(self, new_track, frame_id, update_feature=True):
    """
    Update a matched track
    :type new_track: STrack
    :type frame_id: int
    :type update_feature: bool
    :return:
    """
    self.frame_id = frame_id
    self.tracklet_len += 1

    new_tlwh = new_track.tlwh
    self.mean, self.covariance = self.kalman_filter.update(
        self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)
    )
    self.state = TrackState.Tracked
    self.is_activated = True

    self.score = new_track.score
    if update_feature:
        self.update_features(new_track.curr_feat)
def update_features(self, feat)
Expand source code
def update_features(self, feat):
    feat /= np.linalg.norm(feat)
    self.curr_feat = feat
    if self.smooth_feat is None:
        self.smooth_feat = feat
    else:
        self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat
    self.features.append(feat)
    self.smooth_feat /= np.linalg.norm(self.smooth_feat)