Module narya.linker.multitracker
Expand source code
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
"""
Cloned from https://github.com/Zhongdao/Towards-Realtime-MOT,
with some modifications to :
* the tracking of the ball
* the dual tracking - embedding models
"""
import mxnet as mx
from collections import deque
import numpy as np
import torch
import torch.nn.functional as F
import tensorflow as tf
from .matching import embedding_distance, fuse_motion, linear_assignment, iou_distance
from .kalman_filter import KalmanFilter
from .basetrack import BaseTrack, TrackState
from ..utils.utils import to_torch, to_numpy
from ..utils.linker import ctdet_post_process
from ..models.gluon_models import TrackerModel
from ..models.torch_models import ReIdModel
GLUON_PATH = (
"https://storage.googleapis.com/narya-bucket-1/models/player_tracker.params"
)
GLUON_NAME = "player_tracker.params"
GLUON_TOTAR = False
TORCH_PATH = "https://storage.googleapis.com/narya-bucket-1/models/player_reid.pth"
TORCH_NAME = "player_reid.pth"
TORCH_TOTAR = False
class STrack(BaseTrack):
shared_kalman = KalmanFilter()
def __init__(self, tlwh, score, temp_feat, buffer_size=30):
# wait activate
self._tlwh = np.asarray(tlwh, dtype=np.float)
self.kalman_filter = None
self.mean, self.covariance = None, None
self.is_activated = False
self.score = score
self.tracklet_len = 0
self.smooth_feat = None
self.update_features(temp_feat)
self.features = deque([], maxlen=buffer_size)
self.alpha = 0.9
def update_features(self, feat):
feat /= np.linalg.norm(feat)
self.curr_feat = feat
if self.smooth_feat is None:
self.smooth_feat = feat
else:
self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat
self.features.append(feat)
self.smooth_feat /= np.linalg.norm(self.smooth_feat)
def predict(self):
mean_state = self.mean.copy()
if self.state != TrackState.Tracked:
mean_state[7] = 0
self.mean, self.covariance = self.kalman_filter.predict(
mean_state, self.covariance
)
@staticmethod
def multi_predict(stracks):
if len(stracks) > 0:
multi_mean = np.asarray([st.mean.copy() for st in stracks])
multi_covariance = np.asarray([st.covariance for st in stracks])
for i, st in enumerate(stracks):
if st.state != TrackState.Tracked:
multi_mean[i][7] = 0
multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(
multi_mean, multi_covariance
)
for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
stracks[i].mean = mean
stracks[i].covariance = cov
def activate(self, kalman_filter, frame_id):
"""Start a new tracklet"""
self.kalman_filter = kalman_filter
self.track_id = self.next_id()
self.mean, self.covariance = self.kalman_filter.initiate(
self.tlwh_to_xyah(self._tlwh)
)
self.tracklet_len = 0
self.state = TrackState.Tracked
if frame_id == 1:
self.is_activated = True
# self.is_activated = True
self.frame_id = frame_id
self.start_frame = frame_id
def re_activate(self, new_track, frame_id, new_id=False):
self.mean, self.covariance = self.kalman_filter.update(
self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)
)
self.update_features(new_track.curr_feat)
self.tracklet_len = 0
self.state = TrackState.Tracked
self.is_activated = True
self.frame_id = frame_id
if new_id:
self.track_id = self.next_id()
def update(self, new_track, frame_id, update_feature=True):
"""
Update a matched track
:type new_track: STrack
:type frame_id: int
:type update_feature: bool
:return:
"""
self.frame_id = frame_id
self.tracklet_len += 1
new_tlwh = new_track.tlwh
self.mean, self.covariance = self.kalman_filter.update(
self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)
)
self.state = TrackState.Tracked
self.is_activated = True
self.score = new_track.score
if update_feature:
self.update_features(new_track.curr_feat)
@property
# @jit(nopython=True)
def tlwh(self):
"""Get current position in bounding box format `(top left x, top left y,
width, height)`.
"""
if self.mean is None:
return self._tlwh.copy()
ret = self.mean[:4].copy()
ret[2] *= ret[3]
ret[:2] -= ret[2:] / 2
return ret
@property
# @jit(nopython=True)
def tlbr(self):
"""Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
`(top left, bottom right)`.
"""
ret = self.tlwh.copy()
ret[2:] += ret[:2]
return ret
@staticmethod
# @jit(nopython=True)
def tlwh_to_xyah(tlwh):
"""Convert bounding box to format `(center x, center y, aspect ratio,
height)`, where the aspect ratio is `width / height`.
"""
ret = np.asarray(tlwh).copy()
ret[:2] += ret[2:] / 2
ret[2] /= ret[3]
return ret
def to_xyah(self):
return self.tlwh_to_xyah(self.tlwh)
@staticmethod
# @jit(nopython=True)
def tlbr_to_tlwh(tlbr):
ret = np.asarray(tlbr).copy()
ret[2:] -= ret[:2]
return ret
@staticmethod
# @jit(nopython=True)
def tlwh_to_tlbr(tlwh):
ret = np.asarray(tlwh).copy()
ret[2:] += ret[:2]
return ret
def __repr__(self):
return "OT_{}_({}-{})".format(self.track_id, self.start_frame, self.end_frame)
class JDETracker(object):
def __init__(
self, conf_thres=0.5, track_buffer=30, K=100, frame_rate=30,
):
print("Creating model...")
self.tracker_model = TrackerModel(
pretrained=True, backbone="ssd_512_resnet50_v1_coco"
)
self.reid_model = ReIdModel()
gluon_tracker_params = tf.keras.utils.get_file(
GLUON_NAME, GLUON_PATH, GLUON_TOTAR,
)
torch_reid_pth = tf.keras.utils.get_file(TORCH_NAME, TORCH_PATH, TORCH_TOTAR,)
self.tracker_model.load_weights(gluon_tracker_params)
self.reid_model.load_weights(torch_reid_pth)
self.tracked_stracks = [] # type: list[STrack]
self.lost_stracks = [] # type: list[STrack]
self.removed_stracks = [] # type: list[STrack]
self.last_ball_bbox = None
self.frame_id = 0
self.det_thresh = conf_thres
self.buffer_size = int((frame_rate / 30.0) * track_buffer)
self.max_time_lost = self.buffer_size
self.max_per_image = K
self.kalman_filter = KalmanFilter()
def update(self, im_blob, img0, split_size=None, verbose=True):
self.frame_id += 1
activated_starcks = []
refind_stracks = []
lost_stracks = []
removed_stracks = []
""" Step 1: Network forward, get detections & embeddings"""
cid, score, bbox = self.tracker_model(img0, split_size=split_size)
if split_size is None:
bbox = bbox.asnumpy()
cid = cid.asnumpy()
score = score.asnumpy()
id_feature = self.reid_model._get_embeddings(
img0, bbox, score, cid, self.det_thresh, (512, 512)
)
cid = to_torch(cid)
score = to_torch(score)
bbox = to_torch(bbox)
id_feature = to_torch(id_feature)
detections = torch.cat([bbox, score, cid], dim=2)
detections = detections[0]
ball_detections = torch.cat([bbox, score, cid], dim=2)[0]
id_feature = id_feature[0]
for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
remain_inds = detections[:, indx] > tresh
detections = detections[remain_inds]
id_feature = id_feature[remain_inds]
dets = detections[:, :5]
cnt = 0
for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]):
remain_inds = (
ball_detections[:, indx] > tresh
if cnt == 0
else ball_detections[:, indx] < tresh
)
ball_detections = ball_detections[remain_inds]
cnt += 1
dets_ball = ball_detections[:, :5]
if len(dets_ball) > 0:
best_candidate_ball = np.argmax(dets_ball[:, 4], axis=0)
best_candidate_ball_bbox = STrack.tlbr_to_tlwh(
dets_ball[best_candidate_ball, :4]
)
self.last_ball_bbox = best_candidate_ball_bbox
else:
best_candidate_ball_bbox = None
# vis
"""
for i in range(0, dets.shape[0]):
bbox = dets[i][0:4]
cv2.rectangle(img0, (bbox[0], bbox[1]),
(bbox[2], bbox[3]),
(0, 255, 0), 2)
cv2.imshow('dets', img0)
cv2.waitKey(0)
id0 = id0-1
"""
if len(dets) > 0:
"""Detections"""
detections = [
STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30)
for (tlbrs, f) in zip(dets[:, :5], id_feature)
]
else:
detections = []
""" Add newly detected tracklets to tracked_stracks"""
unconfirmed = []
tracked_stracks = [] # type: list[STrack]
for track in self.tracked_stracks:
if not track.is_activated:
unconfirmed.append(track)
else:
tracked_stracks.append(track)
""" Step 2: First association, with embedding"""
strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)
# Predict the current location with KF
# for strack in strack_pool:
# strack.predict()
STrack.multi_predict(strack_pool)
dists = embedding_distance(strack_pool, detections)
dists = fuse_motion(self.kalman_filter, dists, strack_pool, detections)
matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)
for itracked, idet in matches:
track = strack_pool[itracked]
det = detections[idet]
if track.state == TrackState.Tracked:
track.update(detections[idet], self.frame_id)
activated_starcks.append(track)
else:
track.re_activate(det, self.frame_id, new_id=False)
refind_stracks.append(track)
""" Step 3: Second association, with IOU"""
detections = [detections[i] for i in u_detection]
r_tracked_stracks = [
strack_pool[i]
for i in u_track
if strack_pool[i].state == TrackState.Tracked
]
dists = iou_distance(r_tracked_stracks, detections)
matches, u_track, u_detection = linear_assignment(dists, thresh=0.9)
for itracked, idet in matches:
track = r_tracked_stracks[itracked]
det = detections[idet]
if track.state == TrackState.Tracked:
track.update(det, self.frame_id)
activated_starcks.append(track)
else:
track.re_activate(det, self.frame_id, new_id=False)
refind_stracks.append(track)
for it in u_track:
track = r_tracked_stracks[it]
if not track.state == TrackState.Lost:
track.mark_lost()
lost_stracks.append(track)
"""Deal with unconfirmed tracks, usually tracks with only one beginning frame"""
detections = [detections[i] for i in u_detection]
dists = iou_distance(unconfirmed, detections)
matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.9)
for itracked, idet in matches:
unconfirmed[itracked].update(detections[idet], self.frame_id)
activated_starcks.append(unconfirmed[itracked])
for it in u_unconfirmed:
track = unconfirmed[it]
track.mark_removed()
removed_stracks.append(track)
""" Step 4: Init new stracks"""
for inew in u_detection:
track = detections[inew]
if track.score < self.det_thresh:
continue
track.activate(self.kalman_filter, self.frame_id)
activated_starcks.append(track)
""" Step 5: Update state"""
for track in self.lost_stracks:
if self.frame_id - track.end_frame > self.max_time_lost:
track.mark_removed()
removed_stracks.append(track)
self.tracked_stracks = [
t for t in self.tracked_stracks if t.state == TrackState.Tracked
]
self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)
self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)
self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)
self.lost_stracks.extend(lost_stracks)
self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)
self.removed_stracks.extend(removed_stracks)
self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(
self.tracked_stracks, self.lost_stracks
)
# get scores of lost tracks
output_stracks = [track for track in self.tracked_stracks if track.is_activated]
if verbose:
print("===========Frame {}==========".format(self.frame_id))
print(
"Activated: {}".format([track.track_id for track in activated_starcks])
)
print("Refind: {}".format([track.track_id for track in refind_stracks]))
print("Lost: {}".format([track.track_id for track in lost_stracks]))
print("Removed: {}".format([track.track_id for track in removed_stracks]))
return output_stracks, best_candidate_ball_bbox
def joint_stracks(tlista, tlistb):
exists = {}
res = []
for t in tlista:
exists[t.track_id] = 1
res.append(t)
for t in tlistb:
tid = t.track_id
if not exists.get(tid, 0):
exists[tid] = 1
res.append(t)
return res
def sub_stracks(tlista, tlistb):
stracks = {}
for t in tlista:
stracks[t.track_id] = t
for t in tlistb:
tid = t.track_id
if stracks.get(tid, 0):
del stracks[tid]
return list(stracks.values())
def remove_duplicate_stracks(stracksa, stracksb):
pdist = iou_distance(stracksa, stracksb)
pairs = np.where(pdist < 0.15)
dupa, dupb = list(), list()
for p, q in zip(*pairs):
timep = stracksa[p].frame_id - stracksa[p].start_frame
timeq = stracksb[q].frame_id - stracksb[q].start_frame
if timep > timeq:
dupb.append(q)
else:
dupa.append(p)
resa = [t for i, t in enumerate(stracksa) if not i in dupa]
resb = [t for i, t in enumerate(stracksb) if not i in dupb]
return resa, resb
Functions
def joint_stracks(tlista, tlistb)
-
Expand source code
def joint_stracks(tlista, tlistb): exists = {} res = [] for t in tlista: exists[t.track_id] = 1 res.append(t) for t in tlistb: tid = t.track_id if not exists.get(tid, 0): exists[tid] = 1 res.append(t) return res
def remove_duplicate_stracks(stracksa, stracksb)
-
Expand source code
def remove_duplicate_stracks(stracksa, stracksb): pdist = iou_distance(stracksa, stracksb) pairs = np.where(pdist < 0.15) dupa, dupb = list(), list() for p, q in zip(*pairs): timep = stracksa[p].frame_id - stracksa[p].start_frame timeq = stracksb[q].frame_id - stracksb[q].start_frame if timep > timeq: dupb.append(q) else: dupa.append(p) resa = [t for i, t in enumerate(stracksa) if not i in dupa] resb = [t for i, t in enumerate(stracksb) if not i in dupb] return resa, resb
def sub_stracks(tlista, tlistb)
-
Expand source code
def sub_stracks(tlista, tlistb): stracks = {} for t in tlista: stracks[t.track_id] = t for t in tlistb: tid = t.track_id if stracks.get(tid, 0): del stracks[tid] return list(stracks.values())
Classes
class JDETracker (conf_thres=0.5, track_buffer=30, K=100, frame_rate=30)
-
Expand source code
class JDETracker(object): def __init__( self, conf_thres=0.5, track_buffer=30, K=100, frame_rate=30, ): print("Creating model...") self.tracker_model = TrackerModel( pretrained=True, backbone="ssd_512_resnet50_v1_coco" ) self.reid_model = ReIdModel() gluon_tracker_params = tf.keras.utils.get_file( GLUON_NAME, GLUON_PATH, GLUON_TOTAR, ) torch_reid_pth = tf.keras.utils.get_file(TORCH_NAME, TORCH_PATH, TORCH_TOTAR,) self.tracker_model.load_weights(gluon_tracker_params) self.reid_model.load_weights(torch_reid_pth) self.tracked_stracks = [] # type: list[STrack] self.lost_stracks = [] # type: list[STrack] self.removed_stracks = [] # type: list[STrack] self.last_ball_bbox = None self.frame_id = 0 self.det_thresh = conf_thres self.buffer_size = int((frame_rate / 30.0) * track_buffer) self.max_time_lost = self.buffer_size self.max_per_image = K self.kalman_filter = KalmanFilter() def update(self, im_blob, img0, split_size=None, verbose=True): self.frame_id += 1 activated_starcks = [] refind_stracks = [] lost_stracks = [] removed_stracks = [] """ Step 1: Network forward, get detections & embeddings""" cid, score, bbox = self.tracker_model(img0, split_size=split_size) if split_size is None: bbox = bbox.asnumpy() cid = cid.asnumpy() score = score.asnumpy() id_feature = self.reid_model._get_embeddings( img0, bbox, score, cid, self.det_thresh, (512, 512) ) cid = to_torch(cid) score = to_torch(score) bbox = to_torch(bbox) id_feature = to_torch(id_feature) detections = torch.cat([bbox, score, cid], dim=2) detections = detections[0] ball_detections = torch.cat([bbox, score, cid], dim=2)[0] id_feature = id_feature[0] for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]): remain_inds = detections[:, indx] > tresh detections = detections[remain_inds] id_feature = id_feature[remain_inds] dets = detections[:, :5] cnt = 0 for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]): remain_inds = ( ball_detections[:, indx] > tresh if cnt == 0 else ball_detections[:, indx] < tresh ) ball_detections = ball_detections[remain_inds] cnt += 1 dets_ball = ball_detections[:, :5] if len(dets_ball) > 0: best_candidate_ball = np.argmax(dets_ball[:, 4], axis=0) best_candidate_ball_bbox = STrack.tlbr_to_tlwh( dets_ball[best_candidate_ball, :4] ) self.last_ball_bbox = best_candidate_ball_bbox else: best_candidate_ball_bbox = None # vis """ for i in range(0, dets.shape[0]): bbox = dets[i][0:4] cv2.rectangle(img0, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2) cv2.imshow('dets', img0) cv2.waitKey(0) id0 = id0-1 """ if len(dets) > 0: """Detections""" detections = [ STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30) for (tlbrs, f) in zip(dets[:, :5], id_feature) ] else: detections = [] """ Add newly detected tracklets to tracked_stracks""" unconfirmed = [] tracked_stracks = [] # type: list[STrack] for track in self.tracked_stracks: if not track.is_activated: unconfirmed.append(track) else: tracked_stracks.append(track) """ Step 2: First association, with embedding""" strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) # Predict the current location with KF # for strack in strack_pool: # strack.predict() STrack.multi_predict(strack_pool) dists = embedding_distance(strack_pool, detections) dists = fuse_motion(self.kalman_filter, dists, strack_pool, detections) matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) for itracked, idet in matches: track = strack_pool[itracked] det = detections[idet] if track.state == TrackState.Tracked: track.update(detections[idet], self.frame_id) activated_starcks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) """ Step 3: Second association, with IOU""" detections = [detections[i] for i in u_detection] r_tracked_stracks = [ strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked ] dists = iou_distance(r_tracked_stracks, detections) matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) for itracked, idet in matches: track = r_tracked_stracks[itracked] det = detections[idet] if track.state == TrackState.Tracked: track.update(det, self.frame_id) activated_starcks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) for it in u_track: track = r_tracked_stracks[it] if not track.state == TrackState.Lost: track.mark_lost() lost_stracks.append(track) """Deal with unconfirmed tracks, usually tracks with only one beginning frame""" detections = [detections[i] for i in u_detection] dists = iou_distance(unconfirmed, detections) matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.9) for itracked, idet in matches: unconfirmed[itracked].update(detections[idet], self.frame_id) activated_starcks.append(unconfirmed[itracked]) for it in u_unconfirmed: track = unconfirmed[it] track.mark_removed() removed_stracks.append(track) """ Step 4: Init new stracks""" for inew in u_detection: track = detections[inew] if track.score < self.det_thresh: continue track.activate(self.kalman_filter, self.frame_id) activated_starcks.append(track) """ Step 5: Update state""" for track in self.lost_stracks: if self.frame_id - track.end_frame > self.max_time_lost: track.mark_removed() removed_stracks.append(track) self.tracked_stracks = [ t for t in self.tracked_stracks if t.state == TrackState.Tracked ] self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks) self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks) self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks) self.lost_stracks.extend(lost_stracks) self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks) self.removed_stracks.extend(removed_stracks) self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks( self.tracked_stracks, self.lost_stracks ) # get scores of lost tracks output_stracks = [track for track in self.tracked_stracks if track.is_activated] if verbose: print("===========Frame {}==========".format(self.frame_id)) print( "Activated: {}".format([track.track_id for track in activated_starcks]) ) print("Refind: {}".format([track.track_id for track in refind_stracks])) print("Lost: {}".format([track.track_id for track in lost_stracks])) print("Removed: {}".format([track.track_id for track in removed_stracks])) return output_stracks, best_candidate_ball_bbox
Methods
def update(self, im_blob, img0, split_size=None, verbose=True)
-
Expand source code
def update(self, im_blob, img0, split_size=None, verbose=True): self.frame_id += 1 activated_starcks = [] refind_stracks = [] lost_stracks = [] removed_stracks = [] """ Step 1: Network forward, get detections & embeddings""" cid, score, bbox = self.tracker_model(img0, split_size=split_size) if split_size is None: bbox = bbox.asnumpy() cid = cid.asnumpy() score = score.asnumpy() id_feature = self.reid_model._get_embeddings( img0, bbox, score, cid, self.det_thresh, (512, 512) ) cid = to_torch(cid) score = to_torch(score) bbox = to_torch(bbox) id_feature = to_torch(id_feature) detections = torch.cat([bbox, score, cid], dim=2) detections = detections[0] ball_detections = torch.cat([bbox, score, cid], dim=2)[0] id_feature = id_feature[0] for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]): remain_inds = detections[:, indx] > tresh detections = detections[remain_inds] id_feature = id_feature[remain_inds] dets = detections[:, :5] cnt = 0 for indx, tresh in zip([4, 5], [self.det_thresh, 0.5]): remain_inds = ( ball_detections[:, indx] > tresh if cnt == 0 else ball_detections[:, indx] < tresh ) ball_detections = ball_detections[remain_inds] cnt += 1 dets_ball = ball_detections[:, :5] if len(dets_ball) > 0: best_candidate_ball = np.argmax(dets_ball[:, 4], axis=0) best_candidate_ball_bbox = STrack.tlbr_to_tlwh( dets_ball[best_candidate_ball, :4] ) self.last_ball_bbox = best_candidate_ball_bbox else: best_candidate_ball_bbox = None # vis """ for i in range(0, dets.shape[0]): bbox = dets[i][0:4] cv2.rectangle(img0, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2) cv2.imshow('dets', img0) cv2.waitKey(0) id0 = id0-1 """ if len(dets) > 0: """Detections""" detections = [ STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30) for (tlbrs, f) in zip(dets[:, :5], id_feature) ] else: detections = [] """ Add newly detected tracklets to tracked_stracks""" unconfirmed = [] tracked_stracks = [] # type: list[STrack] for track in self.tracked_stracks: if not track.is_activated: unconfirmed.append(track) else: tracked_stracks.append(track) """ Step 2: First association, with embedding""" strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) # Predict the current location with KF # for strack in strack_pool: # strack.predict() STrack.multi_predict(strack_pool) dists = embedding_distance(strack_pool, detections) dists = fuse_motion(self.kalman_filter, dists, strack_pool, detections) matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) for itracked, idet in matches: track = strack_pool[itracked] det = detections[idet] if track.state == TrackState.Tracked: track.update(detections[idet], self.frame_id) activated_starcks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) """ Step 3: Second association, with IOU""" detections = [detections[i] for i in u_detection] r_tracked_stracks = [ strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked ] dists = iou_distance(r_tracked_stracks, detections) matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) for itracked, idet in matches: track = r_tracked_stracks[itracked] det = detections[idet] if track.state == TrackState.Tracked: track.update(det, self.frame_id) activated_starcks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) for it in u_track: track = r_tracked_stracks[it] if not track.state == TrackState.Lost: track.mark_lost() lost_stracks.append(track) """Deal with unconfirmed tracks, usually tracks with only one beginning frame""" detections = [detections[i] for i in u_detection] dists = iou_distance(unconfirmed, detections) matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.9) for itracked, idet in matches: unconfirmed[itracked].update(detections[idet], self.frame_id) activated_starcks.append(unconfirmed[itracked]) for it in u_unconfirmed: track = unconfirmed[it] track.mark_removed() removed_stracks.append(track) """ Step 4: Init new stracks""" for inew in u_detection: track = detections[inew] if track.score < self.det_thresh: continue track.activate(self.kalman_filter, self.frame_id) activated_starcks.append(track) """ Step 5: Update state""" for track in self.lost_stracks: if self.frame_id - track.end_frame > self.max_time_lost: track.mark_removed() removed_stracks.append(track) self.tracked_stracks = [ t for t in self.tracked_stracks if t.state == TrackState.Tracked ] self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks) self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks) self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks) self.lost_stracks.extend(lost_stracks) self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks) self.removed_stracks.extend(removed_stracks) self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks( self.tracked_stracks, self.lost_stracks ) # get scores of lost tracks output_stracks = [track for track in self.tracked_stracks if track.is_activated] if verbose: print("===========Frame {}==========".format(self.frame_id)) print( "Activated: {}".format([track.track_id for track in activated_starcks]) ) print("Refind: {}".format([track.track_id for track in refind_stracks])) print("Lost: {}".format([track.track_id for track in lost_stracks])) print("Removed: {}".format([track.track_id for track in removed_stracks])) return output_stracks, best_candidate_ball_bbox
class STrack (tlwh, score, temp_feat, buffer_size=30)
-
Expand source code
class STrack(BaseTrack): shared_kalman = KalmanFilter() def __init__(self, tlwh, score, temp_feat, buffer_size=30): # wait activate self._tlwh = np.asarray(tlwh, dtype=np.float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False self.score = score self.tracklet_len = 0 self.smooth_feat = None self.update_features(temp_feat) self.features = deque([], maxlen=buffer_size) self.alpha = 0.9 def update_features(self, feat): feat /= np.linalg.norm(feat) self.curr_feat = feat if self.smooth_feat is None: self.smooth_feat = feat else: self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat self.features.append(feat) self.smooth_feat /= np.linalg.norm(self.smooth_feat) def predict(self): mean_state = self.mean.copy() if self.state != TrackState.Tracked: mean_state[7] = 0 self.mean, self.covariance = self.kalman_filter.predict( mean_state, self.covariance ) @staticmethod def multi_predict(stracks): if len(stracks) > 0: multi_mean = np.asarray([st.mean.copy() for st in stracks]) multi_covariance = np.asarray([st.covariance for st in stracks]) for i, st in enumerate(stracks): if st.state != TrackState.Tracked: multi_mean[i][7] = 0 multi_mean, multi_covariance = STrack.shared_kalman.multi_predict( multi_mean, multi_covariance ) for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): stracks[i].mean = mean stracks[i].covariance = cov def activate(self, kalman_filter, frame_id): """Start a new tracklet""" self.kalman_filter = kalman_filter self.track_id = self.next_id() self.mean, self.covariance = self.kalman_filter.initiate( self.tlwh_to_xyah(self._tlwh) ) self.tracklet_len = 0 self.state = TrackState.Tracked if frame_id == 1: self.is_activated = True # self.is_activated = True self.frame_id = frame_id self.start_frame = frame_id def re_activate(self, new_track, frame_id, new_id=False): self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh) ) self.update_features(new_track.curr_feat) self.tracklet_len = 0 self.state = TrackState.Tracked self.is_activated = True self.frame_id = frame_id if new_id: self.track_id = self.next_id() def update(self, new_track, frame_id, update_feature=True): """ Update a matched track :type new_track: STrack :type frame_id: int :type update_feature: bool :return: """ self.frame_id = frame_id self.tracklet_len += 1 new_tlwh = new_track.tlwh self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh) ) self.state = TrackState.Tracked self.is_activated = True self.score = new_track.score if update_feature: self.update_features(new_track.curr_feat) @property # @jit(nopython=True) def tlwh(self): """Get current position in bounding box format `(top left x, top left y, width, height)`. """ if self.mean is None: return self._tlwh.copy() ret = self.mean[:4].copy() ret[2] *= ret[3] ret[:2] -= ret[2:] / 2 return ret @property # @jit(nopython=True) def tlbr(self): """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., `(top left, bottom right)`. """ ret = self.tlwh.copy() ret[2:] += ret[:2] return ret @staticmethod # @jit(nopython=True) def tlwh_to_xyah(tlwh): """Convert bounding box to format `(center x, center y, aspect ratio, height)`, where the aspect ratio is `width / height`. """ ret = np.asarray(tlwh).copy() ret[:2] += ret[2:] / 2 ret[2] /= ret[3] return ret def to_xyah(self): return self.tlwh_to_xyah(self.tlwh) @staticmethod # @jit(nopython=True) def tlbr_to_tlwh(tlbr): ret = np.asarray(tlbr).copy() ret[2:] -= ret[:2] return ret @staticmethod # @jit(nopython=True) def tlwh_to_tlbr(tlwh): ret = np.asarray(tlwh).copy() ret[2:] += ret[:2] return ret def __repr__(self): return "OT_{}_({}-{})".format(self.track_id, self.start_frame, self.end_frame)
Ancestors
Class variables
Static methods
def multi_predict(stracks)
-
Expand source code
@staticmethod def multi_predict(stracks): if len(stracks) > 0: multi_mean = np.asarray([st.mean.copy() for st in stracks]) multi_covariance = np.asarray([st.covariance for st in stracks]) for i, st in enumerate(stracks): if st.state != TrackState.Tracked: multi_mean[i][7] = 0 multi_mean, multi_covariance = STrack.shared_kalman.multi_predict( multi_mean, multi_covariance ) for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): stracks[i].mean = mean stracks[i].covariance = cov
def tlbr_to_tlwh(tlbr)
-
Expand source code
@staticmethod # @jit(nopython=True) def tlbr_to_tlwh(tlbr): ret = np.asarray(tlbr).copy() ret[2:] -= ret[:2] return ret
def tlwh_to_tlbr(tlwh)
-
Expand source code
@staticmethod # @jit(nopython=True) def tlwh_to_tlbr(tlwh): ret = np.asarray(tlwh).copy() ret[2:] += ret[:2] return ret
def tlwh_to_xyah(tlwh)
-
Convert bounding box to format
(center x, center y, aspect ratio, height)<code>, where the aspect ratio is </code>width / height
.Expand source code
@staticmethod # @jit(nopython=True) def tlwh_to_xyah(tlwh): """Convert bounding box to format `(center x, center y, aspect ratio, height)`, where the aspect ratio is `width / height`. """ ret = np.asarray(tlwh).copy() ret[:2] += ret[2:] / 2 ret[2] /= ret[3] return ret
Instance variables
var tlbr
-
Convert bounding box to format
(min x, min y, max x, max y)
, i.e.,(top left, bottom right)
.Expand source code
@property # @jit(nopython=True) def tlbr(self): """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., `(top left, bottom right)`. """ ret = self.tlwh.copy() ret[2:] += ret[:2] return ret
var tlwh
-
Get current position in bounding box format
(top left x, top left y, width, height)
.Expand source code
@property # @jit(nopython=True) def tlwh(self): """Get current position in bounding box format `(top left x, top left y, width, height)`. """ if self.mean is None: return self._tlwh.copy() ret = self.mean[:4].copy() ret[2] *= ret[3] ret[:2] -= ret[2:] / 2 return ret
Methods
def activate(self, kalman_filter, frame_id)
-
Start a new tracklet
Expand source code
def activate(self, kalman_filter, frame_id): """Start a new tracklet""" self.kalman_filter = kalman_filter self.track_id = self.next_id() self.mean, self.covariance = self.kalman_filter.initiate( self.tlwh_to_xyah(self._tlwh) ) self.tracklet_len = 0 self.state = TrackState.Tracked if frame_id == 1: self.is_activated = True # self.is_activated = True self.frame_id = frame_id self.start_frame = frame_id
def predict(self)
-
Expand source code
def predict(self): mean_state = self.mean.copy() if self.state != TrackState.Tracked: mean_state[7] = 0 self.mean, self.covariance = self.kalman_filter.predict( mean_state, self.covariance )
def re_activate(self, new_track, frame_id, new_id=False)
-
Expand source code
def re_activate(self, new_track, frame_id, new_id=False): self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh) ) self.update_features(new_track.curr_feat) self.tracklet_len = 0 self.state = TrackState.Tracked self.is_activated = True self.frame_id = frame_id if new_id: self.track_id = self.next_id()
def to_xyah(self)
-
Expand source code
def to_xyah(self): return self.tlwh_to_xyah(self.tlwh)
def update(self, new_track, frame_id, update_feature=True)
-
Update a matched track :type new_track: STrack :type frame_id: int :type update_feature: bool :return:
Expand source code
def update(self, new_track, frame_id, update_feature=True): """ Update a matched track :type new_track: STrack :type frame_id: int :type update_feature: bool :return: """ self.frame_id = frame_id self.tracklet_len += 1 new_tlwh = new_track.tlwh self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh) ) self.state = TrackState.Tracked self.is_activated = True self.score = new_track.score if update_feature: self.update_features(new_track.curr_feat)
def update_features(self, feat)
-
Expand source code
def update_features(self, feat): feat /= np.linalg.norm(feat) self.curr_feat = feat if self.smooth_feat is None: self.smooth_feat = feat else: self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat self.features.append(feat) self.smooth_feat /= np.linalg.norm(self.smooth_feat)