Skip to content

Commit 3953163

Browse files
ZhangHandiZhangHandi
and
ZhangHandi
authored
one euro filter and ema smoothing for keypoints (PaddlePaddle#6267)
* one euro filter and ema smoothing for keypoints * change few code Co-authored-by: ZhangHandi <[email protected]>
1 parent 831a431 commit 3953163

File tree

2 files changed

+94
-0
lines changed

2 files changed

+94
-0
lines changed

deploy/python/det_keypoint_unite_infer.py

+82
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,9 @@ def topdown_unite_predict_video(detector,
143143
writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
144144
index = 0
145145
store_res = []
146+
previous_keypoints = None
147+
keypoint_smoothing = KeypointSmoothing(width, height, filter_type=FLAGS.filter_type, alpha=0.8, beta=1)
148+
146149
while (1):
147150
ret, frame = capture.read()
148151
if not ret:
@@ -161,12 +164,20 @@ def topdown_unite_predict_video(detector,
161164
keypoint_res = predict_with_given_det(
162165
frame2, results, topdown_keypoint_detector, keypoint_batch_size,
163166
FLAGS.run_benchmark)
167+
168+
if FLAGS.smooth:
169+
current_keypoints = np.array(keypoint_res['keypoint'][0][0])
170+
smooth_keypoints = keypoint_smoothing.smooth_process(previous_keypoints, current_keypoints)
171+
previous_keypoints = smooth_keypoints
172+
173+
keypoint_res['keypoint'][0][0] = smooth_keypoints.tolist()
164174

165175
im = visualize_pose(
166176
frame,
167177
keypoint_res,
168178
visual_thresh=FLAGS.keypoint_threshold,
169179
returnimg=True)
180+
170181
if save_res:
171182
store_res.append([
172183
index, keypoint_res['bbox'],
@@ -192,6 +203,77 @@ def topdown_unite_predict_video(detector,
192203
json.dump(store_res, wf, indent=4)
193204

194205

206+
class KeypointSmoothing(object):
207+
# The following code are modified from:
208+
# https://github.com/610265158/Peppa_Pig_Face_Engine/blob/7bb1066ad3fbb12697924ba7f9287bf198c15232/lib/core/LK/lk.py
209+
210+
def __init__(self, width, height, filter_type, alpha=0.5, fc_d=1, fc_min=1, beta=0):
211+
super(KeypointSmoothing, self).__init__()
212+
self.image_width = width
213+
self.image_height = height
214+
self.threshold = [0.005, 0.005, 0.005, 0.005, 0.005, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01]
215+
self.filter_type = filter_type
216+
self.alpha = alpha
217+
self.dx_prev_hat = None
218+
self.x_prev_hat = None
219+
self.fc_d = fc_d
220+
self.fc_min = fc_min
221+
self.beta = beta
222+
223+
if self.filter_type == 'one_euro':
224+
self.smooth_func = self.one_euro_filter
225+
elif self.filter_type == 'ema':
226+
self.smooth_func = self.exponential_smoothing
227+
else:
228+
raise ValueError('filter type must be one_euro or ema')
229+
230+
def smooth_process(self, previous_keypoints, current_keypoints):
231+
if previous_keypoints is None:
232+
previous_keypoints = current_keypoints
233+
result = current_keypoints
234+
else:
235+
result = []
236+
num_keypoints = len(current_keypoints)
237+
for i in range(num_keypoints):
238+
result.append(self.smooth(previous_keypoints[i], current_keypoints[i], self.threshold[i]))
239+
return np.array(result)
240+
241+
242+
def smooth(self, previous_keypoint, current_keypoint, threshold):
243+
distance = np.sqrt(np.square((current_keypoint[0] - previous_keypoint[0]) / self.image_width) + np.square((current_keypoint[1] - previous_keypoint[1]) / self.image_height))
244+
if distance < threshold:
245+
result = previous_keypoint
246+
else:
247+
result = self.smooth_func(previous_keypoint, current_keypoint)
248+
return result
249+
250+
251+
def one_euro_filter(self, x_prev, x_cur):
252+
te = 1
253+
self.alpha = self.smoothing_factor(te, self.fc_d)
254+
if self.x_prev_hat is None:
255+
self.x_prev_hat = x_prev
256+
dx_cur = (x_cur - self.x_prev_hat) / te
257+
if self.dx_prev_hat is None:
258+
self.dx_prev_hat = 0
259+
dx_cur_hat = self.exponential_smoothing(self.dx_prev_hat, dx_cur)
260+
261+
fc = self.fc_min + self.beta * np.abs(dx_cur_hat)
262+
self.alpha = self.smoothing_factor(te, fc)
263+
x_cur_hat = self.exponential_smoothing(self.x_prev_hat, x_cur)
264+
self.dx_prev_hat = dx_cur_hat
265+
self.x_prev_hat = x_cur_hat
266+
return x_cur_hat
267+
268+
269+
def smoothing_factor(self, te, fc):
270+
r = 2 * math.pi * fc * te
271+
return r / (r + 1)
272+
273+
def exponential_smoothing(self, x_prev, x_cur):
274+
return self.alpha * x_cur + (1 - self.alpha) * x_prev
275+
276+
195277
def main():
196278
deploy_file = os.path.join(FLAGS.det_model_dir, 'infer_cfg.yml')
197279
with open(deploy_file) as f:

deploy/python/det_keypoint_unite_utils.py

+12
Original file line numberDiff line numberDiff line change
@@ -126,4 +126,16 @@ def argsparser():
126126
"3) rects: list of rect [xmin, ymin, xmax, ymax]"
127127
"4) keypoints: 17(joint numbers)*[x, y, conf], total 51 data in list"
128128
"5) scores: mean of all joint conf"))
129+
parser.add_argument(
130+
'--smooth',
131+
type=ast.literal_eval,
132+
default=False,
133+
help='smoothing keypoints for each frame, new incoming keypoints will be more stable.'
134+
)
135+
parser.add_argument(
136+
'--filter_type',
137+
type=str,
138+
default='one_euro',
139+
help='when set --smooth True, choose filter type you want to use, it can be one_euro or ema.'
140+
)
129141
return parser

0 commit comments

Comments
 (0)