AOMedia AV1 Codec
temporal_filter.h
1 /*
2  * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #ifndef AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
13 #define AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
14 
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
18 
19 struct AV1_COMP;
20 struct ThreadData;
21 // TODO(any): These two variables are only used in avx2, sse2, sse4
22 // implementations, where the block size is still hard coded. This should be
23 // fixed to align with the c implementation.
24 #define BH 32
25 #define BW 32
26 
27 // Block size used in temporal filtering.
28 #define TF_BLOCK_SIZE BLOCK_32X32
29 
30 // Window size for temporal filtering.
31 #define TF_WINDOW_LENGTH 5
32 
33 // Hyper-parameters used to compute filtering weight. These hyper-parameters can
34 // be tuned for a better performance.
35 // 0. A scale factor used in temporal filtering to raise the filter weight from
36 // `double` with range [0, 1] to `int` with range [0, 1000].
37 #define TF_WEIGHT_SCALE 1000
38 // 1. Weight factor used to balance the weighted-average between window error
39 // and block error. The weight is for window error while the weight for block
40 // error is always set as 1.
41 #define TF_WINDOW_BLOCK_BALANCE_WEIGHT 5
42 // 2. Threshold for using q to adjust the filtering weight. Concretely, when
43 // using a small q (high bitrate), we would like to reduce the filtering
44 // strength such that more detailed information can be preserved. Hence, when
45 // q is smaller than this threshold, we will adjust the filtering weight
46 // based on the q-value.
47 #define TF_Q_DECAY_THRESHOLD 20
48 // 3. Normalization factor used to normalize the motion search error. Since the
49 // motion search error can be large and uncontrollable, we will simply
50 // normalize it before using it to compute the filtering weight.
51 #define TF_SEARCH_ERROR_NORM_WEIGHT 20
52 // 4. Threshold for using `arnr_strength` to adjust the filtering strength.
53 // Concretely, users can use `arnr_strength` arguments to control the
54 // strength of temporal filtering. When `arnr_strength` is small enough (
55 // i.e., smaller than this threshold), we will adjust the filtering weight
56 // based on the strength value.
57 #define TF_STRENGTH_THRESHOLD 4
58 // 5. Threshold for using motion search distance to adjust the filtering weight.
59 // Concretely, larger motion search vector leads to a higher probability of
60 // unreliable search. Hence, we would like to reduce the filtering strength
61 // when the distance is large enough. Considering that the distance actually
62 // relies on the frame size, this threshold is also a resolution-based
63 // threshold. Taking 720p videos as an instance, if this field equals to 0.1,
64 // then the actual threshold will be 720 * 0.1 = 72. Similarly, the threshold
65 // for 360p videos will be 360 * 0.1 = 36.
66 #define TF_SEARCH_DISTANCE_THRESHOLD 0.1
67 
68 #define NOISE_ESTIMATION_EDGE_THRESHOLD 50
69 
75 typedef struct {
79  YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS];
95  struct scale_factors sf;
99  double noise_levels[MAX_MB_PLANE];
103  int num_pels;
107  int mb_rows;
111  int mb_cols;
119  int q_factor;
121 
124 // Sum and SSE source vs filtered frame difference returned by
125 // temporal filter.
126 typedef struct {
127  int64_t sum;
128  int64_t sse;
129 } FRAME_DIFF;
130 
131 // Data related to temporal filtering.
132 typedef struct {
133  // Source vs filtered frame error.
134  FRAME_DIFF diff;
135  // Pointer to temporary block info used to store state in temporal filtering
136  // process.
137  MB_MODE_INFO *tmp_mbmi;
138  // Pointer to accumulator buffer used in temporal filtering process.
139  uint32_t *accum;
140  // Pointer to count buffer used in temporal filtering process.
141  uint16_t *count;
142  // Pointer to predictor used in temporal filtering process.
143  uint8_t *pred;
144 } TemporalFilterData;
145 
146 // Data related to temporal filter multi-thread synchronization.
147 typedef struct {
148 #if CONFIG_MULTITHREAD
149  // Mutex lock used for dispatching jobs.
150  pthread_mutex_t *mutex_;
151 #endif // CONFIG_MULTITHREAD
152  // Next temporal filter block row to be filtered.
153  int next_tf_row;
154 } AV1TemporalFilterSync;
155 
156 // Estimates noise level from a given frame using a single plane (Y, U, or V).
157 // This is an adaptation of the mehtod in the following paper:
158 // Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise
159 // estimation using Laplacian operator and adaptive edge detection",
160 // Proc. 3rd International Symposium on Communications, Control and
161 // Signal Processing, 2008, St Julians, Malta.
162 // Inputs:
163 // frame: Pointer to the frame to estimate noise level from.
164 // plane: Index of the plane used for noise estimation. Commonly, 0 for
165 // Y-plane, 1 for U-plane, and 2 for V-plane.
166 // bit_depth: Actual bit-depth instead of the encoding bit-depth of the frame.
167 // Returns:
168 // The estimated noise, or -1.0 if there are too few smooth pixels.
169 double av1_estimate_noise_from_single_plane(const YV12_BUFFER_CONFIG *frame,
170  const int plane,
171  const int bit_depth);
185 void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td,
186  int mb_row);
187 
213 int av1_temporal_filter(struct AV1_COMP *cpi,
214  const int filter_frame_lookahead_idx,
215  FRAME_UPDATE_TYPE update_type, int is_forward_keyframe,
216  int *show_existing_arf);
217 
219 // Helper function to get `q` used for encoding.
220 int av1_get_q(const struct AV1_COMP *cpi);
221 
222 // Allocates memory for members of TemporalFilterData.
223 // Inputs:
224 // tf_data: Pointer to the structure containing temporal filter related data.
225 // num_pels: Number of pixels in the block across all planes.
226 // is_high_bitdepth: Whether the frame is high-bitdepth or not.
227 // Returns:
228 // Nothing will be returned. But the contents of tf_data will be modified.
229 static AOM_INLINE void tf_alloc_and_reset_data(TemporalFilterData *tf_data,
230  int num_pels,
231  int is_high_bitdepth) {
232  tf_data->tmp_mbmi = (MB_MODE_INFO *)malloc(sizeof(*tf_data->tmp_mbmi));
233  memset(tf_data->tmp_mbmi, 0, sizeof(*tf_data->tmp_mbmi));
234  tf_data->accum =
235  (uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum));
236  tf_data->count =
237  (uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count));
238  memset(&tf_data->diff, 0, sizeof(tf_data->diff));
239  if (is_high_bitdepth)
240  tf_data->pred = CONVERT_TO_BYTEPTR(
241  aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred)));
242  else
243  tf_data->pred =
244  (uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
245 }
246 
247 // Setup macroblockd params for temporal filtering process.
248 // Inputs:
249 // mbd: Pointer to the block for filtering.
250 // tf_data: Pointer to the structure containing temporal filter related data.
251 // scale: Scaling factor.
252 // Returns:
253 // Nothing will be returned. Contents of mbd will be modified.
254 static AOM_INLINE void tf_setup_macroblockd(MACROBLOCKD *mbd,
255  TemporalFilterData *tf_data,
256  const struct scale_factors *scale) {
257  mbd->block_ref_scale_factors[0] = scale;
258  mbd->block_ref_scale_factors[1] = scale;
259  mbd->mi = &tf_data->tmp_mbmi;
260  mbd->mi[0]->motion_mode = SIMPLE_TRANSLATION;
261 }
262 
263 // Deallocates the memory allocated for members of TemporalFilterData.
264 // Inputs:
265 // tf_data: Pointer to the structure containing temporal filter related data.
266 // is_high_bitdepth: Whether the frame is high-bitdepth or not.
267 // Returns:
268 // Nothing will be returned.
269 static AOM_INLINE void tf_dealloc_data(TemporalFilterData *tf_data,
270  int is_high_bitdepth) {
271  if (is_high_bitdepth)
272  tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred);
273  free(tf_data->tmp_mbmi);
274  aom_free(tf_data->accum);
275  aom_free(tf_data->count);
276  aom_free(tf_data->pred);
277 }
278 
279 // Helper function to compute number of blocks on either side of the frame.
280 static INLINE int get_num_blocks(const int frame_length, const int mb_length) {
281  return (frame_length + mb_length - 1) / mb_length;
282 }
283 
284 // Saves the state prior to temporal filter process.
285 // Inputs:
286 // mbd: Pointer to the block for filtering.
287 // input_mbmi: Backup block info to save input state.
288 // input_buffer: Backup buffer pointer to save input state.
289 // num_planes: Number of planes.
290 // Returns:
291 // Nothing will be returned. Contents of input_mbmi and input_buffer will be
292 // modified.
293 static INLINE void tf_save_state(MACROBLOCKD *mbd, MB_MODE_INFO ***input_mbmi,
294  uint8_t **input_buffer, int num_planes) {
295  for (int i = 0; i < num_planes; i++) {
296  input_buffer[i] = mbd->plane[i].pre[0].buf;
297  }
298  *input_mbmi = mbd->mi;
299 }
300 
301 // Restores the initial state after temporal filter process.
302 // Inputs:
303 // mbd: Pointer to the block for filtering.
304 // input_mbmi: Backup block info from where input state is restored.
305 // input_buffer: Backup buffer pointer from where input state is restored.
306 // num_planes: Number of planes.
307 // Returns:
308 // Nothing will be returned. Contents of mbd will be modified.
309 static INLINE void tf_restore_state(MACROBLOCKD *mbd, MB_MODE_INFO **input_mbmi,
310  uint8_t **input_buffer, int num_planes) {
311  for (int i = 0; i < num_planes; i++) {
312  mbd->plane[i].pre[0].buf = input_buffer[i];
313  }
314  mbd->mi = input_mbmi;
315 }
316 
318 #ifdef __cplusplus
319 } // extern "C"
320 #endif
321 
322 #endif // AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
TemporalFilterCtx::q_factor
int q_factor
Definition: temporal_filter.h:119
encoder.h
Declares top-level encoder structures and functions.
TemporalFilterCtx
Parameters related to temporal filtering.
Definition: temporal_filter.h:75
av1_apply_temporal_filter_c
void av1_apply_temporal_filter_c(const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, const int num_planes, const double *noise_levels, const MV *subblock_mvs, const int *subblock_mses, const int q_factor, const int filter_strength, const uint8_t *pred, uint32_t *accum, uint16_t *count)
Applies temporal filtering. NOTE that there are various optimised versions of this function called wh...
Definition: temporal_filter.c:541
TemporalFilterCtx::mb_rows
int mb_rows
Definition: temporal_filter.h:107
macroblock::plane
struct macroblock_plane plane[3]
Each of the encoding plane.
Definition: block.h:856
RATE_CONTROL
Rate Control parameters and status.
Definition: ratectrl.h:118
GF_GROUP
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:171
macroblockd::block_ref_scale_factors
const struct scale_factors * block_ref_scale_factors[2]
Definition: blockd.h:685
AOM_PLANE_Y
#define AOM_PLANE_Y
Definition: aom_image.h:199
MultiThreadInfo
Encoder parameters related to multi-threading.
Definition: encoder.h:1375
AOM_PLANE_U
#define AOM_PLANE_U
Definition: aom_image.h:200
AV1_COMP::sf
SPEED_FEATURES sf
Definition: encoder.h:2314
AV1_COMP::td
ThreadData td
Definition: encoder.h:2110
AV1_COMP::gf_frame_index
unsigned char gf_frame_index
Definition: encoder.h:2362
AV1_COMP::compressor_stage
COMPRESSOR_STAGE compressor_stage
Definition: encoder.h:2645
TemporalFilterCtx::filter_frame_idx
int filter_frame_idx
Definition: temporal_filter.h:87
AlgoCfg::arnr_max_frames
int arnr_max_frames
Definition: encoder.h:735
FeatureFlags::cur_frame_force_integer_mv
bool cur_frame_force_integer_mv
Definition: av1_common_int.h:347
macroblockd::bd
int bd
Definition: blockd.h:806
AV1EncoderConfig::kf_cfg
KeyFrameCfg kf_cfg
Definition: encoder.h:822
MultiThreadInfo::num_workers
int num_workers
Definition: encoder.h:1379
AV1_COMP::mt_info
MultiThreadInfo mt_info
Definition: encoder.h:2501
TemporalFilterCtx::num_frames
int num_frames
Definition: temporal_filter.h:83
av1_rc_pick_q_and_bounds
int av1_rc_pick_q_and_bounds(const struct AV1_COMP *cpi, RATE_CONTROL *rc, int width, int height, int gf_index, int *bottom_index, int *top_index)
Picks q and q bounds given the rate control parameters in cpi->rc.
AV1_COMP::gf_group
GF_GROUP gf_group
Definition: encoder.h:2357
AV1_COMP
Top level encoder structure.
Definition: encoder.h:2095
AV1Common::height
int height
Definition: av1_common_int.h:749
av1_tf_do_filtering_row
void av1_tf_do_filtering_row(AV1_COMP *cpi, ThreadData *td, int mb_row)
Does temporal filter for a given macroblock row.
Definition: temporal_filter.c:755
TemporalFilterCtx::is_highbitdepth
int is_highbitdepth
Definition: temporal_filter.h:115
MotionVectorSearchParams::search_site_cfg
search_site_config search_site_cfg[SS_CFG_TOTAL][NUM_DISTINCT_SEARCH_METHODS]
Definition: encoder.h:1756
TemporalFilterCtx::num_pels
int num_pels
Definition: temporal_filter.h:103
AV1_PRIMARY::lookahead
struct lookahead_ctx * lookahead
Definition: encoder.h:2071
AV1_COMP::common
AV1_COMMON common
Definition: encoder.h:2138
TemporalFilterCtx::mb_cols
int mb_cols
Definition: temporal_filter.h:111
AV1_COMP::mv_search_params
MotionVectorSearchParams mv_search_params
Definition: encoder.h:2319
RATE_CONTROL::gfu_boost
int gfu_boost
Definition: ratectrl.h:169
tf_setup_filtering_buffer
static void tf_setup_filtering_buffer(AV1_COMP *cpi, const int filter_frame_lookahead_idx, const int is_second_arf, FRAME_UPDATE_TYPE update_type, int is_forward_keyframe)
Setups the frame buffer for temporal filtering. This fuction determines how many frames will be used ...
Definition: temporal_filter.c:927
AV1_COMP::fn_ptr
aom_variance_fn_ptr_t fn_ptr[BLOCK_SIZES_ALL]
Definition: encoder.h:2347
tf_motion_search
static void tf_motion_search(AV1_COMP *cpi, MACROBLOCK *mb, const YV12_BUFFER_CONFIG *frame_to_filter, const YV12_BUFFER_CONFIG *ref_frame, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, MV *ref_mv, MV *subblock_mvs, int *subblock_mses)
Does motion search for blocks in temporal filtering. This is the first step for temporal filtering....
Definition: temporal_filter.c:84
macroblock::mv_limits
FullMvLimits mv_limits
Limit for the range of motion vectors.
Definition: block.h:1194
AV1_COMP::alt_ref_buffer
YV12_BUFFER_CONFIG alt_ref_buffer
Definition: encoder.h:2378
MB_MODE_INFO::motion_mode
MOTION_MODE motion_mode
The motion mode used by the inter prediction.
Definition: blockd.h:244
AV1_COMP::tf_ctx
TemporalFilterCtx tf_ctx
Definition: encoder.h:2207
RATE_CONTROL::base_frame_target
int base_frame_target
Definition: ratectrl.h:125
SPEED_FEATURES::hl_sf
HIGH_LEVEL_SPEED_FEATURES hl_sf
Definition: speed_features.h:1124
MotionVectorSearchParams::find_fractional_mv_step
fractional_mv_step_fp * find_fractional_mv_step
Definition: encoder.h:1749
yv12_buffer_config
YV12 frame buffer data structure.
Definition: yv12config.h:38
TemporalFilterCtx::check_show_existing
int check_show_existing
Definition: temporal_filter.h:91
macroblock_plane::src
struct buf_2d src
A buffer containing the source frame.
Definition: block.h:117
AV1_COMP::rc
RATE_CONTROL rc
Definition: encoder.h:2294
tf_build_predictor
static void tf_build_predictor(const YV12_BUFFER_CONFIG *ref_frame, const MACROBLOCKD *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, const int num_planes, const struct scale_factors *scale, const MV *subblock_mvs, uint8_t *pred)
Builds predictor for blocks in temporal filtering. This is the second step for temporal filtering,...
Definition: temporal_filter.c:312
AlgoCfg::arnr_strength
int arnr_strength
Definition: encoder.h:740
AV1EncoderConfig::algo_cfg
AlgoCfg algo_cfg
Definition: encoder.h:817
tf_do_filtering
static void tf_do_filtering(AV1_COMP *cpi)
Does temporal filter for a given frame.
Definition: temporal_filter.c:887
av1_lookahead_peek
struct lookahead_entry * av1_lookahead_peek(struct lookahead_ctx *ctx, int index, COMPRESSOR_STAGE stage)
Get a future source buffer to encode.
HIGH_LEVEL_SPEED_FEATURES::second_alt_ref_filtering
int second_alt_ref_filtering
Definition: speed_features.h:355
AV1_COMP::ppi
AV1_PRIMARY * ppi
Definition: encoder.h:2099
MB_MODE_INFO
Stores the prediction/txfm mode of the current coding block.
Definition: blockd.h:216
macroblockd
Variables related to current coding block.
Definition: blockd.h:568
KeyFrameCfg::enable_keyframe_filtering
int enable_keyframe_filtering
Definition: encoder.h:391
macroblock::e_mbd
MACROBLOCKD e_mbd
Decoder's view of current coding block.
Definition: block.h:864
av1_lookahead_depth
unsigned int av1_lookahead_depth(struct lookahead_ctx *ctx, COMPRESSOR_STAGE stage)
Get the number of frames currently in the lookahead queue.
TemporalFilterCtx::noise_levels
double noise_levels[MAX_MB_PLANE]
Definition: temporal_filter.h:99
AV1Common::width
int width
Definition: av1_common_int.h:748
AV1_COMP::oxcf
AV1EncoderConfig oxcf
Definition: encoder.h:2143
AV1Common::seq_params
SequenceHeader seq_params
Definition: av1_common_int.h:953
TemporalFilterCtx::sf
struct scale_factors sf
Definition: temporal_filter.h:95
av1_temporal_filter
int av1_temporal_filter(struct AV1_COMP *cpi, const int filter_frame_lookahead_idx, FRAME_UPDATE_TYPE update_type, int is_forward_keyframe, int *show_existing_arf)
Performs temporal filtering if needed on a source frame. For example to create a filtered alternate r...
macroblockd::plane
struct macroblockd_plane plane[3]
Definition: blockd.h:604
tf_normalize_filtered_frame
static void tf_normalize_filtered_frame(const MACROBLOCKD *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, const int num_planes, const uint32_t *accum, const uint16_t *count, YV12_BUFFER_CONFIG *result_buffer)
Normalizes the accumulated filtering result to produce the filtered frame.
Definition: temporal_filter.c:708
AV1Common::showable_frame
int showable_frame
Definition: av1_common_int.h:870
AV1Common::features
FeatureFlags features
Definition: av1_common_int.h:882
TemporalFilterCtx::frames
YV12_BUFFER_CONFIG * frames[MAX_LAG_BUFFERS]
Definition: temporal_filter.h:79
AV1Common::mi_params
CommonModeInfoParams mi_params
Definition: av1_common_int.h:887
macroblock
Encoder's parameters related to the current coding block.
Definition: block.h:846
macroblockd::mi
MB_MODE_INFO ** mi
Definition: blockd.h:615
RATE_CONTROL::frames_to_key
int frames_to_key
Definition: ratectrl.h:223
AlgoCfg::enable_overlay
bool enable_overlay
Definition: encoder.h:759