64   xnn_operator_t average_pooling_op = NULL;  in xnn_create_average_pooling2d_nhwc_qu8()  local
177   average_pooling_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));  in xnn_create_average_pooling2d_nhwc_qu8()
178   if (average_pooling_op == NULL) {  in xnn_create_average_pooling2d_nhwc_qu8()
194   average_pooling_op->zero_buffer = zero_buffer;  in xnn_create_average_pooling2d_nhwc_qu8()
196   average_pooling_op->padding_top = input_padding_top;  in xnn_create_average_pooling2d_nhwc_qu8()
197   average_pooling_op->padding_right = input_padding_right;  in xnn_create_average_pooling2d_nhwc_qu8()
198   average_pooling_op->padding_bottom = input_padding_bottom;  in xnn_create_average_pooling2d_nhwc_qu8()
199   average_pooling_op->padding_left = input_padding_left;  in xnn_create_average_pooling2d_nhwc_qu8()
201   average_pooling_op->kernel_height = pooling_height;  in xnn_create_average_pooling2d_nhwc_qu8()
202   average_pooling_op->kernel_width = pooling_width;  in xnn_create_average_pooling2d_nhwc_qu8()
203   average_pooling_op->stride_height = stride_height;  in xnn_create_average_pooling2d_nhwc_qu8()
204   average_pooling_op->stride_width = stride_width;  in xnn_create_average_pooling2d_nhwc_qu8()
205   average_pooling_op->dilation_height = 1;  in xnn_create_average_pooling2d_nhwc_qu8()
206   average_pooling_op->dilation_width = 1;  in xnn_create_average_pooling2d_nhwc_qu8()
207   average_pooling_op->channels = channels;  in xnn_create_average_pooling2d_nhwc_qu8()
208   average_pooling_op->input_pixel_stride = input_pixel_stride;  in xnn_create_average_pooling2d_nhwc_qu8()
209   average_pooling_op->output_pixel_stride = output_pixel_stride;  in xnn_create_average_pooling2d_nhwc_qu8()
211   average_pooling_op->input_zero_point = (int32_t) (uint32_t) input_zero_point;  in xnn_create_average_pooling2d_nhwc_qu8()
212   average_pooling_op->output_zero_point = output_zero_point;  in xnn_create_average_pooling2d_nhwc_qu8()
213   average_pooling_op->input_scale = input_scale;  in xnn_create_average_pooling2d_nhwc_qu8()
214   average_pooling_op->output_scale = output_scale;  in xnn_create_average_pooling2d_nhwc_qu8()
215   average_pooling_op->output_min = output_min;  in xnn_create_average_pooling2d_nhwc_qu8()
216   average_pooling_op->output_max = output_max;  in xnn_create_average_pooling2d_nhwc_qu8()
221   average_pooling_op->params.qu8_avgpool =  in xnn_create_average_pooling2d_nhwc_qu8()
227   average_pooling_op->type = xnn_operator_type_average_pooling_nhwc_qu8;  in xnn_create_average_pooling2d_nhwc_qu8()
228   average_pooling_op->ukernel.type = xnn_ukernel_type_average_pooling;  in xnn_create_average_pooling2d_nhwc_qu8()
229   average_pooling_op->flags = flags;  in xnn_create_average_pooling2d_nhwc_qu8()
231   *average_pooling_op_out = average_pooling_op;  in xnn_create_average_pooling2d_nhwc_qu8()
235   xnn_delete_operator(average_pooling_op);  in xnn_create_average_pooling2d_nhwc_qu8()
256   xnn_operator_t average_pooling_op = NULL;  in xnn_create_average_pooling2d_nhwc_f32()  local
348   average_pooling_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));  in xnn_create_average_pooling2d_nhwc_f32()
349   if (average_pooling_op == NULL) {  in xnn_create_average_pooling2d_nhwc_f32()
364   average_pooling_op->zero_buffer = zero_buffer;  in xnn_create_average_pooling2d_nhwc_f32()
366   average_pooling_op->padding_top = input_padding_top;  in xnn_create_average_pooling2d_nhwc_f32()
367   average_pooling_op->padding_right = input_padding_right;  in xnn_create_average_pooling2d_nhwc_f32()
368   average_pooling_op->padding_bottom = input_padding_bottom;  in xnn_create_average_pooling2d_nhwc_f32()
369   average_pooling_op->padding_left = input_padding_left;  in xnn_create_average_pooling2d_nhwc_f32()
371   average_pooling_op->kernel_height = pooling_height;  in xnn_create_average_pooling2d_nhwc_f32()
372   average_pooling_op->kernel_width = pooling_width;  in xnn_create_average_pooling2d_nhwc_f32()
373   average_pooling_op->stride_height = stride_height;  in xnn_create_average_pooling2d_nhwc_f32()
374   average_pooling_op->stride_width = stride_width;  in xnn_create_average_pooling2d_nhwc_f32()
375   average_pooling_op->dilation_height = 1;  in xnn_create_average_pooling2d_nhwc_f32()
376   average_pooling_op->dilation_width = 1;  in xnn_create_average_pooling2d_nhwc_f32()
377   average_pooling_op->channels = channels;  in xnn_create_average_pooling2d_nhwc_f32()
378   average_pooling_op->input_pixel_stride = input_pixel_stride;  in xnn_create_average_pooling2d_nhwc_f32()
379   average_pooling_op->output_pixel_stride = output_pixel_stride;  in xnn_create_average_pooling2d_nhwc_f32()
381   average_pooling_op->type = xnn_operator_type_average_pooling_nhwc_f32;  in xnn_create_average_pooling2d_nhwc_f32()
382   average_pooling_op->params.f32_scaleminmax =  in xnn_create_average_pooling2d_nhwc_f32()
386     average_pooling_op->params.f32_minmax =  in xnn_create_average_pooling2d_nhwc_f32()
388     average_pooling_op->ukernel.type = xnn_ukernel_type_pixelwise_average_pooling;  in xnn_create_average_pooling2d_nhwc_f32()
390     average_pooling_op->ukernel.type = xnn_ukernel_type_average_pooling;  in xnn_create_average_pooling2d_nhwc_f32()
392   average_pooling_op->flags = flags;  in xnn_create_average_pooling2d_nhwc_f32()
394   *average_pooling_op_out = average_pooling_op;  in xnn_create_average_pooling2d_nhwc_f32()
398   xnn_delete_operator(average_pooling_op);  in xnn_create_average_pooling2d_nhwc_f32()
403   xnn_operator_t average_pooling_op,  in setup_average_pooling2d()  argument
423   average_pooling_op->state = xnn_run_state_invalid;  in setup_average_pooling2d()
427       xnn_operator_type_to_string(average_pooling_op->type));  in setup_average_pooling2d()
434       xnn_operator_type_to_string(average_pooling_op->type), input_width, input_height);  in setup_average_pooling2d()
439     average_pooling_op->state = xnn_run_state_skip;  in setup_average_pooling2d()
443   average_pooling_op->input_height = input_height;  in setup_average_pooling2d()
444   average_pooling_op->input_width = input_width;  in setup_average_pooling2d()
445   average_pooling_op->input = input;  in setup_average_pooling2d()
447   const bool tf_same_padding = (average_pooling_op->flags & XNN_FLAG_TENSORFLOW_SAME_PADDING) != 0;  in setup_average_pooling2d()
449     average_pooling_op->output_height = compute_output_dimension_with_tf_same_padding(  in setup_average_pooling2d()
450         input_height, average_pooling_op->stride_height);  in setup_average_pooling2d()
451     average_pooling_op->output_width = compute_output_dimension_with_tf_same_padding(  in setup_average_pooling2d()
452         input_width, average_pooling_op->stride_width);  in setup_average_pooling2d()
454     const uint32_t kernel_height = average_pooling_op->kernel_height;  in setup_average_pooling2d()
455     const uint32_t kernel_width = average_pooling_op->kernel_width;  in setup_average_pooling2d()
457 …(average_pooling_op->output_height - 1) * average_pooling_op->stride_height + kernel_height - inpu…  in setup_average_pooling2d()
459 …(average_pooling_op->output_width - 1) * average_pooling_op->stride_width + kernel_width - input_w…  in setup_average_pooling2d()
460     average_pooling_op->padding_top = total_padding_height / 2;  in setup_average_pooling2d()
461     average_pooling_op->padding_left = total_padding_width / 2;  in setup_average_pooling2d()
462     average_pooling_op->padding_bottom = total_padding_height - average_pooling_op->padding_top;  in setup_average_pooling2d()
463     average_pooling_op->padding_right = total_padding_width - average_pooling_op->padding_left;  in setup_average_pooling2d()
465     average_pooling_op->output_height = compute_output_dimension(  in setup_average_pooling2d()
466         average_pooling_op->padding_top + input_height + average_pooling_op->padding_bottom,  in setup_average_pooling2d()
467         average_pooling_op->kernel_height,  in setup_average_pooling2d()
468         average_pooling_op->stride_height);  in setup_average_pooling2d()
469     average_pooling_op->output_width = compute_output_dimension(  in setup_average_pooling2d()
470         average_pooling_op->padding_left + input_width + average_pooling_op->padding_right,  in setup_average_pooling2d()
471         average_pooling_op->kernel_width,  in setup_average_pooling2d()
472         average_pooling_op->stride_width);  in setup_average_pooling2d()
474   average_pooling_op->output = output;  in setup_average_pooling2d()
476   const size_t output_height = average_pooling_op->output_height;  in setup_average_pooling2d()
477   const size_t output_width = average_pooling_op->output_width;  in setup_average_pooling2d()
478 …const size_t padded_input_width = average_pooling_op->padding_left + input_width + average_pooling…  in setup_average_pooling2d()
479 …const size_t padded_input_height = average_pooling_op->padding_top + input_height + average_poolin…  in setup_average_pooling2d()
480 …if (padded_input_width == average_pooling_op->kernel_width && padded_input_height == average_pooli…  in setup_average_pooling2d()
483 …const size_t input_stride_in_bytes = average_pooling_op->input_pixel_stride << log2_input_element_…  in setup_average_pooling2d()
484     const size_t channels = average_pooling_op->channels;  in setup_average_pooling2d()
485 …average_pooling_op->context.global_average_pooling_nwc = (struct global_average_pooling_nwc_contex…  in setup_average_pooling2d()
487         .zero = average_pooling_op->zero_buffer,  in setup_average_pooling2d()
493         .output_batch_stride = average_pooling_op->output_pixel_stride << log2_output_element_size,  in setup_average_pooling2d()
495 …memcpy(&average_pooling_op->context.global_average_pooling_nwc.params, global_params, global_param…  in setup_average_pooling2d()
496     average_pooling_op->compute.type = xnn_parallelization_type_1d;  in setup_average_pooling2d()
497     average_pooling_op->compute.range[0] = batch_size;  in setup_average_pooling2d()
500 …average_pooling_op->compute.task_1d = (pthreadpool_task_1d_t) xnn_compute_global_average_pooling_n…  in setup_average_pooling2d()
501       average_pooling_op->context.global_average_pooling_nwc.unipass_ukernel = gavgpool->up;  in setup_average_pooling2d()
503 …average_pooling_op->compute.task_1d = (pthreadpool_task_1d_t) xnn_compute_global_average_pooling_n…  in setup_average_pooling2d()
504       average_pooling_op->context.global_average_pooling_nwc.multipass_ukernel = gavgpool->mp;  in setup_average_pooling2d()
508     const size_t pooling_height = average_pooling_op->kernel_height;  in setup_average_pooling2d()
509     const size_t pooling_width = average_pooling_op->kernel_width;  in setup_average_pooling2d()
514     const size_t step_width = min(average_pooling_op->stride_width, pooling_width);  in setup_average_pooling2d()
517     const size_t last_input_height = average_pooling_op->last_input_height;  in setup_average_pooling2d()
518     const size_t last_input_width = average_pooling_op->last_input_width;  in setup_average_pooling2d()
524 …(const void**) xnn_reallocate_memory(average_pooling_op->indirection_buffer, indirection_buffer_si…  in setup_average_pooling2d()
527           indirection_buffer_size, xnn_operator_type_to_string(average_pooling_op->type));  in setup_average_pooling2d()
530       average_pooling_op->indirection_buffer = indirection_buffer;  in setup_average_pooling2d()
532 …xnn_indirection_init_dwconv2d(average_pooling_op, step_height, step_width, log2_input_element_size…  in setup_average_pooling2d()
534       average_pooling_op->last_input = input;  in setup_average_pooling2d()
535       average_pooling_op->last_input_height = input_height;  in setup_average_pooling2d()
536       average_pooling_op->last_input_width = input_width;  in setup_average_pooling2d()
539     const size_t channels = average_pooling_op->channels;  in setup_average_pooling2d()
542 …const size_t output_width_stride = average_pooling_op->output_pixel_stride << log2_output_element_…  in setup_average_pooling2d()
550 …       (float*) xnn_reallocate_memory(average_pooling_op->pixelwise_buffer, pixelwise_buffer_size);  in setup_average_pooling2d()
553             pixelwise_buffer_size, xnn_operator_type_to_string(average_pooling_op->type));  in setup_average_pooling2d()
556         average_pooling_op->pixelwise_buffer = pixelwise_buffer;  in setup_average_pooling2d()
560 …const size_t input_y_start = doz(output_y * average_pooling_op->stride_height, average_pooling_op-…  in setup_average_pooling2d()
562 …min(doz(output_y * average_pooling_op->stride_height + average_pooling_op->kernel_height, average_…  in setup_average_pooling2d()
565 …const size_t input_x_start = doz(output_x * average_pooling_op->stride_width, average_pooling_op->…  in setup_average_pooling2d()
567 …min(doz(output_x * average_pooling_op->stride_width + average_pooling_op->kernel_width, average_po…  in setup_average_pooling2d()
577 …average_pooling_op->context.pixelwise_average_pooling = (struct pixelwise_average_pooling_context)…  in setup_average_pooling2d()
578         .indirect_input = average_pooling_op->indirection_buffer,  in setup_average_pooling2d()
580 ….input_batch_stride = input_height * input_width * average_pooling_op->input_pixel_stride << log2_…  in setup_average_pooling2d()
581         .input_offset = (size_t) ((uintptr_t) input - (uintptr_t) average_pooling_op->last_input),  in setup_average_pooling2d()
582         .pixelwise_buffer = average_pooling_op->pixelwise_buffer,  in setup_average_pooling2d()
590         .zero = average_pooling_op->zero_buffer,  in setup_average_pooling2d()
594       memcpy(&average_pooling_op->context.pixelwise_average_pooling.params, params, params_size);  in setup_average_pooling2d()
596         average_pooling_op->context.pixelwise_average_pooling.unipass_ukernel = pavgpool->up;  in setup_average_pooling2d()
597 …average_pooling_op->compute.task_2d = (pthreadpool_task_2d_t) xnn_compute_pixelwise_average_poolin…  in setup_average_pooling2d()
599         average_pooling_op->context.pixelwise_average_pooling.multipass_ukernel = pavgpool->mp;  in setup_average_pooling2d()
600 …average_pooling_op->compute.task_2d = (pthreadpool_task_2d_t) xnn_compute_pixelwise_average_poolin…  in setup_average_pooling2d()
606       average_pooling_op->context.average_pooling = (struct average_pooling_context) {  in setup_average_pooling2d()
607         .indirect_input = average_pooling_op->indirection_buffer,  in setup_average_pooling2d()
609         .input_offset = (size_t) ((uintptr_t) input - (uintptr_t) average_pooling_op->last_input),  in setup_average_pooling2d()
610 ….input_batch_stride = input_height * input_width * average_pooling_op->input_pixel_stride << log2_…  in setup_average_pooling2d()
617         .zero = average_pooling_op->zero_buffer,  in setup_average_pooling2d()
620         .params.f32 = average_pooling_op->params.f32_scaleminmax,  in setup_average_pooling2d()
622       memcpy(&average_pooling_op->context.average_pooling.params, params, params_size);  in setup_average_pooling2d()
624         average_pooling_op->context.average_pooling.unipass_ukernel = avgpool->up;  in setup_average_pooling2d()
625 … average_pooling_op->compute.task_2d = (pthreadpool_task_2d_t) xnn_compute_average_pooling_unipass;  in setup_average_pooling2d()
627         average_pooling_op->context.average_pooling.multipass_ukernel = avgpool->mp;  in setup_average_pooling2d()
628 …average_pooling_op->compute.task_2d = (pthreadpool_task_2d_t) xnn_compute_average_pooling_multipas…  in setup_average_pooling2d()
631     average_pooling_op->compute.type = xnn_parallelization_type_2d;  in setup_average_pooling2d()
632     average_pooling_op->compute.range[0] = batch_size;  in setup_average_pooling2d()
633     average_pooling_op->compute.range[1] = output_height;  in setup_average_pooling2d()
635   average_pooling_op->state = xnn_run_state_ready;  in setup_average_pooling2d()
641     xnn_operator_t average_pooling_op,  in xnn_setup_average_pooling2d_nhwc_qu8()  argument
649   if (average_pooling_op->type != xnn_operator_type_average_pooling_nhwc_qu8) {  in xnn_setup_average_pooling2d_nhwc_qu8()
652       xnn_operator_type_to_string(average_pooling_op->type));  in xnn_setup_average_pooling2d_nhwc_qu8()
656   assert(average_pooling_op->ukernel.type == xnn_ukernel_type_average_pooling);  in xnn_setup_average_pooling2d_nhwc_qu8()
660   const size_t pooling_size = average_pooling_op->kernel_height * average_pooling_op->kernel_width;  in xnn_setup_average_pooling2d_nhwc_qu8()
662   average_pooling_op->params.qu8_gavgpool =  in xnn_setup_average_pooling2d_nhwc_qu8()
664       -(average_pooling_op->input_zero_point * (int32_t) gavgpool_nrows),  in xnn_setup_average_pooling2d_nhwc_qu8()
665       average_pooling_op->input_scale / (average_pooling_op->output_scale * (float) pooling_size),  in xnn_setup_average_pooling2d_nhwc_qu8()
666       average_pooling_op->output_zero_point,  in xnn_setup_average_pooling2d_nhwc_qu8()
667       average_pooling_op->output_min,  in xnn_setup_average_pooling2d_nhwc_qu8()
668       average_pooling_op->output_max);  in xnn_setup_average_pooling2d_nhwc_qu8()
671     average_pooling_op,  in xnn_setup_average_pooling2d_nhwc_qu8()
679     &average_pooling_op->params.qu8_avgpool,  in xnn_setup_average_pooling2d_nhwc_qu8()
680     sizeof(average_pooling_op->params.qu8_avgpool),  in xnn_setup_average_pooling2d_nhwc_qu8()
681     &average_pooling_op->params.qu8_gavgpool,  in xnn_setup_average_pooling2d_nhwc_qu8()
682     sizeof(average_pooling_op->params.qu8_gavgpool),  in xnn_setup_average_pooling2d_nhwc_qu8()
688     xnn_operator_t average_pooling_op,  in xnn_setup_average_pooling2d_nhwc_f32()  argument
696   if (average_pooling_op->type != xnn_operator_type_average_pooling_nhwc_f32) {  in xnn_setup_average_pooling2d_nhwc_f32()
699       xnn_operator_type_to_string(average_pooling_op->type));  in xnn_setup_average_pooling2d_nhwc_f32()
703   assert(average_pooling_op->ukernel.type == xnn_ukernel_type_average_pooling ||  in xnn_setup_average_pooling2d_nhwc_f32()
704          average_pooling_op->ukernel.type == xnn_ukernel_type_pixelwise_average_pooling);  in xnn_setup_average_pooling2d_nhwc_f32()
706 …const bool is_pixelwise = average_pooling_op->ukernel.type == xnn_ukernel_type_pixelwise_average_p…  in xnn_setup_average_pooling2d_nhwc_f32()
709 …xnn_update_f32_scaleminmax_params(&average_pooling_op->params.f32_scaleminmax, 1.0f / (float) inpu…  in xnn_setup_average_pooling2d_nhwc_f32()
713     average_pooling_op,  in xnn_setup_average_pooling2d_nhwc_f32()
721 …is_pixelwise ? (const void*) &average_pooling_op->params.f32_minmax : (const void*) &average_pooli…  in xnn_setup_average_pooling2d_nhwc_f32()
722 …is_pixelwise ? sizeof(average_pooling_op->params.f32_minmax) : sizeof(average_pooling_op->params.f…  in xnn_setup_average_pooling2d_nhwc_f32()
723     &average_pooling_op->params.f32_scaleminmax,  in xnn_setup_average_pooling2d_nhwc_f32()
724     sizeof(average_pooling_op->params.f32_scaleminmax),  in xnn_setup_average_pooling2d_nhwc_f32()