Layer Normalization

Overview

// global functions

dnnl_status_t DNNL_API dnnl_layer_normalization_forward_primitive_desc_create(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t dst_desc,
    const_dnnl_memory_desc_t stat_desc,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_attr_t attr
    );

dnnl_status_t DNNL_API dnnl_layer_normalization_backward_primitive_desc_create(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t diff_src_desc,
    const_dnnl_memory_desc_t diff_dst_desc,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t stat_desc,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_desc_t hint_fwd_pd,
    const_dnnl_primitive_attr_t attr
    );

dnnl_status_t DNNL_API dnnl_layer_normalization_forward_primitive_desc_create_v2(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t dst_desc,
    const_dnnl_memory_desc_t stat_desc,
    dnnl_data_type_t scale_shift_data_type,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_attr_t attr
    );

dnnl_status_t DNNL_API dnnl_layer_normalization_backward_primitive_desc_create_v2(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t diff_src_desc,
    const_dnnl_memory_desc_t diff_dst_desc,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t stat_desc,
    dnnl_data_type_t diff_scale_shift_data_type,
    dnnl_data_type_t scale_shift_data_type,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_desc_t hint_fwd_pd,
    const_dnnl_primitive_attr_t attr
    );

Detailed Documentation

Global Functions

dnnl_status_t DNNL_API dnnl_layer_normalization_forward_primitive_desc_create(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t dst_desc,
    const_dnnl_memory_desc_t stat_desc,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_attr_t attr
    )

Creates a primitive descriptor for a layer normalization forward propagation primitive.

Note

In-place operation is supported: the dst can refer to the same memory as the src.

Parameters:

primitive_desc	Output primitive_descriptor.
engine	Engine to use.
prop_kind	Propagation kind. Possible values are dnnl_forward_training and dnnl_forward_inference.
src_desc	Source memory descriptor.
dst_desc	Destination memory descriptor.
stat_desc	Memory descriptor for mean and variance. If this parameter is NULL, a zero memory descriptor, or a memory descriptor with format_kind set to dnnl_format_kind_undef, then the memory descriptor for stats is derived from src_desc by removing the last dimension.
epsilon	Layer normalization epsilon parameter.
flags	Layer normalization flags (dnnl_normalization_flags_t).
attr	Primitive attributes (can be NULL).

Returns:

dnnl_success on success and a status describing the error otherwise.

dnnl_status_t DNNL_API dnnl_layer_normalization_backward_primitive_desc_create(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t diff_src_desc,
    const_dnnl_memory_desc_t diff_dst_desc,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t stat_desc,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_desc_t hint_fwd_pd,
    const_dnnl_primitive_attr_t attr
    )

Creates a primitive descriptor for a layer normalization backward propagation primitive.

Note

In-place operation is supported: the diff_dst can refer to the same memory as the diff_src.

Parameters:

primitive_desc	Output primitive_descriptor.
engine	Engine to use.
prop_kind	Propagation kind. Possible values are dnnl_backward_data and dnnl_backward (diffs for all parameters are computed in this case).
diff_src_desc	Diff source memory descriptor.
diff_dst_desc	Diff destination memory descriptor.
src_desc	Source memory descriptor.
stat_desc	Memory descriptor for mean and variance. If this parameter is NULL, a zero memory descriptor, or a memory descriptor with format_kind set to dnnl_format_kind_undef, then the memory descriptor for stats is derived from src_desc by removing the last dimension.
epsilon	Layer normalization epsilon parameter.
flags	Layer normalization flags (dnnl_normalization_flags_t).
hint_fwd_pd	Primitive descriptor for a respective forward propagation primitive.
attr	Primitive attributes (can be NULL).

Returns:

dnnl_success on success and a status describing the error otherwise.

dnnl_status_t DNNL_API dnnl_layer_normalization_forward_primitive_desc_create_v2(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t dst_desc,
    const_dnnl_memory_desc_t stat_desc,
    dnnl_data_type_t scale_shift_data_type,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_attr_t attr
    )

Creates a primitive descriptor for a layer normalization forward propagation primitive with a user-provided data type for the scale and shift memory objects.

Note

In-place operation is supported: the dst can refer to the same memory as the src.

Parameters:

primitive_desc	Output primitive_descriptor.
engine	Engine to use.
prop_kind	Propagation kind. Possible values are dnnl_forward_training and dnnl_forward_inference.
src_desc	Source memory descriptor.
dst_desc	Destination memory descriptor.
stat_desc	Memory descriptor for mean and variance. If this parameter is NULL, a zero memory descriptor, or a memory descriptor with format_kind set to dnnl_format_kind_undef, then the memory descriptor for stats is derived from src_desc by removing the last dimension.
scale_shift_data_type	Data type of scale and shift memory. If neither scale nor shift flag are specified the parameter is ignored.
epsilon	Layer normalization epsilon parameter.
flags	Layer normalization flags (dnnl_normalization_flags_t).
attr	Primitive attributes (can be NULL).

Returns:

dnnl_success on success and a status describing the error otherwise.

dnnl_status_t DNNL_API dnnl_layer_normalization_backward_primitive_desc_create_v2(
    dnnl_primitive_desc_t* primitive_desc,
    dnnl_engine_t engine,
    dnnl_prop_kind_t prop_kind,
    const_dnnl_memory_desc_t diff_src_desc,
    const_dnnl_memory_desc_t diff_dst_desc,
    const_dnnl_memory_desc_t src_desc,
    const_dnnl_memory_desc_t stat_desc,
    dnnl_data_type_t diff_scale_shift_data_type,
    dnnl_data_type_t scale_shift_data_type,
    float epsilon,
    unsigned flags,
    const_dnnl_primitive_desc_t hint_fwd_pd,
    const_dnnl_primitive_attr_t attr
    )

Creates a primitive descriptor for a layer normalization backward propagation primitive with a user-provided data type for the scale and shift memory objects.

Note

In-place operation is supported: the diff_dst can refer to the same memory as the diff_src.

Parameters:

primitive_desc	Output primitive_descriptor.
engine	Engine to use.
prop_kind	Propagation kind. Possible values are dnnl_backward_data and dnnl_backward (diffs for all parameters are computed in this case).
diff_src_desc	Diff source memory descriptor.
diff_dst_desc	Diff destination memory descriptor.
src_desc	Source memory descriptor.
stat_desc	Memory descriptor for mean and variance. If this parameter is NULL, a zero memory descriptor, or a memory descriptor with format_kind set to dnnl_format_kind_undef, then the memory descriptor for stats is derived from src_desc by removing the last dimension.
diff_scale_shift_data_type	Data type of diff scale and shift memory. If neither scale nor shift flag are specified the parameter is ignored.
scale_shift_data_type	Data type of scale and shift memory. If neither scale nor shift flag are specified the parameter is ignored.
epsilon	Layer normalization epsilon parameter.
flags	Layer normalization flags (dnnl_normalization_flags_t).
hint_fwd_pd	Primitive descriptor for a respective forward propagation primitive.
attr	Primitive attributes (can be NULL).

Returns:

dnnl_success on success and a status describing the error otherwise.