.. index:: pair: example; cpu_inference_int8.cpp
.. _doxid-cpu_inference_int8_8cpp-example:

cpu_inference_int8.cpp
======================

This is an example to demonstrate how to build an int8 graph with Graph API and run it on CPU. Annotated version: :ref:`Convolution int8 inference example with Graph API <doxid-graph_cpu_inference_int8_cpp>`

This is an example to demonstrate how to build an int8 graph with Graph API and run it on CPU. Annotated version: :ref:`Convolution int8 inference example with Graph API <doxid-graph_cpu_inference_int8_cpp>`

.. ref-code-block:: cpp

	/*******************************************************************************
	* Copyright 2023-2024 Intel Corporation
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	*     http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*******************************************************************************/
	
	
	
	//[Headers and namespace]
	#include <iostream>
	#include <memory>
	#include <vector>
	#include <unordered_map>
	#include <unordered_set>
	
	#include <assert.h>
	
	#include "oneapi/dnnl/dnnl_graph.hpp"
	
	#include "example_utils.hpp"
	#include "graph_example_utils.hpp"
	
	using namespace :ref:`dnnl::graph <doxid-namespacednnl_1_1graph>`;
	using :ref:`data_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1acddb1dc65b7b4feede7710a719f32227>` = :ref:`logical_tensor::data_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1acddb1dc65b7b4feede7710a719f32227>`;
	using :ref:`layout_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1ad3fcaff44671577e56adb03b770f4867>` = :ref:`logical_tensor::layout_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1ad3fcaff44671577e56adb03b770f4867>`;
	using :ref:`property_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1a037ba7c242d8127d2f42c0c2aef29d82>` = :ref:`logical_tensor::property_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1a037ba7c242d8127d2f42c0c2aef29d82>`;
	using dim = :ref:`logical_tensor::dim <doxid-classdnnl_1_1graph_1_1logical__tensor_1a759c7b96472681049e17716334a2b334>`;
	using dims = :ref:`logical_tensor::dims <doxid-classdnnl_1_1graph_1_1logical__tensor_1a31af724d1ea783a09b6900d69b43ddc7>`;
	//[Headers and namespace]
	
	void simple_pattern_int8() {
	
	    dim N = 8, IC = 256, IH = 56, IW = 56, KH = 1, KW = 1, OC = 64;
	
	    dims conv_input_dims {N, IH, IW, IC};
	    dims conv_weight_dims {KH, KW, IC, OC};
	    dims conv_bias_dims {OC};
	
	    //[Create dequant's logical tensor and the op]
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` dequant0_src_desc {0, data_type::u8};
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` conv_src_desc {1, :ref:`data_type::f32 <doxid-group__dnnl__api__accumulation__mode_1ggad6b8b3ca2e61b8a9703227f4d58ac215a512dc597be7ae761876315165dc8bd2e>`};
	    :ref:`op <doxid-classdnnl_1_1graph_1_1op>` dequant0(2, op::kind::Dequantize, {dequant0_src_desc}, {conv_src_desc},
	            "dequant0");
	    dequant0.:ref:`set_attr <doxid-classdnnl_1_1graph_1_1op_1aecb46529826bf8a47664757a56b92a0d>`<std::string>(:ref:`op::attr::qtype <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a63da59315662c87a47b7a1a4847e675e>`, "per_tensor");
	    dequant0.:ref:`set_attr <doxid-classdnnl_1_1graph_1_1op_1aecb46529826bf8a47664757a56b92a0d>`<std::vector<float>>(:ref:`op::attr::scales <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a8e7bb02b763a2e07d30b4ab24beb7fa1>`, {0.1f});
	    dequant0.:ref:`set_attr <doxid-classdnnl_1_1graph_1_1op_1aecb46529826bf8a47664757a56b92a0d>`<std::vector<int64_t>>(:ref:`op::attr::zps <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a5c284a074767998e9708c3656d41a91c>`, {10});
	    //[Create dequant's logical tensor and the op]
	
	    //[Create dequant's logical tensor and the op.]
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` dequant1_src_desc {3, data_type::s8};
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` conv_weight_desc {
	            4, :ref:`data_type::f32 <doxid-group__dnnl__api__accumulation__mode_1ggad6b8b3ca2e61b8a9703227f4d58ac215a512dc597be7ae761876315165dc8bd2e>`, 4, :ref:`layout_type::undef <doxid-group__dnnl__api__attributes_1ggac7db48f6583aa9903e54c2a39d65438faf31ee5e3824f1f5e5d206bdf3029f22b>`, property_type::constant};
	    :ref:`op <doxid-classdnnl_1_1graph_1_1op>` dequant1(5, op::kind::Dequantize, {dequant1_src_desc},
	            {conv_weight_desc}, "dequant1");
	    dequant1.:ref:`set_attr <doxid-classdnnl_1_1graph_1_1op_1aecb46529826bf8a47664757a56b92a0d>`<std::string>(:ref:`op::attr::qtype <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a63da59315662c87a47b7a1a4847e675e>`, "per_channel");
	    // the memory format of weight is XIO, which indicates channel equals
	    // to 64 for the convolution.
	    std::vector<float> wei_scales(64, 0.1f);
	    dims wei_zps(64, 0);
	    dequant1.set_attr<std::vector<float>>(:ref:`op::attr::scales <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a8e7bb02b763a2e07d30b4ab24beb7fa1>`, wei_scales);
	    dequant1.set_attr<std::vector<int64_t>>(:ref:`op::attr::zps <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a5c284a074767998e9708c3656d41a91c>`, wei_zps);
	    dequant1.set_attr<int64_t>(:ref:`op::attr::axis <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a433169d5d9bcbb6d43f0d288e68f0cad>`, 1);
	    //[Create dequant's logical tensor and the op.]
	
	    //[Create conv's logical tensor and the op]
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` conv_bias_desc {
	            6, :ref:`data_type::f32 <doxid-group__dnnl__api__accumulation__mode_1ggad6b8b3ca2e61b8a9703227f4d58ac215a512dc597be7ae761876315165dc8bd2e>`, 1, :ref:`layout_type::undef <doxid-group__dnnl__api__attributes_1ggac7db48f6583aa9903e54c2a39d65438faf31ee5e3824f1f5e5d206bdf3029f22b>`, property_type::constant};
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` conv_dst_desc {7, :ref:`data_type::f32 <doxid-group__dnnl__api__accumulation__mode_1ggad6b8b3ca2e61b8a9703227f4d58ac215a512dc597be7ae761876315165dc8bd2e>`, :ref:`layout_type::undef <doxid-group__dnnl__api__attributes_1ggac7db48f6583aa9903e54c2a39d65438faf31ee5e3824f1f5e5d206bdf3029f22b>`};
	
	    // create the convolution op
	    :ref:`op <doxid-classdnnl_1_1graph_1_1op>` conv(8, op::kind::Convolution,
	            {conv_src_desc, conv_weight_desc, conv_bias_desc}, {conv_dst_desc},
	            "conv");
	    conv.set_attr<dims>(:ref:`op::attr::strides <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a3372f3d8ac7d6db0997a8fe6b38d549a>`, {1, 1});
	    conv.set_attr<dims>(:ref:`op::attr::pads_begin <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684ad9563b69290681059378cb6b98127310>`, {0, 0});
	    conv.set_attr<dims>(:ref:`op::attr::pads_end <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684ae9dcd3256fd8b6e2b6385091cffe2cd6>`, {0, 0});
	    conv.set_attr<dims>(:ref:`op::attr::dilations <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684acbcf9c952f6e423b94fe04593665b49e>`, {1, 1});
	    conv.set_attr<std::string>(:ref:`op::attr::data_format <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a4abbd547d2eb3887fd8613bb8be33cc5>`, "NXC");
	    conv.set_attr<std::string>(:ref:`op::attr::weights_format <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a51c305464b90b1e5e4092ccfb5e904a7>`, "XIO");
	    conv.set_attr<int64_t>(:ref:`op::attr::groups <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a1471e4e05a4db95d353cc867fe317314>`, 1);
	    //[Create conv's logical tensor and the op]
	
	    //[Create ReLu's logical tensor and the op]
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` relu_dst_desc {9, :ref:`data_type::f32 <doxid-group__dnnl__api__accumulation__mode_1ggad6b8b3ca2e61b8a9703227f4d58ac215a512dc597be7ae761876315165dc8bd2e>`, :ref:`layout_type::undef <doxid-group__dnnl__api__attributes_1ggac7db48f6583aa9903e54c2a39d65438faf31ee5e3824f1f5e5d206bdf3029f22b>`};
	    :ref:`op <doxid-classdnnl_1_1graph_1_1op>` relu(10, op::kind::ReLU, {conv_dst_desc}, {relu_dst_desc}, "relu");
	    //[Create ReLu's logical tensor and the op]
	
	    //[Create Quantize's logical tensor and the op]
	    :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` quant_dst_desc {11, data_type::u8, :ref:`layout_type::undef <doxid-group__dnnl__api__attributes_1ggac7db48f6583aa9903e54c2a39d65438faf31ee5e3824f1f5e5d206bdf3029f22b>`};
	    :ref:`op <doxid-classdnnl_1_1graph_1_1op>` quant(
	            12, op::kind::Quantize, {relu_dst_desc}, {quant_dst_desc}, "quant");
	    quant.:ref:`set_attr <doxid-classdnnl_1_1graph_1_1op_1aecb46529826bf8a47664757a56b92a0d>`<std::string>(:ref:`op::attr::qtype <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a63da59315662c87a47b7a1a4847e675e>`, "per_tensor");
	    quant.:ref:`set_attr <doxid-classdnnl_1_1graph_1_1op_1aecb46529826bf8a47664757a56b92a0d>`<std::vector<float>>(:ref:`op::attr::scales <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a8e7bb02b763a2e07d30b4ab24beb7fa1>`, {0.1f});
	    quant.:ref:`set_attr <doxid-classdnnl_1_1graph_1_1op_1aecb46529826bf8a47664757a56b92a0d>`<std::vector<int64_t>>(:ref:`op::attr::zps <doxid-classdnnl_1_1graph_1_1op_1ac7650c0c15849338f9c558f53ce82684a5c284a074767998e9708c3656d41a91c>`, {10});
	    //[Create Quantize's logical tensor and the op]
	
	    //[Create graph and add ops]
	    :ref:`graph <doxid-classdnnl_1_1graph_1_1graph>` g(:ref:`dnnl::engine::kind::cpu <doxid-structdnnl_1_1engine_1a2635da16314dcbdb9bd9ea431316bb1aad9747e2da342bdb995f6389533ad1a3d>`);
	
	    g.add_op(dequant0);
	    g.add_op(dequant1);
	    g.add_op(conv);
	    g.add_op(relu);
	    g.add_op(quant);
	    //[Create graph and add ops]
	
	    g.finalize();
	
	    //[Get partition]
	    auto partitions = g.get_partitions();
	    //[Get partition]
	
	    // Check partitioning results to ensure the examples works. Users do
	    // not need to follow this step.
	    assert(partitions.size() == 1);
	
	    //[Create engine]
	    :ref:`allocator <doxid-classdnnl_1_1graph_1_1allocator>` alloc {};
	    :ref:`dnnl::engine <doxid-structdnnl_1_1engine>` eng
	            = :ref:`make_engine_with_allocator <doxid-group__dnnl__graph__api__engine_1ga42ac93753b2a12d14b29704fe3b0b2fa>`(:ref:`dnnl::engine::kind::cpu <doxid-structdnnl_1_1engine_1a2635da16314dcbdb9bd9ea431316bb1aad9747e2da342bdb995f6389533ad1a3d>`, 0, alloc);
	    :ref:`dnnl::stream <doxid-structdnnl_1_1stream>` strm {eng};
	    //[Create engine]
	
	    // Mapping from logical tensor id to output tensors
	    // used to the connection relationship between partitions (e.g partition 0's
	    // output tensor is fed into partition 1)
	    std::unordered_map<size_t, tensor> global_outputs_ts_map;
	
	    // Memory buffers bound to the partition input/output tensors
	    // that helps manage the lifetime of these tensors
	    std::vector<std::shared_ptr<void>> data_buffer;
	
	    // Mapping from id to queried logical tensor from compiled partition
	    // used to record the logical tensors that are previously enabled with
	    // ANY layout
	    std::unordered_map<size_t, logical_tensor> id_to_queried_logical_tensors;
	
	    // This is a helper function which helps decide which logical tensor is
	    // needed to be set with `dnnl::graph::logical_tensor::layout_type::any`
	    // layout.
	    // This function is not a part to Graph API, but similar logic is
	    // essential for Graph API integration to achieve best performance.
	    // Typically, users need implement the similar logic in their code.
	    std::unordered_set<size_t> ids_with_any_layout;
	    set_any_layout(partitions, ids_with_any_layout);
	
	    // Mapping from logical tensor id to the concrete shapes.
	    // In practical usage, concrete shapes and layouts are not given
	    // until compilation stage, hence need this mapping to mock the step.
	    std::unordered_map<size_t, dims> concrete_shapes {
	            {0, conv_input_dims}, {3, conv_weight_dims}, {6, conv_bias_dims}};
	
	    // Compile and execute the partitions, including the following steps:
	    //
	    // 1. Update the input/output logical tensors with concrete shape and layout
	    // 2. Compile the partition
	    // 3. Update the output logical tensors with queried ones after compilation
	    // 4. Allocate memory and bind the data buffer for the partition
	    // 5. Execute the partition
	    //
	    // Although they are not part of the APIs, these steps are essential for
	    // the integration of Graph API., hence users need to implement similar
	    // logic.
	    for (const auto &:ref:`partition <doxid-classdnnl_1_1graph_1_1partition>` : partitions) {
	
	        if (!:ref:`partition <doxid-classdnnl_1_1graph_1_1partition>`.:ref:`is_supported <doxid-classdnnl_1_1graph_1_1partition_1ad80536833d69e2660c496adbd9ec0aa3>`()) {
	            std::cout << "cpu_inference_int8: Got unsupported partition, users "
	                         "need handle the operators by themselves."
	                      << std::endl;
	            continue;
	        }
	        std::vector<logical_tensor> inputs = :ref:`partition <doxid-classdnnl_1_1graph_1_1partition>`.:ref:`get_input_ports <doxid-classdnnl_1_1graph_1_1partition_1a415319dcb89d9e1d77bd4b7b0058df52>`();
	        std::vector<logical_tensor> outputs = :ref:`partition <doxid-classdnnl_1_1graph_1_1partition>`.:ref:`get_output_ports <doxid-classdnnl_1_1graph_1_1partition_1aaa4abecc6e09f417742402ab207a1e6d>`();
	
	        // Update input logical tensors with concrete shape and layout
	        for (auto &input : inputs) {
	            const auto id = input.get_id();
	            // If the tensor is an output of another partition,
	            // use the cached logical tensor
	            if (id_to_queried_logical_tensors.find(id)
	                    != id_to_queried_logical_tensors.end())
	                input = id_to_queried_logical_tensors[id];
	            else
	                // Create logical tensor with strided layout
	                input = :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` {id, input.:ref:`get_data_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1aaea19b3ce4512e5f2e1d0c68d9f0677f>`(),
	                        concrete_shapes[id], layout_type::strided};
	        }
	
	        // Update output logical tensors with concrete shape and layout
	        for (auto &output : outputs) {
	            const auto id = output.get_id();
	            output = :ref:`logical_tensor <doxid-classdnnl_1_1graph_1_1logical__tensor>` {id, output.:ref:`get_data_type <doxid-classdnnl_1_1graph_1_1logical__tensor_1aaea19b3ce4512e5f2e1d0c68d9f0677f>`(),
	                    :ref:`DNNL_GRAPH_UNKNOWN_NDIMS <doxid-group__dnnl__graph__api__logical__tensor_1ga49497533d28f67dc4cce08fe210bf4bf>`, // set output dims to unknown
	                    ids_with_any_layout.count(id) ? :ref:`layout_type::any <doxid-group__dnnl__api__fpmath__mode_1gga0ad94cbef13dce222933422bfdcfa725a100b8cad7cf2a56f6df78f171f97a1ec>`
	                                                  : layout_type::strided};
	        }
	
	        //[Compile partition]
	        :ref:`compiled_partition <doxid-classdnnl_1_1graph_1_1compiled__partition>` cp = :ref:`partition <doxid-classdnnl_1_1graph_1_1partition>`.:ref:`compile <doxid-classdnnl_1_1graph_1_1partition_1a5c2af93c65a09c9d0a1507571ada0318>`(inputs, outputs, eng);
	        //[Compile partition]
	
	        // Update output logical tensors with queried one
	        for (auto &output : outputs) {
	            const auto id = output.get_id();
	            output = cp.:ref:`query_logical_tensor <doxid-classdnnl_1_1graph_1_1compiled__partition_1a85962826e94cc3cefb3c19c0fadc4e09>`(id);
	            id_to_queried_logical_tensors[id] = output;
	        }
	
	        // Allocate memory for the partition, and bind the data buffers with
	        // input and output logical tensors
	        std::vector<tensor> inputs_ts, outputs_ts;
	        allocate_graph_mem(inputs_ts, inputs, data_buffer,
	                global_outputs_ts_map, eng, /*is partition input=*/true);
	        allocate_graph_mem(outputs_ts, outputs, data_buffer,
	                global_outputs_ts_map, eng, /*is partition input=*/false);
	
	        //[Execute compiled partition]
	        cp.:ref:`execute <doxid-classdnnl_1_1graph_1_1compiled__partition_1a558ed47b3cbc5cc2167001da3faa0339>`(strm, inputs_ts, outputs_ts);
	        //[Execute compiled partition]
	    }
	
	    // wait for all compiled partition's execution finished
	    strm.:ref:`wait <doxid-structdnnl_1_1stream_1a59985fa8746436057cf51a820ef8929c>`();
	
	    std::cout << "Graph:" << std::endl
	              << " [dq0_src]   [dq1_src]" << std::endl
	              << "    |            |" << std::endl
	              << " dequant0    dequant1" << std::endl
	              << "       \\      /" << std::endl
	              << "         conv" << std::endl
	              << "          |" << std::endl
	              << "         relu" << std::endl
	              << "          |" << std::endl
	              << "        quant" << std::endl
	              << "          |" << std::endl
	              << "     [quant_dst]" << std::endl
	              << "Note:" << std::endl
	              << " '[]' represents a logical tensor, which refers to "
	                 "inputs/outputs of the graph. "
	              << std::endl;
	}
	
	int main(int argc, char **argv) {
	    return handle_example_errors({:ref:`engine::kind::cpu <doxid-structdnnl_1_1engine_1a2635da16314dcbdb9bd9ea431316bb1aad9747e2da342bdb995f6389533ad1a3d>`}, simple_pattern_int8);
	}