diff --git a/CMakeLists.txt b/CMakeLists.txt index 769a35318d9d..a6d8c75c5151 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,6 +106,7 @@ if(MSVC) set(CMAKE_SUPPRESS_REGENERATION ON) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /bigobj") if(USE_MSVC_MT) foreach(flag_var diff --git a/include/tvm/relay/qnn/attrs.h b/include/tvm/relay/qnn/attrs.h index c5213fe07471..17317098a81d 100644 --- a/include/tvm/relay/qnn/attrs.h +++ b/include/tvm/relay/qnn/attrs.h @@ -32,6 +32,25 @@ namespace tvm { namespace relay { namespace qnn { +/*! \brief Attribute for qnn add operator */ +struct QnnAddAttrs : public tvm::AttrsNode { + std::string rounding; + + TVM_DECLARE_ATTRS(QnnAddAttrs, "relay.attrs.QnnAddAttrs") { + TVM_ATTR_FIELD(rounding).set_default("UPWARD").describe( + "Defines the rounding direction when the value is midway between" + "two representable values. There are two 3 modes - UPWARD, TONEAREST" + "or TFLITE. UP/TONEAREST modes behave exactly same except at the" + "midpoints between the two representable values. At the midpoint," + "UPWARD rounds towards positive infinity (for example -1.5 will be" + "rounded to -1). TONEAREST is the standard rounding where the" + "value is rounded away from zero at midpoints (for example, -1.5" + "rounds to -2). More context can be found at following glibc manual" + "https://www.gnu.org/software/libc/manual/html_node/Rounding.html." + "TFLITE mode is more complicated, referring to tflite implementation."); + } +}; + /*! \brief Attribute for requantize operator */ struct RequantizeAttrs : public tvm::AttrsNode { int axis; @@ -46,14 +65,15 @@ struct RequantizeAttrs : public tvm::AttrsNode { .set_default(-1); TVM_ATTR_FIELD(rounding).set_default("UPWARD").describe( "Defines the rounding direction when the value is midway between" - "two representable values. There are two supported modes - UPWARD" - "or TONEAREST. Both modes behave exactly same except at the" + "two representable values. There are two 3 modes - UPWARD, TONEAREST" + "or TFLITE. UP/TONEAREST modes behave exactly same except at the" "midpoints between the two representable values. At the midpoint," "UPWARD rounds towards positive infinity (for example -1.5 will be" "rounded to -1). TONEAREST is the standard rounding where the" "value is rounded away from zero at midpoints (for example, -1.5" - "rounds to -2). More context can be found at following gblic manual" - "https://www.gnu.org/software/libc/manual/html_node/Rounding.html."); + "rounds to -2). More context can be found at following glibc manual" + "https://www.gnu.org/software/libc/manual/html_node/Rounding.html." + "TFLITE mode is more complicated, referring to tflite implementation."); TVM_ATTR_FIELD(out_dtype) .set_default(NullValue()) .describe("Output data type, set to explicit type under mixed precision setting"); diff --git a/include/tvm/tir/analysis.h b/include/tvm/tir/analysis.h index e5b2c2b6957c..c953641916fe 100644 --- a/include/tvm/tir/analysis.h +++ b/include/tvm/tir/analysis.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,55 @@ struct ExprDeepEqual { TVM_DLL bool operator()(const PrimExpr& lhs, const PrimExpr& rhs) const; }; +#define PLUS_ONE(OP) \ + void VisitExpr_(const OP* op) final { num_symbols_++; } + +#define PLUS_ONE_BINARY(OP) \ + void VisitExpr_(const OP* op) final { \ + num_symbols_++; \ + VisitExpr(op->a); \ + VisitExpr(op->b); \ + } + +/*! + * \brief Calculate the expresion complexity based on number of symbols it contains. + */ +class ExprComplexity : public ExprVisitor { + public: + TVM_DLL size_t Eval(const PrimExpr& expr) { + VisitExpr(expr); + return num_symbols_; + } + + PLUS_ONE_BINARY(AddNode) + PLUS_ONE_BINARY(SubNode) + PLUS_ONE_BINARY(MulNode) + PLUS_ONE_BINARY(DivNode) + PLUS_ONE_BINARY(ModNode) + PLUS_ONE_BINARY(FloorDivNode) + PLUS_ONE_BINARY(FloorModNode) + PLUS_ONE_BINARY(MinNode) + PLUS_ONE_BINARY(MaxNode) + PLUS_ONE_BINARY(EQNode) + PLUS_ONE_BINARY(NENode) + PLUS_ONE_BINARY(LTNode) + PLUS_ONE_BINARY(LENode) + PLUS_ONE_BINARY(GTNode) + PLUS_ONE_BINARY(GENode) + PLUS_ONE_BINARY(AndNode) + PLUS_ONE_BINARY(OrNode) + PLUS_ONE(VarNode) + PLUS_ONE(FloatImmNode) + PLUS_ONE(IntImmNode) + void VisitExpr_(const NotNode* op) final { + num_symbols_++; + VisitExpr(op->a); + } + + private: + size_t num_symbols_{0}; +}; + /*! * \brief Find undefined vars in the statement. * \param stmt The function to be checked. diff --git a/include/tvm/topi/nn/pooling.h b/include/tvm/topi/nn/pooling.h index 8c30e673b304..bcaa24bc8d2e 100644 --- a/include/tvm/topi/nn/pooling.h +++ b/include/tvm/topi/nn/pooling.h @@ -149,27 +149,52 @@ inline Tensor pool_impl(const Tensor& x, const Array& kernel_size, "tensor", "pool_sum"); // TVM compute for dividing the reduced window sum by kernel size. - return tvm::te::compute( - out_shape, - [&](const Array& output) { - Array indices; - for (const Var& var : output) indices.push_back(var); - if (count_include_pad) { - return div(pool_sum(indices), (kernel_height * kernel_width)); - } else { - PrimExpr h_start = output[height_axis] * stride_height - pad_top; - PrimExpr w_start = output[width_axis] * stride_width - pad_left; - - PrimExpr h_end = min(h_start + kernel_height, height); - PrimExpr w_end = min(w_start + kernel_width, width); - h_start = max(h_start, make_const(DataType::DataType::Int(32), 0)); - w_start = max(w_start, make_const(DataType::DataType::Int(32), 0)); - PrimExpr divide_factor = max((h_end - h_start) * (w_end - w_start), - make_const(DataType::DataType::Int(32), 1)); - return div(pool_sum(indices), divide_factor); - } - }, - "tensor", kElementWise); + if (x->dtype.code() == DataType::kInt || x->dtype.code() == DataType::kUInt) { + return tvm::te::compute( + out_shape, + [&](const Array& output) { + Array indices; + for (const Var& var : output) indices.push_back(var); + if (count_include_pad) { + PrimExpr kernel_size = kernel_height * kernel_width; + PrimExpr up_rounder = floordiv(kernel_size, 2); + return floordiv(pool_sum(indices) + up_rounder, kernel_size); + } else { + PrimExpr h_start = output[height_axis] * stride_height - pad_top; + PrimExpr w_start = output[width_axis] * stride_width - pad_left; + PrimExpr h_end = min(h_start + kernel_height, height); + PrimExpr w_end = min(w_start + kernel_width, width); + h_start = max(h_start, make_const(DataType::DataType::Int(32), 0)); + w_start = max(w_start, make_const(DataType::DataType::Int(32), 0)); + PrimExpr divide_factor = max((h_end - h_start) * (w_end - w_start), + make_const(DataType::DataType::Int(32), 1)); + PrimExpr up_rounder = floordiv(divide_factor, 2); + return floordiv(pool_sum(indices) + up_rounder, divide_factor); + } + }, + "tensor", kElementWise); + } else { + return tvm::te::compute( + out_shape, + [&](const Array& output) { + Array indices; + for (const Var& var : output) indices.push_back(var); + if (count_include_pad) { + return div(pool_sum(indices), (kernel_height * kernel_width)); + } else { + PrimExpr h_start = output[height_axis] * stride_height - pad_top; + PrimExpr w_start = output[width_axis] * stride_width - pad_left; + PrimExpr h_end = min(h_start + kernel_height, height); + PrimExpr w_end = min(w_start + kernel_width, width); + h_start = max(h_start, make_const(DataType::DataType::Int(32), 0)); + w_start = max(w_start, make_const(DataType::DataType::Int(32), 0)); + PrimExpr divide_factor = max((h_end - h_start) * (w_end - w_start), + make_const(DataType::DataType::Int(32), 1)); + return div(pool_sum(indices), divide_factor); + } + }, + "tensor", kElementWise); + } } else { LOG(ERROR) << "Unrecognized pool_type: " << pool_type; return x; diff --git a/python/tvm/relay/frontend/tflite.py b/python/tvm/relay/frontend/tflite.py index 1b593ad8dea3..6fb946438658 100644 --- a/python/tvm/relay/frontend/tflite.py +++ b/python/tvm/relay/frontend/tflite.py @@ -50,7 +50,7 @@ def __init__(self, tensor_idx, tensor, buffer, qnn_params=None): class OperatorConverter(object): """Operator Converted for converting TFLite ops to Relay ops""" - def __init__(self, model, subgraph, exp_tab): + def __init__(self, model, subgraph, exp_tab, rounding): try: from tflite.BuiltinOperator import BuiltinOperator @@ -66,6 +66,7 @@ def __init__(self, model, subgraph, exp_tab): self.activation_fn_type = build_str_map(ActivationFunctionType()) self.builtin_options = build_str_map(BuiltinOptions()) self.prefetched_nodes = {} + self.rounding = rounding # Add more operators self.convert_map = { @@ -570,6 +571,7 @@ def convert_reshape(self, op): input_zero_point=input_tensor.qnn_params["zero_point"], output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, ) @@ -840,6 +842,7 @@ def convert_relu(self, op): input_zero_point=input_tensor.qnn_params["zero_point"], output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, ) @@ -915,6 +918,7 @@ def convert_relu6(self, op): input_zero_point=input_tensor.qnn_params["zero_point"], output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, ) @@ -986,6 +990,7 @@ def convert_relu_n1_to_1(self, op): input_zero_point=input_tensor.qnn_params["zero_point"], output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, ) @@ -1228,16 +1233,30 @@ def _convert_elemwise(self, relay_op, op, ignore_qnn_params=False): if not ignore_qnn_params and lhs_tensor.qnn_params: assert rhs_tensor.qnn_params, "Both tensors should be quantized." assert output_tensor.qnn_params, "Output tensor should be quantized." - out = relay_op( - lhs=lhs_expr, - rhs=rhs_expr, - lhs_scale=lhs_tensor.qnn_params["scale"], - lhs_zero_point=lhs_tensor.qnn_params["zero_point"], - rhs_scale=rhs_tensor.qnn_params["scale"], - rhs_zero_point=rhs_tensor.qnn_params["zero_point"], - output_scale=output_tensor.qnn_params["scale"], - output_zero_point=output_tensor.qnn_params["zero_point"], - ) + has_tflite_rounding_mode = [_qnn.op.add] + if relay_op in has_tflite_rounding_mode: + out = relay_op( + lhs=lhs_expr, + rhs=rhs_expr, + lhs_scale=lhs_tensor.qnn_params["scale"], + lhs_zero_point=lhs_tensor.qnn_params["zero_point"], + rhs_scale=rhs_tensor.qnn_params["scale"], + rhs_zero_point=rhs_tensor.qnn_params["zero_point"], + output_scale=output_tensor.qnn_params["scale"], + output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, + ) + else: + out = relay_op( + lhs=lhs_expr, + rhs=rhs_expr, + lhs_scale=lhs_tensor.qnn_params["scale"], + lhs_zero_point=lhs_tensor.qnn_params["zero_point"], + rhs_scale=rhs_tensor.qnn_params["scale"], + rhs_zero_point=rhs_tensor.qnn_params["zero_point"], + output_scale=output_tensor.qnn_params["scale"], + output_zero_point=output_tensor.qnn_params["zero_point"], + ) else: out = relay_op(lhs_expr, rhs_expr) @@ -1732,6 +1751,7 @@ def _convert_reduce(self, relay_op, op): input_zero_point=input_tensor.qnn_params["zero_point"], output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, ) @@ -1904,6 +1924,7 @@ def convert_fully_connected(self, op): input_zero_point=new_input_zero_point, output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, ) @@ -2157,6 +2178,7 @@ def convert_conv(self, op, conv_type): input_zero_point=new_input_zero_point, output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, axis=3, ) @@ -2933,6 +2955,7 @@ def convert_transpose_conv(self, op): input_zero_point=new_input_zero_point, output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, axis=3, ) @@ -2965,6 +2988,7 @@ def convert_quantize(self, op): input_zero_point=input_tensor.qnn_params["zero_point"], output_scale=output_tensor.qnn_params["scale"], output_zero_point=output_tensor.qnn_params["zero_point"], + rounding=self.rounding, out_dtype=output_tensor_type_str, ) return out @@ -3457,9 +3481,11 @@ def get_scalar_from_constant(expr): isinstance(expr, _expr.Constant) and not expr.data.shape ), "Expr is not a constant scalar." value = expr.data.asnumpy() - assert value.dtype == np.dtype(np.int32) or value.dtype == np.dtype( - np.float32 - ), "value must be float32/int32" + assert ( + value.dtype == np.dtype(np.int32) + or value.dtype == np.dtype(np.float32) + or value.dtype == np.dtype(np.float64) + ), "value must be float32/float64/int32" return np.asscalar(value) @@ -3577,7 +3603,7 @@ def _input_type(model): return shape_dict, dtype_dict -def from_tflite(model, shape_dict=None, dtype_dict=None): +def from_tflite(model, shape_dict=None, dtype_dict=None, rounding="TFLITE"): """Convert from tflite model into compatible relay Function. Parameters @@ -3591,6 +3617,9 @@ def from_tflite(model, shape_dict=None, dtype_dict=None): dtype_dict : dict of str to str Input types of the model. + rounding : str + Rounding mode for tflite model + Returns ------- mod : tvm.IRModule @@ -3637,7 +3666,7 @@ def from_tflite(model, shape_dict=None, dtype_dict=None): exp_tab.set_expr(model_input_name, _expr.var(model_input_name, shape=shape, dtype=dtype)) # op code in model - op_converter = OperatorConverter(model, subgraph, exp_tab) + op_converter = OperatorConverter(model, subgraph, exp_tab, rounding) op_converter.check_unsupported_ops() op_converter.convert_op_to_relay() diff --git a/python/tvm/relay/qnn/op/qnn.py b/python/tvm/relay/qnn/op/qnn.py index a5892f331f06..7d1d2d24ac36 100644 --- a/python/tvm/relay/qnn/op/qnn.py +++ b/python/tvm/relay/qnn/op/qnn.py @@ -409,7 +409,15 @@ def conv2d_transpose( def add( - lhs, rhs, lhs_scale, lhs_zero_point, rhs_scale, rhs_zero_point, output_scale, output_zero_point + lhs, + rhs, + lhs_scale, + lhs_zero_point, + rhs_scale, + rhs_zero_point, + output_scale, + output_zero_point, + rounding="UPWARD", ): """Quantized addition with numpy-style broadcasting. @@ -439,6 +447,9 @@ def add( output_zero_point: relay.Expr The zero point of output quantized expr. + rounding: str, optional + rounding mode of qnn add + Returns ------- result : relay.Expr @@ -454,6 +465,7 @@ def add( rhs_zero_point, output_scale, output_zero_point, + rounding, ) diff --git a/python/tvm/relay/quantize/quantize.py b/python/tvm/relay/quantize/quantize.py index 8f7333051a4c..d53bae3c221d 100644 --- a/python/tvm/relay/quantize/quantize.py +++ b/python/tvm/relay/quantize/quantize.py @@ -177,7 +177,7 @@ def qconfig(**kwargs): is None, which means will try to call all operartors' annotate rewrite function. - rounding: "UPWARD" or "TONEAREST" + rounding: "UPWARD" or "TONEAREST" or "TFLITE" Rounding direction for fixed point multiplications. partition_conversions: 'disabled', 'enabled', or 'fully_integral' diff --git a/src/arith/canonical_simplify.cc b/src/arith/canonical_simplify.cc index ba549959ac98..8a0d4758a374 100644 --- a/src/arith/canonical_simplify.cc +++ b/src/arith/canonical_simplify.cc @@ -544,7 +544,12 @@ class CanonicalSimplifier::Impl : public RewriteSimplifier::Impl { explicit Impl(Analyzer* parent) : Rewriter(parent) {} PrimExpr CanonicalSimplify(PrimExpr expr) { - expr = operator()(expr); + // same as kMaxFusedOps. + // avoid long compile time of tflite quantized model + constexpr static size_t kMaxPrimOps = 256; + if (tir::ExprComplexity().Eval(expr) < kMaxPrimOps) { + expr = operator()(expr); + } return expr; } diff --git a/src/arith/rewrite_simplify.cc b/src/arith/rewrite_simplify.cc index a58e4433dadd..e25d2b0d44ad 100644 --- a/src/arith/rewrite_simplify.cc +++ b/src/arith/rewrite_simplify.cc @@ -1598,10 +1598,15 @@ PrimExpr RewriteSimplifier::operator()(const PrimExpr& expr) { // Run simplification in post order PrimExpr res = expr; int max_iter = 2; - for (int i = 0; i < max_iter; ++i) { - PrimExpr new_expr = impl_->operator()(res); - if (new_expr.same_as(res)) return res; - res = new_expr; + // same as kMaxFusedOps. + // avoid long compile time of tflite quantized model + constexpr static size_t kMaxPrimOps = 256; + if (tir::ExprComplexity().Eval(expr) < kMaxPrimOps) { + for (int i = 0; i < max_iter; ++i) { + PrimExpr new_expr = impl_->operator()(res); + if (new_expr.same_as(res)) return res; + res = new_expr; + } } return res; } diff --git a/src/arith/solve_linear_inequality.cc b/src/arith/solve_linear_inequality.cc index dd9044833546..5c124551b39e 100644 --- a/src/arith/solve_linear_inequality.cc +++ b/src/arith/solve_linear_inequality.cc @@ -39,55 +39,6 @@ namespace arith { using namespace tvm::runtime; using namespace tvm::tir; -#define PLUS_ONE(OP) \ - void VisitExpr_(const OP* op) final { num_symbols_++; } - -#define PLUS_ONE_BINARY(OP) \ - void VisitExpr_(const OP* op) final { \ - num_symbols_++; \ - VisitExpr(op->a); \ - VisitExpr(op->b); \ - } - -/*! - * \brief Calculate the expresion complexity based on number of symbols it contains. - */ -class ExprComplexity : public ExprVisitor { - public: - size_t Eval(const PrimExpr& expr) { - VisitExpr(expr); - return num_symbols_; - } - - PLUS_ONE_BINARY(AddNode) - PLUS_ONE_BINARY(SubNode) - PLUS_ONE_BINARY(MulNode) - PLUS_ONE_BINARY(DivNode) - PLUS_ONE_BINARY(ModNode) - PLUS_ONE_BINARY(FloorDivNode) - PLUS_ONE_BINARY(FloorModNode) - PLUS_ONE_BINARY(MinNode) - PLUS_ONE_BINARY(MaxNode) - PLUS_ONE_BINARY(EQNode) - PLUS_ONE_BINARY(NENode) - PLUS_ONE_BINARY(LTNode) - PLUS_ONE_BINARY(LENode) - PLUS_ONE_BINARY(GTNode) - PLUS_ONE_BINARY(GENode) - PLUS_ONE_BINARY(AndNode) - PLUS_ONE_BINARY(OrNode) - PLUS_ONE(VarNode) - PLUS_ONE(FloatImmNode) - PLUS_ONE(IntImmNode) - void VisitExpr_(const NotNode* op) final { - num_symbols_++; - VisitExpr(op->a); - } - - private: - size_t num_symbols_{0}; -}; - struct ExprLess { bool operator()(const PrimExpr& l, const PrimExpr& r) const { return ExprComplexity().Eval(l) < ExprComplexity().Eval(r); diff --git a/src/relay/qnn/op/add.cc b/src/relay/qnn/op/add.cc index b0dc3e4af5c4..9b0db3bd6591 100644 --- a/src/relay/qnn/op/add.cc +++ b/src/relay/qnn/op/add.cc @@ -30,9 +30,11 @@ namespace tvm { namespace relay { namespace qnn { +TVM_REGISTER_NODE_TYPE(QnnAddAttrs); + /* * \brief Canonicalizes the QNN add op. - * \param attrs The empty attribute. + * \param attrs The QNN add attrs. * \param new_args The new mutated args to the call node. * \param arg_types The types of input and output. * \return The sequence of Relay ops for add op. @@ -42,9 +44,55 @@ Expr QnnAddCanonicalize(const Attrs& attrs, const Array& new_args, // Get the args. QnnBinaryOpArguments args(new_args); + // Get the attrs. + const QnnAddAttrs* add_attrs = attrs.as(); + ICHECK(add_attrs != nullptr); + auto& rounding = add_attrs->rounding; + // Get the input dtype and shape. QnnBinaryOpTensorType input_type(arg_types, 0); + if (rounding == "TFLITE") { + double lhs_scale_val = GetScalarFromConstant(args.lhs_scale); + double rhs_scale_val = GetScalarFromConstant(args.rhs_scale); + double out_scale_val = GetScalarFromConstant(args.output_scale); + double twice_max_input_scale = 2 * std::max(lhs_scale_val, rhs_scale_val); + double real_lhs_scale_val = lhs_scale_val / twice_max_input_scale; + double real_rhs_scale_val = rhs_scale_val / twice_max_input_scale; + double real_out_scale_val = twice_max_input_scale / ((1 << 20) * out_scale_val); + + auto real_lhs_scale = MakeConstantScalar(DataType::Float(64), real_lhs_scale_val); + auto real_rhs_scale = MakeConstantScalar(DataType::Float(64), real_rhs_scale_val); + auto real_out_scale = MakeConstantScalar(DataType::Float(64), real_out_scale_val); + auto one_scalar = MakeConstantScalar(DataType::Float(64), 1); + auto zero_scalar = MakeConstantScalar(DataType::Int(32), 0); + auto left_shift_scalar = MakeConstantScalar(DataType::Int(32), 1 << 20); + + Expr adapted_lhs = Cast(args.lhs, DataType::Int(32)); + if (!IsEqualScalar(args.lhs_zero_point, zero_scalar)) { + adapted_lhs = Subtract(adapted_lhs, Cast(args.lhs_zero_point, DataType::Int(32))); + } + adapted_lhs = Multiply(adapted_lhs, left_shift_scalar); + + Expr adapted_rhs = Cast(args.rhs, DataType::Int(32)); + if (!IsEqualScalar(args.rhs_zero_point, zero_scalar)) { + adapted_rhs = Subtract(adapted_rhs, Cast(args.rhs_zero_point, DataType::Int(32))); + } + adapted_rhs = Multiply(adapted_rhs, left_shift_scalar); + + auto requantized_lhs = Requantize(adapted_lhs, input_type.shape, real_lhs_scale, zero_scalar, + one_scalar, zero_scalar, DataType::Int(32), rounding); + + auto requantized_rhs = Requantize(adapted_rhs, input_type.shape, real_rhs_scale, zero_scalar, + one_scalar, zero_scalar, DataType::Int(32), rounding); + + auto output = Add(requantized_lhs, requantized_rhs); + output = Requantize(output, input_type.shape, real_out_scale, zero_scalar, one_scalar, + args.output_zero_point, DataType::Int(32), rounding); + // Go back to lower precision. + return ConvertDtype(output, input_type.dtype); + } + // FIXME (anijain2305) - The lowering can be further optimized. Instead of inserting requantize in // the start, we can insert requantize at the end if both input tensors have same qnn params. In // that case, we can first add the tensors, subtract the zero point, and requantize at the end. @@ -86,9 +134,23 @@ Expr QnnAddCanonicalize(const Attrs& attrs, const Array& new_args, return ConvertDtype(output, input_type.dtype); } +Expr MakeQnnAdd(Expr lhs, Expr rhs, Expr lhs_scale, Expr lhs_zero_point, Expr rhs_scale, + Expr rhs_zero_point, Expr output_scale, Expr output_zero_point, + std::string rounding) { + auto attrs = make_object(); + attrs->rounding = std::move(rounding); + + static const Op& op = Op::Get("qnn.add"); + return Call(op, + {lhs, rhs, lhs_scale, lhs_zero_point, rhs_scale, rhs_zero_point, output_scale, + output_zero_point}, + Attrs(attrs), {}); +} + // QNN Addition operator. -QNN_REGISTER_BINARY_OP("add") +QNN_REGISTER_BINARY_OP_WITH_BODY("add", MakeQnnAdd) .describe("Elementwise add with with broadcasting for quantized tensors.") + .set_attrs_type() .set_support_level(11) .set_attr("FTVMQnnCanonicalize", QnnAddCanonicalize); diff --git a/src/relay/qnn/op/concatenate.cc b/src/relay/qnn/op/concatenate.cc index 59a519d66436..3a2f81836747 100644 --- a/src/relay/qnn/op/concatenate.cc +++ b/src/relay/qnn/op/concatenate.cc @@ -60,7 +60,11 @@ bool QnnConcatenateRel(const Array& types, int num_inputs, const Attrs& at if (input_scale.as()) { return false; } - ICHECK(IsScalarType(input_scale, DataType::Float(32))); // input_scales[idx] + + const auto* input_scale_type = input_scale.as(); + ICHECK(input_scale_type && IsScalarType(input_scale, input_scale_type->dtype) && + (input_scale_type->dtype == DataType::Float(32) || + input_scale_type->dtype == DataType::Float(64))); // input_scale[idx] } const auto* input_zero_points_tuple = types[2].as(); @@ -85,8 +89,13 @@ bool QnnConcatenateRel(const Array& types, int num_inputs, const Attrs& at return false; } } - ICHECK(IsScalarType(types[3], DataType::Float(32))); // output_scale - ICHECK(IsScalarType(types[4], DataType::Int(32))); // output_zero_point + + const auto* output_scale_type = types[3].as(); + ICHECK(output_scale_type && IsScalarType(types[3], output_scale_type->dtype) && + (output_scale_type->dtype == DataType::Float(32) || + output_scale_type->dtype == DataType::Float(64))); // output_scale + + ICHECK(IsScalarType(types[4], DataType::Int(32))); // output_zero_point // Collect the input tensor and output tensor devoid of scale and zero points to reuse Relay // Concatenate infer type function. diff --git a/src/relay/qnn/op/convolution.cc b/src/relay/qnn/op/convolution.cc index 21335ec2fb34..af1cb43df442 100644 --- a/src/relay/qnn/op/convolution.cc +++ b/src/relay/qnn/op/convolution.cc @@ -64,21 +64,26 @@ bool QnnConv2DRel(const Array& types, int num_inputs, const Attrs& attrs, return false; } } - ICHECK(IsScalarType(types[2], DataType::Int(32))); // input_zero_point - ICHECK(IsScalarType(types[3], DataType::Int(32))); // weight_zero_point - ICHECK(IsScalarType(types[4], DataType::Float(32))); // input_scale + ICHECK(IsScalarType(types[2], DataType::Int(32))); // input_zero_point + ICHECK(IsScalarType(types[3], DataType::Int(32))); // weight_zero_point + const auto* input_scale_type = types[4].as(); + ICHECK(input_scale_type && IsScalarType(types[4], input_scale_type->dtype) && + (input_scale_type->dtype == DataType::Float(32) || + input_scale_type->dtype == DataType::Float(64))); // input_scale // Kernel scale can be a vector of length output_channels or a scalar. if (param->groups == 1) { size_t axis = param->kernel_layout.operator std::string().find('O'); ICHECK(axis != std::string::npos) << "Kernel layout attribute is not defined"; - AssignType(types[5], DataType::Float(32), weight->shape[axis], reporter); // weight_scale + const auto* kernel_scale_type = types[5].as(); + AssignType(types[5], kernel_scale_type->dtype, weight->shape[axis], reporter); // weight_scale } else { // Here, total number of output channels depend on depth multiplier. size_t o_axis = param->kernel_layout.operator std::string().find('O'); size_t i_axis = param->kernel_layout.operator std::string().find('I'); ICHECK(o_axis != std::string::npos || i_axis != std::string::npos) << "Kernel layout attribute is not defined"; - AssignType(types[5], DataType::Float(32), weight->shape[i_axis] * weight->shape[o_axis], + const auto* kernel_scale_type = types[5].as(); + AssignType(types[5], kernel_scale_type->dtype, weight->shape[i_axis] * weight->shape[o_axis], reporter); // weight_scale } diff --git a/src/relay/qnn/op/dequantize.cc b/src/relay/qnn/op/dequantize.cc index 724441e0c523..daea02682223 100644 --- a/src/relay/qnn/op/dequantize.cc +++ b/src/relay/qnn/op/dequantize.cc @@ -59,8 +59,11 @@ bool DequantizeRel(const Array& types, int num_inputs, const Attrs& attrs, ICHECK_GE(axis, 0) << "axis " << dequantize_attrs->axis << " is out of range"; // Check and assign types for scale and zero points. - AssignType(types[1], DataType::Float(32), data->shape[axis], reporter); // scale - AssignType(types[2], DataType::Int(32), data->shape[axis], reporter); // zero point + const auto* input_scale_type = types[1].as(); + ICHECK(input_scale_type && (input_scale_type->dtype == DataType::Float(32) || + input_scale_type->dtype == DataType::Float(64))); + AssignType(types[1], input_scale_type->dtype, data->shape[axis], reporter); // scale + AssignType(types[2], DataType::Int(32), data->shape[axis], reporter); // zero point const Array oshape = data->shape; // assign output type, output will always be float 32. @@ -104,7 +107,8 @@ Expr DequantizeLower(const Expr& input_tensor, const Expr& input_scale, } auto shift = Subtract(Cast(input_tensor, DataType::Int(32)), expanded_input_zero_point); - auto scaled_output = Multiply(Cast(shift, DataType::Float(32)), expanded_input_scale); + auto scaled_output = + Multiply(Cast(shift, DataType::Float(32)), Cast(expanded_input_scale, DataType::Float(32))); return scaled_output; } diff --git a/src/relay/qnn/op/mul.cc b/src/relay/qnn/op/mul.cc index 781114cc5f5a..a058312d0488 100644 --- a/src/relay/qnn/op/mul.cc +++ b/src/relay/qnn/op/mul.cc @@ -50,6 +50,7 @@ Expr QnnMulCanonicalize(const Attrs& attrs, const Array& new_args, // data types const auto int32_dtype = DataType::Int(32); const auto float32_dtype = DataType::Float(32); + const auto float64_dtype = DataType::Float(64); /* A tensor multiplication c = a * b can be written in terms of respective @@ -79,10 +80,29 @@ Expr QnnMulCanonicalize(const Attrs& attrs, const Array& new_args, auto output = Multiply(lhs_shifted, rhs_shifted); // Get the adjusted new scale and zero points. - float lhs_scale_float = GetScalarFromConstant(args.lhs_scale); - float rhs_scale_float = GetScalarFromConstant(args.rhs_scale); - float new_scale_float = lhs_scale_float * rhs_scale_float; - auto new_input_scale = MakeConstantScalar(float32_dtype, new_scale_float); + auto lhs_scale_dtype = GetDataTypeFromConstant(args.lhs_scale); + auto rhs_scale_dtype = GetDataTypeFromConstant(args.rhs_scale); + double lhs_scale_val = -1.f, rhs_scale_val = -1.f; + if (lhs_scale_dtype == DataType::Float(32)) { + lhs_scale_val = GetScalarFromConstant(args.lhs_scale); + } else if (lhs_scale_dtype == DataType::Float(64)) { + lhs_scale_val = GetScalarFromConstant(args.lhs_scale); + } + + if (rhs_scale_dtype == DataType::Float(32)) { + rhs_scale_val = GetScalarFromConstant(args.rhs_scale); + } else if (rhs_scale_dtype == DataType::Float(64)) { + rhs_scale_val = GetScalarFromConstant(args.rhs_scale); + } + CHECK(lhs_scale_val != -1.f && rhs_scale_val != -1.f); + + auto new_scale_val = lhs_scale_val * rhs_scale_val; + Constant new_input_scale; + if (lhs_scale_dtype == DataType::Float(32) || rhs_scale_dtype == DataType::Float(32)) { + new_input_scale = MakeConstantScalar(float32_dtype, new_scale_val); + } else { + new_input_scale = MakeConstantScalar(float64_dtype, new_scale_val); + } auto new_input_zero_point = zero_scalar; // Requantize to get Q_c diff --git a/src/relay/qnn/op/op_common.h b/src/relay/qnn/op/op_common.h index 0f77db4f501a..714d024f647c 100644 --- a/src/relay/qnn/op/op_common.h +++ b/src/relay/qnn/op/op_common.h @@ -184,12 +184,22 @@ static inline bool QnnBroadcastRel(const Array& types, int num_inputs, con return false; } } - ICHECK(IsScalarType(types[2], DataType::Float(32))); // lhs_scale - ICHECK(IsScalarType(types[3], DataType::Int(32))); // lhs_zero_point - ICHECK(IsScalarType(types[4], DataType::Float(32))); // rhs_scale - ICHECK(IsScalarType(types[5], DataType::Int(32))); // rhs_zero_point - ICHECK(IsScalarType(types[6], DataType::Float(32))); // output_scale - ICHECK(IsScalarType(types[7], DataType::Int(32))); // output_zero_point + + const auto* lhs_scale_type = types[2].as(); + const auto* rhs_scale_type = types[4].as(); + const auto* out_scale_type = types[6].as(); + ICHECK(lhs_scale_type && IsScalarType(types[2], lhs_scale_type->dtype) && + (lhs_scale_type->dtype == DataType::Float(32) || + lhs_scale_type->dtype == DataType::Float(64))); // lhs_scale + ICHECK(rhs_scale_type && IsScalarType(types[4], rhs_scale_type->dtype) && + (rhs_scale_type->dtype == DataType::Float(32) || + rhs_scale_type->dtype == DataType::Float(64))); // rhs_scale + ICHECK(out_scale_type && IsScalarType(types[6], out_scale_type->dtype) && + (out_scale_type->dtype == DataType::Float(32) || + out_scale_type->dtype == DataType::Float(64))); // output_scale + ICHECK(IsScalarType(types[3], DataType::Int(32))); // lhs_zero_point + ICHECK(IsScalarType(types[5], DataType::Int(32))); // rhs_zero_point + ICHECK(IsScalarType(types[7], DataType::Int(32))); // output_zero_point // Collect the input tensor and output tensor devoid of scale and zero points to reuse Relay // BroadcastRel infer type function. @@ -207,29 +217,33 @@ static inline bool QnnBroadcastRel(const Array& types, int num_inputs, con * * \param OpName the name of registry. */ + +#define QNN_REGISTER_BINARY_OP_WITH_BODY(OpName, Body) \ + TVM_REGISTER_GLOBAL("relay.qnn.op._make." OpName).set_body_typed(Body); \ + RELAY_REGISTER_OP("qnn." OpName) \ + .set_num_inputs(kNumQnnBinaryOpInputs) \ + .add_argument("lhs", "Tensor", "The left hand side quantized tensor.") \ + .add_argument("rhs", "Tensor", "The right hand side quantized tensor.") \ + .add_argument("lhs_scale", "Tensor", "The scale of the lhs tensor.") \ + .add_argument("lhs_zero_point", "Tensor", "The zero_point of the lhs tensor.") \ + .add_argument("rhs_scale", "Tensor", "The scale of the rhs tensor.") \ + .add_argument("rhs_zero_point", "Tensor", "The zero_point of the rhs tensor.") \ + .add_argument("output_scale", "Tensor", "The scale of the output tensor.") \ + .add_argument("output_zero_point", "Tensor", "The zero_point of the output tensor.") \ + .add_type_rel("QnnBroadcast", QnnBroadcastRel) \ + .set_attr("TNonComputational", true) \ + .set_attr("FInferCorrectLayout", QnnBinaryBroadcastLayout) + #define QNN_REGISTER_BINARY_OP(OpName) \ - TVM_REGISTER_GLOBAL("relay.qnn.op._make." OpName) \ - .set_body_typed([](Expr lhs, Expr rhs, Expr lhs_scale, Expr lhs_zero_point, Expr rhs_scale, \ - Expr rhs_zero_point, Expr output_scale, Expr output_zero_point) { \ + QNN_REGISTER_BINARY_OP_WITH_BODY( \ + OpName, [](Expr lhs, Expr rhs, Expr lhs_scale, Expr lhs_zero_point, Expr rhs_scale, \ + Expr rhs_zero_point, Expr output_scale, Expr output_zero_point) { \ static const Op& op = Op::Get("qnn." OpName); \ return Call(op, \ {lhs, rhs, lhs_scale, lhs_zero_point, rhs_scale, rhs_zero_point, output_scale, \ output_zero_point}, \ Attrs(), {}); \ - }); \ - RELAY_REGISTER_OP("qnn." OpName) \ - .set_num_inputs(kNumQnnBinaryOpInputs) \ - .add_argument("lhs", "Tensor", "The left hand side quantized tensor.") \ - .add_argument("rhs", "Tensor", "The right hand side quantized tensor.") \ - .add_argument("lhs_scale", "Tensor", "The scale of the lhs tensor.") \ - .add_argument("lhs_zero_point", "Tensor", "The zero_point of the lhs tensor.") \ - .add_argument("rhs_scale", "Tensor", "The scale of the rhs tensor.") \ - .add_argument("rhs_zero_point", "Tensor", "The zero_point of the rhs tensor.") \ - .add_argument("output_scale", "Tensor", "The scale of the output tensor.") \ - .add_argument("output_zero_point", "Tensor", "The zero_point of the output tensor.") \ - .add_type_rel("QnnBroadcast", QnnBroadcastRel) \ - .set_attr("TNonComputational", true) \ - .set_attr("FInferCorrectLayout", QnnBinaryBroadcastLayout) + }) } // namespace qnn } // namespace relay diff --git a/src/relay/qnn/op/quantize.cc b/src/relay/qnn/op/quantize.cc index 9829834f43a3..5e7da0728169 100644 --- a/src/relay/qnn/op/quantize.cc +++ b/src/relay/qnn/op/quantize.cc @@ -57,8 +57,11 @@ bool QuantizeRel(const Array& types, int num_inputs, const Attrs& attrs, ICHECK_GE(axis, 0) << "axis " << quantize_attrs->axis << " is out of range"; // Check and assign types for scale and zero points. - AssignType(types[1], DataType::Float(32), data->shape[axis], reporter); // scale - AssignType(types[2], DataType::Int(32), data->shape[axis], reporter); // zero point + const auto* scale_type = types[1].as(); + ICHECK(scale_type && + (scale_type->dtype == DataType::Float(32) || scale_type->dtype == DataType::Float(64))); + AssignType(types[1], scale_type->dtype, data->shape[axis], reporter); // scale + AssignType(types[2], DataType::Int(32), data->shape[axis], reporter); // zero point const Array oshape = data->shape; const DataType out_dtype = quantize_attrs->out_dtype; @@ -109,7 +112,7 @@ Expr QuantizeLower(const Expr& input_tensor, const Expr& output_scale, const int32_t min_val = GetQmin(out_dtype); const int32_t max_val = GetQmax(out_dtype); - auto scale_data = Divide(input_tensor, expanded_output_scale); + auto scale_data = Divide(input_tensor, Cast(expanded_output_scale, DataType::Float(32))); auto add_zero_point = Cast(Round(Add(scale_data, Cast(expanded_output_zero_point, DataType::Float(32)))), DataType::Int(32)); diff --git a/src/relay/qnn/op/requantize.cc b/src/relay/qnn/op/requantize.cc index 2ae879595659..9fb5c932821e 100644 --- a/src/relay/qnn/op/requantize.cc +++ b/src/relay/qnn/op/requantize.cc @@ -133,7 +133,13 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, const Expr& input_zero_point, const Expr& output_scale, const Expr& output_zero_point, const RequantizeAttrs* param, const Array& input_shape, const DataType& out_dtype) { - auto tensor = Cast(input_tensor, DataType::Int(32)); + bool input_is_int32 = false; + if ((input_tensor.as() || input_tensor.as()) && + input_tensor->checked_type_.defined()) { + auto tensor_type = input_tensor->checked_type().as(); + if (tensor_type && tensor_type->dtype == DataType::Int(32)) input_is_int32 = true; + } + auto tensor = input_is_int32 ? input_tensor : Cast(input_tensor, DataType::Int(32)); // 1) Subtract the input_zero_point auto zero_scalar = MakeConstantScalar(DataType::Int(32), 0); if (!IsEqualScalar(input_zero_point, zero_scalar)) { @@ -145,12 +151,25 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, // the whole tensor. For per-channel (aka per-axis), there is a vector of scales for the input // tensor. Depending on the quantization type, the fixed point multiplication routing is called. auto scaled_int32_t = tensor; - float output_scale_float = GetScalarFromConstant(output_scale); + auto out_scale_dtype = GetDataTypeFromConstant(output_scale); + double output_scale_val = -1.0f; + if (out_scale_dtype == DataType::Float(64)) { + output_scale_val = GetScalarFromConstant(output_scale); + } else if (out_scale_dtype == DataType::Float(32)) { + output_scale_val = GetScalarFromConstant(output_scale); + } + ICHECK_GE(output_scale_val, 0.0f); + + auto in_scale_dtype = GetDataTypeFromConstant(input_scale); if (IsConstScalar(input_scale)) { // This is per-tensor quantization. Single scale. - float input_scale_float = GetScalarFromConstant(input_scale); - double double_multiplier = - static_cast(input_scale_float) / static_cast(output_scale_float); + double input_scale_val = -1.0f; + if (in_scale_dtype == DataType::Float(64)) { + input_scale_val = GetScalarFromConstant(input_scale); + } else if (in_scale_dtype == DataType::Float(32)) { + input_scale_val = GetScalarFromConstant(input_scale); + } + double double_multiplier = input_scale_val / output_scale_val; // Skip if input and output scales are same. if (!IsEqualScalar(input_scale, output_scale)) { int32_t fixed_point_multiplier, shift; @@ -162,19 +181,28 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, // the FixedPointMultiply operator scaled_int32_t = (is_upward_rounding - ? FixedPointMultiply(scaled_int32_t, fixed_point_multiplier, shift) - : FixedPointMultiplyToNearest(scaled_int32_t, double_multiplier, input_shape)); + ? relay::FixedPointMultiply(scaled_int32_t, fixed_point_multiplier, shift) + : qnn::FixedPointMultiply(scaled_int32_t, double_multiplier, input_shape, + param->rounding)); } } else { // This is per-channel (per=axis) quantization. std::vector double_multipliers; - auto input_axis_scales = GetFloatVectorFromConstant(input_scale); - for (auto input_axis_scale : input_axis_scales) { - double multiplier = - static_cast(input_axis_scale) / static_cast(output_scale_float); - double_multipliers.push_back(multiplier); + if (in_scale_dtype == DataType::Float(64)) { + auto input_axis_scales = GetDoubleVectorFromConstant(input_scale); + for (auto input_axis_scale : input_axis_scales) { + double multiplier = input_axis_scale / output_scale_val; + double_multipliers.push_back(multiplier); + } + } else if (in_scale_dtype == DataType::Float(32)) { + auto input_axis_scales = GetFloatVectorFromConstant(input_scale); + for (auto input_axis_scale : input_axis_scales) { + double multiplier = input_axis_scale / output_scale_val; + double_multipliers.push_back(multiplier); + } } + ICHECK_GT(double_multipliers.size(), 0); int axis = param->axis; axis = (axis == -1) ? input_shape.size() - 1 : axis; scaled_int32_t = FixedPointMultiplyPerChannel(scaled_int32_t, double_multipliers, input_shape, @@ -239,9 +267,10 @@ Expr RequantizeQnnCanonicalize(const Attrs& attrs, const Array& new_args, auto out_dtype = out_tensor_type->dtype; // Check rounding validity. - ICHECK(param->rounding == "UPWARD" || param->rounding == "TONEAREST") - << "QNN requantize supports two rounding modes - UPWARD and " - << "TONEAREST"; + ICHECK(param->rounding == "UPWARD" || param->rounding == "TONEAREST" || + param->rounding == "TFLITE") + << "QNN requantize supports 3 rounding modes - UPWARD, " + << "TONEAREST and TFLITE"; return RequantizeLower(quantized_data, input_scale, input_zero_point, output_scale, output_zero_point, param, input_shape, out_dtype); } @@ -283,11 +312,17 @@ bool RequantizeRel(const Array& types, int num_inputs, const Attrs& attrs, ICHECK_GE(axis, 0) << "axis " << requantize_attrs->axis << " is out of range"; // Check and assign types for scale and zero points. - AssignType(types[1], DataType::Float(32), data->shape[axis], reporter); // input_scale - AssignType(types[2], DataType::Int(32), data->shape[axis], reporter); // input_zero_pt + const auto* input_scale_type = types[1].as(); + ICHECK(input_scale_type && (input_scale_type->dtype == DataType::Float(32) || + input_scale_type->dtype == DataType::Float(64))); + AssignType(types[1], input_scale_type->dtype, data->shape[axis], reporter); // input_scale + AssignType(types[2], DataType::Int(32), data->shape[axis], reporter); // input_zero_pt // For now, requantize output tensor is limited to full tensor uniform quantization. - ICHECK(IsScalarType(types[3], DataType::Float(32))); // output_scale - ICHECK(IsScalarType(types[4], DataType::Int(32))); // output_zero_point + const auto* output_scale_type = types[3].as(); + ICHECK(output_scale_type && IsScalarType(types[3], output_scale_type->dtype) && + (output_scale_type->dtype == DataType::Float(32) || + output_scale_type->dtype == DataType::Float(64))); // output_scale + ICHECK(IsScalarType(types[4], DataType::Int(32))); // output_zero_point const Array oshape = data->shape; // assign output type diff --git a/src/relay/qnn/utils.cc b/src/relay/qnn/utils.cc index 982efa0a61c1..2df55198dd50 100644 --- a/src/relay/qnn/utils.cc +++ b/src/relay/qnn/utils.cc @@ -24,12 +24,52 @@ #include "utils.h" +#include + #include "../transforms/pattern_utils.h" namespace tvm { namespace relay { namespace qnn { +/* \brief This function implements the rounding part of ARMv7 NEON VQRDMULH + * instruction. For code reuse, the multiplied tensor is directly passed in + * as parameter. + */ +Expr SaturatingRoundingDoublingHigh32(const Expr& input_tensor, const Expr& multiplier_expr, + const Expr& scaled_tensor, + const Array& input_shape, + bool possible_to_overflow = true) { + DataType hp_dtype = DataType::Int(64); + DataType lp_dtype = DataType::Int(32); + int64_t pos_nudge_value = (1ll << 30); + int64_t neg_nudge_value = 1 - (1ll << 30); + auto pos_nudge = MakeConstantScalar(hp_dtype, pos_nudge_value); + auto neg_nudge = MakeConstantScalar(hp_dtype, neg_nudge_value); + auto pos_nudge_t = Full(pos_nudge, input_shape, hp_dtype); + auto neg_nudge_t = Full(neg_nudge, input_shape, hp_dtype); + + auto dividend = MakeConstantScalar(hp_dtype, 1ll << 31); + + auto zero_t = Zeros(input_shape, hp_dtype); + auto nudged_tensor_t = + Add(scaled_tensor, Where(GreaterEqual(scaled_tensor, zero_t), pos_nudge_t, neg_nudge_t)); + auto high32_t = Cast(Divide(nudged_tensor_t, dividend), lp_dtype); + + if (possible_to_overflow) { + auto int32_min = MakeConstantScalar(lp_dtype, std::numeric_limits::min()); + auto int32_max = MakeConstantScalar(lp_dtype, std::numeric_limits::max()); + auto int32_max_t = Full(int32_max, input_shape, lp_dtype); + auto int32_min_t = Full(int32_min, input_shape, lp_dtype); + + auto overflow_t = + LogicalAnd(Equal(input_tensor, int32_min_t), Equal(multiplier_expr, int32_min_t)); + return Where(overflow_t, int32_max_t, high32_t); + } else { + return high32_t; + } +} + std::pair GetFixedPointMultiplierShift(double double_multiplier) { int32_t significand, exponent; if (double_multiplier == 0.) { @@ -56,11 +96,12 @@ std::pair GetFixedPointMultiplierShift(double double_multiplie return std::make_pair(significand, exponent); } -Expr FixedPointMultiplyToNearest(Expr tensor, double multiplier, - const Array& input_shape) { +Expr FixedPointMultiply(Expr tensor, double multiplier, const Array& input_shape, + const std::string& rounding) { // Choose high precision datatype to be int64. This is for avoiding overflow // in multiplication of two int32 values. DataType hp_dtype = DataType::Int(64); + DataType lp_dtype = DataType::Int(32); tensor = Cast(tensor, hp_dtype); // 1) Calculating the integer multiplier and integer shift @@ -81,26 +122,53 @@ Expr FixedPointMultiplyToNearest(Expr tensor, double multiplier, // (from the right, rightmost bit is bit 0). The computation is performed in // higher precision to avoid overflow in multiplying two int32 values. Expr scalar = MakeConstantScalar(hp_dtype, fixed_point_multiplier); - tensor = Multiply(tensor, scalar); + Expr scaled_tensor = Multiply(tensor, scalar); // 4) Find the rounding scalar. This depends on where the final decimal // point sits. As we will be right shifting the multiplied_t, we need to // first calculate the total_right_shift. int total_right_shift = right_shift + 31; - int64_t pos_rounding_value = (1ll << (total_right_shift - 1)); - Expr round_scalar; + // This lambda function gathers some shared logic in "TONEAREST" and "TFLITE" + // rounding scheme, which calculates a rounder tensor according to the sign + // of values in the tensor to be rounded. + auto nearest_rounding_scalar = [&](const Expr& input_tensor, int right_shift, + DataType dtype) -> Expr { + int64_t pos_rounding_value = (1ll << (right_shift - 1)); + auto pos_rounder = MakeConstantScalar(dtype, pos_rounding_value); + auto neg_rounder = MakeConstantScalar(dtype, pos_rounding_value - 1); + auto pos_rounder_t = Full(pos_rounder, input_shape, dtype); + auto neg_rounder_t = Full(neg_rounder, input_shape, dtype); - auto pos_rounder = MakeConstantScalar(hp_dtype, pos_rounding_value); - auto neg_rounder = MakeConstantScalar(hp_dtype, pos_rounding_value - 1); - auto pos_rounder_t = Full(pos_rounder, input_shape, hp_dtype); - auto neg_rounder_t = Full(neg_rounder, input_shape, hp_dtype); + auto zero_t = Zeros(input_shape, dtype); + return Where(GreaterEqual(input_tensor, zero_t), pos_rounder_t, neg_rounder_t); + }; - auto zero_t = Zeros(input_shape, hp_dtype); - round_scalar = Where(GreaterEqual(tensor, zero_t), pos_rounder_t, neg_rounder_t); + Expr round_scalar; + if (rounding == "TONEAREST") { + round_scalar = nearest_rounding_scalar(scaled_tensor, total_right_shift, hp_dtype); + } else if (rounding == "TFLITE") { + auto scalar_t = Full(scalar, input_shape, hp_dtype); + bool possible_to_overflow = fixed_point_multiplier == std::numeric_limits::min(); + auto high32_t = SaturatingRoundingDoublingHigh32(tensor, scalar_t, scaled_tensor, input_shape, + possible_to_overflow); + + if (right_shift <= 0) { + scaled_tensor = high32_t; + } else { + auto zero_t = Zeros(input_shape, lp_dtype); + round_scalar = nearest_rounding_scalar(high32_t, right_shift, lp_dtype); + scaled_tensor = Add(high32_t, round_scalar); + auto rshift_expr = MakeConstantScalar(lp_dtype, right_shift); + scaled_tensor = RightShift(scaled_tensor, rshift_expr); + } + return scaled_tensor; + } else { + LOG(FATAL) << "Rounding mode " << rounding << " not supported."; + } // Add the rounding scalar. - tensor = Add(tensor, round_scalar); + tensor = Add(scaled_tensor, round_scalar); // 5) Simply right shift the result to get the final output. tensor = RightShift(tensor, MakeConstantScalar(hp_dtype, total_right_shift)); @@ -121,29 +189,36 @@ Expr FixedPointMultiplyPerChannel(Expr tensor, std::vector multipliers, // Choose high precision datatype to be int64. This is for avoiding overflow // in multiplication of two int32 values. DataType hp_dtype = DataType::Int(64); + DataType lp_dtype = DataType::Int(32); tensor = Cast(tensor, hp_dtype); // 1) Calculating the integer multiplier and integer shift. These are calculated per axis/per // channel. - std::vector fixed_pt_multipliers, lshifts, rshifts; - bool is_lshift_required = false; + std::vector fixed_pt_multipliers, lshifts, rshifts; + bool lshift_required = false; + bool rshift_required = false; + bool possible_to_overflow = false; for (auto multiplier : multipliers) { - int32_t fixed_pt_multiplier, shift; + int64_t fixed_pt_multiplier, shift; std::tie(fixed_pt_multiplier, shift) = GetFixedPointMultiplierShift(multiplier); - int lshift = shift > 0 ? shift : 0; - int rshift = shift > 0 ? 0 : -shift; + int64_t lshift = shift > 0 ? shift : 0; + int64_t rshift = shift > 0 ? 0 : -shift; fixed_pt_multipliers.push_back(fixed_pt_multiplier); lshifts.push_back(lshift); rshifts.push_back(rshift); - is_lshift_required = is_lshift_required | (lshift != 0); + lshift_required |= (lshift != 0); + rshift_required |= (rshift != 0); + possible_to_overflow |= (fixed_pt_multiplier == std::numeric_limits::min()); } // 2) Multiply the integer multiplier. Convert lefts shifts into expr and multiply. - if (is_lshift_required) { + if (lshift_required) { auto lshift_expr = MakeConstantTensor(hp_dtype, {n_channels}, lshifts); auto exp_lshift_expr = ExpandBiasToMatchAxis(lshift_expr, n_dim, {channel_axis}); tensor = LeftShift(tensor, exp_lshift_expr); } + auto rshift_expr = MakeConstantTensor(lp_dtype, {n_channels}, rshifts); + auto exp_rshift_expr = ExpandBiasToMatchAxis(rshift_expr, n_dim, {channel_axis}); // 3) Perform the multiplication in higher precision. // The scalar is a fixed point value of int32 where the decimal point is @@ -154,41 +229,70 @@ Expr FixedPointMultiplyPerChannel(Expr tensor, std::vector multipliers, auto fixed_pt_multiplier_expr = MakeConstantTensor(hp_dtype, {n_channels}, fixed_pt_multipliers); auto exp_fixed_pt_multiplier_expr = ExpandBiasToMatchAxis(fixed_pt_multiplier_expr, n_dim, {channel_axis}); - tensor = Multiply(tensor, exp_fixed_pt_multiplier_expr); + auto scaled_tensor = Multiply(tensor, exp_fixed_pt_multiplier_expr); // 4) Find the rounding scalar. This depends on where the final decimal point sits. As we will be // right shifting the multiplied_t, we need to first calculate the total_rshift. Further, we can // calculate the pos and neg rounding offset. - std::vector pos_rounding_values, neg_rounding_values, total_rshifts; + std::vector pos_rounding_values, total_rshifts; for (auto rshift : rshifts) { - int total_rshift = rshift + 31; + int64_t total_rshift = rshift + 31; total_rshifts.push_back(total_rshift); pos_rounding_values.push_back((1ll << (total_rshift - 1))); - neg_rounding_values.push_back((1ll << (total_rshift - 1)) - 1); } + + // This lambda function gathers some shared logic in "TONEAREST" and "TFLITE" + // rounding scheme, which calculates a rounder tensor according to the sign + // of values in the tensor to be rounded. + auto nearest_rounding_tensor = [&](const Expr& input_tensor, const std::vector& rshifts, + DataType dtype) -> Expr { + std::vector pos_rounding_values, neg_rounding_values; + for (auto rshift : rshifts) { + int64_t pos_rounding_val = rshift > 0 ? (1ll << (rshift - 1)) : 0; + int64_t neg_rounding_val = rshift > 0 ? ((1ll << (rshift - 1)) - 1) : 0; + pos_rounding_values.push_back(pos_rounding_val); + neg_rounding_values.push_back(neg_rounding_val); + } + // Make a Relay expr from positive and negative rounding offset values. + auto pos_rounding_value_expr = MakeConstantTensor(dtype, {n_channels}, pos_rounding_values); + auto exp_pos_rounding_value_expr = + ExpandBiasToMatchAxis(pos_rounding_value_expr, n_dim, {channel_axis}); + auto pos_rounder = BroadCastTo(exp_pos_rounding_value_expr, input_shape); + auto neg_rounding_value_expr = MakeConstantTensor(dtype, {n_channels}, neg_rounding_values); + auto exp_neg_rounding_value_expr = + ExpandBiasToMatchAxis(neg_rounding_value_expr, n_dim, {channel_axis}); + auto neg_rounder = BroadCastTo(exp_neg_rounding_value_expr, input_shape); + auto zero_t = Zeros(input_shape, dtype); + return Where(GreaterEqual(input_tensor, zero_t), pos_rounder, neg_rounder); + }; + // Make a Relay expr from positive and negative rounding offset values. auto pos_rounding_value_expr = MakeConstantTensor(hp_dtype, {n_channels}, pos_rounding_values); auto exp_pos_rounding_value_expr = ExpandBiasToMatchAxis(pos_rounding_value_expr, n_dim, {channel_axis}); - auto neg_rounding_value_expr = MakeConstantTensor(hp_dtype, {n_channels}, neg_rounding_values); - auto exp_neg_rounding_value_expr = - ExpandBiasToMatchAxis(neg_rounding_value_expr, n_dim, {channel_axis}); Expr round_scalar; if (rounding == "UPWARD") { round_scalar = exp_pos_rounding_value_expr; } else if (rounding == "TONEAREST") { - // To satisfy where op shape requirements, the rounding values are broadcasted. - auto pos_rounder = BroadCastTo(exp_pos_rounding_value_expr, input_shape); - auto neg_rounder = BroadCastTo(exp_neg_rounding_value_expr, input_shape); - - auto zero_t = Zeros(input_shape, hp_dtype); - round_scalar = Where(GreaterEqual(tensor, zero_t), pos_rounder, neg_rounder); + round_scalar = nearest_rounding_tensor(scaled_tensor, total_rshifts, hp_dtype); + } else if (rounding == "TFLITE") { + auto high32_t = SaturatingRoundingDoublingHigh32( + tensor, exp_fixed_pt_multiplier_expr, scaled_tensor, input_shape, possible_to_overflow); + if (!rshift_required) { + return high32_t; + } else { + auto zero_t = Zeros(input_shape, lp_dtype); + round_scalar = nearest_rounding_tensor(high32_t, rshifts, lp_dtype); + scaled_tensor = Add(high32_t, round_scalar); + return RightShift(scaled_tensor, exp_rshift_expr); + } } else { LOG(FATAL) << "Rounding mode " << rounding << " not supported."; } + // Add the rounding scalar. - tensor = Add(tensor, round_scalar); + tensor = Add(scaled_tensor, round_scalar); // 5) Simply right shift the result to get the final output. auto total_rshift_expr = MakeConstantTensor(hp_dtype, {n_channels}, total_rshifts); diff --git a/src/relay/qnn/utils.h b/src/relay/qnn/utils.h index 23759a52ec41..36b0b28b763f 100644 --- a/src/relay/qnn/utils.h +++ b/src/relay/qnn/utils.h @@ -134,8 +134,8 @@ static inline int64_t get_const_int(const tvm::PrimExpr& x) { * 2) Round the result. * 3) Right shift the result */ -Expr FixedPointMultiplyToNearest(Expr tensor, double multiplier, - const Array& input_shape); +Expr FixedPointMultiply(Expr tensor, double multiplier, const Array& input_shape, + const std::string& rounding = "TONEAREST"); /* * \brief Fixed point multiplication between integer tensor with floating point @@ -226,6 +226,27 @@ static inline std::vector GetFloatVectorFromConstant(const Expr& expr) { return vals; } +static inline std::vector GetDoubleVectorFromConstant(const Expr& expr) { + const auto* n = expr.as(); + std::vector vals; + ICHECK(n) << "Expr must be a constant expr - " << AsText(expr, false); + int64_t num_elems = 1; + auto shape = n->data.Shape(); + for (size_t i = 0; i < shape.size(); i++) { + num_elems *= shape[i]; + } + for (int64_t i = 0; i < num_elems; i++) { + vals.push_back(static_cast(n->data->data)[i]); + } + return vals; +} + +static inline DataType GetDataTypeFromConstant(const Expr& expr) { + const auto* n = expr.as(); + ICHECK(n) << "Expr must be a constant expr - " << AsText(expr, false); + return n->tensor_type()->dtype; +} + } // namespace qnn } // namespace relay } // namespace tvm diff --git a/src/relay/quantize/realize.cc b/src/relay/quantize/realize.cc index 2716c6e65f65..f7a1faa83a5f 100644 --- a/src/relay/quantize/realize.cc +++ b/src/relay/quantize/realize.cc @@ -118,7 +118,8 @@ inline Expr MulAndDiv(Expr data, float s1, float s2, DataType dtype, std::tie(fixed_point_multiplier, shift) = qnn::GetFixedPointMultiplierShift(factor); data = relay::FixedPointMultiply(data, fixed_point_multiplier, shift); } else { - data = qnn::FixedPointMultiplyToNearest(data, factor, data_shape); + ICHECK(cfg->rounding == "TONEAREST" || cfg->rounding == "TFLITE"); + data = qnn::FixedPointMultiply(data, factor, data_shape, cfg->rounding); } return Cast(data, dtype); @@ -177,8 +178,8 @@ Expr QuantizeRealize(const Call& ref_call, const Array& new_args, const Ob qnn::GetFixedPointMultiplierShift(idom_scale_imm / odom_scale_imm); data = relay::FixedPointMultiply(data, fixed_point_multiplier, shift); } else { - data = qnn::FixedPointMultiplyToNearest(data, idom_scale_imm / odom_scale_imm, - ref_call->type_as()->shape); + data = qnn::FixedPointMultiply(data, idom_scale_imm / odom_scale_imm, + ref_call->type_as()->shape, cfg->rounding); } data = Cast(Clip(data, clip_min_imm, clip_max_imm), n->dtype); return QRealizeIntExpr(data, dom_scale, n->dtype); diff --git a/src/relay/transforms/pattern_utils.h b/src/relay/transforms/pattern_utils.h index bc0fcc9f2988..74ae019b8bb6 100644 --- a/src/relay/transforms/pattern_utils.h +++ b/src/relay/transforms/pattern_utils.h @@ -674,6 +674,21 @@ static inline Expr BroadCastTo(Expr data, Array shape) { return MakeBroadCastTo(data, CheckConstantShapeArrayInteger(shape)); } +static inline Expr Greater(Expr lhs, Expr rhs) { + static const Op& op = Op::Get("greater"); + return Call(op, {lhs, rhs}, Attrs(), {}); +} + +static inline Expr Equal(Expr lhs, Expr rhs) { + static const Op& op = Op::Get("equal"); + return Call(op, {lhs, rhs}, Attrs(), {}); +} + +static inline Expr LogicalAnd(Expr lhs, Expr rhs) { + static const Op& op = Op::Get("logical_and"); + return Call(op, {lhs, rhs}, Attrs(), {}); +} + Expr StopFusion(Expr data); Expr CastHint(Expr data, DataType dtype); diff --git a/tests/python/contrib/test_arm_compute_lib/test_add.py b/tests/python/contrib/test_arm_compute_lib/test_add.py index d7abc5c414fb..4cc1fb6627f5 100644 --- a/tests/python/contrib/test_arm_compute_lib/test_add.py +++ b/tests/python/contrib/test_arm_compute_lib/test_add.py @@ -73,6 +73,8 @@ def _get_expected_codegen(shape, dtype, op_name, qnn_params): "dtype": [[dtype]], }, } + if dtype == "uint8": + node["attrs"]["rounding"] = [["UPWARD"]] return [*inputs, node] diff --git a/tests/python/frontend/tflite/test_forward.py b/tests/python/frontend/tflite/test_forward.py index 0d02c15f2eb8..2cd8c4ed4158 100644 --- a/tests/python/frontend/tflite/test_forward.py +++ b/tests/python/frontend/tflite/test_forward.py @@ -3826,11 +3826,9 @@ def test_forward_qnn_inception_v1_net(): tflite_output = run_tflite_graph(tflite_model_buf, data) tflite_predictions = np.squeeze(tflite_output) - tflite_sorted_labels = tflite_predictions.argsort()[-3:][::-1] tvm_output = run_tvm_graph(tflite_model_buf, data, "input") tvm_predictions = np.squeeze(tvm_output) - tvm_sorted_labels = tvm_predictions.argsort()[-3:][::-1] - tvm.testing.assert_allclose(tvm_sorted_labels, tflite_sorted_labels) + tvm.testing.assert_allclose(tvm_predictions, tflite_predictions, rtol=0, atol=0) def test_forward_qnn_mobilenet_v1_net(): @@ -3843,18 +3841,13 @@ def test_forward_qnn_mobilenet_v1_net(): with open(tflite_model_file, "rb") as f: tflite_model_buf = f.read() - # Test image. Checking the labels because the requantize implementation is different between - # TFLite and Relay. This cause final output numbers to mismatch. So, testing accuracy via - # labels. Also, giving a real image, instead of random inputs. data = get_real_image(224, 224) tflite_output = run_tflite_graph(tflite_model_buf, data) - tflite_predictions = np.squeeze(tflite_output) - tflite_sorted_labels = tflite_predictions.argsort()[-3:][::-1] + tflite_predictions = np.squeeze(tflite_output).astype("int32") tvm_output = run_tvm_graph(tflite_model_buf, data, "input") - tvm_predictions = np.squeeze(tvm_output) - tvm_sorted_labels = tvm_predictions.argsort()[-3:][::-1] - tvm.testing.assert_allclose(tvm_sorted_labels, tflite_sorted_labels) + tvm_predictions = np.squeeze(tvm_output).astype("int32") + tvm.testing.assert_allclose(tvm_predictions, tflite_predictions, rtol=0, atol=0) def test_forward_qnn_mobilenet_v2_net(): @@ -3867,18 +3860,13 @@ def test_forward_qnn_mobilenet_v2_net(): with open(tflite_model_file, "rb") as f: tflite_model_buf = f.read() - # Test image. Checking the labels because the requantize implementation is different between - # TFLite and Relay. This cause final output numbers to mismatch. So, testing accuracy via - # labels. Also, giving a real image, instead of random inputs. data = get_real_image(224, 224) tflite_output = run_tflite_graph(tflite_model_buf, data) - tflite_predictions = np.squeeze(tflite_output) - tflite_sorted_labels = tflite_predictions.argsort()[-3:][::-1] + tflite_predictions = np.squeeze(tflite_output).astype("int32") tvm_output = run_tvm_graph(tflite_model_buf, data, "input") - tvm_predictions = np.squeeze(tvm_output) - tvm_sorted_labels = tvm_predictions.argsort()[-3:][::-1] - tvm.testing.assert_allclose(tvm_sorted_labels, tflite_sorted_labels) + tvm_predictions = np.squeeze(tvm_output).astype("int32") + tvm.testing.assert_allclose(tvm_predictions, tflite_predictions, rtol=0, atol=0) ####################################################################### @@ -3908,11 +3896,9 @@ def test_forward_qnn_mobilenet_v3_net(): tflite_output = run_tflite_graph(tflite_model_buf, data) tflite_predictions = np.squeeze(tflite_output) - tflite_sorted_labels = tflite_predictions.argsort()[-3:][::-1] tvm_output = run_tvm_graph(tflite_model_buf, data, "input") tvm_predictions = np.squeeze(tvm_output) - tvm_sorted_labels = tvm_predictions.argsort()[-3:][::-1] - tvm.testing.assert_allclose(tvm_sorted_labels, tflite_sorted_labels) + tvm.testing.assert_allclose(tvm_predictions, tflite_predictions, rtol=0, atol=0) def test_forward_tflite2_qnn_resnet50(): @@ -4283,7 +4269,7 @@ def test_prevent_tensorflow_dynamic_range(): # End to End Sparse models test_forward_sparse_mobilenet_v1() test_forward_sparse_mobilenet_v2() - + # # End to End quantized test_forward_qnn_inception_v1_net() test_forward_qnn_mobilenet_v1_net() @@ -4292,7 +4278,7 @@ def test_prevent_tensorflow_dynamic_range(): # with Tflite 1.15.2 test_forward_qnn_mobilenet_v3_net() test_forward_qnn_coco_ssd_mobilenet_v1() - + # # TFLite 2.1.0 quantized tests test_forward_tflite2_qnn_resnet50() test_forward_tflite2_qnn_inception_v1() diff --git a/tests/python/relay/test_op_level2.py b/tests/python/relay/test_op_level2.py index 1a1f451f4c74..923de390f38c 100644 --- a/tests/python/relay/test_op_level2.py +++ b/tests/python/relay/test_op_level2.py @@ -1003,8 +1003,18 @@ def test_pool2d(): _test_pool2d(relay.nn.max_pool2d, np.max, pool_size=2, strides=2, padding=0) _test_pool2d(relay.nn.avg_pool2d, np.mean) _test_pool2d(relay.nn.avg_pool2d, np.mean, pool_size=2, strides=2, padding=0) - _test_pool2d_int(relay.nn.avg_pool2d, np.mean, "int32") - _test_pool2d_int(relay.nn.avg_pool2d, np.mean, "uint16") + + def mean_integer(int_array, axis, keepdims=False): + sum_array = np.sum(int_array, axis=axis, keepdims=keepdims) + input_shape = np.shape(int_array) + kernel_size = 1 + for a in axis: + kernel_size = kernel_size * input_shape[a] + sum_array += kernel_size // 2 + return sum_array // kernel_size + + _test_pool2d_int(relay.nn.avg_pool2d, mean_integer, "int32") + _test_pool2d_int(relay.nn.avg_pool2d, mean_integer, "uint16") _test_global_pool2d(relay.nn.global_max_pool2d, np.max) _test_global_pool2d(relay.nn.global_avg_pool2d, np.mean) diff --git a/tests/python/relay/test_op_level5.py b/tests/python/relay/test_op_level5.py index 6d7d401d706b..669a311d4d04 100644 --- a/tests/python/relay/test_op_level5.py +++ b/tests/python/relay/test_op_level5.py @@ -151,6 +151,9 @@ def verify_crop_and_resize( func = relay.Function([img, bx, bx_idx], z) for target, ctx in tvm.testing.enabled_targets(): + # LLVM11 on A100 has bug with nvptx backend, disable it. + if target.startswith("nvptx"): + continue for kind in ["graph", "debug"]: intrp = relay.create_executor(kind, ctx=ctx, target=target) op_res = intrp.evaluate(func)(image_data, boxes, box_indices) diff --git a/tests/python/relay/test_op_qnn_concatenate.py b/tests/python/relay/test_op_qnn_concatenate.py index 55836dc1ee52..e6f8328aeb76 100644 --- a/tests/python/relay/test_op_qnn_concatenate.py +++ b/tests/python/relay/test_op_qnn_concatenate.py @@ -28,8 +28,8 @@ def test_same_io_qnn_params(): axis = 0 x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype) y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype) - x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") - y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") + x_scale = relay.const(2.933666e-8, "float32") + y_scale = relay.const(2.933666e-8, "float32") zero = relay.const(0, "int32") x = relay.var("x", shape=(1, 64), dtype=data_dtype) @@ -62,8 +62,8 @@ def test_different_io_qnn_params(): x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype) y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype) - x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") - y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") + x_scale = relay.const(2.933666e-8, "float32") + y_scale = relay.const(2.933666e-8, "float32") x_zero_point = relay.const(3, "int32") y_zero_point = relay.const(4, "int32") @@ -97,8 +97,8 @@ def test_few_same_io_qnn_params(): x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype) y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype) - x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") - y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") + x_scale = relay.const(2.933666e-8, "float32") + y_scale = relay.const(2.933666e-8, "float32") x_zero_point = relay.const(0, "int32") y_zero_point = relay.const(1, "int32") @@ -132,8 +132,8 @@ def test_same_i_qnn_params(): x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype) y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype) - x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") - y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), "float32") + x_scale = relay.const(2.933666e-8, "float32") + y_scale = relay.const(2.933666e-8, "float32") x_zero_point = relay.const(0, "int32") y_zero_point = relay.const(0, "int32") diff --git a/tests/python/relay/test_op_qnn_requantize.py b/tests/python/relay/test_op_qnn_requantize.py index f40a08711451..223ffc9f86a8 100644 --- a/tests/python/relay/test_op_qnn_requantize.py +++ b/tests/python/relay/test_op_qnn_requantize.py @@ -21,7 +21,7 @@ from tvm import relay from tvm.contrib import graph_runtime -roundings = ["UPWARD", "TONEAREST"] +roundings = ["UPWARD", "TONEAREST", "TFLITE"] def verify(mod, goldens): @@ -106,7 +106,10 @@ def test_downscale(): # Try positive values # 8 corresponds to 0.5, resulting in 1 golden_data = np.arange(0, 32, 1).astype("int32") - golden_output = np.repeat([0, 1, 2], [8, 16, 8]) + if rounding == "TFLITE": + golden_output = np.repeat([0, 1, 2], [7, 16, 9]) + else: + golden_output = np.repeat([0, 1, 2], [8, 16, 8]) verify(mod, (golden_data, golden_output)) # Try negative values @@ -131,7 +134,10 @@ def test_downscale(): # Try positive values # 2I corresponds to 0.5, resulting in 1 golden_data = np.arange(0, 32, 1).astype("int32") - golden_output = np.repeat([0, 1, 2, 3, 4, 5, 6, 7, 8], [2, 4, 4, 4, 4, 4, 4, 4, 2]) + if rounding == "TFLITE": + golden_output = np.repeat([0, 1, 2, 3, 4, 5, 6, 7, 8], [1, 4, 4, 4, 4, 4, 4, 4, 3]) + else: + golden_output = np.repeat([0, 1, 2, 3, 4, 5, 6, 7, 8], [2, 4, 4, 4, 4, 4, 4, 4, 2]) verify(mod, (golden_data, golden_output)) # Try negative values @@ -160,7 +166,10 @@ def test_downscale(): # Try positive values # 8 corresponds to 0.5, resulting in 1 golden_data = np.arange(0, 32, 1).astype("int32") - golden_output = np.repeat([0, 1, 2], [8, 16, 8]) + if rounding == "TFLITE": + golden_output = np.repeat([0, 1, 2], [7, 16, 9]) + else: + golden_output = np.repeat([0, 1, 2], [8, 16, 8]) verify(mod, (golden_data, golden_output)) # Try uint8 in_dtyope and uint8 out_dtype @@ -176,7 +185,10 @@ def test_downscale(): # Try positive values # 8 corresponds to 0.5, resulting in 1 golden_data = np.arange(0, 32, 1).astype("int32") - golden_output = np.repeat([0, 1, 2], [8, 16, 8]) + if rounding == "TFLITE": + golden_output = np.repeat([0, 1, 2], [7, 16, 9]) + else: + golden_output = np.repeat([0, 1, 2], [8, 16, 8]) verify(mod, (golden_data, golden_output)) @@ -307,7 +319,10 @@ def test_zero_point(): # Try positive values # 8 corresponds to 0.5, resulting in 1 golden_data = np.arange(0, 32, 1).astype("int32") - golden_output = np.repeat([0, 1, 2], [8, 16, 8]) + if rounding == "TFLITE": + golden_output = np.repeat([0, 1, 2], [7, 16, 9]) + else: + golden_output = np.repeat([0, 1, 2], [8, 16, 8]) golden_output = np.add(1, golden_output) verify(mod, (golden_data, golden_output)) @@ -335,7 +350,10 @@ def test_zero_point(): # Try positive values golden_data = np.arange(32, 64, 1).astype("int32") - golden_output = np.repeat([2, 3, 4], [8, 16, 8]) + if rounding == "TFLITE": + golden_output = np.repeat([2, 3, 4], [7, 16, 9]) + else: + golden_output = np.repeat([2, 3, 4], [8, 16, 8]) golden_output = np.subtract(golden_output, 1) verify(mod, (golden_data, golden_output)) @@ -351,31 +369,187 @@ def test_zero_point(): def test_per_channel_same_scale(): # Have same scales, everything within range - golden_data = np.arange(-5, 5, 1).astype("int32").reshape((5, 2)) - golden_output = golden_data + golden_data = np.arange(-32, 32, 1).astype("int32").reshape((32, 2)) for rounding in roundings: mod = get_mod( - data_shape=(5, 2), + data_shape=(32, 2), data_dtype="int32", out_dtype="int8", - input_scale=[0.5, 0.5], - output_scale=0.5, + input_scale=[1, 32], + output_scale=16, axis=1, rounding=rounding, ) + + if rounding == "UPWARD": + golden_output = np.array( + [ + -2, + -62, + -2, + -58, + -2, + -54, + -2, + -50, + -1, + -46, + -1, + -42, + -1, + -38, + -1, + -34, + -1, + -30, + -1, + -26, + -1, + -22, + -1, + -18, + 0, + -14, + 0, + -10, + 0, + -6, + 0, + -2, + 0, + 2, + 0, + 6, + 0, + 10, + 0, + 14, + 1, + 18, + 1, + 22, + 1, + 26, + 1, + 30, + 1, + 34, + 1, + 38, + 1, + 42, + 1, + 46, + 2, + 50, + 2, + 54, + 2, + 58, + 2, + 62, + ] + ).reshape(32, 2) + else: + golden_output = np.array( + [ + -2, + -62, + -2, + -58, + -2, + -54, + -2, + -50, + -2, + -46, + -1, + -42, + -1, + -38, + -1, + -34, + -1, + -30, + -1, + -26, + -1, + -22, + -1, + -18, + -1, + -14, + 0, + -10, + 0, + -6, + 0, + -2, + 0, + 2, + 0, + 6, + 0, + 10, + 0, + 14, + 1, + 18, + 1, + 22, + 1, + 26, + 1, + 30, + 1, + 34, + 1, + 38, + 1, + 42, + 1, + 46, + 2, + 50, + 2, + 54, + 2, + 58, + 2, + 62, + ] + ).reshape(32, 2) verify(mod, (golden_data, golden_output)) # Change axis - golden_data = np.arange(-10, 10, 1).astype("int32").reshape((2, 2, 5)) - golden_output = golden_data + golden_data = np.arange(-20, 20, 2).astype("int32").reshape((2, 2, 5)) + golden_output = np.array( + [-20, -18, -16, -14, -12, -5, -4, -3, -2, -1, 0, 2, 4, 6, 8, 5, 6, 7, 8, 9] + ).reshape((2, 2, 5)) for rounding in roundings: mod = get_mod( data_shape=(2, 2, 5), data_dtype="int32", out_dtype="int8", - input_scale=[0.5, 0.5], + input_scale=[0.5, 0.25], + output_scale=0.5, + axis=1, + rounding=rounding, + ) + verify(mod, (golden_data, golden_output)) + + # Have input scale > output scale + golden_data = np.arange(-5, 5, 1).astype("int32").reshape((5, 2)) + golden_output = np.array([-10, -2, -6, -1, -2, 0, 2, 1, 6, 2]).reshape((5, 2)) + + for rounding in roundings: + mod = get_mod( + data_shape=(5, 2), + data_dtype="int32", + out_dtype="int8", + input_scale=[1.0, 0.25], output_scale=0.5, axis=1, rounding=rounding,