o %Tḩ@sddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlm Z Gd d d eZ!dS) N)onnx_pb) BaseQuantizerQuantizationParams) TensorData) ONNXModel)TENSOR_NAME_QUANT_SUFFIXQuantizationModeQuantizedValueQuantizedValueType __producer__ __version__add_infer_metadataattribute_to_kwargcompute_scale_zpcompute_scale_zp_float8 find_by_nameget_qmin_qmax_for_qTypeget_qrange_for_qType ms_domain&save_and_reload_model_with_shape_infertensor_proto_to_array)CreateOpQuantizerc@s eZdZ d:ddZddZddZdd Zd d Zd d ZddZ ddZ d;ddZ ddZ ddZ ddZddZdd$d%Zd&d'Zd;d(d)Z   * d?d+d,Z -   * d@d.d/ZdAd0d1Z - dBd2d3Zd4d5Zd6d7Zd8d9ZdS)C ONNXQuantizerNc  Cszt|||||||| | | | |sE|jt|jj}dd|jjD|_|jdd|jj D|jdd|jj Dt ||_||_ ||_ |jdk|_d|jvoZ|jd|_g|_d|_i|_|jdd|jj D|jd d|jj D|jjjjD]} |jd d| j Dq|j tvrtd |j ||_d |_d |_d|_d|_i|_|j|_ dS)NcSi|]}|j|qSname).0virrh/home/air/segue/gemini/back/venv/lib/python3.10/site-packages/onnxruntime/quantization/onnx_quantizer.py Gz*ONNXQuantizer.__init__..cSrrrrotrrr r!Hr"cSrrrritrrr r!Ir" MatMulConstBOnly/cSi|]}|jdqSrrr#rrr r!Ur"cSr*r+rr%rrr r!Vr"cSsi|]}|dqSr+r)r output_namerrr r!Xszunsupported quantization mode fixed_quantization_range_uint8fixed_quantization_range_int8 fixed_zero fixed_zero_zp)!r__init__modelreplace_gemm_with_matmulrgraph value_info value_infosupdateoutputinputrmodestatic opset_versionfuse_dynamic_quant extra_optionsq_matmul_const_b_only new_nodes graph_scope tensor_namesnoder ValueErrorcalculate_quantization_paramsquantization_paramsfixed_qrange_uint8_namefixed_qrange_int8_namefixed_zero_namefixed_zero_zp_namequantized_value_mapget_non_initializer_inputsgenerated_value_names)selfr2 per_channel reduce_ranger:r; weight_qTypeactivation_qType tensors_rangenodes_to_quantizenodes_to_excludeop_types_to_quantizer>rCrrr r1&sP      zONNXQuantizer.__init__c Cs~tjj|d|jjjd}t|t||j|j|j |j |j |j |j |j|j|j|j }||_|j|d|_||jjjS)z generate submodel for the subgraph, so that we re-utilize current quantization implementation. quantize the submodel update subgraph and set it back to node onnx-quantizer) producer_name opset_importsr))onnxhelper make_modelr2 opset_importrrrOrPr:r;rQrRrSrTrUrVr>parentrAquantize_modelr4)rNsubgraph graph_key warped_model sub_quantizerrrr quantize_subgraphos0 zONNXQuantizer.quantize_subgraphc Cs dd|jD}t|dkr|S|jr|jn |jdt|j}i}|jD]M}|jtjjkr@|j| |j |d|ji}n.|jtjj krjg}|j D]}| | ||d|jdt|gqL|j|i}nt|}||q&tjj|j|j|jfd|ji|S)z| Check subgraph, if any, quantize it and replace it. return new_nodes added for quantizing subgraph cSs,g|]}|jtjjks|jtjjkr|qSr)typerZAttributeProtoGRAPHGRAPHS)rattrrrr s z>ONNXQuantizer.quantize_node_with_sub_graph..r _node_count_:r) attributelenrop_typer@rerZrfrgrdgrhgraphsextendrr7r[ make_noder9r8) rNrC graph_attrs node_namekwargsrikvvaluer`rrr quantize_node_with_sub_graphs0 " "   $z*ONNXQuantizer.quantize_node_with_sub_graphcCstdd|jDS)zQ Detect if model already has QuantizeLinear or DequantizeLinear. css$|] }|jdkp |jdkVqdS)QuantizeLinearDequantizeLinearN)rorrCrrr s z.ONNXQuantizer.has_QDQ_nodes..)anyr2nodes)rNrrr has_QDQ_nodesszONNXQuantizer.has_QDQ_nodescCs2t||jdur dS|jdur|j|SdS)NTF)rr2 initializerr^find_initializer_in_path)rNinitializer_namerrr rs   z&ONNXQuantizer.find_initializer_in_pathcCs2|j||D]}|jD]}|j|q qdSN)r@rrr8rMadd)rNrrCr,rrr add_new_nodess  zONNXQuantizer.add_new_nodesc CsD|r td|jD]2}|jr||}t|j}t ||}| t |t|jD]}|j|j D]}|j |q6q.q||jd|jj|j|jdurq|j\}}t|dkrqtdt|t|jj_t|jj_dd|jjjD}|sdd|jD} | r|jjj} d| _t| _|jjS) NzPlease check if the model is already quantized. Note you don't need to quantize a QAT model. OnnxRuntime support to run QAT model directly.rCrz0Invalid model with unknown initializers/tensors.cSsg|] }|jtkr|qSr)domainr)ropsetrrr rjz0ONNXQuantizer.quantize_model..cSsg|] }|jdkr|qS)z com.microsoft)rr|rrr rjrr) rloggingwarningr2renable_subgraph_quantizationryrnr@rquantizeranger8rMr_dequantize_outputsr4 ClearFieldrCrrr^clean_initializers RuntimeErrorstrr rXr producer_versionr]versionrr) rNrCnumber_of_existing_new_nodes op_quantizerir,_initializers_not_foundms_opsetms_nodesrrrr r_s@       zONNXQuantizer.quantize_modelcCs8d|jvrtd||jd|jdStd|d)NDefaultTensorTypezDget_tensor_type returns DefaultTensorType for tensor name %r, use %dz)Unable to find data type for weight_name=a7. shape_inference failed to return a type probably this node is from a different domain or using an input produced by such an operator. This may happen if you quantize a model already quantized. You may use extra_options `DefaultTensorType` to indicate the default weight type, usually `onnx.TensorProto.FLOAT`.)r>rinforrN tensor_namerrr _get_default_tensor_types   z&ONNXQuantizer._get_default_tensor_typeFcCst||j}|dur|jS||jvr2|j|}|jdr2|r-|jjjdkr-| |S|jjjS|j r:|j durC|rA| |SdS|j |}|durO|S|j ra|j ra|j |}|dura|S|rh| |SdS)N tensor_typer)rr2r data_typer6reHasFieldr elem_typerrr^is_valid_quantize_weightget_tensor_type)rNr mandatoryweightrotyperesrrr rs.          zONNXQuantizer.get_tensor_typecCs||r ||S||jvr8|j|}|jdr)|jjjtjj tjj fvr)dSt d|d|jddS|j rD|jrD|j|St d|ddS) NrTzr?r@rrr quantize_weights zONNXQuantizer.quantize_weightTc Csg}g} g} g} |D]e} |j| } | |jvr/|j| }||j| |j| |jq | sA| d|d| dq t| |j}|dur|j re|re| |j |r[|j n|j ||\}}}n|||rm|j n|j |\}}}| || |||q || r8|j| d|j|j}|dur|j| }||jvr|j|}|dsJd|d|jdsJd|d|jjj}n||jvsJd |d |j|}|j|| |j |d }|durd S|r||n| ||d }|jdkr| |j||jd| |jdq | |jd||jd| |jdq |jdurf|jj|| g||||dd\}}}}| |d||d| |dq t d| d|j!| | || fS)a Given a node, this function quantizes the inputs as follows: - If input is an initializer, quantize the initializer data, replace old initializer with new initializer - Else, add QuantizeLinear nodes to perform quantization parameter node: node being quantized in NodeProto format. parameter indices: input indices to quantize. return: (List of quantized input names, List of zero point names used for input quantization, List of scale names used for input quantization, List of new QuantizeLinear nodes created) rNrrez value_info=z has no type.rz is not a tensor.zshape inference failed for zF and attribute 'tensor_names' does not have any value for this tensor.r)NNNNrrzrrT)r=rPr>r?r@z!Invalid tensor name to quantize: z @graph scope)"r9rKrrr r(rr2rrOquantize_weight_per_channelrrQrRquantize_initializerr:find_node_by_namer@r4r6rrerrrBr#rrrror8r^rBrDrA)rNrCr<r=rPr>r?r@ scale_nameszero_point_namesquantized_input_namesrr node_inputr8r q_weight_namer rr!rr5rquantize_input_nodesparent_quantized_input_namesparent_zero_point_namesparent_scale_namesrrrr __quantize_inputss                               zONNXQuantizer.__quantize_inputsc Csj|j|jvr|j|j}|j|j|jfS|||||\}}}t|j|||tjd}||j|j<|||fS)a :param weight: TensorProto initializer :param qType: type to quantize to :param keep_float_weight: Whether to quantize the weight. In some cases, we only want to qunatize scale and zero point. If keep_float_weight is False, quantize the weight, or don't quantize the weight. :return: quantized weight name, zero point name, scale name N) rrKr(r rquantize_initializer_implr r r*) rNrrrPkeep_float_weightr8rMr rrrr rG\s&    z"ONNXQuantizer.quantize_initializerc Csd||jvr|j|}|j|j|jfS||||||\}}} t||| |tjd}||j|<||| fSr)rKr(r r quantize_weight_per_channel_implr r r*) rNr-rQ channel_axisrPrTr8rMr rrrr rF}s&     z)ONNXQuantizer.quantize_weight_per_channelcCs||jvrj||jvrj|j|}t|j|j}|jjjdks*|jjjdkr9|dur9|dus9tj |j dks9J|d}|j ||j |j }|dura|j|j|jg}tjd||g|}|S||jdksjJdS)a Given a value (input/output) which is quantized, add a DequantizeLinear node to dequantize it back to float32 or float16 parameter value_name: value to dequantize parameter new_nodes_list: List of new nodes created before processing current node return: None if there is already a DequantizeLinear node that dequantizes it A DequantizeLinear node otherwise rWNr_DequantizeLinearr{r)rKrMrrr2rrXrZ numpy_helperto_arrayr+rHr@r4r(r r[rsr8)rN value_namer8 scale_init dqlinear_name dqlinear_nodedqlinear_inputsdequantize_noderrr _dequantize_values&  zONNXQuantizer._dequantize_valuecCs6|jjD]}||j}|dur|j|qdS)z Dequantize output if it is quantized parameter new_nodes_list: List of new nodes created before processing current node return: List of new nodes created N)r2r4r8r`rr@r)rNr8r_rrr rs   z!ONNXQuantizer._dequantize_outputscCs@|jdurdS|i}|jD]}|j|}t|ts)tdt|d|d|jj|id}|j}d|vr=|dj }d|vrOd|vrO|d|d}}nE|t j j kr`t ||jd\}}n4|d |jd }|d |jd} |d |j} |d d} t|| | d\} } t|| | | | |j\}}t|||d||<q|S)Nrrr) default_valrrrrrminrrmax symmetricrPF)rPrd)rrr)rSadjust_tensor_rangesrrrretensor_quant_overridesget_per_tensor_overridesrRrrZr FLOAT8E4M3FNravg_stdget range_valueis_activation_symmetricrrmin_real_ranger)rNrFrtdquant_overridesrzerorrbrcrdrPqminqmaxrrr rEs0       z+ONNXQuantizer.calculate_quantization_paramsr)F)NN)NNN)r%)FFrF)TFFrF)FF)TF)__name__ __module__ __qualname__r1rdryrrrr_rrrrrrrr#r$r9r:rCrDrBrGrFr`rrErrrr r%sR I"- T ^< F 1    &  ' r)"rnumpyrrZonnx.numpy_helperrrbase_quantizerrr calibrater onnx_modelr quant_utilsrr r r r r rrrrrrrrrrregistryrrrrrr s   H