o 3Ih<2@s|ddlmZddlZddlmZmZddlmZmZmZm Z ddlm Z ddl m Z eeZGdddZGd d d ZdS) ) getLoggerN) array_equalndarray) NodeProto TensorProtohelper numpy_helper)onnx_pb) OnnxModelc @s.eZdZdefddZdedeeeffddZd1defd d Z d2ded e d ed Bded BfddZ defddZ defddZ eddZed3defddZdeded BfddZed4defddZed ejfd!d"Zed5dedefd$d%Zd&e fd'd(Zd)d*Zd+d,Zd-d.Zd/d0Zd S)6 FusionUtilsmodelcCs ||_dSN)r )selfr re/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/onnxruntime/transformers/fusion_utils.py__init__s zFusionUtils.__init__ input_namereturncCsn|j|}|dur&|jjjtjkr&||\}}t d|dd|fSt d|d|dud|fS)NzCasted graph input z to int32TzDid not cast graph input z to int32: found F) r find_graph_inputtype tensor_type elem_typerINT32cast_input_to_int32loggerdebug)rr graph_input cast_output cast_noderrrcast_graph_input_to_int32s z%FusionUtils.cast_graph_input_to_int32int32cCsf|d|}|dkrttj}n|dkrttj}n|dkr$ttj}ntd||||}||fS)N_r float32float16z"Invalid target_type: {target_type})intrrFLOATFLOAT16 ValueError add_cast_node)rr target_type output_nameto_typerrrr cast_inputs    zFusionUtils.cast_inputNr+r* graph_namec Cs|dur |d|}|g}|dur|j}||vr,||}|r,|jdkr,|jdg}tjd||gd}|jtd|g|jj ||d|S)N _cast_to_Castr)inputsoutputsto)r-) r output_name_to_nodeop_typeinputr make_node attributeextendmake_attributeadd_node) rrr+r*r3r-r0 parent_noderrrrr(.s  zFusionUtils.add_cast_nodecCs ||dS)Nr )r,)rrrrrrIs zFusionUtils.cast_input_to_int32cCs|j}||}|D]4}|jdkr?d}|jD]}|jdkr*|jttjkr*d}nq|r?|j d}|j ||j ||q dS)Nr/Fr2Tr) r input_name_to_nodesr4r7nameir$rroutput remove_nodereplace_input_of_all_nodes)rrr<nodesnodeis_int32attr*rrrremove_cast_int32Ls     zFusionUtils.remove_cast_int32cCs|d}|j||vr%|||j|vr%||j||t||j|}||j|<||vr7||||S|g||<|S)Nr)r5removelenappend)rCr>new_input_namer<old_input_referencerrrupdate_node_input[s   zFusionUtils.update_node_inputrc Cs<|j|}|j|}t||||}|dko|| } | S)a Before: (input)-->parent-->node-->(output) After: (input)-->parent--> | +----->node-->(output) This function returns a flag whether the parent node can be removed. r)r5r rLfind_graph_output) r rCr;r<node_input_indexparent_input_indexold_input_namerJrKparent_can_be_removedrrr skip_parentks zFusionUtils.skip_parentrCcCsV|jdvsJt|jdkr|j|jdSd}|jD] }|jdkr(t|}q|S)N)Squeeze Unsqueezeaxes) r4rHr5r get_constant_valuer7r=rget_attribute_value)rrCrVattrrrrget_squeeze_or_unsqueeze_axess   z)FusionUtils.get_squeeze_or_unsqueeze_axesattribute_namecCsR|}|jD] }|j|krt|}qt|tr%t|ttfo$t||ddS||kS)aVerify that a node has expected value for an attribute. Args: node (NodeProto): a node to check attribute_name (str): name of attribute expected_value (Any): expected value of the attribute default_value (Any, optional): default value if the attribute does not exist. Defaults to None. Returns: bool: whether the check is passed or not F equal_nan)r7r=rrX isinstancelistrr)rCr[expected_value default_valuevaluerYrrrcheck_node_attributes    z FusionUtils.check_node_attributetensorcCst|tjstdt|t|jdks|jtjjkr!td|j r@t t j |j dd|j}t |ddg}||_ |Std) zTranspose a 2-D INT8 TensorProto Args: tensor (TensorProto): tensor to be transposed Returns: tensor (TensorProto): transposed tensor z3Expected input type is an ONNX TensorProto but got z'Only INT8 2-D tensors can be transposedint8)dtyperUrzonly raw buffer supported)r^ onnx_protorr'rrHdims data_typeINT8raw_datanumpyreshape frombuffer transposetobytes)rd int32_dataint32_transposed_datarrrtranspose_2d_int8_tensors  z$FusionUtils.transpose_2d_int8_tensorTcCs|jdvrtd|j||jd}|durdS|jdkp,|jdko,|jddk}|r3|s3dSt|jdkrQuantizeLinearDequantizeLinearz+Provided node is not a Q/DQ node. Op Type: rUNFrreT) r4rrrWr5ndimshaperHrmall)rCr "allow_per_tensor_quantization_onlyscalescale_has_single_element zero_pointrrrcheck_qdq_node_for_fusions " z%FusionUtils.check_qdq_node_for_fusion input_indexcCsRt|j|ks J|j|j|}t|tr%t|ttfo$t||ddS||kS)a7Verify that a node has expected input value Args: node (NodeProto): a node to check input_index (int): index of its input to be verified expected_value (Any): expected value of the input Returns: bool: whether the check is passed or not Fr\)rHr5r rWr^r_rr)rrCrr`rbrrrcheck_node_input_values   z"FusionUtils.check_node_input_valuecCsg}|j}|jD] }|jdkr,|jd|vr,|j|jd|jd||q |rB|j|t dt |ddSdS)z>Remove Identity nodes, except those right before graph output.IdentityrzRemoved z Identity nodesN) r get_graphs_output_namesrBr4r?rAr5rI remove_nodesrinforH)rnodes_to_removegraph_output_namesrCrrrremove_identity_nodess    z!FusionUtils.remove_identity_nodescC|jdSr )r remove_cascaded_cast_nodesrrrrr z&FusionUtils.remove_cascaded_cast_nodescCrr )r remove_useless_cast_nodesrrrrr rz%FusionUtils.remove_useless_cast_nodescCs@|jjdd}|dur dSg}|jD]0}|jdkrD||jd}||jd}|rD|rD||krDtd|j d|| |q|rt |j }t |j }|D]F}tt |j|@rtt |j|@st|j|jddkr|j|jd|jdnqW|j|jd|jd|j|qWdSdS) ziRemove reshape node that is not needed based on symbolic shape inference: input and output has same shapeT)updateNReshaperzRemove reshape node z* since its input shape is same as output: rU)r infer_runtime_shaperBr4get_edge_shaper5r?rrr=rIsetget_graphs_input_namesrboolrHr<replace_output_of_all_nodesrAr@)r shape_inferrrC input_shape output_shapegraph_input_namesrrrrremove_useless_reshape_nodess8  z(FusionUtils.remove_useless_reshape_nodes)r )NNN)rrr )T)__name__ __module__ __qualname__r rstrtuplerrr,r$r(rrF staticmethodrLrRrrrZrcrhrrtr~rrrrrrrrrr sD    * r c@s(eZdZeddededefddZdS) NumpyHelperFrd fill_zerosrcCs$|r t|jt|jdSt|S)N)rxrg)rrirtensor_dtype_to_np_dtyperjrto_array)rdrrrrr3s   zNumpyHelper.to_arrayN)F)rrrrrrrrrrrrr2sr)loggingrrmrronnxrrrrr rh onnx_modelr rrr rrrrrs   $