o 3Ih@sddlmZddlZddlmZddlmZddlm Z m Z ddl m Z m Z mZmZddlmZeeZGdd d ZGd d d eZdS) ) getLoggerN)Fusion)AttentionMaskFormat) FusionUtils NumpyHelper) NodeProto TensorProtohelper numpy_helper) OnnxModelc@sReZdZdZdefddZdefddZdd Zd d Z d e d e dBfddZ dS) AttentionMask: Fuse Attention subgraph into one Attention node. modelcCs2||_i|_i|_t||_tj|_||_ dSN) r mask_indice mask_castedrutilsr MaskIndexEnd mask_formatget_opset_version opset_version)selfrri/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/onnxruntime/transformers/fusion_attention.py__init__s  zAttentionMask.__init__rcCs ||_dSr)r)rrrrrset_mask_format s zAttentionMask.set_mask_formatcCs*||jvr||j|ksJ||j|<dSr)r)rmask mask_indexrrrset_mask_indice#s zAttentionMask.set_mask_indicecCs t|jdks Jtt|jSNr)lenrnextiter)rrrrget_first_mask(szAttentionMask.get_first_maskmask_2dreturnNc Cs`|jtjkrdS||jvr|j|S|j|r!|j|\}}n |j|\}}d}|r2||j |<|jtj kr?||j|<|S|j d}|j dkrlt jd|g|g|j ddd}|jt ddgt d d gn7d }|j|dur|jt j|tjdgdgd d t jd||g|g|j ddd}|jt d d g|j|||j|<|S)NTr ReduceSum MaskReduceSuminputsoutputsnameaxeskeepdimsrort_const_1_reduce_sum_axesFr, data_typedimsvalsraw)rrNoMaskrrfind_graph_inputrcast_graph_input_to_int32cast_input_to_int32rr create_node_namerr make_node attributeextendmake_attributeget_initializeradd_initializer make_tensorrINT64add_node)rr$casted input_name _cast_node output_namemask_index_node axes_namerrr process_mask,sV          $    zAttentionMask.process_mask) __name__ __module__ __qualname____doc__r rrrrr#strrJrrrrr s r c(s,eZdZdZdddddgfdededed edBd ed ed ee ffd d Z de de eeffddZ de de eeffddZde fddZde fddZde de de fddZde d e d!e fd"d#Zd$e d%e dBd&e dBd'e de dBf d(d)Zd*e d+e d,e d$e d%e dBd&e dBde e e e ffd-d.Z / /  / / / / dAd*e d+e e BdBd,e e BdBd$e d%e dBd&e dBdeded0e d1e de d2ede de d3e d4e d5ede dBf$d6d7Z / / / / /  dBd8e dBd*e d+e d,e d$e d%e d&e deded9e d0e d:e de de d3e d4e d;edBdZd?d@ZZS)CFusionAttentionr NFSkipLayerNormalizationLayerNormalizationr hidden_size num_headsattention_maskuse_multi_head_attention!disable_multi_head_attention_biassearch_op_typesc sh|rdnd}t|||||_||_|r|nt||_||_||_d|_d|_ d|_ d|_ d|_ dS)NMultiHeadAttention AttentionT) superrrSrTr rUrVrWmask_filter_valuenum_heads_warninghidden_size_warning shape_infershape_infer_done) rrrSrTrUrVrWrXattention_op_name __class__rrrls  zFusionAttention.__init__concatr%cCst|jdkr;|j|jd}|j|jd}t|tjr;|jdkr;t|tjr;|jdkr;|d|d|dfS|j|j fS)aU Detect num_heads and hidden_size from Concat node in the following subgraph: SkipLayerNormalization or EmbedLayerNormalization / | MatMul Shape | | Add Gather(indices=0) | | | Unsqueeze | | | Concat (*, -1, 12, 64) | / Reshape | Transpose r.r) r inputrget_constant_value isinstancenpndarraysizerTrS)rrdrT head_sizerrr)get_num_heads_and_hidden_size_from_concats     z9FusionAttention.get_num_heads_and_hidden_size_from_concat reshape_qcCs*|j|jd}|dur1|j|d}|dur"|jdkr"||Std|jd|j|j fSt |t j rIt |dksI|ddksI|ddkrUtd ||j|j fS|d}|d}||}|jdkry||jkry|jrytd |j|d |_|j dkr||j kr|jrtd |j |d |_||fS) zDetect num_heads and hidden_size from a reshape node. Args: reshape_q (NodeProto): reshape node for Q Returns: Tuple[int, int]: num_heads and hidden_size r.NConcatz%s is not initializer.rerfrrgzGq_shape_value=%s. Expected value are like [0, 0, num_heads, head_size].z>--num_heads is %d. Detected value is %d. Using detected value.Fz@--hidden_size is %d. Detected value is %d. Using detected value.)rrirh get_parentop_typerologgerdebugrTrSrjrkrlr r]warningr^)rrp q_shape_valuerdrTrnrSrrrget_num_heads_and_hidden_sizes:       z-FusionAttention.get_num_heads_and_hidden_sizeadd_qkcCs|js|jjdd|_d|_|jdurdS|j|jd}|j|jd}|dus/|dur7td|dS||krCtd|dS|jdS)NT)updaterr.zone of the inputs of %s is Nonez)the shape of two inputs of %s is not same)r`rinfer_runtime_shaper_get_edge_shaperhrtru)rry input_0_shape input_1_shaperrrget_add_qk_strs    zFusionAttention.get_add_qk_strcsdttfdd|j}t|dkrSt|dks J|jd}tjdfddt|j Dg|dd }|j ||j |j |<S) N_maskcs|jdkSr)output)node)mask_output_namerrsz0FusionAttention.reshape_add_qk..r.rrqcsg|]}qSrr).0_)ryrr sz2FusionAttention.reshape_add_qk..r*r+r,axis) listfilter nodes_to_addr rr:r r;rangerTappendthis_graph_namenode_name_to_graph_name)rry concat_nodeconcat_node_nameconcat_add_qk_fp32r)ryrrreshape_add_qks     zFusionAttention.reshape_add_qkpast_kpast_vc Cs|jd}|jd}|ddd}|ddd}tjd|g|g|dgd}tjd|g|g|dgd}|j||j||j|j|<|j|j|<|jd} |dd ddd d } tjd||g| g| dd } |j| |j|j| <| S) zConcatenate past_k and past_v inputs to create past_kv input. Args: past_k (str): name of past K value past_v (str): name of past V value Returns: kv_output_name (str): name of past KV value Unsqueeze_5d.rr)r*r+r,r-rqz.valuez.kv_value_kvr) rr:replacer r;rrrr) rrrunsqueeze_k_nameunsqueeze_v_name k_5d_name v_5d_namek_5dv_5drkv_output_name concat_kvrrrrsD      zFusionAttention.concat_kvpresent_k_namepresent_v_namekv_nodec Csd\}}|j|}|j|}|dur(tjtjddd|d}|j||j|dur@tjtjddd|d}|j||j|jd}|jd} t j d||g|g|dd } t j d||g|g| dd } |j | |j | |j|j |<|j|j | <dS) a?Split kv_node containing present KV values into separate present K and present V values. Args: present_k_name (str): name of output to store present K value in present_v_name (str): name of output to store present V value in kv_node (str): name of present KV values )index_0index_1Nrint64)dtyper,r.Gatherr)rr?r from_arrayrkarrayr@rr:r r;rrr) rrrrk_indexv_indexk_dimv_dim gather_k_name gather_v_name present_k present_vrrrsplit_kv8s:      zFusionAttention.split_kvq_addk_addv_add name_prefixcCs|j|jdp|j|jd}t|}t|}t|}|dur<|j|jdp6|j|jd} t| }|durW|j|jdpQ|j|jd} t| }tj|||fdd} dt|j } |d} |j | |j | g| d| S)Nr.rrrg _qkv_biasr,r2r3r4) rr?rhrto_arrayrk zeros_likestackprodshaper@r2)rrrrrq_biasqbkbvbk_biasv_biasqkv_bias qkv_bias_dim bias_namerrrcreate_combined_qkv_biases($   $ $ z(FusionAttention.create_combined_qkv_biasq_matmulk_matmulv_matmulc#Cs"|jd}|jd|jdkr|jd|jdksJ|j|jd}|j|jd} |j|jd} t|} t| } t| } | j| jkrR| j| jksTJ| jd}tj| | | fdd |d|f}|d}|j ||j |jd|jdg|d|d}t j d|jd|g|g|d }|j|j|<|g}|d }|j |tjdgdgd d |d }|j |tjdg|gd d |d}|j |tjdgd|gd d |d}|j |tjdgd|gd d |d}|j |tjdgdgd d |d}t j d||||g|g|jdd }|j|j|j<|d}t j d||||g|g|jdd }|j|j|j<|d}t j d||||g|g|jdd }|j|j|j<|}|} |}!||||g|jr|dur|j|jdrodnd}"tt|j|j|"r||jd|"<|}|||j|j|j<|dur|j|jdrdnd}"tt|j|j|"r||jd|"<|} |||j|j|j<|dur|j|jdrdnd}"tt|j|j|"r||jd|"<|}!|||j|j|j<|j||| |!fS)aCreate packed QKV MatMul node before MultiHeadAttention node. This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path Returns: q_output (NodeProto): Slice node for Q k_output (NodeProto): Slice node for K v_output (NodeProto): Slice node for V MatMulrr.rrg _qkv_weightr_qkv_outr)_q_start_indexFr1_k_start_index_v_start_indexrf_end_of_qkv_index_qkv_last_axis_q_outSlice_k_out_v_outN)rr:rhr?rrrrkrreshaper@r2r r;rrrrBr,r=rWanyrr)#rrrrrrrmatmul_node_nameq_weightk_weightv_weightqwkwvwd qkv_weightqkv_weight_nameqkv_matmul_output qkv_matmul qkv_nodes q_slice_name k_slice_name v_slice_nameend_of_qkv_nameqkv_last_axis_nameq_slice_outputq_slicek_slice_outputk_slicev_slice_outputv_sliceq_outputk_outputv_outputinitializer_inputrrrcreate_packed_qkv_matmul_nodes ,    "                   z-FusionAttention.create_packed_qkv_matmul_noderkey_padding_maskunidirectionalrr packed_qkvcCs,|dksJ|dkr||dkrtd||dSdd|jjD}|jd}g}|rM|||||||\}}}||jd|jd|jdgn`t |t r~t |t r~|j rl||jd|jd|jdgnA||jd|jd|jdgn/t |t rt |t r||vr||vr|j r||jd||gn||jd||gndS|j s| ||||}||n|d| r|r|| | | |gn | s| r|| | g| g}|r|r|||gtjd|||d }d |_|jtd || r|jtd t| |d|S) aCreate a MultiHeadAttention node. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. output (str): output name of MHA key_padding_mask (str): name of key padding mask add_qk (str): name of add after Q x K' unidirectional (bool): whether to apply causal attention mask automatically or not past_k (str): name of past K value - (batch_size, num_heads, past_sequence_length, head_size) past_v (str): name of past V value - (batch_size, num_heads, past_sequence_length, head_size) present_k (str): name of present K value - (batch_size, num_heads, sequence_length, head_size) present_v (str): name of present V value - (batch_size, num_heads, sequence_length, head_size) packed_qkv (bool): whether to combine MatMuls from Q, K, V paths Note: This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Returns: Union[NodeProto, None]: the node created or None if failed. r9input hidden size %d is not a multiple of num of heads %dNcSsh|]}|jqSrr)rrrrr HzBFusionAttention.create_multihead_attention_node..rZrrYr) com.microsoftrTr)rtrurgraphrhr:rr=rrjrrWrOrrr r;domainr<r>intincrease_counter)rrrrrrrrTrSrrryrrrrrrgraph_input_names mha_node_name mha_inputsrrrr mha_outputsmha_noderrrcreate_multihead_attention_nodesl /  $$$   z/FusionAttention.create_multihead_attention_noder first_input add_qk_strscalecausalc5Csj|dksJ| dkr| |dkrtd| |dSd}|dur)|dur)|dur)d}|j|jd}|j|jd}|j|jd}d\}}}|r|j|jdp\|j|jd}|j|jdpn|j|jd}|j|jdp|j|jd}|r|r|r|sdS|durt|jdddSt|}t|}t|}|j|jksJ|jd}|jd}|jd}||kr|ksJJ| dkr| |krt d | |d} |j|jkrd} t |jdd}!t |jdd}"t |jdd}#d}$| rt j |||fdd }%|!|"|#}$nt j |||fdd }%d |!}$d}&d}'|rt|}(t|})t|}*t |(j}+t |)j},t |*j}-|+|,kr_|!ksbJJ|-|#ksiJ| r}t j |(|)|*fdd }'|+|,|-}&nt j |(|)|*fdd }'d |+}&|jd }.|js|j|.d |j|t|$g|%d|r|j|.d|jt|&g|'d|jr| rtddS|jd|jd|jd|.dg}/|dur|/|tjd|/| g|.d}0|dnr| |.d |r|.dndg}/|dur |/|n|/d| o|}1|1r!|| |}2|/|2| r1|1s,|/d|/| | g}3|rT|rT|dddddd}4|3|4||||4tjd |/|3|.d}0|d d|0_|0jtd|g|r~|0jtddg|dur|0jtd|g| r|0jtd|!|"|#gg|jdur|0jtdt |jg|0S)a>Create an Attention node. Args: mask_index (str | None): mask input q_matmul (NodeProto): MatMul node in fully connection for Q k_matmul (NodeProto): MatMul node in fully connection for K v_matmul (NodeProto): MatMul node in fully connection for V q_add (NodeProto): Add bias node in fully connection for Q k_add (NodeProto): Add bias node in fully connection for K v_add (NodeProto): Add bias node in fully connection for V num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. first_input (str): first input name output (str): output name add_qk_str (str): name of Add node after Q x K' past_k (str): name of input for past K value past_v (str): name of input for past V value present_k (str): name of output to store present K value present_v (str): name of output to store present V value scale: scale before softmax causal: whether it is uni-directional mask. Returns: Union[NodeProto, None]: the node created or None if failed. rrNTFr.)NNNzl is not an initializer. Please set do_constant_folding=True in torch.onnx.export to unblock attention fusionzInput hidden size (%d) is not same as weight matrix dimension of q,k,v (%d). Please provide a correct input hidden size or pass in 0rrgrZrrrzVMultiHeadAttention does not support relative_position_bias: cannot fuse the attention.rYr)rz.key_keyrrrrTrrqkv_hidden_sizesr\)!rtrurr?rhprintrrrrvrkr concatenaterr:rVr@r2rrrr r;r rrrrr<r=r>r\float)5rrrrrrrrrTrSrrrrrrrrrhas_biasrrrrrrrrr qw_in_size kw_in_size vw_in_sizeis_qkv_diff_dims qw_out_size kw_out_size vw_out_sizeqkv_weight_dimrrrrrr q_bias_shape k_bias_shape v_bias_shapeattention_node_nameattention_inputsattention_node past_existspast_kvattention_outputs present_kvrrrcreate_attention_nodes . $$$                                 z%FusionAttention.create_attention_nodec;CsV|}|}|jdkr|j|dd}|dur|}ndS|j|gdgd}d}|dur4|\} } } } } n|j|gdgd}|durK|\} }} } ndSg} t|jD]\}}||vr]qT||djdkrgqT| |qTt| dkrudS| d}|j|d d}|dur||jd}|durt|d kr|d}|jdkr|jd}n,dS|durt|d kr|jd}ndS|jdkr||}|D] }|jdkr|jd}q||}|jd krt|jd kr|jd}||}dd|D}| ddkrdS|j| gdgd}|durt ddS|\} } }}d}d}d}d}gdgdfgdgdfgdgdfgdgdfgdgdfgdgd fd!}d} | D]6\}!}"|j| |"d|"d} | durjqS|!d"krrd#}n|!d$krzd#}n|!d%krd#}n|!d&krd#}| durt d'dSd}#d}$d}%d}&|r| \} }%}$} n$|r| \} }#}%}$n|r| \} } }$n|r| \} }#}$}&} n| \} }#} }$|&p|$}&|j|&gdgd(}'|'dur|j|&gd)gd*}'|'durt d+dS|'d,}(|'d-})|'d.}*|$}+|r'|j|$d d/gddg},|,dur#t d0dS|,\}+} |j|+gd|r3dnddddg}-|-durW|j|$gd1gd2}-|-durWt d3dS|-d-}.|-d.}/d}0d4}1|r|j |%gd5gdfgd6gdfgd7gd8fg|\} }0} n_|r|j |%gd9gd8fgd6gdfg|\} }0} |#dur||#}1|1durt d:|#dSn+|rn'|j |#gd;gd<fgd=gd>fgd?gd@fgdAgdBfg|\} }0} |s|0durt dCdS|s0t|0dkr0|j|0d\} }2|2dus"t|2tjr"|2jdkr"t|2dkr$dSt|2dDkr0t|2|_|jd|kr%|*jd|kr'|/jd|kr)|sV|j|0d.jdnd}3|dur_| n| }4||(\}5}6|5dksr|6dkryt dEdS|j|3|*|/||)|.||5|6||4jd|1dF }7|7durdS|j|7|j|j|7j<|dur|jd}8dG|8}9|jdH|8t j!d gdd|5t"|6|5gddI}:|j#t$%dJ|4jd|:jg|9gdK|8|j|9|jd<|j&'|4| | g|j&'| |j&'|j(s|'n|'dd.|j&'|j(s |-n|-dd.|j&'|j(s|n|dd.d#|_)dSdSdSdS)LNrRAddr)r-rReshape Transposer)NNrrr)r-Einsumr/r)r.Nrrr.MulrfrQrecSsg|]}|jqSr)rs)rchildrrrrrz(FusionAttention.fuse..rrg)r/r.r-r)r.rrNz&fuse_attention: failed to match v pathF)Softmaxr-Divr)rrNr)r4r-r1r)r4Whererr5)rrrfr)r4r-r6r)rrrrf)r4r5r)rrr)r4r-rr1Sqrt)rrNrr.)path1path2path3path4path5sdpar:Tr;r<r=z'fuse_attention: failed to match qk path)rrrN)r5r/r.r-r)rrrrNz&fuse_attention: failed to match q pathrr7z/fuse_attention: failed to match mul sqrt q path)r/r/r.r-r)r.rrrNz&fuse_attention: failed to match k pathr)Expandr.Equal)rArr)Castr@r.rA)rrrr)rBrArrz6fuse_attention: failed to verify shape inference of %s)r1SubrBrr)Nrr.rr)r1rCrr)Nrr.r)r6rBrCr@rr)Nrrr.rr)r6rBrCrBr@rr)Nrrr.rrrz)fuse_attention: failed to match mask pathizmFailed to detect num_heads and hidden_size for Attention fusion. Please specify those parameters in argument.) rrrrrrrrTrSrrredge_modified_shape_modified_tensorr1r.reshape_modified_)*rsr match_parentmatch_parent_path enumeraterhrrr countrtruitemsmatch_parent_pathsrget_constant_inputrjrkrlrmrr\rUrJrxrvr,rrrr,r@rrBrrCr r;nodes_to_remover=rV prune_graph);rrinput_name_to_nodesoutput_name_to_nodenormalize_node start_nodeadd_before_layernormr einsum_noder reshape_qkv transpose_qkv matmul_qkv other_inputs_i node_input root_inputmul_before_layernorm mul_childrenlayernorm_nodechildrenr3 parent_nodechildren_typesv_nodesadd_vmatmul_v is_distillis_distill_addis_no_mask_attentionis_sdpaqk_pathsqk_nodeskvry matmul_qkwhere_qkafter_qq_nodesrpadd_qmatmul_qafter_k mul_k_nodesk_nodesadd_kmatmul_k mask_nodesrmul_valrattention_last_node q_num_heads q_hidden_sizenew_node unique_indexnew_edge shape_tensorrrrfusevs                                             0            zFusionAttention.fuse)rrFrrrrF)rrrrrNF)rKrLrMrNr rr boolrrOrrtuplerorxrrrrrrrrr,r __classcell__rrrbrrPgs( .7-                 krP)loggingrnumpyrk fusion_baserfusion_optionsr fusion_utilsrronnxrrr r onnx_modelr rKrtr rPrrrrs    V