o 3IhL0@sHddlmZddlmZGdddZGdddeZGdddZd S) )ArgumentParser)Enumc@seZdZdZdZdZdZdS)AttentionMaskFormatrN)__name__ __module__ __qualname__ MaskIndexEndMaskIndexEndAndStart AttentionMaskNoMaskrrg/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/onnxruntime/transformers/fusion_options.pyr s rc@s4eZdZdZdZdZdZddZddZd d Z d S) AttentionOpType AttentionMultiHeadAttentionGroupQueryAttentionPagedAttentioncCs|jSNvalueselfrrr__str__szAttentionOpType.__str__cCs t|jSr)hashrrrrr__hash__! zAttentionOpType.__hash__cCs |j|jkSrr)rotherrrr__eq__$ zAttentionOpType.__eq__N) rr r rrrrrrr rrrrrs rc@sVeZdZdZddZdddZddZd efd d Ze d d Z e de fddZ dS) FusionOptionsz'Options of fusion in graph optimizationcCsd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_|dkr4d|_tj|_|dkrAtj|_n|dkrItj|_d|_|dvrgd|_d|_d|_d|_d|_d|_d|_dSdS)NTFclipbertvitunetvaer#) enable_geluenable_layer_normenable_attentionenable_rotary_embeddingsuse_multi_head_attention!disable_multi_head_attention_biasenable_skip_layer_normenable_embed_layer_normenable_bias_skip_layer_normenable_bias_geluenable_gelu_approximationenable_qordered_matmulenable_shape_inferenceenable_gemm_fast_gelugroup_norm_channels_lastrr attention_mask_formatr rattention_op_typeenable_nhwc_convenable_group_normenable_skip_group_normenable_bias_splitgeluenable_packed_qkvenable_packed_kvenable_bias_add)r model_typerrr__init__+s@  zFusionOptions.__init__TcCs|rtj|_dStj|_dSr)rr r8r )r use_raw_maskrrruse_raw_attention_maskZs  z$FusionOptions.use_raw_attention_maskcCs tj|_dSr)rrr8rrrrdisable_attention_mask`r!z$FusionOptions.disable_attention_mask attn_op_typecCs ||_dSr)r9)rrFrrrset_attention_op_typecrz#FusionOptions.set_attention_op_typecCs6t|j}|jr d|_|jrd|_|jrd|_|jrd|_ |j r#d|_ |j r)d|_ |j r/d|_|jr5d|_|jr;d|_|jrAd|_|jrGd|_|jrMd|_|jrU|d|jr]|d|jrd||jdvr|jrod|_|jrud|_|jr{d|_ |j!rd|_"|j#rd|_$|j%rd|_&|j'rd|_(|j)rd|_*|S)NFTr&)+r"rA disable_gelur)disable_layer_normr*disable_rotary_embeddingsr,disable_attentionr+r-disable_skip_layer_normr/disable_embed_layer_normr0disable_bias_skip_layer_normr1disable_bias_gelur2r3disable_shape_inferencer5r6use_mask_indexrDno_attention_maskrEuse_group_norm_channels_firstr7disable_nhwc_convr:disable_group_normr;disable_skip_group_normr<disable_bias_splitgelur=disable_packed_qkvr>disable_packed_kvr?disable_bias_addr@)argsoptionsrrrparsefsb    zFusionOptions.parseparsercCs|jddddd|jdd|jddddd|jdd |jd ddd d|jdd |jd dddd|jdd|jddddd|jdd|jddddd|jdd|jddddd|jdd|jddddd|jdd|jddddd|jdd|jdddd d|jdd!|jd"ddd#d|jdd$|jd%ddd&d|jdd'|jd(ddd)d|jdd*|jd+ddd,d|jdd-|jd.ddd/d|jdd0|jd1ddd2d|jdd3|jd4ddd5d|jdd6|jd7ddd8d|jdd9|jd:ddd;d|jdd<|jd=ddd>d|jdd?|jd@dddAd|jddB|jdCdddDd|jddE|jdFdddGddS)HNz--disable_attentionF store_truezdisable Attention fusion)requiredactionhelp)rKz--disable_skip_layer_normz%disable SkipLayerNormalization fusion)rLz--disable_embed_layer_normz&disable EmbedLayerNormalization fusion)rMz--disable_bias_skip_layer_normz2disable Add Bias and SkipLayerNormalization fusion)rNz--disable_bias_geluz)disable Add Bias and Gelu/FastGelu fusion)rOz--disable_layer_normz!disable LayerNormalization fusion)rIz--disable_geluzdisable Gelu fusion)rHz--enable_gelu_approximationz+enable Gelu/BiasGelu to FastGelu conversion)r3z--disable_shape_inferencez disable symbolic shape inference)rPz--enable_gemm_fast_geluzenable GemmfastGelu fusion)r6z--use_mask_indexzWuse mask index to activate fused attention to speed up. It requires right-side padding!)rQz--use_raw_attention_maskzuse raw attention mask. Use this option if your input is not right-side padding. This might deactivate fused attention and get worse performance.)rDz--no_attention_maskz1no attention mask. Only works for model_type=bert)rRz--use_multi_head_attentionzUse MultiHeadAttention instead of Attention operator for testing purpose. Note that MultiHeadAttention might be slower than Attention when qkv are not packed. )r-z--disable_group_normz9not fuse GroupNorm. Only works for model_type=unet or vae)rUz--disable_skip_group_normzPnot fuse Add + GroupNorm to SkipGroupNorm. Only works for model_type=unet or vae)rVz--disable_packed_kvz[not use packed kv for cross attention in MultiHeadAttention. Only works for model_type=unet)rYz--disable_packed_qkvz[not use packed qkv for self attention in MultiHeadAttention. Only works for model_type=unet)rXz--disable_bias_addz0not fuse BiasAdd. Only works for model_type=unet)rZz--disable_bias_splitgeluz6not fuse BiasSplitGelu. Only works for model_type=unet)rWz--disable_nhwc_convz:Do not use NhwcConv. Only works for model_type=unet or vae)rTz--use_group_norm_channels_firstznUse channels_first (NCHW) instead of channels_last (NHWC) for GroupNorm. Only works for model_type=unet or vae)rSz--disable_rotary_embeddingsz5Do not fuse rotary embeddings into RotaryEmbedding op) add_argument set_defaults)r^rrr add_argumentss@                       zFusionOptions.add_argumentsN)T) rr r __doc__rBrDrErrG staticmethodr]rrerrrrr"(s / 5r"N)argparserenumrrrr"rrrrs