o Xxi"C@sddlmZmZddlZddlZddlZddlZddlmZm Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZddlmZmZmZe eZee je Zee je dZe ee!edZ"d Z#d Z$d Z%d Z&d Z'dZ(dZ)dZ*dAddZ+ddZ,ddZ-ddZ.ddZ/dBddZ0dBddZ1dBd d!Z2d"d#Z3d$d%Z4d&d'Z5dCd)d*Z6d+d,Z7d-d.Z8d/d0Z9d1d2Z:Gd3d4d4eZ;Gd5d6d6e;ZZ?Gd;d<dd>e<ZAGd?d@d@eBZCdS)D)ABCabstractmethodN)packunpack) lru_cache)Matchercreate_minimum_transducer compileFSTz0%(asctime)s %(name)s - %(levelname)s %(message)sz fst_data%d.pyzentries_extra%d.pyzentries_compact%d.pyzentries_buckets.pyzconnections%d.pyz chardef.pyz unknowns.pyz user_fst.datazuser_entries.datacCs ttj|t||dddS)NT)binary)_save_as_moduleospathjoinMODULE_FST_DATA)datadirpartrI/home/air/biblejyuku/back/venv/lib/python3.10/site-packages/janome/dic.py save_fstdata3s rcCs4ttj|t||ttj|t||dSN)_start_entries_as_moduler rrMODULE_ENTRIES_COMPACTMODULE_ENTRIES_EXTRA)r bucket_idx morph_offsetrrrstart_save_entries7srcCs0ttj|t|ttj|t|dSr)_end_entries_as_moduler rrrr)rrrrrend_save_entries<srcCs8ttj|t|||ttj|t|||dSr)_save_entry_as_module_compactr rrr_save_entry_as_module_extrar)rrmorph_identryrrr save_entryAsr$cCsttj|t|dSr)r r rrMODULE_ENTRIES_BUCKETS)rbucketsrrrsave_entry_bucketsFr'.cCsTt|dd}d}tddD]}ttj|t|||||||7}qdS)Nrr)lenranger r rrMODULE_CONNECTIONS) connectionsr bucket_sizeoffsetirrrsave_connectionsJs r3cCttj|t|dSr)r r rrMODULE_CHARDEFS)chardefsrrrr save_chardefsUr(r7cCr4r)r r rrMODULE_UNKNOWNS)unknownsrrrr save_unknownsYr(r:cCsN|sdSt|d|}|||WddS1s wYdS)Nwb)gzipopenwriteflush)filer compresslevelfrrr_save]s   "rCcCsLtj|sdSt|d}|}|WdS1swYdS)Nrb)r rexistsr<r=read)r@rBrrrr_loades $rGcCs8zt||}Wn tyYdSwt|tjdBS)N)pkgutilget_dataIOErrorzlib decompress MAX_WBITS)packageresourcerawdatarrr_load_package_datams  rRFcCs|sdSt|d5}|d|r'|d|t|d|dn |t|dd|WddS1sAwYdS)NwzDATA="asciiz\\\)r=r>base64 b64encodedecodestrreplacer?)r@rr rBrrrr us     "r c Cstdd|}t|d9}t|d}|d|d|d|dWdn1s0wYWddSWddS1sHwYdS)N\.py$_idx.pyrSzDATA={z "offset": z, "positions": [)resubr=r>)r@morph_id_offsetidx_filerBf_idxrrrrs    "rc Cstdd|}t|d8}t|d}|d|d||Wdn1s/wYWddSWddS1sGwYdS)Nr\r]az} z]} )r^r_r=r>r?)r@rarBrbrrrrs     "rc Cstdd|}t|dY}t|d;}|d||}||dd|ddd |d |d |d f}|||d Wdn1sPwYWddSWddS1shwYdS)Nr\r]rc%d:(,zu'%s',%4d,%4d,%5drunicode_escaperUrr*r+),r^r_r=r>tellencoderYr@r"r#rarBrbpossrrrr s$    "r c Cs&tdd|}t|d}}t|d_}|d||}||dd|ddd |d dd |d dd |d dd |d dd |ddd f}|||dWdn1stwYWddSWddS1swYdS)Nr\r]rcrdrez#u'%s',u'%s',u'%s',u'%s',u'%s',u'%s'rfrU rgrhrkrrrr!s(    "r!c@s4eZdZdZeddZeddZeddZdS) Dictionaryz Base dictionary class cCdSrr)selfrmmatcherrrrlookupzDictionary.lookupcCrurrrvnumrrr lookup_extraryzDictionary.lookup_extracCrurrrvid1id2rrrget_trans_costryzDictionary.get_trans_costN)__name__ __module__ __qualname____doc__rrxr|rrrrrrts  rtc@s0eZdZdZddZddZddZdd Zd S) RAMDictionaryz RAM dictionary class cCs||_||_dSr)entriesr/)rvrr/rrr__init__s zRAMDictionary.__init__cCs||\}}|s gSz g}|D]}td|d}||f|j|ddq|WStyUtdtd|tdt|t t dYdSw)NIrrn=Cannot load dictionary data. The dictionary may be corrupted?input=outputs=r) runrappendr ExceptionloggererrorrZ traceback format_excsysexit)rvrmrwmatchedoutputsreser{rrrrxs    zRAMDictionary.lookupcCsFz |j|ddWSty"tdttdYdSw)Nrnrr)rrrrrrrrrzrrrr|s  zRAMDictionary.lookup_extracC|j||Srr/r}rrrrzRAMDictionary.get_trans_costN)rrrrrrxr|rrrrrrs  rc@sTeZdZdZddZddZedddd Zed dd d Zd dZ ddZ dS)MMapDictionaryz MMap dictionary class cCs&||_||_||_||_||_dSr)entries_compactkeys bucket_ranges entries_extra open_filesr/)rvrrrr/rrrrs   zMMapDictionary.__init__cCs||\}}|s gSzg}|D]}td|d}||f||q|WStyQtdtd|tdt|t t dYdSw)Nrrrrrr) rrr _find_entryrrrrZrrrr)rvrmrwrrmatched_entriesridxrrrrxs   zMMapDictionary.lookupi maxsizecsttfdd|j}|j|\}}|d}|d|d}|d|}|d}|d} | d} | d} | d} | d } |||d t||| t|| | t|| | f}|S) Nc|dko |dkSNrrrbrrrz,MMapDictionary._find_entry..r1 positionsr*s',rnrrorf)nextfilterrrfindrYint)rvrbucketmmmm_idxrel_idx_pos1s_pos1e_pos2s_pos2e_pos3s_pos3e_pos4s_pos4e_entryrrrr s"  zMMapDictionary._find_entrycsZzttfdd|j}|j|\}}|d}|d|d}|d|}|d}|d|} | d} |d| } | d} |d| } | d}|d|}|d}|d|}|||d ||| d || | d || | d |||d |||d fWStytd td t t d YdSw) Ncrrrrrrrr$rz-MMapDictionary.lookup_extra..r1rr*s',u'rns')rfz8Cannot load extra info. The dictionary may be corrupted?zidx=r) rrrrrrYrrrrrrr)rvrrrrrrrrrrrrr_pos5s_pos5e_pos6s_pos6errrr|!s>         zMMapDictionary.lookup_extracCrrrr}rrrr?rzMMapDictionary.get_trans_costcCsV|jD]\}}|q|jr|jD]\}}|q|jD]}|q"dSr)rvaluescloserr)rvrrfprrr__del__Bs    zMMapDictionary.__del__N) rrrrrrxrrr|rrrrrrrs   rc@sBeZdZdZddZeddddZdd Zd d Zd d Z dS)UnknownsDictionaryz5 Dictionary class for handling unknown words cCs|d|_|d|_||_dSr)char_categories char_rangesr9)rvr6r9rrrrQs   zUnknownsDictionary.__init__rrcCsdi}|jD]$}|d|kr|dkr)nq|d}d|vr#|dng}|||<q|s0dgi}|S)Nfromtocate compat_catesDEFAULT)r)rvcr chr_ranger compate_catesrrrget_char_categoriesVs  z&UnknownsDictionary.get_char_categoriescC||jvr |j|dSdS)NINVOKEFrrvrrrrunknown_invoked_alwaysb z)UnknownsDictionary.unknown_invoked_alwayscCr)NGROUPFrrrrrunknown_groupinggrz#UnknownsDictionary.unknown_groupingcCr)NLENGTHrrrrrunknown_lengthlrz!UnknownsDictionary.unknown_lengthN) rrrrrrrrrrrrrrrLs   rcs\eZdZdZdfdd ZfddZeddZed d Zed d Z dddZ Z S)UserDictionaryz, User dictionary class (on-the-fly) Ncs:t||||\}}t|||g|_t|g|_dS)a Initialize user defined dictionary object. :param user_dict: user dictionary file (CSV format) :param enc: character encoding :param type: user dictionary type. supported types are 'ipadic' and 'simpledic' :param connections: connection cost matrix. expected value is SYS_DIC.connections :param progress_handler: handler mainly to indicate progress, implementation of ProgressHandler .. seealso:: http://mocobeta.github.io/janome/en/#use-with-user-defined-dictionary N)r build_dicsuperr compiledFSTrrw)rv user_dictenctyper/progress_handlerfst_datar __class__rrrws zUserDictionary.__init__ct||jSrrrxrwrvrmrrrrxzUserDictionary.lookupc CsZ|d\ }}}}}}}} } } } } }d|||| g}|t|t|t||| | | | |f S)z;Convert IPADIC formatted string to an user dictionary entryre)splitrr)clslinesurfaceleft_idright_idcost pos_major pos_minor1 pos_minor2 pos_minor3 infl_type infl_form base_formreadingphoneticpart_of_speechrrrline_to_entry_ipadics   z#UserDictionary.line_to_entry_ipadicc Cs:|d\}}}d|dddg}|ddd|dd|||f S)z?Convert simpledict formatted string to an user dictionary entryre*ri`y)rr)rrrrrrrrrline_to_entry_simpledicsz&UserDictionary.line_to_entry_simplediccCsXg}i}t|d|}|r1t||d}|jtdd|DddWdn1s,wYtj||d2}|D]'} | } || } t|} || ddt d | f| || <|rb| q;Wdn1smwY|rx| t |} t|t|ksJ|r|jt| d dt | |r|j ndd \} }|r| t|}||fS) Nline_to_entry_)encodingcss|]}dVqdS)rNr).0rrrr sz+UserDictionary.build_dic..z Reading user dictionary from CSV)totaldescrutf8rz!Running create_minimum_transducer) on_progress)getattrr=on_startsumiorstripr,rrjrr  on_completesortedr r )rrr dict_typersurfacesr line_to_entryrBrr#r"inputs processedfstrrrrrsN  zUserDictionary.build_dicrscCstj|rtj|std|tj|s#tj|tdddttj|t |j d|ttj|t t |j|dS)z Save compressed compiled dictionary data. :param to_dir: directory to save dictionary data :compressionlevel: (Optional) gzip compression level. default is 9 zNot a directory : 0755rr)moderN)r rrEisdirrmakedirsrrCrFILE_USER_FST_DATArFILE_USER_ENTRIES_DATApickledumpsr)rvto_dircompressionlevelrrrsaves  "zUserDictionary.saver)rs) rrrrrrx classmethodrrrr" __classcell__rrrrrrs    2rcs8eZdZdZfddZfddZeddZZS)CompiledUserDictionaryz* User dictionary class (compiled) cs,t|\}}t||t|g|_dSr)r% load_dictrrrrw)rvdic_dirr/rrrrrrszCompiledUserDictionary.__init__crrrrrrrrxrzCompiledUserDictionary.lookupcCsXtj|r tj|std|ttj|t}t ttj|t }||fS)NzNo such directory : ) r rrErrrGrrrloadsr)rr'rrrrrr&s z CompiledUserDictionary.load_dict) rrrrrrxr#r&r$rrrrr%s   r%c@seZdZddZdS)LoadingDictionaryErrorcCs d|_dS)NzECannot load dictionary data. Try mmap mode for very large dictionary.)message)rvrrrrs zLoadingDictionaryError.__init__N)rrrrrrrrr)s r))r)r))F)Dabcrrr rrr<structrrrloggingrr^rIrLrW functoolsrrrr r getLoggerrrsetLevelWARN StreamHandlerhandler Formatter formatter setFormatter addHandlerrrrr%r.r5r8rrrrrr$r'r3r7r:rCrGrRr rrr r!rtrrobjectrrr%rr)rrrrsl             &\&l