U "c%@s8ddlZddlZGdddZGdddZddgZdS)Nc@s"eZdZdZddZdddZdS) Perlunipropsa3 This class is used to read lists of characters from the Perl Unicode Properties (see http://perldoc.perl.org/perluniprops.html). The files in the perluniprop.zip are extracted using the Unicode::Tussle module from http://search.cpan.org/~bdfoy/Unicode-Tussle-1.11/lib/Unicode/Tussle.pm cCsPtjtjtd|_dddddddd d d d d ddddddddddg|_dS)Nz/data/perluniprops/ZClose_PunctuationZCurrency_SymbolZIsAlnumZIsAlphaZIsLowerZIsNZIsScZIsSoZIsUpperZLine_SeparatorNumberZOpen_PunctuationZ PunctuationZ SeparatorZSymbolZLowercase_LetterZTitlecase_LetterZUppercase_LetterZIsPfZIsPiZ CJKSymbolsCJK)ospathdirnameabspath__file__datadirZavailable_categoriesselfr 5/tmp/pip-unpacked-wheel-43qgbez0/sacremoses/corpus.py__init__s2zPerluniprops.__init__Nccs:tjdd|d}td|}|dD] }|Vq*dS)u This module returns a list of characters from the Perl Unicode Properties. They are very useful when porting Perl tokenizers to Python. >>> from sacremoses.corpus import Perluniprops >>> pup = Perluniprops() >>> list(pup.chars('Open_Punctuation'))[:5] == [u'(', u'[', u'{', u'༺', u'༼'] True >>> list(pup.chars('Currency_Symbol'))[:5] == [u'$', u'¢', u'£', u'¤', u'¥'] True >>> pup.available_categories[:5] ['Close_Punctuation', 'Currency_Symbol', 'IsAlnum', 'IsAlpha', 'IsLower'] :return: a generator of characters given the specific unicode character category dataZ perlunipropsz.txt sacremosesutf-8N)rrjoinpkgutilget_datadecode)r category relative_path binary_datachr r rchars.s zPerluniprops.chars)N)__name__ __module__ __qualname____doc__rrr r r rrsrc@s"eZdZdZddZdddZdS) NonbreakingPrefixesz This is a class to read the nonbreaking prefixes textfiles from the Moses Machine Translation toolkit. These lists are used in the Python port of the Moses' word tokenizer. c'Cstjtjtd|_dddddddd d d d d ddddddddddddddddddd d!d"d#d$d%d&d'd(&|_|jd)d*|jDdS)+Nz/data/nonbreaking_prefixes/asZbncacsdeelenesetfifrgaZguhihuisitknltlvmlZmnimrnlorpaplptroruskslsvtateZtdtZyuezh)&ZassameseZbengalicatalanczechgermangreekenglishspanishestonianfinnishfrenchZirishZgujaratiZhindi hungarian icelandicitalianZkannada lithuanianZlatvianZ malayalamZmanipuriZmarathidutchZoriyaZpunjabipolish portugueseromanianrussianslovak slovenianswedishZtamilZteluguZtetumZ cantonesechinesecSsi|] }||qSr r .0vr r r xsz0NonbreakingPrefixes.__init__..) rrrrr r available_langsupdatevaluesr r r rrKsT)zNonbreakingPrefixes.__init__N#ccs||jkrd|j|g}n(|dkr>ddt|jD}ndg}|D]P}tjdd|}td|}|d  D] }| }|rv| |sv|VqvqHdS) u This module returns a list of nonbreaking prefixes for the specified language(s). >>> from sacremoses.corpus import NonbreakingPrefixes >>> nbp = NonbreakingPrefixes() >>> list(nbp.words('en'))[:10] == [u'A', u'B', u'C', u'D', u'E', u'F', u'G', u'H', u'I', u'J'] True >>> list(nbp.words('ta'))[:5] == ['ர', 'ூ', 'திரு', 'ஏ', 'பீ'] True :return: a generator words for the specified language(s). nonbreaking_prefix.NcSsg|] }d|qS)r`r rXr r r sz-NonbreakingPrefixes.words..znonbreaking_prefix.enrZnonbreaking_prefixesrr) r\setr^rrrrrr splitlinesstrip startswith)r langZignore_lines_startswith filenamesfilenamerrliner r rwordszs   zNonbreakingPrefixes.words)Nr_)rrrrrrjr r r rr Ds/r )rrrr __all__r r r rs<X