diff options
Diffstat (limited to 'sci-libs/tokenizers/tokenizers-0.15.2.ebuild')
| -rw-r--r-- | sci-libs/tokenizers/tokenizers-0.15.2.ebuild | 339 |
1 files changed, 339 insertions, 0 deletions
diff --git a/sci-libs/tokenizers/tokenizers-0.15.2.ebuild b/sci-libs/tokenizers/tokenizers-0.15.2.ebuild new file mode 100644 index 000000000000..d2da8b88ac9b --- /dev/null +++ b/sci-libs/tokenizers/tokenizers-0.15.2.ebuild @@ -0,0 +1,339 @@ +# Copyright 2023-2024 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +# Autogenerated by pycargoebuild 0.10 + +EAPI=8 + +DISTUTILS_USE_PEP517=maturin +PYTHON_COMPAT=( python3_{9..12} ) +DISTUTILS_EXT=1 + +CRATES=" + adler@1.0.2 + aho-corasick@1.1.2 + aho-corasick@1.1.3 + anes@0.1.6 + anstream@0.6.11 + anstream@0.6.13 + anstyle-parse@0.2.3 + anstyle-query@1.0.2 + anstyle-wincon@3.0.2 + anstyle@1.0.6 + assert_approx_eq@1.1.0 + autocfg@1.1.0 + autocfg@1.2.0 + base64@0.13.1 + base64@0.21.7 + bit-set@0.5.3 + bit-vec@0.6.3 + bitflags@1.3.2 + bitflags@2.4.2 + bitflags@2.5.0 + bumpalo@3.15.4 + cast@0.3.0 + cc@1.0.83 + cc@1.0.90 + cfg-if@1.0.0 + ciborium-io@0.2.2 + ciborium-ll@0.2.2 + ciborium@0.2.2 + clap@4.5.0 + clap@4.5.4 + clap_builder@4.5.0 + clap_builder@4.5.2 + clap_derive@4.5.0 + clap_derive@4.5.4 + clap_lex@0.7.0 + colorchoice@1.0.0 + console@0.15.8 + core-foundation-sys@0.8.6 + core-foundation@0.9.4 + crc32fast@1.4.0 + criterion-plot@0.5.0 + criterion@0.5.1 + crossbeam-deque@0.8.5 + crossbeam-epoch@0.9.18 + crossbeam-utils@0.8.19 + crunchy@0.2.2 + darling@0.14.4 + darling_core@0.14.4 + darling_macro@0.14.4 + derive_builder@0.12.0 + derive_builder_core@0.12.0 + derive_builder_macro@0.12.0 + dirs-sys@0.4.1 + dirs@5.0.1 + either@1.10.0 + encode_unicode@0.3.6 + env_logger@0.10.2 + errno@0.3.8 + esaxx-rs@0.1.10 + fancy-regex@0.13.0 + fastrand@2.0.1 + fastrand@2.0.2 + flate2@1.0.28 + fnv@1.0.7 + foreign-types-shared@0.1.1 + foreign-types@0.3.2 + form_urlencoded@1.2.1 + getrandom@0.2.12 + half@2.4.0 + heck@0.4.1 + heck@0.5.0 + hermit-abi@0.3.5 + hermit-abi@0.3.9 + hf-hub@0.3.2 + humantime@2.1.0 + ident_case@1.0.1 + idna@0.5.0 + indicatif@0.17.8 + indoc@2.0.4 + indoc@2.0.5 + instant@0.1.12 + is-terminal@0.4.12 + itertools@0.10.5 + itertools@0.11.0 + itertools@0.12.1 + itoa@1.0.10 + itoa@1.0.11 + js-sys@0.3.69 + lazy_static@1.4.0 + libc@0.2.153 + libredox@0.1.3 + linux-raw-sys@0.4.13 + lock_api@0.4.11 + log@0.4.20 + log@0.4.21 + macro_rules_attribute-proc_macro@0.2.0 + macro_rules_attribute@0.2.0 + matrixmultiply@0.3.8 + memchr@2.7.1 + memchr@2.7.2 + memoffset@0.9.0 + memoffset@0.9.1 + minimal-lexical@0.2.1 + miniz_oxide@0.7.2 + monostate-impl@0.1.11 + monostate@0.1.11 + native-tls@0.2.11 + ndarray@0.15.6 + nom@7.1.3 + num-complex@0.4.5 + num-integer@0.1.46 + num-traits@0.2.18 + number_prefix@0.4.0 + numpy@0.20.0 + once_cell@1.19.0 + onig@6.4.0 + onig_sys@69.8.1 + oorandom@11.1.3 + openssl-macros@0.1.1 + openssl-probe@0.1.5 + openssl-sys@0.9.102 + openssl@0.10.64 + option-ext@0.2.0 + parking_lot@0.12.1 + parking_lot_core@0.9.9 + paste@1.0.14 + percent-encoding@2.3.1 + pkg-config@0.3.29 + pkg-config@0.3.30 + plotters-backend@0.3.5 + plotters-svg@0.3.5 + plotters@0.3.5 + portable-atomic@1.6.0 + ppv-lite86@0.2.17 + proc-macro2@1.0.78 + proc-macro2@1.0.79 + pyo3-build-config@0.20.2 + pyo3-build-config@0.20.3 + pyo3-ffi@0.20.2 + pyo3-ffi@0.20.3 + pyo3-macros-backend@0.20.2 + pyo3-macros-backend@0.20.3 + pyo3-macros@0.20.2 + pyo3-macros@0.20.3 + pyo3@0.20.2 + pyo3@0.20.3 + quote@1.0.35 + rand@0.8.5 + rand_chacha@0.3.1 + rand_core@0.6.4 + rawpointer@0.2.1 + rayon-cond@0.3.0 + rayon-core@1.12.1 + rayon@1.8.1 + rayon@1.10.0 + redox_syscall@0.4.1 + regex-automata@0.4.5 + regex-syntax@0.8.2 + redox_users@0.4.5 + regex-automata@0.4.6 + regex-syntax@0.8.3 + regex@1.10.3 + regex@1.10.4 + ring@0.17.8 + rustc-hash@1.1.0 + rustix@0.38.31 + rustix@0.38.32 + rustls-pki-types@1.4.1 + rustls-webpki@0.102.2 + rustls@0.22.3 + ryu@1.0.16 + ryu@1.0.17 + scopeguard@1.2.0 + same-file@1.0.6 + schannel@0.1.23 + scopeguard@1.2.0 + security-framework-sys@2.10.0 + security-framework@2.10.0 + serde@1.0.196 + serde@1.0.197 + serde_derive@1.0.196 + serde_derive@1.0.197 + serde_json@1.0.113 + serde_json@1.0.115 + smallvec@1.13.1 + smallvec@1.13.2 + spin@0.9.8 + spm_precompiled@0.1.4 + strsim@0.10.0 + strsim@0.11.0 + strsim@0.11.1 + subtle@2.5.0 + syn@1.0.109 + syn@2.0.48 + syn@2.0.58 + target-lexicon@0.12.13 + target-lexicon@0.12.14 + tempfile@3.10.0 + tempfile@3.10.1 + termcolor@1.4.1 + thiserror-impl@1.0.56 + thiserror-impl@1.0.58 + thiserror@1.0.56 + thiserror@1.0.58 + tinytemplate@1.2.1 + tinyvec@1.6.0 + tinyvec_macros@0.1.1 + unicode-bidi@0.3.15 + unicode-ident@1.0.12 + unicode-normalization-alignments@0.1.12 + unicode-normalization@0.1.23 + unicode-segmentation@1.11.0 + unicode-width@0.1.11 + unicode_categories@0.1.1 + unindent@0.2.3 + untrusted@0.9.0 + ureq@2.9.6 + url@2.5.0 + utf8parse@0.2.1 + vcpkg@0.2.15 + walkdir@2.5.0 + wasi@0.11.0+wasi-snapshot-preview1 + wasm-bindgen-backend@0.2.92 + wasm-bindgen-macro-support@0.2.92 + wasm-bindgen-macro@0.2.92 + wasm-bindgen-shared@0.2.92 + wasm-bindgen@0.2.92 + web-sys@0.3.69 + webpki-roots@0.26.1 + winapi-i686-pc-windows-gnu@0.4.0 + winapi-util@0.1.6 + winapi-x86_64-pc-windows-gnu@0.4.0 + winapi@0.3.9 + windows-sys@0.48.0 + windows-sys@0.52.0 + windows-targets@0.48.5 + windows-targets@0.52.0 + windows-targets@0.52.4 + windows_aarch64_gnullvm@0.48.5 + windows_aarch64_gnullvm@0.52.0 + windows_aarch64_gnullvm@0.52.4 + windows_aarch64_msvc@0.48.5 + windows_aarch64_msvc@0.52.0 + windows_aarch64_msvc@0.52.4 + windows_i686_gnu@0.48.5 + windows_i686_gnu@0.52.0 + windows_i686_gnu@0.52.4 + windows_i686_msvc@0.48.5 + windows_i686_msvc@0.52.0 + windows_i686_msvc@0.52.4 + windows_x86_64_gnu@0.48.5 + windows_x86_64_gnu@0.52.0 + windows_x86_64_gnu@0.52.4 + windows_x86_64_gnullvm@0.48.5 + windows_x86_64_gnullvm@0.52.0 + windows_x86_64_gnullvm@0.52.4 + windows_x86_64_msvc@0.48.5 + windows_x86_64_msvc@0.52.0 + windows_x86_64_msvc@0.52.4 + zeroize@1.7.0 +" + +inherit cargo distutils-r1 + +DESCRIPTION="Implementation of today's most used tokenizers" +HOMEPAGE="https://github.com/huggingface/tokenizers" +SRC_URI=" + https://github.com/huggingface/${PN}/archive/refs/tags/v${PV}.tar.gz + -> ${P}.gh.tar.gz + ${CARGO_CRATE_URIS} +" + +LICENSE="Apache-2.0" +# Dependent crate licenses +LICENSE+=" + Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0 + Unicode-DFS-2016 +" +SLOT="0" +KEYWORDS="~amd64" +RESTRICT="test" + +BDEPEND="dev-python/setuptools-rust[${PYTHON_USEDEP}]" + +distutils_enable_tests pytest + +QA_FLAGS_IGNORED="/usr/bin/cli .*/site-packages/tokenizers/.*so" + +src_unpack() { + cargo_src_unpack +} + +src_prepare() { + default + cd bindings/python + distutils-r1_src_prepare +} + +src_configure() { + cd tokenizers + cargo_src_configure + cd ../bindings/python + distutils-r1_src_configure +} + +src_compile() { + cd tokenizers + cargo_src_compile + cd ../bindings/python + distutils-r1_src_compile +} + +src_test() { + cd tokenizers + # Tests do not work + #cargo_src_test + cd ../bindings/python + # Need dataset module + #distutils-r1_src_test +} + +src_install() { + cd tokenizers + cargo_src_install + cd ../bindings/python + distutils-r1_src_install +} |