diff --git a/modin/pandas/groupby.py b/modin/pandas/groupby.py
index bfa425913f3..0103b5bce5d 100644
--- a/modin/pandas/groupby.py
+++ b/modin/pandas/groupby.py
@@ -22,6 +22,7 @@
 import pandas.core.common as com
 import pandas.core.groupby
 from pandas._libs import lib
+from pandas.api.types import is_scalar
 from pandas.core.apply import reconstruct_func
 from pandas.core.dtypes.common import (
     is_datetime64_any_dtype,
@@ -904,19 +905,40 @@ def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs)

         do_relabel = None
         if isinstance(func, dict) or func is None:
-            relabeling_required, func_dict, new_columns, order = reconstruct_func(
+            # the order from `reconstruct_func` cannot be used correctly if there
+            # is more than one columnar partition, since for correct use all columns
+            # must be available within one partition.
+            old_kwargs = dict(kwargs)
+            relabeling_required, func_dict, new_columns, _ = reconstruct_func(
                 func, **kwargs
             )

             if relabeling_required:

                 def do_relabel(obj_to_relabel):  # noqa: F811
-                    new_order, new_columns_idx = order, pandas.Index(new_columns)
+                    # unwrap nested labels into one level tuple
+                    result_labels = [None] * len(old_kwargs)
+                    for idx, labels in enumerate(old_kwargs.values()):
+                        if is_scalar(labels) or callable(labels):
+                            result_labels[idx] = (
+                                labels if not callable(labels) else labels.__name__
+                            )
+                            continue
+                        new_elem = []
+                        for label in labels:
+                            if is_scalar(label) or callable(label):
+                                new_elem.append(
+                                    label if not callable(label) else label.__name__
+                                )
+                            else:
+                                new_elem.extend(label)
+                        result_labels[idx] = tuple(new_elem)
+
+                    new_order = obj_to_relabel.columns.get_indexer(result_labels)
+                    new_columns_idx = pandas.Index(new_columns)
                     if not self._as_index:
                         nby_cols = len(obj_to_relabel.columns) - len(new_columns_idx)
-                        new_order = np.concatenate(
-                            [np.arange(nby_cols), new_order + nby_cols]
-                        )
+                        new_order = np.concatenate([np.arange(nby_cols), new_order])
                         by_cols = obj_to_relabel.columns[:nby_cols]
                         if by_cols.nlevels != new_columns_idx.nlevels:
                             by_cols = by_cols.remove_unused_levels()

diff --git a/modin/pandas/test/test_groupby.py b/modin/pandas/test/test_groupby.py
index bd27c3ce885..ff0e2bc5d61 100644
--- a/modin/pandas/test/test_groupby.py
+++ b/modin/pandas/test/test_groupby.py
@@ -3087,6 +3087,60 @@ def test_groupby_named_aggregation():
     )

+def test_groupby_several_column_partitions():
+    # see details in #6948
+    columns = [
+        "l_returnflag",
+        "l_linestatus",
+        "l_discount",
+        "l_extendedprice",
+        "l_quantity",
+    ]
+    modin_df, pandas_df = create_test_dfs(
+        np.random.randint(0, 100, size=(1000, len(columns))), columns=columns
+    )
+
+    pandas_df["a"] = (pandas_df.l_extendedprice) * (1 - (pandas_df.l_discount))
+    # to create another column partition
+    modin_df["a"] = (modin_df.l_extendedprice) * (1 - (modin_df.l_discount))
+
+    eval_general(
+        modin_df,
+        pandas_df,
+        lambda df: df.groupby(["l_returnflag", "l_linestatus"])
+        .agg(
+            sum_qty=("l_quantity", "sum"),
+            sum_base_price=("l_extendedprice", "sum"),
+            sum_disc_price=("a", "sum"),
+            # sum_charge=("b", "sum"),
+            avg_qty=("l_quantity", "mean"),
+            avg_price=("l_extendedprice", "mean"),
+            avg_disc=("l_discount", "mean"),
+            count_order=("l_returnflag", "count"),
+        )
+        .reset_index(),
+    )
+
+
+def test_groupby_named_agg():
+    # from pandas docs
+
+    data = {
+        "A": [1, 1, 2, 2],
+        "B": [1, 2, 3, 4],
+        "C": [0.362838, 0.227877, 1.267767, -0.562860],
+    }
+    modin_df, pandas_df = create_test_dfs(data)
+    eval_general(
+        modin_df,
+        pandas_df,
+        lambda df: df.groupby("A").agg(
+            b_min=pd.NamedAgg(column="B", aggfunc="min"),
+            c_sum=pd.NamedAgg(column="C", aggfunc="sum"),
+        ),
+    )
+
+
 ### TEST GROUPBY WARNINGS ###

Poor Performance on TPC-H Queries
In a benchmark project ([link](https://github.com/hesamshahrokhi/modin_tpch_benchmark)), I tried to compare the performance of Pandas and Modin (on Ray and Dask) on simple TPC-H queries (Q1 and Q6) written in Pandas. The results ([link](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt)) show that the ```read_table``` API is 2x faster than its Pandas equivalent on Ray. However, the same API on Dask, and also the individual query run times on both backends are much slower than Pandas. Moreover, the results of Q1 are not correct.

Could you please help me to understand why this is the case? I tried to use the suggested configurations and imported the data using ```read_table``` which is officially supported by Modin.

Hi @hesamshahrokhi! Thanks for the question!

The method you use to initialize dask is not intended for Modin:
https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/e7cf88cf5d6152c32593193a94c837f5fde4f298/tpch_dask.py#L12C17-L12C32. The `processes` parameter should be `True`. Could I also ask why you initialize the dask yourself rather than relying on the initialization that Modin does implicitly?

> Moreover, the results of Q1 are not correct.

Let's see what's wrong. Could you please provide more information?
Hi @anmyachev
Thanks for your quick response!

I did the explicit initialization of Dask and Ray because of a runtime warning that asked me to do so. Now, based on your suggestion, I entirely removed the explicit initializations for [Ray](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/tpch_ray.py#L14-L15) and [Dask](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/tpch_dask.py#L11-L12). But, after re-running the benchmark, I still see poor performance from Modin (on both backends). Here are the updated benchmark [results](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt) and the mentioned warnings for [Ray](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L22-L26) and [Dask](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L48-L53).

Regarding the correctness issue in Q1, that still exists, I would refer you to the correct answer by [Pandas](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L5-L10) and the incorrect answers by Modin on [Ray](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L31-L36) and [Dask](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L57-L62). In the Modin outputs, the titles for these columns ```sum_disc_price, sum_charge, avg_disc, and count_order``` are not match to their relevant values.
>  of a runtime warning that asked me to do so.

This was not the most obvious warning, which in fact simply showed that Modin initializes the engine itself. We have already removed it in Modin 0.27.0.

What version of Modin are you using? Can you use version 0.27.0? Perhaps the bug has already been fixed and the performance is better.

> Regarding the correctness issue in Q1, that still exists, I would refer you to the correct answer by [Pandas](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L5-L10) and the incorrect answers by Modin on [Ray](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L31-L36) and [Dask](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L57-L62). In the Modin outputs, the titles for these columns sum_disc_price, sum_charge, avg_disc, and count_order are not match to their relevant values.

We'll look at this problem.
I'm able to reproduce the problem with the following reproducer (for Modin 0.27.0):
```python
import numpy as np
import modin.pandas as pd
from modin.pandas.test.utils import df_equals

np.random.seed(42)

columns = ['l_returnflag', 'l_linestatus', 'l_discount', 'l_extendedprice', 'l_quantity', 'b', 'a'] # , 'l_extendedprice'] first fail
columns = ['l_returnflag', 'l_linestatus', 'l_discount', 'l_extendedprice', 'l_quantity']
df = pd.DataFrame(np.random.randint(0, 100, size=(1000, len(columns))), columns=columns)
df["a"] = ((df.l_extendedprice) * (1 - (df.l_discount)))
df['b'] = ((df.l_extendedprice) * (1 - (df.l_discount))) * 1.94

agg_df = df \
    .groupby(['l_returnflag', 'l_linestatus']) \
    .agg(
        sum_qty=("l_quantity", "sum"),
        sum_base_price=("l_extendedprice", "sum"),
        sum_disc_price=("a", "sum"),
        sum_charge=("b", "sum"),
        avg_qty=("l_quantity", "mean"),
        avg_price=("l_extendedprice", "mean"),
        avg_disc=("l_discount", "mean"),
        count_order=("l_returnflag", "count"),
    ).reset_index()

agg_df_pd = df._to_pandas() \
    .groupby(['l_returnflag', 'l_linestatus']) \
    .agg(
        sum_qty=("l_quantity", "sum"),
        sum_base_price=("l_extendedprice", "sum"),
        sum_disc_price=("a", "sum"),
        sum_charge=("b", "sum"),
        avg_qty=("l_quantity", "mean"),
        avg_price=("l_extendedprice", "mean"),
        avg_disc=("l_discount", "mean"),
        count_order=("l_returnflag", "count"),
    ).reset_index()

df_equals(agg_df, agg_df_pd)

```
> > of a runtime warning that asked me to do so.
>
> This was not the most obvious warning, which in fact simply showed that Modin initializes the engine itself. We have already removed it in Modin 0.27.0.
>
> What version of Modin are you using? Can you use version 0.27.0? Perhaps the bug has already been fixed and the performance is better.

I changed my modin version to 0.27.0. The warnings are gone but the performance and correctness issues still exist.

>
> > Regarding the correctness issue in Q1, that still exists, I would refer you to the correct answer by [Pandas](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L5-L10) and the incorrect answers by Modin on [Ray](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L31-L36) and [Dask](https://github.com/hesamshahrokhi/modin_tpch_benchmark/blob/main/Results_TPCH_SF1.txt#L57-L62). In the Modin outputs, the titles for these columns sum_disc_price, sum_charge, avg_disc, and count_order are not match to their relevant values.
>
> We'll look at this problem.

Thanks :)


diff --git a/modin/core/dataframe/pandas/partitioning/partition_manager.py b/modin/core/dataframe/pandas/partitioning/partition_manager.py
index f7b3899550c..2ccbd1de47f 100644
--- a/modin/core/dataframe/pandas/partitioning/partition_manager.py
+++ b/modin/core/dataframe/pandas/partitioning/partition_manager.py
@@ -869,14 +869,7 @@ def split_pandas_df_into_partitions(
         put_func = cls._partition_class.put
         # even a full-axis slice can cost something (https://github.com/pandas-dev/pandas/issues/55202)
         # so we try not to do it if unnecessary.
-        # FIXME: it appears that this optimization doesn't work for Unidist correctly as it
-        # doesn't explicitly copy the data when putting it into storage (as the rest engines do)
-        # causing it to eventially share memory with a pandas object that was provided by user.
-        # Everything works fine if we do this column slicing as pandas then would set some flags
-        # to perform in COW mode apparently (and so it wouldn't crash our tests).
-        # @YarShev promised that this will be eventially fixed on Unidist's side, but for now there's
-        # this hacky condition
-        if col_chunksize >= len(df.columns) and Engine.get() != "Unidist":
+        if col_chunksize >= len(df.columns):
             col_parts = [df]
         else:
             col_parts = [

diff --git a/modin/pandas/test/test_general.py b/modin/pandas/test/test_general.py
index 7a445657d2d..72f33d55d80 100644
--- a/modin/pandas/test/test_general.py
+++ b/modin/pandas/test/test_general.py
@@ -971,3 +971,15 @@ def make_frame(lib):
 def test_get(key):
     modin_df, pandas_df = create_test_dfs({"col0": [0, 1]})
     eval_general(modin_df, pandas_df, lambda df: df.get(key))
+
+
+def test_df_immutability():
+    """
+    Verify that modifications of the source data doesn't propagate to Modin's DataFrame objects.
+    """
+    src_data = pandas.DataFrame({"a": [1]})
+
+    md_df = pd.DataFrame(src_data)
+    src_data.iloc[0, 0] = 100
+
+    assert md_df._to_pandas().iloc[0, 0] == 1

Remove unidist specific workaround in `.from_pandas()`
Since unidist 0.6.0 (modin-project/unidist#409) it now always copy the input data, so the workaround can be removed
https://github.com/modin-project/modin/blob/01c529cf06cfaf412b5725f41c81a5f914b44b95/modin/core/dataframe/pandas/partitioning/partition_manager.py#L872-L880


diff --git a/lib/ansible/modules/dnf5.py b/lib/ansible/modules/dnf5.py
index df4ee206748716..b157158514ffab 100644
--- a/lib/ansible/modules/dnf5.py
+++ b/lib/ansible/modules/dnf5.py
@@ -358,6 +358,15 @@

 def is_installed(base, spec):
     settings = libdnf5.base.ResolveSpecSettings()
+    # Disable checking whether SPEC is a binary -> `/usr/(s)bin/<SPEC>`,
+    # this prevents scenarios like the following:
+    #   * the `sssd-common` package is installed and provides `/usr/sbin/sssd` binary
+    #   * the `sssd` package is NOT installed
+    #   * due to `set_with_binaries(True)` being default `is_installed(base, "sssd")` would "unexpectedly" return True
+    # If users wish to target the `sssd` binary they can by specifying the full path `name=/usr/sbin/sssd` explicitly
+    # due to settings.set_with_filenames(True) being default.
+    settings.set_with_binaries(False)
+
     installed_query = libdnf5.rpm.PackageQuery(base)
     installed_query.filter_installed()
     match, nevra = installed_query.resolve_pkg_spec(spec, settings, True)

diff --git a/changelogs/fragments/84259-dnf5-latest-fix.yml b/changelogs/fragments/84259-dnf5-latest-fix.yml
new file mode 100644
index 00000000000000..40f6ddb740860f
--- /dev/null
+++ b/changelogs/fragments/84259-dnf5-latest-fix.yml
@@ -0,0 +1,2 @@
+bugfixes:
+  - "dnf5 - fix installing a package using ``state=latest`` when a binary of the same name as the package is already installed (https://github.com/ansible/ansible/issues/84259)"
diff --git a/test/integration/targets/dnf/tasks/repo.yml b/test/integration/targets/dnf/tasks/repo.yml
index 6ab8fa1a3b3cdd..ec31fe4a4ae39e 100644
--- a/test/integration/targets/dnf/tasks/repo.yml
+++ b/test/integration/targets/dnf/tasks/repo.yml
@@ -517,3 +517,50 @@
       dnf:
         name: provides_foo*
         state: absent
+
+# https://github.com/ansible/ansible/issues/84259
+- name: test installing a package named `package-name` while a package providing `/usr/sbin/package-name` is installed
+  block:
+    - dnf:
+        name: package-name
+        state: absent
+
+    - dnf:
+        name: provides-binary
+        state: present
+
+    - dnf:
+        name: package-name
+        state: latest
+      register: dnf_result
+
+    - assert:
+        that:
+          - dnf_result is changed
+  always:
+    - name: Clean up
+      dnf:
+        name:
+          - provides-binary
+          - package-name
+        state: absent
+
+- name: test installing a package that provides a binary by specifying the binary name
+  block:
+    - dnf:
+        name: provides-binary
+        state: absent
+
+    - dnf:
+        name: /usr/sbin/package-name
+        state: present
+      register: dnf_result
+
+    - assert:
+        that:
+          - dnf_result is changed
+  always:
+    - name: Clean up
+      dnf:
+        name: provides-binary
+        state: absent
diff --git a/test/integration/targets/setup_rpm_repo/library/create_repo.py b/test/integration/targets/setup_rpm_repo/library/create_repo.py
index 5acf2397195754..6fffe5ad90b487 100644
--- a/test/integration/targets/setup_rpm_repo/library/create_repo.py
+++ b/test/integration/targets/setup_rpm_repo/library/create_repo.py
@@ -15,8 +15,10 @@
     from rpmfluff.make import make_gif
     from rpmfluff.sourcefile import GeneratedSourceFile
     from rpmfluff.rpmbuild import SimpleRpmBuild
+    from rpmfluff.utils import expectedArch
     from rpmfluff.yumrepobuild import YumRepoBuild
 except ImportError:
+    expectedArch = None  # define here to avoid NameError as it is used on top level in SPECS
     HAS_RPMFLUFF = False

@@ -30,6 +32,7 @@ class RPM:
     recommends: list[str] | None = None
     requires: list[str] | None = None
     file: str | None = None
+    binary: str | None = None

 SPECS = [
@@ -58,6 +61,8 @@ class RPM:
     RPM(name='broken-b', version='1.0', requires=['broken-a = 1.2.3-1']),
     RPM(name='broken-c', version='1.0', requires=['broken-c = 1.2.4-1']),
     RPM(name='broken-d', version='1.0', requires=['broken-a']),
+    RPM(name='provides-binary', version='1.0', arch=[expectedArch], binary='/usr/sbin/package-name'),
+    RPM(name='package-name', version='1.0'),
 ]

@@ -81,10 +86,13 @@ def create_repo():
                 )
             )

+        if spec.binary:
+            pkg.add_simple_compilation(installPath=spec.binary)
+
         pkgs.append(pkg)

     repo = YumRepoBuild(pkgs)
-    repo.make('noarch', 'i686', 'x86_64')
+    repo.make('noarch', 'i686', 'x86_64', expectedArch)

     for pkg in pkgs:
         pkg.clean()

DNF5 state: latest fails when installing
### Summary

When using `ansible.builtin.dnf` with `state: latest` the previous behavior was to install and update all packages.
Fedora 41 uses DNF5 which does not install the package but instead fails.
A Bug report is filed on the DNF5 repo: https://github.com/rpm-software-management/dnf5/issues/1824
But this may not be reverted because this may be intended behavior.

### Issue Type

Bug Report

### Component Name

dnf

### Ansible Version

```console
$ ansible --version
ansible [core 2.17.5]
  config file = ...
  configured module search path = ['...', '/usr/share/ansible/plugins/modules']
  ansible python module location = ...
  ansible collection location = ...:/usr/share/ansible/collections
  executable location = ...
  python version = 3.12.7 (main, Oct  1 2024, 02:05:46) [GCC 13.3.0] (...)
  jinja version = 3.1.4
  libyaml = True
```

### Configuration

```console
# if using a version older than ansible-core 2.12 you should omit the '-t all'
$ ansible-config dump --only-changed -t all
ansible-config [core 2.17.5]
  config file = [...]/ansible.cfg
  configured module search path = ['[...]/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
  ansible python module location = [...]/python3.12/site-packages/ansible
  ansible collection location = [...]/.ansible/collections:/usr/share/ansible/collections
  executable location = [...]/bin/ansible-config
  python version = 3.12.7 (main, Oct  1 2024, 02:05:46) [GCC 13.3.0] ([...]/bin/python3.12)
  jinja version = 3.1.4
  libyaml = True
Using [...]/inventory/ansible.cfg as config file
CONFIG_FILE() = [...]/ansible.cfg
DEFAULT_HOST_LIST([...]/ansible.cfg) = ['[...]/hosts.yml']
DEFAULT_VERBOSITY(/[...]/ansible.cfg) = 2
EDITOR(env: EDITOR) = /usr/bin/nano
PERSISTENT_COMMAND_TIMEOUT([...]/ansible.cfg) = 30
PERSISTENT_CONNECT_TIMEOUT([...]/ansible.cfg) = 30
```

### OS / Environment

Fedora 41

### Steps to Reproduce

<!--- Paste example playbooks or commands between quotes below -->
```yaml (paste below)

```
- name: Install sssd
  ansible.builtin.dnf:
    name:
      - sssd
    state: latest

### Expected Results

SSSD is expected to be installed.
Documentation: https://docs.ansible.com/ansible/latest/collections/ansible/builtin/dnf_module.html#parameter-state
> Whether to install (present, latest), or remove (absent) a package.

`latest` should install and update the packages

### Actual Results

```console
{
  "failures": [
    "Packages for argument 'sssd' available, but not installed."
  ],
  "rc": 1,
  "msg": "Failed to install some of the specified packages",
  "invocation": {
    "module_args": {
      "name": [
        "sssd"
      ],
      "state": "latest",
      "allow_downgrade": false,
      "autoremove": false,
      "bugfix": false,
      "cacheonly": false,
      "disable_gpg_check": false,
      "disable_plugin": [],
      "disablerepo": [],
      "download_only": false,
      "enable_plugin": [],
      "enablerepo": [],
      "exclude": [],
      "installroot": "/",
      "install_repoquery": true,
      "install_weak_deps": true,
      "security": false,
      "skip_broken": false,
      "update_cache": false,
      "update_only": false,
      "validate_certs": true,
      "sslverify": true,
      "lock_timeout": 30,
      "allowerasing": false,
      "nobest": false,
      "conf_file": null,
      "disable_excludes": null,
      "download_dir": null,
      "list": null,
      "releasever": null
    }
  },
  "_ansible_no_log": false,
  "changed": false
}
```

### Code of Conduct

- [X] I agree to follow the Ansible Code of Conduct

Files identified in the description:

* [`lib/ansible/plugins/action/dnf.py`](https://github.com/ansible/ansible/blob/devel/lib/ansible/plugins/action/dnf.py)
* [`lib/ansible/modules/dnf.py`](https://github.com/ansible/ansible/blob/devel/lib/ansible/modules/dnf.py)

If these files are incorrect, please update the `component name` section of the description or use the [component](https://github.com/ansible/ansibotmini#commands) bot command.

<!--- boilerplate: components_banner --->

I don't seem to be able to reproduce this.

Preparing a target host:
```
% podman run --name container --rm -it --entrypoint bash fedora:41
[root@2049d5586296 /]# dnf install python3-libdnf5
...
```

Running the task provided in the reproducer:
```
% ansible container -m dnf -a "name=sssd state=latest"
container | CHANGED => {
    "ansible_facts": {
        "pkg_mgr": "dnf5"
    },
    "changed": true,
    "msg": "",
    "rc": 0,
    "results": [
        "Installed: sssd-2.10.0-1.fc41.aarch64",
        "Installed: sssd-ad-2.10.0-1.fc41.aarch64",
        "Installed: sssd-common-2.10.0-1.fc41.aarch64",
        "Installed: sssd-ipa-2.10.0-1.fc41.aarch64",
        "Installed: sssd-krb5-2.10.0-1.fc41.aarch64",
        "Installed: sssd-ldap-2.10.0-1.fc41.aarch64",
        "Installed: sssd-proxy-2.10.0-1.fc41.aarch64",
        "Installed: dbus-libs-1:1.14.10-4.fc41.aarch64",
        "Installed: libbasicobjects-0.1.1-57.fc41.aarch64",
        "Installed: libcollection-0.7.0-57.fc41.aarch64",
        "Installed: libdhash-0.5.0-57.fc41.aarch64",
        "Installed: libini_config-1.3.1-57.fc41.aarch64",
        "Installed: libref_array-0.1.5-57.fc41.aarch64",
        "Installed: libsss_certmap-2.10.0-1.fc41.aarch64",
        "Installed: libsss_idmap-2.10.0-1.fc41.aarch64",
        "Installed: libtalloc-2.4.2-4.fc41.aarch64",
        "Installed: libtdb-1.4.12-3.fc41.aarch64",
        "Installed: libtevent-0.16.1-4.fc41.aarch64",
        "Installed: sssd-common-pac-2.10.0-1.fc41.aarch64",
        "Installed: sssd-krb5-common-2.10.0-1.fc41.aarch64",
        "Installed: c-ares-1.33.0-1.fc41.aarch64",
        "Installed: sssd-client-2.10.0-1.fc41.aarch64",
        "Installed: systemd-256.7-1.fc41.aarch64",
        "Installed: libipa_hbac-2.10.0-1.fc41.aarch64",
        "Installed: libpath_utils-0.2.1-57.fc41.aarch64",
        "Installed: cyrus-sasl-gssapi-2.1.28-27.fc41.aarch64",
        "Installed: jansson-2.13.1-10.fc41.aarch64",
        "Installed: libsss_nss_idmap-2.10.0-1.fc41.aarch64",
        "Installed: dbus-1:1.14.10-4.fc41.aarch64",
        "Installed: libfdisk-2.40.2-4.fc41.aarch64",
        "Installed: libseccomp-2.5.5-2.fc41.aarch64",
        "Installed: systemd-pam-256.7-1.fc41.aarch64",
        "Installed: dbus-broker-36-4.fc41.aarch64",
        "Installed: dbus-common-1:1.14.10-4.fc41.noarch",
        "Installed: libldb-2:4.21.1-7.fc41.aarch64",
        "Installed: lmdb-libs-0.9.33-2.fc41.aarch64",
        "Installed: samba-client-libs-2:4.21.1-7.fc41.aarch64",
        "Installed: avahi-libs-0.8-29.fc41.aarch64",
        "Installed: libicu-74.2-2.fc41.aarch64",
        "Installed: libwbclient-2:4.21.1-7.fc41.aarch64",
        "Installed: samba-common-2:4.21.1-7.fc41.noarch",
        "Installed: samba-common-libs-2:4.21.1-7.fc41.aarch64",
        "Installed: libsmbclient-2:4.21.1-7.fc41.aarch64",
        "Installed: libnl3-3.11.0-1.fc41.aarch64",
        "Installed: glibc-gconv-extra-2.40-9.fc41.aarch64",
        "Installed: libsss_sudo-2.10.0-1.fc41.aarch64",
        "Installed: bind-utils-32:9.18.30-1.fc41.aarch64",
        "Installed: bind-libs-32:9.18.30-1.fc41.aarch64",
        "Installed: jemalloc-5.3.0-7.fc41.aarch64",
        "Installed: protobuf-c-1.5.0-4.fc41.aarch64",
        "Installed: fstrm-0.6.1-11.fc41.aarch64",
        "Installed: libmaxminddb-1.11.0-1.fc41.aarch64",
        "Installed: adcli-0.9.2-7.fc41.aarch64",
        "Installed: libxkbcommon-1.7.0-4.fc41.aarch64",
        "Installed: xkeyboard-config-2.42-2.fc41.noarch",
        "Installed: libbpf-2:1.4.7-1.fc41.aarch64",
        "Installed: diffutils-3.10-8.fc41.aarch64",
        "Installed: kmod-libs-33-1.fc41.aarch64",
        "Installed: cryptsetup-libs-2.7.5-1.fc41.aarch64",
        "Installed: device-mapper-libs-1.02.199-4.fc41.aarch64",
        "Installed: device-mapper-1.02.199-4.fc41.aarch64",
        "Installed: qrencode-libs-4.1.1-8.fc41.aarch64",
        "Installed: systemd-networkd-256.7-1.fc41.aarch64",
        "Installed: systemd-resolved-256.7-1.fc41.aarch64",
        "Installed: logrotate-3.22.0-2.fc41.aarch64",
        "Installed: libuv-1:1.49.2-1.fc41.aarch64",
        "Installed: systemd-udev-256.7-1.fc41.aarch64",
        "Installed: kbd-2.6.4-4.fc41.aarch64",
        "Installed: kmod-33-1.fc41.aarch64",
        "Installed: kbd-legacy-2.6.4-4.fc41.noarch",
        "Installed: kbd-misc-2.6.4-4.fc41.noarch",
        "Installed: pkgconf-pkg-config-2.3.0-1.fc41.aarch64",
        "Installed: pkgconf-2.3.0-1.fc41.aarch64",
        "Installed: pkgconf-m4-2.3.0-1.fc41.noarch",
        "Installed: libpkgconf-2.3.0-1.fc41.aarch64",
        "Installed: tpm2-tss-4.1.3-3.fc41.aarch64",
        "Installed: libfido2-1.15.0-2.fc41.aarch64",
        "Installed: libcbor-0.11.0-2.fc41.aarch64",
        "Installed: glibc-2.40-9.fc41.aarch64",
        "Installed: glibc-common-2.40-9.fc41.aarch64",
        "Installed: glibc-minimal-langpack-2.40-9.fc41.aarch64",
        "Removed: glibc-2.40-3.fc41.aarch64",
        "Removed: glibc-common-2.40-3.fc41.aarch64",
        "Removed: glibc-minimal-langpack-2.40-3.fc41.aarch64"
    ]
}
```

I suspect that the cause of your issue is certain unknown set of packages being installed on the target system that somehow trips the `is_installed` check into thinking `sssd` is already installed and trying to upgrade it instead of installing it. What is the output of `# rpm -qa sssd-*` on the target system prior to running your playbook?
Thank you for your help. `rpm -qa sssd-*` yields:
```
sssd-nfs-idmap-2.10.0-1.fc41.x86_64
sssd-client-2.10.0-1.fc41.x86_64
sssd-common-2.10.0-1.fc41.x86_64
sssd-krb5-common-2.10.0-1.fc41.x86_64
sssd-kcm-2.10.0-1.fc41.x86_64
```

This is on a fresh Fedora 41 _Workstation_ Installation.
What are my options to circumvent this problem? Is this an ansible or a dnf bug?


diff --git a/altair/expr/core.py b/altair/expr/core.py
index 429766b8a..cb89391f4 100644
--- a/altair/expr/core.py
+++ b/altair/expr/core.py
@@ -1,10 +1,20 @@
 from __future__ import annotations

-from typing import Any, Union
-from typing_extensions import TypeAlias
+import datetime as dt
+from typing import TYPE_CHECKING, Any, Literal, Union

 from altair.utils import SchemaBase

+if TYPE_CHECKING:
+    import sys
+
+    from altair.vegalite.v5.schema._typing import Map, PrimitiveValue_T
+
+    if sys.version_info >= (3, 10):
+        from typing import TypeAlias
+    else:
+        from typing_extensions import TypeAlias
+

 class DatumType:
     """An object to assist in building Vega-Lite Expressions."""
@@ -38,10 +48,40 @@ def _js_repr(val) -> str:
         return "null"
     elif isinstance(val, OperatorMixin):
         return val._to_expr()
+    elif isinstance(val, dt.date):
+        return _from_date_datetime(val)
     else:
         return repr(val)

+def _from_date_datetime(obj: dt.date | dt.datetime, /) -> str:
+    """
+    Parse native `datetime.(date|datetime)` into a `datetime expression`_ string.
+
+    **Month is 0-based**
+
+    .. _datetime expression:
+        https://vega.github.io/vega/docs/expressions/#datetime
+    """
+    fn_name: Literal["datetime", "utc"] = "datetime"
+    args: tuple[int, ...] = obj.year, obj.month - 1, obj.day
+    if isinstance(obj, dt.datetime):
+        if tzinfo := obj.tzinfo:
+            if tzinfo is dt.timezone.utc:
+                fn_name = "utc"
+            else:
+                msg = (
+                    f"Unsupported timezone {tzinfo!r}.\n"
+                    "Only `'UTC'` or naive (local) datetimes are permitted.\n"
+                    "See https://altair-viz.github.io/user_guide/generated/core/altair.DateTime.html"
+                )
+                raise TypeError(msg)
+        us = obj.microsecond
+        ms = us if us == 0 else us // 1_000
+        args = *args, obj.hour, obj.minute, obj.second, ms
+    return FunctionExpression(fn_name, args)._to_expr()
+
+
 # Designed to work with Expression and VariableParameter
 class OperatorMixin:
     def _to_expr(self) -> str:
@@ -237,4 +277,6 @@ def __repr__(self) -> str:
         return f"{self.group}[{self.name!r}]"

-IntoExpression: TypeAlias = Union[bool, None, str, float, OperatorMixin, dict[str, Any]]
+IntoExpression: TypeAlias = Union[
+    "PrimitiveValue_T", dt.date, dt.datetime, OperatorMixin, "Map"
+]

diff --git a/tests/expr/test_expr.py b/tests/expr/test_expr.py
index 15757b09f..b2200184c 100644
--- a/tests/expr/test_expr.py
+++ b/tests/expr/test_expr.py
@@ -1,5 +1,6 @@
 from __future__ import annotations

+import datetime as dt
 import operator
 import sys
 from inspect import classify_class_attrs, getmembers, signature
@@ -188,3 +189,43 @@ def test_expression_function_nostring():

     with pytest.raises(ValidationError):
         expr(["foo", "bah"])  # pyright: ignore
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        (dt.date(2000, 1, 1), "datetime(2000,0,1)"),
+        (dt.datetime(2000, 1, 1), "datetime(2000,0,1,0,0,0,0)"),
+        (dt.datetime(2001, 1, 1, 9, 30, 0, 2999), "datetime(2001,0,1,9,30,0,2)"),
+        (
+            dt.datetime(2003, 5, 1, 1, 30, tzinfo=dt.timezone.utc),
+            "utc(2003,4,1,1,30,0,0)",
+        ),
+    ],
+    ids=["date", "datetime (no time)", "datetime (microseconds)", "datetime (UTC)"],
+)
+def test_expr_datetime(value: Any, expected: str) -> None:
+    r_datum = datum.date >= value
+    assert isinstance(r_datum, Expression)
+    assert repr(r_datum) == f"(datum.date >= {expected})"
+
+
+@pytest.mark.parametrize(
+    "tzinfo",
+    [
+        dt.timezone(dt.timedelta(hours=2), "UTC+2"),
+        dt.timezone(dt.timedelta(hours=1), "BST"),
+        dt.timezone(dt.timedelta(hours=-7), "pdt"),
+        dt.timezone(dt.timedelta(hours=-3), "BRT"),
+        dt.timezone(dt.timedelta(hours=9), "UTC"),
+        dt.timezone(dt.timedelta(minutes=60), "utc"),
+    ],
+)
+def test_expr_datetime_unsupported_timezone(tzinfo: dt.timezone) -> None:
+    datetime = dt.datetime(2003, 5, 1, 1, 30)
+
+    result = datum.date == datetime
+    assert repr(result) == "(datum.date === datetime(2003,4,1,1,30,0,0))"
+
+    with pytest.raises(TypeError, match=r"Unsupported timezone.+\n.+UTC.+local"):
+        datum.date == datetime.replace(tzinfo=tzinfo)  # noqa: B015

Support `datetime.(date|datetime)` in `Expression`(s)
### What is your suggestion?

- Target expressions [datetime](https://vega.github.io/vega/docs/expressions/#datetime), [utc](https://vega.github.io/vega/docs/expressions/#utc)

**See #3650 for detail**


diff --git a/sympy/core/basic.py b/sympy/core/basic.py
index 62ccd3ed5508..f9f04c728245 100644
--- a/sympy/core/basic.py
+++ b/sympy/core/basic.py
@@ -2075,6 +2075,11 @@ def rewrite(self, *args, deep=True, **hints):
         pattern = args[:-1]
         rule = args[-1]

+        # Special case: map `abs` to `Abs`
+        if rule is abs:
+            from sympy.functions.elementary.complexes import Abs
+            rule = Abs
+
         # support old design by _eval_rewrite_as_[...] method
         if isinstance(rule, str):
             method = "_eval_rewrite_as_%s" % rule

diff --git a/sympy/core/tests/test_basic.py b/sympy/core/tests/test_basic.py
index 44faf93162f0..d452af7e2a8e 100644
--- a/sympy/core/tests/test_basic.py
+++ b/sympy/core/tests/test_basic.py
@@ -17,6 +17,9 @@
 from sympy.integrals.integrals import Integral
 from sympy.functions.elementary.exponential import exp
 from sympy.testing.pytest import raises, warns_deprecated_sympy
+from sympy.functions.elementary.complexes import Abs, sign
+from sympy.functions.elementary.piecewise import Piecewise
+from sympy.core.relational import Eq

 b1 = Basic()
 b2 = Basic(b1)
@@ -331,3 +334,10 @@ class A(Symbol, Generic[T]):

     class B(A[T]):
         pass
+
+
+def test_rewrite_abs():
+    # https://github.com/sympy/sympy/issues/27323
+    x = Symbol('x')
+    assert sign(x).rewrite(abs) == sign(x).rewrite(Abs)
+    assert sign(x).rewrite(abs) == Piecewise((0, Eq(x, 0)), (x / Abs(x), True))

rewrite(abs) should call rewrite(Abs)
```py
>>> sign(x).rewrite(abs)
sign(x)
>>> sign(x).rewrite(Abs)
Piecewise((0, Eq(x, 0)), (x/Abs(x), True))
```

This is because `rewrite(x)` just dispatches to the method called `_eval_rewrite_as_x`, and the method here is called `_eval_rewrite_as_Abs` since `Abs` is the name of the class. But most people use `abs`, the built-in function which dispatches to `Abs`. So it should be special-cased in rewrite to call `Abs`.


diff --git a/sympy/physics/wigner.py b/sympy/physics/wigner.py
index 3dcd2f8f4690..5874acbb61c8 100644
--- a/sympy/physics/wigner.py
+++ b/sympy/physics/wigner.py
@@ -571,6 +571,8 @@ def wigner_6j(j_1, j_2, j_3, j_4, j_5, j_6, prec=None):
     algebra system [Rasch03]_.

     """
+    j_1, j_2, j_3, j_4, j_5, j_6 = map(sympify, \
+                [j_1, j_2, j_3, j_4, j_5, j_6])
     res = (-1) ** int(j_1 + j_2 + j_4 + j_5) * \
         racah(j_1, j_2, j_5, j_4, j_3, j_6, prec)
     return res
@@ -627,6 +629,8 @@ def wigner_9j(j_1, j_2, j_3, j_4, j_5, j_6, j_7, j_8, j_9, prec=None):
     for finite precision arithmetic and only useful for a computer
     algebra system [Rasch03]_.
     """
+    j_1, j_2, j_3, j_4, j_5, j_6, j_7, j_8, j_9 = map(sympify, \
+                [j_1, j_2, j_3, j_4, j_5, j_6, j_7, j_8, j_9])
     imax = int(min(j_1 + j_9, j_2 + j_6, j_4 + j_8) * 2)
     imin = imax % 2
     sumres = 0

diff --git a/sympy/physics/tests/test_clebsch_gordan.py b/sympy/physics/tests/test_clebsch_gordan.py
index 2085d4a13479..e4313e3e412d 100644
--- a/sympy/physics/tests/test_clebsch_gordan.py
+++ b/sympy/physics/tests/test_clebsch_gordan.py
@@ -67,6 +67,10 @@ def test_clebsch_gordan_numpy():

 def test_wigner():
+    try:
+        import numpy as np
+    except ImportError:
+        skip("numpy not installed")
     def tn(a, b):
         return (a - b).n(64) < S('1e-64')
     assert tn(wigner_9j(1, 1, 1, 1, 1, 1, 1, 1, 0, prec=64), Rational(1, 18))
@@ -74,6 +78,8 @@ def tn(a, b):
         70)/(246960*sqrt(105)) - 365/(3528*sqrt(70)*sqrt(105))
     assert wigner_6j(5, 5, 5, 5, 5, 5) == Rational(1, 52)
     assert tn(wigner_6j(8, 8, 8, 8, 8, 8, prec=64), Rational(-12219, 965770))
+    assert wigner_6j(1, 1, 1, 1.0, np.float64(1.0), 1) == Rational(1, 6)
+    assert wigner_6j(3.0, np.float32(3), 3.0, 3, 3, 3) == Rational(-1, 14)
     # regression test for #8747
     half = S.Half
     assert wigner_9j(0, 0, 0, 0, half, half, 0, half, half) == half
@@ -89,6 +95,14 @@ def tn(a, b):
                       2, 4, 4,
                       5, 7 * half, 7 * half)
             == -sqrt(Rational(3481, 5042614500)))
+    assert (wigner_9j(5, 5, 5.0,
+                      np.float64(5.0), 5, 5,
+                      5, 5, 5)
+            == 0)
+    assert (wigner_9j(1.0, 2.0, 3.0,
+                      3, 2, 1,
+                      2, 1, 3)
+            == -4*sqrt(70)/11025)

 def test_gaunt():

wigner_6j and wigner_9j violating numpy behavioral subtyping
Dear Developers,

I get an error when I run the following code snippet using wigner_6j.

```
import numpy as np
from sympy.physics.wigner import wigner_6j

y = np.float64(5.0)
x = wigner_6j(5, 5, y, 5, 5, 5)
```

This produces the following error message:

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[4], line 5
      2 from sympy.physics.wigner import wigner_6j
      4 y = np.float64(5.0)
----> 5 x = wigner_6j(5, 5, y, 5, 5, 5)

File ~\code\.venv\jlab\lib\site-packages\sympy\physics\wigner.py:572, in wigner_6j(j_1, j_2, j_3, j_4, j_5, j_6, prec)
    479 def wigner_6j(j_1, j_2, j_3, j_4, j_5, j_6, prec=None):
    480     r"""
    481     Calculate the Wigner 6j symbol `\operatorname{Wigner6j}(j_1,j_2,j_3,j_4,j_5,j_6)`.
    482
   (...)
    569
    570     """
    571     res = (-1) ** int(j_1 + j_2 + j_4 + j_5) * \
--> 572         racah(j_1, j_2, j_5, j_4, j_3, j_6, prec)
    573     return res

File ~\code\.venv\jlab\lib\site-packages\sympy\physics\wigner.py:451, in racah(aa, bb, cc, dd, ee, ff, prec)
    398 def racah(aa, bb, cc, dd, ee, ff, prec=None):
    399     r"""
    400     Calculate the Racah symbol `W(a,b,c,d;e,f)`.
    401
   (...)
    449     - Jens Rasch (2009-03-24): initial version
    450     """
--> 451     prefac = _big_delta_coeff(aa, bb, ee, prec) * \
    452         _big_delta_coeff(cc, dd, ee, prec) * \
    453         _big_delta_coeff(aa, cc, ff, prec) * \
    454         _big_delta_coeff(bb, dd, ff, prec)
    455     if prefac == 0:
    456         return S.Zero

File ~\code\.venv\jlab\lib\site-packages\sympy\physics\wigner.py:372, in _big_delta_coeff(aa, bb, cc, prec)
    369 # the triangle test will only pass if a) all 3 values are ints or
    370 # b) 1 is an int and the other two are half-ints
    371 if not int_valued(aa + bb - cc):
--> 372     raise ValueError("j values must be integer or half integer and fulfill the triangle relation")
    373 if not int_valued(aa + cc - bb):
    374     raise ValueError("j values must be integer or half integer and fulfill the triangle relation")

ValueError: j values must be integer or half integer and fulfill the triangle relation
```

A similar issue is found for wigner_9j. A similar issue was found for the wigner_3j method in #27219, and was fixed in [#27228](https://github.com/sympy/sympy/pull/27228).

As an aside maybe there could be some underlying shared function to fix wigner_3j, wigner_6j, wigner_9j, clebsch_gordan, etc., Instead of repeatedly patching.


diff --git a/sympy/core/sympify.py b/sympy/core/sympify.py
index 0024d0a70a68..716833f15aa7 100644
--- a/sympy/core/sympify.py
+++ b/sympy/core/sympify.py
@@ -100,9 +100,14 @@ def _convert_numpy_types(a, **sympify_args):
         prec = np.finfo(a).nmant + 1
         # E.g. double precision means prec=53 but nmant=52
         # Leading bit of mantissa is always 1, so is not stored
-        p, q = a.as_integer_ratio()
-        a = mlib.from_rational(p, q, prec)
-        return Float(a, precision=prec)
+        if np.isposinf(a):
+            return Float('inf')
+        elif np.isneginf(a):
+            return Float('-inf')
+        else:
+            p, q = a.as_integer_ratio()
+            a = mlib.from_rational(p, q, prec)
+            return Float(a, precision=prec)

 @overload

diff --git a/sympy/core/tests/test_sympify.py b/sympy/core/tests/test_sympify.py
index fc87fa448351..40be30c25d58 100644
--- a/sympy/core/tests/test_sympify.py
+++ b/sympy/core/tests/test_sympify.py
@@ -883,3 +883,10 @@ def test_issue_21536():
     assert u.is_Add and set(u.args) == {4*x, 2}
     assert v.is_Add and set(v.args) == {6*x, 6}
     assert sympify(["x+3*x+2", "2*x+4*x+2+4"]) == [u, v]
+
+def test_issue_27284():
+    if not numpy:
+        skip("numpy not installed.")
+
+    assert Float(numpy.float32(float('inf'))) == S.Infinity
+    assert Float(numpy.float32(float('-inf'))) == S.NegativeInfinity

Converting from `numpy.float32(float('inf'))` throws
sympy==1.13.3
numpy==2.1.3
python 3.10.15

```python
import numpy
import sympy
sympy.Float(np.float32(float('inf')))
```
thows with:
```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File ".../lib/python3.10/site-packages/sympy/core/numbers.py", line 805, in __new__
    num = _convert_numpy_types(num)
  File ".../lib/python3.10/site-packages/sympy/core/sympify.py", line 93, in _convert_numpy_types
    p, q = a.as_integer_ratio()
OverflowError: cannot convert Infinity to integer ratio
```

The code here should be improved:
https://github.com/sympy/sympy/blob/efa1521b638e4f7f2b9a18e0280a0667a4bb16c8/sympy/core/sympify.py#L88-L105
I like to work on this issue.


diff --git a/sympy/integrals/heurisch.py b/sympy/integrals/heurisch.py
index 344edf250a2e..2a1b61c27da3 100644
--- a/sympy/integrals/heurisch.py
+++ b/sympy/integrals/heurisch.py
@@ -1,7 +1,8 @@
 from __future__ import annotations

-from itertools import permutations
+from collections import defaultdict
 from functools import reduce
+from itertools import permutations

 from sympy.core.add import Add
 from sympy.core.basic import Basic
@@ -503,7 +504,16 @@ def heurisch(f, x, rewrite=False, hints=None, mappings=None, retries=3,
         # optimizing the number of permutations of mapping              #
         assert mapping[-1][0] == x # if not, find it and correct this comment
         unnecessary_permutations = [mapping.pop(-1)]
-        mappings = permutations(mapping)
+        # only permute types of objects and let the ordering
+        # of types take care of the order of replacement
+        types = defaultdict(list)
+        for i in mapping:
+            types[type(i)].append(i)
+        mapping = [types[i] for i in types]
+        def _iter_mappings():
+            for i in permutations(mapping):
+                yield [j for i in i for j in i]
+        mappings = _iter_mappings()
     else:
         unnecessary_permutations = unnecessary_permutations or []

diff --git a/sympy/integrals/tests/test_heurisch.py b/sympy/integrals/tests/test_heurisch.py
index 2b4ffa0684f0..e220132f582f 100644
--- a/sympy/integrals/tests/test_heurisch.py
+++ b/sympy/integrals/tests/test_heurisch.py
@@ -2,7 +2,7 @@
 from sympy.core.add import Add
 from sympy.core.function import (Derivative, Function, diff)
 from sympy.core.numbers import (I, Rational, pi)
-from sympy.core.relational import Ne
+from sympy.core.relational import Eq, Ne
 from sympy.core.symbol import (Symbol, symbols)
 from sympy.functions.elementary.exponential import (LambertW, exp, log)
 from sympy.functions.elementary.hyperbolic import (asinh, cosh, sinh, tanh)
@@ -12,6 +12,8 @@
 from sympy.functions.special.bessel import (besselj, besselk, bessely, jn)
 from sympy.functions.special.error_functions import erf
 from sympy.integrals.integrals import Integral
+from sympy.logic.boolalg import And
+from sympy.matrices import Matrix
 from sympy.simplify.ratsimp import ratsimp
 from sympy.simplify.simplify import simplify
 from sympy.integrals.heurisch import components, heurisch, heurisch_wrapper
@@ -365,3 +367,33 @@ def f(x):
     Uz = integrate(f(z), z)
     Ut = integrate(f(t), t)
     assert Ut == Uz.subs(z, t)
+
+
+def test_heurisch_complex_erf_issue_26338():
+    r = symbols('r', real=True)
+    a = exp(-r**2/(2*(2 - I)**2))
+    assert heurisch(a, r, hints=[]) is None  # None, not a wrong soln
+    a = sqrt(pi)*erf((1 + I)/2)/2
+    assert integrate(exp(-I*r**2/2), (r, 0, 1)) == a - I*a
+
+    a = exp(-x**2/(2*(2 - I)**2))
+    assert heurisch(a, x, hints=[]) is None  # None, not a wrong soln
+    a = sqrt(pi)*erf((1 + I)/2)/2
+    assert integrate(exp(-I*x**2/2), (x, 0, 1)) == a - I*a
+
+
+def test_issue_15498():
+    Z0 = Function('Z0')
+    k01, k10, t, s= symbols('k01 k10 t s', real=True, positive=True)
+    m = Matrix([[exp(-k10*t)]])
+    _83 = Rational(83, 100)  # 0.83 works, too
+    [a, b, c, d, e, f, g] = [100, 0.5, _83, 50, 0.6, 2, 120]
+    AIF_btf = a*(d*e*(1 - exp(-(t - b)/e)) + f*g*(1 - exp(-(t - b)/g)))
+    AIF_atf = a*(d*e*exp(-(t - b)/e)*(exp((c - b)/e) - 1
+        ) + f*g*exp(-(t - b)/g)*(exp((c - b)/g) - 1))
+    AIF_sym = Piecewise((0, t < b), (AIF_btf, And(b <= t, t < c)), (AIF_atf, c <= t))
+    aif_eq = Eq(Z0(t), AIF_sym)
+    f_vec = Matrix([[k01*Z0(t)]])
+    integrand = m*m.subs(t, s)**-1*f_vec.subs(aif_eq.lhs, aif_eq.rhs).subs(t, s)
+    solution = integrate(integrand[0], (s, 0, t))
+    assert solution is not None  # does not hang and takes less than 10 s
diff --git a/sympy/integrals/tests/test_integrals.py b/sympy/integrals/tests/test_integrals.py
index 6fa978f3bfaa..860580896807 100644
--- a/sympy/integrals/tests/test_integrals.py
+++ b/sympy/integrals/tests/test_integrals.py
@@ -1148,7 +1148,7 @@ def test_issue_3940():
     assert integrate(exp(-x**2 + I*c*x), x) == \
         -sqrt(pi)*exp(-c**2/4)*erf(I*c/2 - x)/2
     assert integrate(exp(a*x**2 + b*x + c), x) == \
-        sqrt(pi)*exp(c)*exp(-b**2/(4*a))*erfi(sqrt(a)*x + b/(2*sqrt(a)))/(2*sqrt(a))
+        sqrt(pi)*exp(c - b**2/(4*a))*erfi((2*a*x + b)/(2*sqrt(a)))/(2*sqrt(a))

     from sympy.core.function import expand_mul
     from sympy.abc import k

Potential hang in monomials during integration
In the example given below, the `intermonomials` function is called from the `heurisch` function of the integration process. The problem I encounter is that before calling `internomomials`, the script subs some expression with polynomials. The expressions look something like this, `(27000.0 - 6902.92767267848*exp(-1.66666666666667*s) - 24100.2086229868*exp(-s/120))*exp(k10*s)` and the mapping it uses to sub over this example would be, `[(exp(k10*s), _x1), (exp(-s/120), _x0), (exp(-1.66666666666667*s), _x3), (s, _x2)] `. As you can see, the third exponential is exactly the second one at the 200th power. So the subbed expression would come out as, `_x1*(-6902.92767267848*_x0**200 - 24100.2086229868*_x0 + 27000.0)`, which isn't false, but it causes the `intermonomials` to go on for hours (maybe days, I have never seen it end) in the `for variable in item:` loop at line 85.
```
import sympy as sy

Z0 = sy.Function('Z0')
Z1 = sy.Function('Z1')
k01, k10, t, s= sy.symbols('k01 k10 t s', real=True, positive=True)

m = sy.Matrix([[sy.exp(-k10*t)]])

[a, b, c, d, e, f, g] = [100, 0.5, 0.83, 50, 0.6, 2, 120]
AIF_btf = a*(d*e*(1-sy.exp(-(t-b)/e))+f*g*(1-sy.exp(-(t-b)/g)))
AIF_atf = a*(d*e*sy.exp(-(t-b)/e)*(sy.exp((c-b)/e)-1)+f*g*sy.exp(-(t-b)/g)*(sy.exp((c-b)/g)-1))
AIF_sym = sy.Piecewise((0, t < b), (AIF_btf, sy.And(b<=t, t<c)), (AIF_atf, c<=t))

aif_eq = sy.Eq(Z0(t), AIF_sym)
c_vec  = sy.Matrix([[Z1(t)]])
f_vec  = sy.Matrix([[k01*Z0(t)]])

integrand = m*(m.subs(t, s)**-1)*f_vec.subs(aif_eq.lhs, aif_eq.rhs).subs(t, s)
solution  = sy.integrate(integrand[0], (s, 0, t))
```
This behavior can be "hotfixed" in the `_subterms` function of `heurisch` by replacing the `return expr.subs(mapping)` of line 460 by `return expr.xreplace(dict(mapping))` but I know this probably would be far from an ideal fix. Maybe this doesn't actually need to be fixed and I'm just doing something wrong, too.

Now here is the interesting part, I first talked about this issue and #15494 in the gitter chat thinking they weren't related at all. But for some reason I don't understand, when applying the fix I just talked about, the behavior from #15494 disappeared. Also, since @normalhuman talked about how SymPy prefers working with known rationals, I tried replacing `[a, b, c, d, e, f, g] = [100, 0.5, 0.83, 50, 0.6, 2, 120]` with `[a, b, c, d, e, f, g] = [100, sy.Rational('0.5'), 0.83, 50, sy.Rational('0.6'), 2, 120]` and got NaN as the result of the integral. I then needed to use one of the fixes I proposed in #15494 to get the good result, but it all worked out in the end. Seeing as both issues seem related only in my code, I'm assuming my script is the source of the problem, but I've been searching for a while without success.

> [a, b, c, d, e, f, g] = [100, sy.Rational('0.5'), 0.83, 50, sy.Rational('0.6'), 2, 120]

If there is a single floating point number `0.83`, then all numbers are considered inexact. That will be much slower than computing with exact numbers as the arithmetic laws are not valid, and therefore coefficients are treated as expressions in `heurisch`.
Actually, it seems that `nan` is created by `manualintegrate` as a result of division by zero. In principle, `heurisch` should be able to compute the integral but it does not succeed in this case, probably because of some unlucky choices in its cache. It looks like following patch would fix `manualintegrate`:
```
--- a/sympy/integrals/manualintegrate.py
+++ b/sympy/integrals/manualintegrate.py
@@ -130,6 +130,8 @@ def find_substitutions(integrand, symbol, u_var):
     results = []

     def test_subterm(u, u_diff):
+        if u_diff == 0:
+            return False
         substituted = integrand / u_diff
         if symbol not in substituted.free_symbols:
             # replaced everything already
```
> Actually, it seems that `nan` is created by `manualintegrate` as a result of division by zero.

Yes, I referenced this issue already in #15494. I had proposed two initial fixes, one being quite close to what you just wrote. I think I like your format better though.

Should we go ahead and make a PR for it?


diff --git a/src/transformers/pipelines/text_classification.py b/src/transformers/pipelines/text_classification.py
index 21ca70c2ac50aa..dadb29c386b41e 100644
--- a/src/transformers/pipelines/text_classification.py
+++ b/src/transformers/pipelines/text_classification.py
@@ -40,7 +40,8 @@ class ClassificationFunction(ExplicitEnum):
             The function to apply to the model outputs in order to retrieve the scores. Accepts four different values:

             - `"default"`: if the model has a single label, will apply the sigmoid function on the output. If the model
-              has several labels, will apply the softmax function on the output.
+              has several labels, will apply the softmax function on the output. In case of regression tasks, will not
+              apply any function on the output.
             - `"sigmoid"`: Applies the sigmoid function on the output.
             - `"softmax"`: Applies the softmax function on the output.
             - `"none"`: Does not apply any function on the output.""",
@@ -69,7 +70,8 @@ class TextClassificationPipeline(Pipeline):
     `"sentiment-analysis"` (for classifying sequences according to positive or negative sentiments).

     If multiple classification labels are available (`model.config.num_labels >= 2`), the pipeline will run a softmax
-    over the results. If there is a single label, the pipeline will run a sigmoid over the result.
+    over the results. If there is a single label, the pipeline will run a sigmoid over the result. In case of regression
+    tasks (`model.config.problem_type == "regression"`), will not apply any function on the output.

     The models that this pipeline can use are models that have been fine-tuned on a sequence classification task. See
     the up-to-date list of available models on
@@ -135,6 +137,7 @@ def __call__(self, inputs, **kwargs):
                 If this argument is not specified, then it will apply the following functions according to the number
                 of labels:

+                - If problem type is regression, will not apply any function on the output.
                 - If the model has a single label, will apply the sigmoid function on the output.
                 - If the model has several labels, will apply the softmax function on the output.

@@ -192,7 +195,9 @@ def postprocess(self, model_outputs, function_to_apply=None, top_k=1, _legacy=Tr
         # the more natural result containing the list.
         # Default value before `set_parameters`
         if function_to_apply is None:
-            if self.model.config.problem_type == "multi_label_classification" or self.model.config.num_labels == 1:
+            if self.model.config.problem_type == "regression":
+                function_to_apply = ClassificationFunction.NONE
+            elif self.model.config.problem_type == "multi_label_classification" or self.model.config.num_labels == 1:
                 function_to_apply = ClassificationFunction.SIGMOID
             elif self.model.config.problem_type == "single_label_classification" or self.model.config.num_labels > 1:
                 function_to_apply = ClassificationFunction.SOFTMAX

diff --git a/tests/pipelines/test_pipelines_text_classification.py b/tests/pipelines/test_pipelines_text_classification.py
index 4956cb8aed132d..417b5f2c95c1e2 100644
--- a/tests/pipelines/test_pipelines_text_classification.py
+++ b/tests/pipelines/test_pipelines_text_classification.py
@@ -108,6 +108,12 @@ def test_small_model_pt(self):
             ],
         )

+        # Do not apply any function to output for regression tasks
+        # hack: changing problem_type artifically (so keep this test at last)
+        text_classifier.model.config.problem_type = "regression"
+        outputs = text_classifier("This is great !")
+        self.assertEqual(nested_simplify(outputs), [{"label": "LABEL_0", "score": 0.01}])
+
     @require_torch
     def test_accepts_torch_device(self):
         text_classifier = pipeline(

Default behaviour in `TextClassificationPipeline` for regression problem type
### Feature request

The `AutoModelForSequenceClassification` class also supports problem_type=regression. I am not sure if it is popularly used but I believe that function_to_apply in `TextClassificationPipeline` should behave as if it is set to "none"  for this case?

### Motivation

Currently, the "sigmoid" function is also applied for regression tasks which, in my opinion, might not be the correct default behaviour.

### Your contribution

I believe this can be handled by adding an additional condition for `self.model.config.problem_type == "regression"` before [this](https://github.com/huggingface/transformers/blob/38f9f10dd9240619ea17fb6c7acb51b3bc592232/src/transformers/pipelines/text_classification.py#L195) line.

cc @Rocketknight1 WDYT?
Hi @subhalingamd, when doing regression tasks, `num_labels` should usually be 0, right? If that is set, then the function to applied will be `NONE`.
Hi @Rocketknight1, that doesn't seem to be the case. The regression score is represented by LABEL_0 by default.
Indeed, `num_labels == 1` is used as the criteria for assigning problem_type as regression [[example](https://github.com/huggingface/transformers/blob/7bae833728c76345caa04a334368684ed2e77f66/src/transformers/models/bert/modeling_bert.py#L1716)].
@subhalingamd hm, good point! Let me see what we can do. Do you have an example of a text regression model on the Hub I can test with?
Actually, after investigating a bit more, I think checking for `problem_type == "regression"` as you suggested is the right strategy. Would you be willing to make that PR?
sure, i can do that
Great, thank you!


diff --git a/src/transformers/models/splinter/tokenization_splinter.py b/src/transformers/models/splinter/tokenization_splinter.py
index 2859497ba882c2..ffa135556aa47d 100644
--- a/src/transformers/models/splinter/tokenization_splinter.py
+++ b/src/transformers/models/splinter/tokenization_splinter.py
@@ -137,6 +137,7 @@ def __init__(
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
+            question_token=question_token,
             tokenize_chinese_chars=tokenize_chinese_chars,
             strip_accents=strip_accents,
             **kwargs,

diff --git a/tests/models/splinter/test_tokenization_splinter.py b/tests/models/splinter/test_tokenization_splinter.py
new file mode 100644
index 00000000000000..4c6d295e8a8281
--- /dev/null
+++ b/tests/models/splinter/test_tokenization_splinter.py
@@ -0,0 +1,174 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+from tests.test_tokenization_common import TokenizerTesterMixin
+from transformers import SplinterTokenizerFast, is_tf_available, is_torch_available
+from transformers.models.splinter import SplinterTokenizer
+from transformers.testing_utils import get_tests_dir, slow
+
+
+SAMPLE_VOCAB = get_tests_dir("fixtures/vocab.txt")
+
+
+if is_torch_available():
+    FRAMEWORK = "pt"
+elif is_tf_available():
+    FRAMEWORK = "tf"
+else:
+    FRAMEWORK = "jax"
+
+
+class SplinterTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
+    tokenizer_class = SplinterTokenizer
+    rust_tokenizer_class = SplinterTokenizerFast
+    space_between_special_tokens = False
+    test_rust_tokenizer = False
+    test_sentencepiece_ignore_case = False
+    pre_trained_model_path = "tau/splinter-base"
+
+    # Copied from transformers.models.siglip.SiglipTokenizationTest.setUp
+    def setUp(self):
+        super().setUp()
+        tokenizer = SplinterTokenizer(SAMPLE_VOCAB)
+        tokenizer.vocab["[UNK]"] = len(tokenizer.vocab)
+        tokenizer.vocab["[QUESTION]"] = len(tokenizer.vocab)
+        tokenizer.vocab["."] = len(tokenizer.vocab)
+        tokenizer.add_tokens("this is a test thou shall not determine rigor truly".split())
+        tokenizer.save_pretrained(self.tmpdirname)
+
+    def get_tokenizer(self, **kwargs) -> SplinterTokenizer:
+        return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs)
+
+    def get_rust_tokenizer(self, **kwargs) -> SplinterTokenizerFast:
+        return self.rust_tokenizer_class.from_pretrained(self.tmpdirname, **kwargs)
+
+    # Copied from transformers.models.siglip.SiglipTokenizationTest.test_get_vocab
+    def test_get_vocab(self):
+        vocab_keys = list(self.get_tokenizer().get_vocab().keys())
+        self.assertEqual(vocab_keys[0], "[PAD]")
+        self.assertEqual(vocab_keys[1], "[SEP]")
+        self.assertEqual(vocab_keys[2], "[MASK]")
+
+    # Copied from transformers.models.siglip.SiglipTokenizationTest.test_convert_token_and_id
+    def test_convert_token_and_id(self):
+        token = "[PAD]"
+        token_id = 0
+
+        self.assertEqual(self.get_tokenizer()._convert_token_to_id(token), token_id)
+        self.assertEqual(self.get_tokenizer()._convert_id_to_token(token_id), token)
+
+    def test_question_token_id(self):
+        tokenizer = self.get_tokenizer()
+        self.assertEqual(tokenizer.question_token_id, tokenizer.convert_tokens_to_ids(tokenizer.question_token))
+
+    # Copied from transformers.models.siglip.SiglipTokenizationTest.test_full_tokenizer
+    def test_full_tokenizer(self):
+        tokenizer = self.get_tokenizer()
+        test_str = "This is a test"
+
+        unk_token = tokenizer.unk_token
+        unk_token_id = tokenizer._convert_token_to_id_with_added_voc(unk_token)
+
+        expected_tokens = test_str.lower().split()
+        tokenizer.add_tokens(expected_tokens)
+        tokens = tokenizer.tokenize(test_str)
+        self.assertListEqual(tokens, expected_tokens)
+
+        # test with out of vocabulary string
+        tokens = tokenizer.tokenize(test_str + " oov")
+        self.assertListEqual(tokens, expected_tokens + [unk_token])
+
+        expected_token_ids = [13, 14, 15, 16, unk_token_id]
+        token_ids = tokenizer.convert_tokens_to_ids(tokens)
+        self.assertListEqual(token_ids, expected_token_ids)
+
+        tokenizer = self.get_tokenizer(basic_tokenize=False)
+        expected_token_ids = [13, 14, 15, 16, unk_token_id]
+        token_ids = tokenizer.convert_tokens_to_ids(tokens)
+        self.assertListEqual(token_ids, expected_token_ids)
+
+    # Copied from transformers.models.siglip.SiglipTokenizationTest.test_rust_and_python_full_tokenizers
+    def test_rust_and_python_full_tokenizers(self):
+        tokenizer = self.get_tokenizer()
+        rust_tokenizer = self.get_rust_tokenizer()
+
+        sequence = "I need to test this rigor"
+        tokens = tokenizer.tokenize(sequence, add_special_tokens=False)
+        rust_tokens = rust_tokenizer.tokenize(sequence, add_special_tokens=False)
+        self.assertListEqual(tokens, rust_tokens)
+
+        ids = tokenizer.encode(sequence)
+        rust_ids = rust_tokenizer.encode(sequence)
+        self.assertListEqual(ids, rust_ids)
+
+    # Copied from transformers.models.siglip.SiglipTokenizationTest.test_max_length
+    def test_max_length(self):
+        max_length = 20
+        tokenizer = self.get_tokenizer()
+        texts = ["this is a test", "I have pizza for lunch"]
+        tokenized = tokenizer(
+            text_target=texts,
+            max_length=max_length,
+            padding="max_length",
+            truncation=True,
+            return_tensors=FRAMEWORK,
+        )
+        self.assertEqual(len(tokenized["input_ids"]), len(texts))
+        self.assertEqual(len(tokenized["input_ids"][0]), max_length)
+        self.assertEqual(len(tokenized["input_ids"][1]), max_length)
+        self.assertEqual(len(tokenized["attention_mask"][0]), max_length)
+        self.assertEqual(len(tokenized["attention_mask"][1]), max_length)
+        self.assertEqual(len(tokenized["token_type_ids"][0]), max_length)
+        self.assertEqual(len(tokenized["token_type_ids"][1]), max_length)
+
+    # Copied from transformers.models.siglip.SiglipTokenizationTest.test_tokenizer_integration
+    # fmt:skip
+    @slow
+    def test_tokenizer_integration(self):
+        tokenizer = SplinterTokenizer.from_pretrained("tau/splinter-base", max_length=10)
+        texts = [
+            "The cat sat on the windowsill, watching birds in the garden.",
+            "She baked a delicious cake for her sister's birthday party.",
+            "The sun set over the horizon, painting the sky with vibrant colors.",
+        ]
+        # fmt:off
+        expected_token_id_list = [
+            [101, 1109, 5855, 2068, 1113, 1103, 3751, 7956, 117, 2903, 4939, 1107, 1103, 4605, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1153, 19983, 170, 13108, 10851, 1111, 1123, 2104, 112, 188, 5913, 1710, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1109, 3336, 1383, 1166, 1103, 11385, 117, 3504, 1103, 3901, 1114, 18652, 5769, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+        ]
+        # fmt:on
+        for text, expected_token_ids in zip(texts, expected_token_id_list):
+            input_ids = tokenizer(text, padding="max_length").input_ids
+            self.assertListEqual(input_ids, expected_token_ids)
+
+    def test_special_tokens_mask_input_pairs(self):
+        tokenizers = self.get_tokenizers(do_lower_case=False)
+        for tokenizer in tokenizers:
+            with self.subTest(f"{tokenizer.__class__.__name__}"):
+                sequence_0 = "Encode this."
+                sequence_1 = "This one too please."
+                encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False)
+                encoded_sequence += tokenizer.encode(sequence_1, add_special_tokens=False)
+                encoded_sequence_dict = tokenizer.encode_plus(
+                    sequence_0,
+                    sequence_1,
+                    add_special_tokens=True,
+                    return_special_tokens_mask=True,
+                )
+                encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
+                special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
+                # splinter tokenizer always add cls, question_suffix, and 2 separators
+                # while in special_token_mask it does not seems to do that
+                self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special) - 2)

Add missing tokenizer test files [:building_construction: in progress]
# 🚀 Add missing tokenizer test files
Several tokenizers currently have no associated tests. I think that adding the test file for one of these tokenizers could be a very good way to make a first contribution to transformers.

## Tokenizers concerned

### not yet claimed
_none_

### claimed

- [ ] LED @nnlnr
- [ ] Flaubert @anmolsjoshi
- [ ] Electra @Rajathbharadwaj
- [ ] ConvBert @elusenji
- [ ] RemBert @IMvision12
- [ ] Splinter @ashwinjohn3

### with an ongoing PR
_none_

### with an accepted PR

- [x] Longformer @tgadeliya #17677
- [x] MobileBert @leondz #16896
- [x] RetriBert @mpoemsl #17017

## How to contribute?

1. Claim a tokenizer

    a. Choose a tokenizer from the list of "not yet claimed" tokenizers

    b. Check that no one in the messages for this issue has indicated that they care about this tokenizer

    c. Put a message in the issue that you are handling this tokenizer

2. Create a local development setup (if you have not already done it)

    I refer you to section ["start-contributing-pull-requests" of the Contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests) where everything is explained. Don't be afraid with step 5. For this contribution you will only need to test locally the tests you add.

3. Follow the instructions on the readme inside [the `templates/adding_a_missing_tokenization_test` folder](https://github.com/huggingface/transformers/tree/main/templates/adding_a_missing_tokenization_test) to generate the template with cookie cutter for the new test file you will be adding. Don't forget to move the new test file at the end of the template generation to the sub-folder named after the model for which you are adding the test file in the `tests` folder. Some details about questionnaire - assuming that the name of the lowcase model is `brand_new_bert`:
    -   "has_slow_class": Set true there is a `tokenization_brand_new_bert.py` file in the folder `src/transformers/models/brand_new_bert`
    -   "has_fast_class": Set true there is a `tokenization_brand_new_bert_fast.py` file the folder `src/transformers/models/brand_new_bert`.
    -   "slow_tokenizer_use_sentencepiece": Set true if the tokenizer defined in the `tokenization_brand_new_bert.py` file uses sentencepiece. If this tokenizer don't have a ``tokenization_brand_new_bert.py` file set False.

4. Complete the `setUp` method in the generated test file, you can take inspiration for how it is done for the other tokenizers.

5. Try to run all the added tests. It is possible that some tests will not pass, so it will be important to understand why, sometimes the common test is not suited for a tokenizer and sometimes a tokenizer can have a bug. You can also look at what is done in similar tokenizer tests, if there are big problems or you don't know what to do we can discuss this in the PR (step 7.).

6. (Bonus) Try to get a good understanding of the tokenizer to add custom tests to the tokenizer

7. Open an PR with the new test file added, remember to fill in the RP title and message body (referencing this PR) and request a review from @LysandreJik and @SaulLu.

## Tips

Do not hesitate to read the questions / answers in this issue :newspaper:

Hi, I would like to add tests for `Longformer` tokenizer
@SaulLu I would like to add tests for Flaubert
Hey I would like to contribute for `Electra`,Pointers please!
Thank you all for offering your help!

@Rajathbharadwaj ,sure! what do you need help with? Do you need more details on any of the steps listed in the main post?
Hi, first time contributor here-could I add tests for ```Splinter```?
Is anyone else encountering this error with the cookiecutter command? my dev environment set up seemed to have went all fine...
Also I had run the command inside the ```tests/splinter``` directory

![Screenshot 2022-04-11 172638](https://user-images.githubusercontent.com/72678356/162786844-3a67e9da-6d98-4e9f-a5b4-c35242a30cd8.jpg)

@faiazrahman , thank you so much for working on this! Regarding your issue, if you're in the `tests/splinter` folder, can you try to run `cookiecutter ../../templates/adding_a_missing_tokenization_test/` ?

You should have a newly created folder `cookiecutter-template-BrandNewBERT` inside `tests/splinter`. :slightly_smiling_face:

If that's the case, you'll need after to do something like:

```
mv cookiecutter-template-BrandNewBERT/test_tokenization_brand_new_bert.py .
rm -r cookiecutter-template-BrandNewBERT/
```
Keep me posted :smile:
Thanks so much @SaulLu turns out it was due to not recognizing my installed cookiecutter so i sorted it out there. 👍
Hi @anmolsjoshi, @tgadeliya, @Rajathbharadwaj and @farahdian,

Just a quick message to see how things are going for you and if you have any problems. If you do, please share them! :hugs:
Thanks @SaulLu ! I've been exploring the tokenization test files in the repo just trying to figure out which ones would be a good basis for writing a tokenization test for splinter... if you have any guidance on this it would be super helpful!
Hey @SaulLu my apologies, been a bit busy. I'll get started ASAP, however, I still didn't understand where exactly I should run the `cookie cutter`

Help on this would be helpful 😄
Hi @farahdian ,

Thank you very much for the update! To know where you stand, have you done step 3)? Is it for step 4) that you are looking for a similar tokenizer? :slightly_smiling_face:
Hi @Rajathbharadwaj ,

Thank you for the update too!

> I still didn't understand where exactly I should run the cookie cutter

You can run the `cookie cutter` command anywhere as long as the command is followed by the path to the [folder `adding_a_missing_tokenization_test`](https://github.com/huggingface/transformers/tree/main/templates/adding_a_missing_tokenization_test) in the transformers repo that you have cloned locally.

When you run the command, it will create a new folder at the location you are. In this folder you will find a base for the python test file that you need to move inside the `tests/electra` folder of the transformers local clone. Once this file is moved, you can delete the folder that was created by the cookie cutter command.

Below is an example of the sequence of bash commands I would personally use:
```bash
(base) username@hostname:~$ cd ~/repos
(base) username@hostname:~/repos$ git clone git@github.com:huggingface/transformers.git
[Install my development setup]
(transformers-dev) username@hostname:~/repos$ cookiecutter transformers/templates/adding_a_missing_tokenization_test/
[Answer the questionnaire]
(transformers-dev) username@hostname:~/repos$ mv cookiecutter-template-Electra/test_tokenization_electra.py transformers/tests/Electra
(transformers-dev) username@hostname:~/repos$ rm -r cookiecutter-template-Electra/
```

Hope that'll help you :smile:
Appreciate your patience @SaulLu ! Yup I've done step 3 and generated a test tokenization file with cookiecutter. Now onto working on the setUp method 😄
@farahdian , this is indeed a very good question: finding the closest tokenizer to draw inspiration from and identifying the important difference with that tokenizer is the most interesting part.

For that there are several ways to start:

1. Identify the high level features of the tokenizer by looking at the contents of the model's "reference" checkpoint files (listed inside the `PRETRAINED_VOCAB_FILES_MAP` global variables in the tokenizer's files) on the hub. A similar model would most likely store the tokenizer vocabulary in the same way (with only a `vocab` file, with both a `vocab` and a `merges` files, with a `sentencepiece` binary file or with only a `tokenizer.json` file).
2. Read the high level explanation of the model in the transformers documentation (e.g. for [Splinter](https://huggingface.co/docs/transformers/model_doc/splinter))
3. Read the paper corresponding to the model
4. Look at the implementation in transformers lib
5. Look at the original implementation of the model (often mentioned in the paper)
6. Look at the discussions on the PR in which the model was added

For the model you're in charge @farahdian:

- Transformers's doc mention that:

  > Use [SplinterTokenizer](https://huggingface.co/docs/transformers/v4.18.0/en/model_doc/splinter#transformers.SplinterTokenizer) (rather than [BertTokenizer](https://huggingface.co/docs/transformers/v4.18.0/en/model_doc/bert#transformers.BertTokenizer)), as it already contains this special token. Also, its default behavior is to use this token when two sequences are given (for example, in the run_qa.py script).

- Splinter's paper mention that:

  > Splinter-base shares the same architecture (transformer encoder (Vaswani et al., 2017)), **vocabulary (cased wordpieces)**, and number of parameters (110M) with SpanBERT-base (Joshi et al., 2020).

- And SpanBERT's paper mention that:

  > We reimplemented BERT’s model and pre-training method in fairseq (Ott et al., 2019). We used the model configuration of BERT large as in Devlin et al. (2019) and also pre-trained all our models on the same corpus: BooksCorpus and English Wikipedia using **cased Wordpiece tokens**.

 - and the vocabulary files of `bert-base-cased` ([vocab file](https://huggingface.co/bert-base-cased/raw/main/vocab.txt)) and of `splinter-base` ([vocab file](https://huggingface.co/tau/splinter-base/raw/main/vocab.txt)) look very similar

Given these mentions, it seems that Splinter's tokenizer is very similar to Bert's one. It would be interesting to confirm this impression and to understand all the differences between SplinterTokenizer and BertTokenizer so that it is well reflected in the test :slightly_smiling_face:
> Hi @Rajathbharadwaj ,
>
> Thank you for the update too!
>
> > I still didn't understand where exactly I should run the cookie cutter
>
> You can run the `cookie cutter` command anywhere as long as the command is followed by the path to the [folder `adding_a_missing_tokenization_test`](https://github.com/huggingface/transformers/tree/main/templates/adding_a_missing_tokenization_test) in the transformers repo that you have cloned locally.
>
> When you run the command, it will create a new folder at the location you are. In this folder you will find a base for the python test file that you need to move inside the `tests/electra` folder of the transformers local clone. Once this file is moved, you can delete the folder that was created by the cookie cutter command.
>
> Below is an example of the sequence of bash commands I would personally use:
>
> ```shell
> (base) username@hostname:~$ cd ~/repos
> (base) username@hostname:~/repos$ git clone git@github.com:huggingface/transformers.git
> [Install my development setup]
> (transformers-dev) username@hostname:~/repos$ cookiecutter transformers/templates/adding_a_missing_tokenization_test/
> [Answer the questionnaire]
> (transformers-dev) username@hostname:~/repos$ mv cookiecutter-template-Electra/test_tokenization_electra.py transformers/tests/Electra
> (transformers-dev) username@hostname:~/repos$ rm -r cookiecutter-template-Electra/
> ```
>
> Hope that'll help you smile

Thank you so much @SaulLu
I understood now, however, I am skeptical about  `slow_tokenizer_use_sentencepiece` question, but I set it to True as it had the `tokenization_electra.py` file but I didn't understand

 >  "Set true if the tokenizer defined in the tokenization_brand_new_bert.py file uses sentencepiece"

 So did I select correctly? Or should I set it to False? Apologies for asking so many questions 😄

However now I've started adding tests for `Electra` will keep you posted if I run into something I don't understand.

Thanks for helping once again!
Hi @SaulLu,
I think my case the easiest one, because `Longformer` model uses actually the same tokenizer as `RoBERTa` with no differences. So, I adapted tests(small refactor and changes) from `RoBERTa` tokenizer and prepare branch with tests. Nevertheless, I really want to dive deeper and study code of `TokenizerTesterMixin` and  if after that I will find some untested behaviour, I will add new tests.
But I think I have one doubt, that you can resolve. Are you anticipating from `Longformer` tests to have different toy tokenizer example than in `RoBERTa` tests? Or maybe I should write my own tests from scratch?

@Rajathbharadwaj , I'm happy to help! Especially as your questions will surely be useful for other people

> however, I am skeptical about slow_tokenizer_use_sentencepiece question, but I set it to True as it had the tokenization_electra.py file but I didn't understand
  > "Set true if the tokenizer defined in the tokenization_brand_new_bert.py file uses sentencepiece"
So did I select correctly? Or should I set it to False? Apologies for asking so many questions smile

Some `XxxTokenizer` (without the Fast at the end, implemented in the `tokenization_xxx.py` file), use a backend based on the [sentencepiece](https://github.com/google/sentencepiece) library. For example `T5Tokenizer` uses a backend based on sentencepiece: you can see this import at the beginning of the `tokenization_t5.py` file:
https://github.com/huggingface/transformers/blob/3dd57b15c561bc26eb0cde8e6c766e7533284b0f/src/transformers/models/t5/tokenization_t5.py#L24
and you can see that the backend is instantiated here:
https://github.com/huggingface/transformers/blob/3dd57b15c561bc26eb0cde8e6c766e7533284b0f/src/transformers/models/t5/tokenization_t5.py#L151-L152

On the contrary, [BertTokenizer](https://github.com/huggingface/transformers/blob/main/src/transformers/models/bert/tokenization_bert.py) for example does not use a sentencepiece backend.

I hope this helped you!
Hi @tgadeliya ,

Thanks for the update!

> But I think I have one doubt, that you can resolve. Are you anticipating from Longformer tests to have different toy tokenizer example than in RoBERTa tests? Or maybe I should write my own tests from scratch?

In your case, I wouldn't be surprised if Longformer uses the same tokenizer as RoBERTa. In this case, it seems legitimate to use the same toy tokenizer. Maybe the only check you can do to confirm this hypothesis is comparing the vocabularies of the 'main" checkpoints of both models:
```bash
!wget https://huggingface.co/allenai/longformer-base-4096/raw/main/merges.txt
!wget https://huggingface.co/allenai/longformer-base-4096/raw/main/vocab.json
!wget https://huggingface.co/roberta-base/raw/main/merges.txt
!wget https://huggingface.co/roberta-base/raw/main/vocab.json

!diff merges.txt merges.txt.1
!diff vocab.json vocab.json.1
```
Turn out the result confirms it!
Hi, I'm happy to take `MobileBert`
I'd like to work on ConvBert.
> Identify the high level features of the tokenizer by looking at the contents of the model's "reference" checkpoint files (listed inside the PRETRAINED_VOCAB_FILES_MAP global variables in the tokenizer's files) on the hub. A similar model would most likely store the tokenizer vocabulary in the same way (with only a vocab file, with both a vocab and a merges files, with a sentencepiece binary file or with only a tokenizer.json file).

@SaulLu  I'm having trouble identifying ConvBert's 'reference' checkpoint files on the hub. Would you kindly provide more guidance on this?
Hi @elusenji ,

In the `src/transformers/models/convbert/tokenization_convbert.py` file you can find the global variable `PRETRAINED_VOCAB_FILES_MAP`:
https://github.com/huggingface/transformers/blob/6d90d76f5db344e333ec184cc6f414abe2aa6559/src/transformers/models/convbert/tokenization_convbert.py#L24-L30

In particular [YituTech/conv-bert-base](https://huggingface.co/YituTech/conv-bert-base) is a reference checkpoint for ConvBert.

Is this what you were having trouble with? :relaxed:
Yes, this helps!
Hi @SaulLu, I am happy to write tests for `RemBert`. Thanks.
Hi @SaulLu, I would like to work on `RetriBert`.
Hi @SaulLu, I'd be happy to work on `LED` - Thanks!!
Thanks @SaulLu, I'm working on this `RemBert` :)
Hello to all!

Two first PRs have been merged into master! Many thanks to @leondz and @mpoemsl! :confetti_ball:

@nnlnr, @anmolsjoshi, @tgadeliya, @Rajathbharadwaj, @farahdian, @elusenji, and @danhphan, I wanted to take this opportunity to ask you how things are going for you? Are you experiencing any particular difficulties?
@SaulLu, I've made some progress. Would it be okay to send in a work-in-progress pull request?
^was wondering if I could do the same, could use a second pair of eyes on it
Yes sure!
Hi @SaulLu
Apologies for the delayed response - I've been making some progress with ```LED``` and will be hopefully submitting a WIP-PR in the coming week. Thanks for following up
Hi  @nnlnr, @anmolsjoshi, @Rajathbharadwaj, @elusenji and @danhphan,

I come to the news to know how the integration of the tests is going for you :hugs:
Hi @SaulLu, Can I work on Splinter if no one is working it? I believe it's not claimed yet
Hi @ashwinjohn3,

Absolutely! Don't hesitate if you are having difficulties
@SaulLu Thank you so much. Will do :)
Hi @SaulLu , sorry for late response and being quite slow. I am still working on RemBert and will try to finish it soon in the coming weeks. Thank you.
@SaulLu are there any tokenizers left???
Hi @IMvision12, I am busy on the deadline of a couple of other projects, so can you work on `RemBert`? Thanks!
Yeah sure @danhphan Thanks.
Thank you @IMvision12 !
Seems like a bit late to the party 😅. Is there any tokenizer not listed here that I can write tests for? Or maybe if some tokenizer becomes available here. Please let me know @SaulLu I would love to contribute 😀
Unfortunately, I don't have much time left to help with transformers now. But let me ping @ArthurZucker for visibility
Hey @y3sar thanks for wanting to contribute. I think that the RemBert tests PR was close, you can probably take that over if you want!
Other tests that might be missing:

- [ ] ./tests/models/flaubert
- [ ] ./tests/models/convbert
- [ ] ./tests/models/splinter
- [ ] ./tests/models/gpt_neox
- [ ] ./tests/models/rembert
@ArthurZucker thanks for your reply. I will start working on RemBert tests.
hey @ArthurZucker, I'm happy to have a look at contributing to a few of these. I'll start off with `gpt_neox` 🙂
Hi. Are the tests still open for contribution? Thanks
@ArthurZucker some of the claimed tokenizers are dormant.  Can I take in one of them? If so, can you let me know which one.

cc: @SaulLu
Hey all! 🤗
If you don't find a PR open for any model feel free to do so.
If a PR is inactive for quite some time, just ping the author to make sure he is alright with you taking over or if he still want to contribute ! (unless it's inactive for more than 2 months I think it's alright to work on it) 👍🏻
Feel free to take Splinter Model! Unfortunately, I am not work on it anymore!
@ashwinjohn3 thanks!

I'll work on splinter as well.
In that case, I will check rembert, convbert and choose one to work on. I also assume that you guys are working on gpx_ and splinter.
> In that case, I will check rembert, convbert and choose one to work on. I also assume that you guys are working on gpx_ and splinter.

@ENate Hi , there is already PR for rembert.  Hence, please refrain from duplication. :)
Hi @nileshkokane01 Thanks :) Will surely do.
Hi, guys! Looks like only longformer, mobilebert and rembert have been merged. Can I try the other ones?
@nileshkokane01 hi! are you still working on splinter?
@ENate hi! are you working on convbert?
@logvinata you can take splinter. I'm not working on it anymore.
Hi @logvinata yes, I am still going to work on it. I was off for a while but will soon open a PR on it.
Hello, i decided to start my first contribution on the flaubert part. I encoutered an assertion error when running the following class:

![image](https://github.com/huggingface/transformers/assets/50299842/3b3f602d-1b6f-4071-ba60-5df87cb9fac8)

Assertion error:
![image](https://github.com/huggingface/transformers/assets/50299842/72b47d54-e239-4e00-95d4-93bb79c0ffc9)

I tried much things but the "do_lowercase" parameter was never found in the signature of the tokenizer, until i tried to propagate it in the FlaubertTokenizer class, and the test passed:
![image](https://github.com/huggingface/transformers/assets/50299842/4b8732c5-d79e-48e8-9a8b-32f1dc1b85f6)

Am i missing something ? Is there a workaround ?

Thanks in advance,
Bastien
cc @ArthurZucker
Could you open a PR? It will be easier for me


diff --git a/src/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py
index 1853d2de4560ea..f04eaae4525de9 100644
--- a/src/transformers/tokenization_utils.py
+++ b/src/transformers/tokenization_utils.py
@@ -480,6 +480,7 @@ def added_tokens_decoder(self, value: Dict[int, Union[AddedToken, str]]) -> Dict

             self._added_tokens_decoder[index] = AddedToken(token) if isinstance(token, str) else token
             self._added_tokens_encoder[str(token)] = index
+        self._update_total_vocab_size()

     def get_added_vocab(self) -> Dict[str, int]:
         """
@@ -494,10 +495,17 @@ def get_added_vocab(self) -> Dict[str, int]:

     def __len__(self):
         """
-        Size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because otherwise if
-        there is a hole in the vocab, we will add tokenizers at a wrong index.
+        Size of the full vocabulary with the added tokens.
         """
-        return len(set(self.get_vocab().keys()))
+        return self.total_vocab_size
+
+    def _update_total_vocab_size(self):
+        """
+        Update the size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because
+        otherwise if there is a hole in the vocab, we will add tokenizers at a wrong index. This operation is slow and
+        is only updated when adding tokens.
+        """
+        self.total_vocab_size = len(self.get_vocab())

     def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
         """
@@ -574,6 +582,7 @@ def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_to
                 logger.info(f"Adding {token} to the vocabulary")

         self._update_trie()
+        self._update_total_vocab_size()
         return added_tokens

     def _update_trie(self, unique_no_split_tokens: Optional[str] = []):

diff --git a/tests/tokenization/test_tokenization_utils.py b/tests/tokenization/test_tokenization_utils.py
index 7ff6b29629ea6d..f97ef6a630221d 100644
--- a/tests/tokenization/test_tokenization_utils.py
+++ b/tests/tokenization/test_tokenization_utils.py
@@ -284,3 +284,15 @@ def test_instantiation_from_tokenizers_json_file(self):
         with tempfile.TemporaryDirectory() as tmpdirname:
             bert_tokenizer.save(os.path.join(tmpdirname, "tokenizer.json"))
             PreTrainedTokenizerFast(tokenizer_file=os.path.join(tmpdirname, "tokenizer.json"))
+
+    def test_len_tokenizer(self):
+        for tokenizer_class in [BertTokenizer, BertTokenizerFast]:
+            with self.subTest(f"{tokenizer_class}"):
+                tokenizer = tokenizer_class.from_pretrained("bert-base-uncased")
+                added_tokens_size = len(tokenizer.added_tokens_decoder)
+                self.assertEqual(len(tokenizer), tokenizer.vocab_size)
+
+                tokenizer.add_tokens(["<test_token>"])
+                self.assertEqual(len(tokenizer), tokenizer.vocab_size + 1)
+                self.assertEqual(len(tokenizer.added_tokens_decoder), added_tokens_size + 1)
+                self.assertEqual(len(tokenizer.added_tokens_encoder), added_tokens_size + 1)

DataCollatorForLanguageModeling is (unnecessary) slow
### System Info

- `transformers` version: 4.43.1
- Platform: Linux-3.10.0-1062.18.1.el7.x86_64-x86_64-with-glibc2.17
- Python version: 3.11.3
- Huggingface_hub version: 0.23.4
- Safetensors version: 0.4.3
- Accelerate version: 0.32.1
- Accelerate config: 	not found
- PyTorch version (GPU?): 1.13.0+cu117 (False)
- Tensorflow version (GPU?): not installed (NA)
- Flax version (CPU?/GPU?/TPU?): not installed (NA)
- Jax version: not installed
- JaxLib version: not installed
- Using distributed or parallel set-up in script?: No

### Who can help?

@ArthurZucker

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [X] My own task or dataset (give details below)

### Reproduction

```
from transformers import AutoTokenizer
from transformers import DataCollatorForLanguageModeling

tokzr = AutoTokenizer.from_pretrained('bert-base-multilingual-cased', use_fast=False)

masker = DataCollatorForLanguageModeling(tokzr)
data = ['this is a very boring test sentence'] * 1000

for line in data:
    token_ids = tokzr.encode(line, return_tensors='pt')[0]
    #input_text, output_labels = masker.torch_mask_tokens(token_ids.view(1, -1))
```

### Expected behavior

I would expect this code to be not much slower when uncommenting the masking. However, uncommenting it makes the code 5 times slower. I have tried to profile where this happens, and it seems like it is mainly happening in finding the random words to replace with, as it gets the length of the vocabulary there which takes a long time (https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L854). I am not sure how to make getting the length faster, but perhaps it can just be saved/cached?

Was curious about this issue, so tried gathering some statistics on time spent in each line of [torch_mask_tokens](https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py) (`transformers` commit ID `85a1269e19af022e04bc2aad82572cd5a9e8cdd9`) and this is what the statistics look like:

```
Code block 'torch_mask_tokens': 18.94850 ms
Code block 'Line 833 to 842' took: 0.04782 ms
Code block 'Line 844 to 846' took: 0.05190 ms
Code block 'Line 849 to 850' took: 0.02048 ms
Code block 'Line 853 to 855' took: 0.01268 ms
Code block 'Line 854' took: 18.43938 ms
Code block 'Line 855' took: 0.06434 ms
```

Measured this using [linetimer](https://github.com/justasb/linetimer). Line 854 is:
```
random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
```

and is taking 97% of the execution time. And within Line 854, if we split it into:

```
                with CodeTimer("854: get tokenizer length"):
                    tok_len = len(self.tokenizer)
                with CodeTimer("854: generate randints"):
                    random_words = torch.randint(tok_len, labels.shape, dtype=torch.long)
```

we see:

```
Code block '854: get tokenizer length' took: 18.28698 ms
Code block '854: generate randints' took: 0.03528 ms
```

You are right that this could be significantly sped up if this one thing is saved/cached.
The length is being calculated in the [PreTrainedTokenizer's __len__ function](https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils.py#L495) function.  This function is doing `return len(set(self.get_vocab().keys()))` each time to calculate the length - which is why it is taking that long.

It looks like this tokenizer class is setup such that new tokens can be added via the `_add_tokens` function. We could probably do something like:

- add a new attribute `self.vocab_size` to the `PreTreainedTokenizer` class when it is `init`ed.
- update `self.vocab_size` whenever `_add_tokens` is called.
- modify the `len` function to just `return self.vocab_size`
I am actually curious whether the set has any effect at all. Shouldnt the keys already be unique?, shouldn't this call always have an equal result to `len(self.get_vocab)`, which is probably much faster?
@itazap if you have bandwidth do you want to take a look at what @rohitdwivedula offers?

cc @ArthurZucker
I'll take a look into a `vocab_size` param @rohitdwivedula 😄 Will get back to you later today!