diff --git a/django/contrib/auth/management/__init__.py b/django/contrib/auth/management/__init__.py
index b29a980cb2d5..c40f2aa69dd2 100644
--- a/django/contrib/auth/management/__init__.py
+++ b/django/contrib/auth/management/__init__.py
@@ -46,6 +46,13 @@ def create_permissions(
     if not app_config.models_module:
         return

+    try:
+        Permission = apps.get_model("auth", "Permission")
+    except LookupError:
+        return
+    if not router.allow_migrate_model(using, Permission):
+        return
+
     # Ensure that contenttypes are created for this app. Needed if
     # 'django.contrib.auth' is in INSTALLED_APPS before
     # 'django.contrib.contenttypes'.
@@ -62,28 +69,15 @@ def create_permissions(
     try:
         app_config = apps.get_app_config(app_label)
         ContentType = apps.get_model("contenttypes", "ContentType")
-        Permission = apps.get_model("auth", "Permission")
     except LookupError:
         return

-    if not router.allow_migrate_model(using, Permission):
-        return
-
-    # This will hold the permissions we're looking for as
-    # (content_type, (codename, name))
-    searched_perms = []
-    # The codenames and ctypes that should exist.
-    ctypes = set()
-    for klass in app_config.get_models():
-        # Force looking up the content types in the current database
-        # before creating foreign keys to them.
-        ctype = ContentType.objects.db_manager(using).get_for_model(
-            klass, for_concrete_model=False
-        )
+    models = list(app_config.get_models())

-        ctypes.add(ctype)
-        for perm in _get_all_permissions(klass._meta):
-            searched_perms.append((ctype, perm))
+    # Grab all the ContentTypes.
+    ctypes = ContentType.objects.db_manager(using).get_for_models(
+        *models, for_concrete_models=False
+    )

     # Find all the Permissions that have a content_type for a model we're
     # looking for.  We don't need to check for codenames since we already have
@@ -91,20 +85,22 @@ def create_permissions(
     all_perms = set(
         Permission.objects.using(using)
         .filter(
-            content_type__in=ctypes,
+            content_type__in=set(ctypes.values()),
         )
         .values_list("content_type", "codename")
     )

     perms = []
-    for ct, (codename, name) in searched_perms:
-        if (ct.pk, codename) not in all_perms:
-            permission = Permission()
-            permission._state.db = using
-            permission.codename = codename
-            permission.name = name
-            permission.content_type = ct
-            perms.append(permission)
+    for model in models:
+        ctype = ctypes[model]
+        for codename, name in _get_all_permissions(model._meta):
+            if (ctype.pk, codename) not in all_perms:
+                permission = Permission()
+                permission._state.db = using
+                permission.codename = codename
+                permission.name = name
+                permission.content_type = ctype
+                perms.append(permission)

     Permission.objects.using(using).bulk_create(perms)
     if verbosity >= 2:

diff --git a/tests/auth_tests/test_management.py b/tests/auth_tests/test_management.py
index 0cc56b6760d7..5765c500346a 100644
--- a/tests/auth_tests/test_management.py
+++ b/tests/auth_tests/test_management.py
@@ -1528,7 +1528,7 @@ class CreatePermissionsMultipleDatabasesTests(TestCase):

     def test_set_permissions_fk_to_using_parameter(self):
         Permission.objects.using("other").delete()
-        with self.assertNumQueries(6, using="other") as captured_queries:
+        with self.assertNumQueries(4, using="other") as captured_queries:
             create_permissions(apps.get_app_config("auth"), verbosity=0, using="other")
         self.assertIn("INSERT INTO", captured_queries[-1]["sql"].upper())
         self.assertGreater(Permission.objects.using("other").count(), 0)

Optimize post-migrate permission creation
Description

(last modified by Adam Johnson)
I have often seen django.contrib.auth.management.create_permissions() take a significant amount of time in test run profiles. It can be optimized by batching more of its operations, including making ContentTypeManager.get_for_models() use batch creation.
For a comparison, I profiled 1518 of Django’s tests in modules called “models”:
$ python -m cProfile -o profile runtests.py --parallel 1 *model*
$ python -m pstats profile <<< 'sort cumtime
stats 10000' | less
Before optimization stats:
Total 11,938,857 function calls taking 5.349 seconds.
88 calls to create_permissions() take 456ms, ~8.5% of the total time.
After optimization stats:
Total 11,359,071 function calls taking 5.035 seconds.
88 calls to create_permissions() now take 239ms, ~4.7% of the toal time.
217ms and 579,786 function calls saved.
Optimization is limited because the post_migrate signal runs once per migrated app config, so there’s no chance to bulk create *all* content types and permissions at once. If we introduced a new “all migrated apps” signal, that could reduce runtime further by batching all creation.

['Accepting following an initial review of the patch which looks sensible. Setting as patch needs improvement due to the comments raised by David and Mariusz.', 1714134972.0]
['I repeated the profiling with the latest version of the patch, on top of the latest main commit. The numbers are similar. Before optimization stats: Total 12,387,798 function calls taking 5.589 seconds. 88 calls to create_permissions() take 483ms, ~8.6% of the total time. After optimization stats: Total 11,797,519 function calls taking 5.207 seconds. 88 calls to create_permissions() take 241ms, ~4.6% of the total time. 590,279 function calls and 242ms saved.', 1714750043.0]


diff --git a/django/db/models/options.py b/django/db/models/options.py
index ed7be7dd7a84..68a7228cbea6 100644
--- a/django/db/models/options.py
+++ b/django/db/models/options.py
@@ -5,6 +5,7 @@
 from django.apps import apps
 from django.conf import settings
 from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured
+from django.core.signals import setting_changed
 from django.db import connections
 from django.db.models import AutoField, Manager, OrderWrt, UniqueConstraint
 from django.db.models.query_utils import PathInfo
@@ -230,6 +231,9 @@ def contribute_to_class(self, cls, name):
                 self.db_table, connection.ops.max_name_length()
             )

+        if self.swappable:
+            setting_changed.connect(self.setting_changed)
+
     def _format_names(self, objs):
         """App label/class name interpolation for object names."""
         names = {"app_label": self.app_label.lower(), "class": self.model_name}
@@ -399,7 +403,7 @@ def verbose_name_raw(self):
         with override(None):
             return str(self.verbose_name)

-    @property
+    @cached_property
     def swapped(self):
         """
         Has this model been swapped out for another? If so, return the model
@@ -427,6 +431,10 @@ def swapped(self):
                     return swapped_for
         return None

+    def setting_changed(self, *, setting, **kwargs):
+        if setting == self.swappable and "swapped" in self.__dict__:
+            del self.swapped
+
     @cached_property
     def managers(self):
         managers = []

diff --git a/tests/model_meta/models.py b/tests/model_meta/models.py
index bc69d61a59cd..20a75baf4f2e 100644
--- a/tests/model_meta/models.py
+++ b/tests/model_meta/models.py
@@ -166,6 +166,11 @@ class Relating(models.Model):
     people_hidden = models.ManyToManyField(Person, related_name="+")

+class Swappable(models.Model):
+    class Meta:
+        swappable = "MODEL_META_TESTS_SWAPPED"
+
+
 # ParentListTests models
 class CommonAncestor(models.Model):
     pass
diff --git a/tests/model_meta/tests.py b/tests/model_meta/tests.py
index 0aa04d760d19..93883b5cf1c3 100644
--- a/tests/model_meta/tests.py
+++ b/tests/model_meta/tests.py
@@ -3,7 +3,7 @@
 from django.core.exceptions import FieldDoesNotExist
 from django.db.models import CharField, Field, ForeignObjectRel, ManyToManyField
 from django.db.models.options import EMPTY_RELATION_TREE, IMMUTABLE_WARNING
-from django.test import SimpleTestCase
+from django.test import SimpleTestCase, override_settings

 from .models import (
     AbstractPerson,
@@ -16,6 +16,7 @@
     Relating,
     Relation,
     SecondParent,
+    Swappable,
 )
 from .results import TEST_RESULTS

@@ -233,6 +234,31 @@ def test_gettext(self):
         self.assertEqual(Person._meta.verbose_name_raw, "Person")

+class SwappedTests(SimpleTestCase):
+    def test_plain_model_none(self):
+        self.assertIsNone(Relation._meta.swapped)
+
+    def test_unset(self):
+        self.assertIsNone(Swappable._meta.swapped)
+
+    def test_set_and_unset(self):
+        with override_settings(MODEL_META_TESTS_SWAPPED="model_meta.Relation"):
+            self.assertEqual(Swappable._meta.swapped, "model_meta.Relation")
+        self.assertIsNone(Swappable._meta.swapped)
+
+    def test_setting_none(self):
+        with override_settings(MODEL_META_TESTS_SWAPPED=None):
+            self.assertIsNone(Swappable._meta.swapped)
+
+    def test_setting_non_label(self):
+        with override_settings(MODEL_META_TESTS_SWAPPED="not-a-label"):
+            self.assertEqual(Swappable._meta.swapped, "not-a-label")
+
+    def test_setting_self(self):
+        with override_settings(MODEL_META_TESTS_SWAPPED="model_meta.swappable"):
+            self.assertIsNone(Swappable._meta.swapped)
+
+
 class RelationTreeTests(SimpleTestCase):
     all_models = (Relation, AbstractPerson, BasePerson, Person, ProxyPerson, Relating)

Cache Model._meta.swapped
Description

Another candidate for caching, like #35232 before.
The Model._meta.swapped property returns the model that this one has been swapped for. Since most models are not swappable (only auth.User is officially swappable), it returns None in nearly all cases.
I found this property was the most called function in Django when profiling a subset of Django’s tests, with:
$ python -m cProfile -o profile runtests.py --parallel 1 *model*
$ python -m pstats profile <<< 'sort ncalls
stats 10000' | less
This showed 439,484 calls to swapped, taking 29ms of the 5.597s test run, or 0.5% of the total runtime.
After adding @cached_property, this is reduced to 3,653 calls, rounding down to 0ms.

Thank you Adam! I a little on the fence on this one but let's give it a chance. Could you please a test in the PR to cover for the new (non trivial) logic?


diff --git a/django/utils/log.py b/django/utils/log.py
index fd0cc1bdc1ff..a25b97a7d5a4 100644
--- a/django/utils/log.py
+++ b/django/utils/log.py
@@ -92,6 +92,13 @@ def __init__(self, include_html=False, email_backend=None, reporter_class=None):
         )

     def emit(self, record):
+        # Early return when no email will be sent.
+        if (
+            not settings.ADMINS
+            # Method not overridden.
+            and self.send_mail.__func__ is AdminEmailHandler.send_mail
+        ):
+            return
         try:
             request = record.request
             subject = "%s (%s IP): %s" % (

diff --git a/tests/logging_tests/tests.py b/tests/logging_tests/tests.py
index 20d2852fde00..610bdc112434 100644
--- a/tests/logging_tests/tests.py
+++ b/tests/logging_tests/tests.py
@@ -1,6 +1,7 @@
 import logging
 from contextlib import contextmanager
 from io import StringIO
+from unittest import mock

 from admin_scripts.tests import AdminScriptTestCase

@@ -470,6 +471,26 @@ def test_emit_no_form_tag(self):
         self.assertIn('<div id="traceback">', body_html)
         self.assertNotIn("<form", body_html)

+    @override_settings(ADMINS=[])
+    def test_emit_no_admins(self):
+        handler = AdminEmailHandler()
+        record = self.logger.makeRecord(
+            "name",
+            logging.ERROR,
+            "function",
+            "lno",
+            "message",
+            None,
+            None,
+        )
+        with mock.patch.object(
+            handler,
+            "format_subject",
+            side_effect=AssertionError("Should not be called"),
+        ):
+            handler.emit(record)
+        self.assertEqual(len(mail.outbox), 0)
+

 class SettingsConfigTest(AdminScriptTestCase):
     """
diff --git a/tests/view_tests/tests/test_defaults.py b/tests/view_tests/tests/test_defaults.py
index 415a9a8c6746..66bc1da16889 100644
--- a/tests/view_tests/tests/test_defaults.py
+++ b/tests/view_tests/tests/test_defaults.py
@@ -123,7 +123,7 @@ def test_bad_request(self):
     )
     def test_custom_bad_request_template(self):
         response = self.client.get("/raises400/")
-        self.assertIs(response.wsgi_request, response.context[-1].request)
+        self.assertIs(response.wsgi_request, response.context.request)

     @override_settings(
         TEMPLATES=[

AdminEmailHandler wastes work when ADMINS isn’t set
Description

AdminEmailHandler.emit() does a lot of work to assemble the message it passes to mail_admins. If settings.ADMINS is empty, mail_admins() returns instantly, wasting all the message-creation work. It’s quite common to not configure ADMINS, whether in lieu of more advanced tools like Sentry, or during tests.
In a quick benchmark on my M1 Mac Pro on Python 3.11, the overhead is ~2.5ms:
In [1]: import logging
In [2]: logger = logging.getLogger('django')
In [3]: %timeit logger.error("Yada")
...
2.78 ms ± 75.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
In [4]: logger = logging.getLogger('example')
In [5]: %timeit logger.error("Yada")
...
8.37 µs ± 38.9 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each)
This can be avoided by adding an initial check to AdminEmailHandler.emit().


diff --git a/django/db/models/fields/related_descriptors.py b/django/db/models/fields/related_descriptors.py
index a2f00eb172c4..c7848ee63a3e 100644
--- a/django/db/models/fields/related_descriptors.py
+++ b/django/db/models/fields/related_descriptors.py
@@ -195,6 +195,9 @@ def get_prefetch_querysets(self, instances, querysets=None):
         else:
             query = {"%s__in" % self.field.related_query_name(): instances}
         queryset = queryset.filter(**query)
+        # There can be only one object prefetched for each instance so clear
+        # ordering if the query allows it without side effects.
+        queryset.query.clear_ordering()

         # Since we're going to assign directly in the cache,
         # we must manage the reverse relation cache manually.
@@ -469,6 +472,9 @@ def get_prefetch_querysets(self, instances, querysets=None):
         instances_dict = {instance_attr(inst): inst for inst in instances}
         query = {"%s__in" % self.related.field.name: instances}
         queryset = queryset.filter(**query)
+        # There can be only one object prefetched for each instance so clear
+        # ordering if the query allows it without side effects.
+        queryset.query.clear_ordering()

         # Since we're going to assign directly in the cache,
         # we must manage the reverse relation cache manually.

diff --git a/tests/prefetch_related/test_prefetch_related_objects.py b/tests/prefetch_related/test_prefetch_related_objects.py
index ca1f904c5205..eea9a7fff78c 100644
--- a/tests/prefetch_related/test_prefetch_related_objects.py
+++ b/tests/prefetch_related/test_prefetch_related_objects.py
@@ -1,7 +1,7 @@
 from django.db.models import Prefetch, prefetch_related_objects
 from django.test import TestCase

-from .models import Author, Book, Reader
+from .models import Author, Book, House, Reader, Room

 class PrefetchRelatedObjectsTests(TestCase):
@@ -33,6 +33,17 @@ def setUpTestData(cls):
         cls.reader1.books_read.add(cls.book1, cls.book4)
         cls.reader2.books_read.add(cls.book2, cls.book4)

+        cls.house1 = House.objects.create(name="b1", address="1")
+        cls.house2 = House.objects.create(name="b2", address="2")
+
+        cls.room1 = Room.objects.create(name="a1", house=cls.house1)
+        cls.room2 = Room.objects.create(name="a2", house=cls.house2)
+
+        cls.house1.main_room = cls.room1
+        cls.house1.save()
+        cls.house2.main_room = cls.room2
+        cls.house2.save()
+
     def test_unknown(self):
         book1 = Book.objects.get(id=self.book1.id)
         with self.assertRaises(AttributeError):
@@ -58,20 +69,75 @@ def test_m2m_reverse(self):

     def test_foreignkey_forward(self):
         authors = list(Author.objects.all())
-        with self.assertNumQueries(1):
+        with self.assertNumQueries(1) as ctx:
             prefetch_related_objects(authors, "first_book")
+        self.assertNotIn("ORDER BY", ctx.captured_queries[0]["sql"])

         with self.assertNumQueries(0):
             [author.first_book for author in authors]

+        authors = list(Author.objects.all())
+        with self.assertNumQueries(1) as ctx:
+            prefetch_related_objects(
+                authors,
+                Prefetch("first_book", queryset=Book.objects.order_by("-title")),
+            )
+        self.assertNotIn("ORDER BY", ctx.captured_queries[0]["sql"])
+
     def test_foreignkey_reverse(self):
         books = list(Book.objects.all())
-        with self.assertNumQueries(1):
+        with self.assertNumQueries(1) as ctx:
             prefetch_related_objects(books, "first_time_authors")
+        self.assertIn("ORDER BY", ctx.captured_queries[0]["sql"])

         with self.assertNumQueries(0):
             [list(book.first_time_authors.all()) for book in books]

+        books = list(Book.objects.all())
+        with self.assertNumQueries(1) as ctx:
+            prefetch_related_objects(
+                books,
+                Prefetch(
+                    "first_time_authors",
+                    queryset=Author.objects.order_by("-name"),
+                ),
+            )
+        self.assertIn("ORDER BY", ctx.captured_queries[0]["sql"])
+
+    def test_one_to_one_forward(self):
+        houses = list(House.objects.all())
+        with self.assertNumQueries(1) as ctx:
+            prefetch_related_objects(houses, "main_room")
+        self.assertNotIn("ORDER BY", ctx.captured_queries[0]["sql"])
+
+        with self.assertNumQueries(0):
+            [house.main_room for house in houses]
+
+        houses = list(House.objects.all())
+        with self.assertNumQueries(1) as ctx:
+            prefetch_related_objects(
+                houses,
+                Prefetch("main_room", queryset=Room.objects.order_by("-name")),
+            )
+        self.assertNotIn("ORDER BY", ctx.captured_queries[0]["sql"])
+
+    def test_one_to_one_reverse(self):
+        rooms = list(Room.objects.all())
+        with self.assertNumQueries(1) as ctx:
+            prefetch_related_objects(rooms, "main_room_of")
+        self.assertNotIn("ORDER BY", ctx.captured_queries[0]["sql"])
+
+        with self.assertNumQueries(0):
+            [room.main_room_of for room in rooms]
+
+        rooms = list(Room.objects.all())
+        with self.assertNumQueries(1) as ctx:
+            prefetch_related_objects(
+                rooms,
+                Prefetch("main_room_of", queryset=House.objects.order_by("-name")),
+            )
+        self.assertNotIn("ORDER BY", ctx.captured_queries[0]["sql"])
+
     def test_m2m_then_m2m(self):
         """A m2m can be followed through another m2m."""
         authors = list(Author.objects.all())

Elide ordering of prefetch querysets for single valued relationships
Description

(last modified by Laurent Lyaudet)
While the ordering of multi-valued relationships must be preserved when prefetching relationships is it unnecessary when using prefetch_related against single valued relationships.
For example, given the following models
class Author(models.Model):
	name = models.CharField(max_length=200)
	class Meta:
		ordering = ["name"]
class Book(models.Model):
	title = models.CharField(max_length=200)
	author = models.ForeignKey(Author, related_name="books", on_delete=models.CASCADE)
	class Meta:
		ordering = ["title"]
The ordering of an author's books in Author.objects.prefetch_related("books") has a significance as multiple books might be associated with each authors.
It's not the case for a book's author in Book.objects.prefetch_related("author") through as the relationship can only contain a single author and there is a single way to order the members of a singleton.
In other words sorted([element], key=sort_func) will result in [element] for any sort_func.
This property holds true for all the single valued relationships that the ORM supports (backward and forward 1:1 and forward 1:M) which allows the prefetching to elide any predefined ordering safely to avoid an unnecessary and possibly expensive ordering defined for the related model queryset.
Currently the prefetch of authors will use the order by and add useless load on the DB server.
It would be useful to remove this order by.
#ClimateChangeBrake

['Meta.ordering is working as expected, please \u200brefer to its documentation and associated warning Ordering is not a free operation. Each field you add to the ordering incurs a cost to your database. Each foreign key you add will implicitly include all of its default orderings as well. If you don\'t want this implicit behaviour then don\'t use Meta.ordering. If you want to keep using it but not for particular prefetches than use Prefetch objects with a queryset that explicitly calls order_by() to disable ordering. B.objects.prefetch_related(Prefetch("a", A.objects.order_by()))', 1710530134.0]
["Again a fast and without thought answer. I already know for this solution with Prefetch. Continue bashing good ideas because you don't like people giving them. I'll applaude at the end. There is no way it is useful to keep an order by when you do a query SELECT * FROM a WHERE a.id IN (.....100 or more ids here) ORDER BY name; then add the result in the cache of B objects. What you reject without thought yields a speed-up of 10 to 15 % on very big prefetches...", 1710531280.0]
["Please refrain from assuming bad faith from triagers regarding the resolution of this ticket. The provided resolution was a reflected based on your report details and in no way based on your persona. What do you suggest should happen for the thousands of projects out there that rely on prefetch_related to return results in a way that respects Meta.ordering? We can't simply make the behaviour of prefetch_related inconsistent with the normal behaviour or related manager access because it performs poorly when defined against an non-indexed field. I think the documentation warning I referred to is unfortunately all we can do to warn about this behaviour. Either use Meta.ordering and be prepared to deal with its implicit footguns or don't use it and use order_by where appropriate. Whether Meta.ordering should exist in the first place is debatable as it's at the origin of many unexpected behaviour with other features of the ORM (aggregation comes to mind) but making prefetch_related special case it would not only be backward incompatible but inconsistent with how the rest of the framework treats it.", 1710538813.0]
['I spent my night on it but I was able to make a patch, and I don\'t think there will be any regression. Consider the following models in some project TestNoOrderByForForeignKeyPrefetches and some app test_no_order_by models.py file: from django.db import models class A(models.Model): name = models.CharField(max_length=200) class Meta: ordering = ["name"] class B(models.Model): name = models.CharField(max_length=200) a = models.ForeignKey(A, related_name="bs", on_delete=models.CASCADE) class Meta: ordering = ["name"] Then consider the following command TestNoOrderByForForeignKeyPrefetches/test_no_order_by/management/commands/test_no_order_by_command.py : from django.core.management.base import BaseCommand from django.db import connection from django.db.models import Prefetch, QuerySet, RawQuerySet from django.db.models.fields.related_descriptors import ( ForwardManyToOneDescriptor, ReverseOneToOneDescriptor, ) from TestNoOrderByForForeignKeyPrefetches.test_no_order_by.models import A, B old_prefetch_init = Prefetch.__init__ def new_prefetch_init(self, *args, **kwargs): result = old_prefetch_init(self, *args, **kwargs) if self.queryset is not None: self.queryset._do_not_modify_order_by = True return result Prefetch.__init__ = new_prefetch_init old_get_prefetch_querysets_forward_many_to_one = ForwardManyToOneDescriptor.get_prefetch_querysets old_get_prefetch_querysets_reverse_one_to_one = ReverseOneToOneDescriptor.get_prefetch_querysets def get_prefetch_querysets_forward_many_to_one(self, *args, **kwargs): result = old_get_prefetch_querysets_forward_many_to_one(self, *args, **kwargs) if not hasattr(result[0], \'_do_not_modify_order_by\'): result = (result[0].order_by(), *result[1:]) return result def get_prefetch_querysets_reverse_one_to_one(self, *args, **kwargs): result = old_get_prefetch_querysets_reverse_one_to_one(self, *args, **kwargs) if not hasattr(result[0], \'_do_not_modify_order_by\'): result = (result[0].order_by(), *result[1:]) return result ForwardManyToOneDescriptor.get_prefetch_querysets = get_prefetch_querysets_forward_many_to_one ReverseOneToOneDescriptor.get_prefetch_querysets = get_prefetch_querysets_reverse_one_to_one old_clone_queryset = QuerySet._clone def new_clone_queryset(self): result = old_clone_queryset(self) if hasattr(self, \'_do_not_modify_order_by\'): result._do_not_modify_order_by = True return result QuerySet._clone = new_clone_queryset old_clone_raw_queryset = RawQuerySet._clone def new_clone_raw_queryset(self): result = old_clone_raw_queryset(self) if hasattr(self, \'_do_not_modify_order_by\'): result._do_not_modify_order_by = True return result RawQuerySet._clone = new_clone_raw_queryset class Command(BaseCommand): help = "Test" def handle(self, *args, **options): B.objects.all().delete() A.objects.all().delete() a1 = A.objects.create(name="a1") a2 = A.objects.create(name="a2") a3 = A.objects.create(name="a3") a4 = A.objects.create(name="a4") a5 = A.objects.create(name="a5") a6 = A.objects.create(name="a6") a7 = A.objects.create(name="a7") b1 = B.objects.create(a=a1, name="b1") b2 = B.objects.create(a=a2, name="b2") b3 = B.objects.create(a=a3, name="b3") b4 = B.objects.create(a=a4, name="b4") b5 = B.objects.create(a=a5, name="b5") b6 = B.objects.create(a=a6, name="b6") b7 = B.objects.create(a=a7, name="b7") bs = list(B.objects.all().prefetch_related("a")) a_s = list(A.objects.all().prefetch_related("bs")) bs = list(B.objects.all().prefetch_related( Prefetch( "a", queryset=A.objects.order_by("-name") ), )) a_s = list(A.objects.all().prefetch_related( Prefetch( "bs", queryset=B.objects.order_by("-name") ), )) print(connection.queries) If you launch the command with python3 manage.py test_no_order_by_command, you will see that there are 8 SELECT after the 14 INSERT and that there is only 7 ORDER BY on them as requested. I will prepare a PR.', 1710549223.0]
['\u200bPR', 1710551387.0]
['Here is the PR, I will improve it when requested : \u200bhttps://github.com/django/django/pull/17984 :) I still have doubts about keeping the order by even with manual Prefetch. I need to verify if it is possible to bypass the filter by id.', 1710555004.0]
["Laurent, thanks for this patch, however I agree with Simon. I appreciate you'd like to reopen the ticket, but please \u200bfollow the triaging guidelines with regards to wontfix tickets and take this to DevelopersMailingList.", 1710567356.0]


diff --git a/django/urls/resolvers.py b/django/urls/resolvers.py
index 5f9941dd65e8..1b26aed8c112 100644
--- a/django/urls/resolvers.py
+++ b/django/urls/resolvers.py
@@ -128,9 +128,6 @@ def get_ns_resolver(ns_pattern, resolver, converters):

 class LocaleRegexDescriptor:
-    def __init__(self, attr):
-        self.attr = attr
-
     def __get__(self, instance, cls=None):
         """
         Return a compiled regular expression based on the active language.
@@ -140,15 +137,23 @@ def __get__(self, instance, cls=None):
         # As a performance optimization, if the given regex string is a regular
         # string (not a lazily-translated string proxy), compile it once and
         # avoid per-language compilation.
-        pattern = getattr(instance, self.attr)
+        pattern = instance._regex
         if isinstance(pattern, str):
-            instance.__dict__["regex"] = instance._compile(pattern)
+            instance.__dict__["regex"] = self._compile(pattern)
             return instance.__dict__["regex"]
         language_code = get_language()
         if language_code not in instance._regex_dict:
-            instance._regex_dict[language_code] = instance._compile(str(pattern))
+            instance._regex_dict[language_code] = self._compile(str(pattern))
         return instance._regex_dict[language_code]

+    def _compile(self, regex):
+        try:
+            return re.compile(regex)
+        except re.error as e:
+            raise ImproperlyConfigured(
+                f'"{regex}" is not a valid regular expression: {e}'
+            ) from e
+

 class CheckURLMixin:
     def describe(self):
@@ -164,12 +169,11 @@ def _check_pattern_startswith_slash(self):
         """
         Check that the pattern does not begin with a forward slash.
         """
-        regex_pattern = self.regex.pattern
         if not settings.APPEND_SLASH:
             # Skip check as it can be useful to start a URL pattern with a slash
             # when APPEND_SLASH=False.
             return []
-        if regex_pattern.startswith(("/", "^/", "^\\/")) and not regex_pattern.endswith(
+        if self._regex.startswith(("/", "^/", "^\\/")) and not self._regex.endswith(
             "/"
         ):
             warning = Warning(
@@ -186,7 +190,7 @@ def _check_pattern_startswith_slash(self):

 class RegexPattern(CheckURLMixin):
-    regex = LocaleRegexDescriptor("_regex")
+    regex = LocaleRegexDescriptor()

     def __init__(self, regex, name=None, is_endpoint=False):
         self._regex = regex
@@ -219,8 +223,7 @@ def check(self):
         return warnings

     def _check_include_trailing_dollar(self):
-        regex_pattern = self.regex.pattern
-        if regex_pattern.endswith("$") and not regex_pattern.endswith(r"\$"):
+        if self._regex.endswith("$") and not self._regex.endswith(r"\$"):
             return [
                 Warning(
                     "Your URL pattern {} uses include with a route ending with a '$'. "
@@ -232,15 +235,6 @@ def _check_include_trailing_dollar(self):
         else:
             return []

-    def _compile(self, regex):
-        """Compile and return the given regular expression."""
-        try:
-            return re.compile(regex)
-        except re.error as e:
-            raise ImproperlyConfigured(
-                '"%s" is not a valid regular expression: %s' % (regex, e)
-            ) from e
-
     def __str__(self):
         return str(self._regex)

@@ -250,7 +244,7 @@ def __str__(self):
 )

-def _route_to_regex(route, is_endpoint=False):
+def _route_to_regex(route, is_endpoint):
     """
     Convert a path pattern into a regular expression. Return the regular
     expression and a dictionary mapping the capture names to the converters.
@@ -296,15 +290,36 @@ def _route_to_regex(route, is_endpoint=False):
     return "".join(parts), converters

+class LocaleRegexRouteDescriptor:
+    def __get__(self, instance, cls=None):
+        """
+        Return a compiled regular expression based on the active language.
+        """
+        if instance is None:
+            return self
+        # As a performance optimization, if the given route is a regular string
+        # (not a lazily-translated string proxy), compile it once and avoid
+        # per-language compilation.
+        if isinstance(instance._route, str):
+            instance.__dict__["regex"] = re.compile(instance._regex)
+            return instance.__dict__["regex"]
+        language_code = get_language()
+        if language_code not in instance._regex_dict:
+            instance._regex_dict[language_code] = re.compile(
+                _route_to_regex(str(instance._route), instance._is_endpoint)[0]
+            )
+        return instance._regex_dict[language_code]
+
+
 class RoutePattern(CheckURLMixin):
-    regex = LocaleRegexDescriptor("_route")
+    regex = LocaleRegexRouteDescriptor()

     def __init__(self, route, name=None, is_endpoint=False):
         self._route = route
+        self._regex, self.converters = _route_to_regex(str(route), is_endpoint)
         self._regex_dict = {}
         self._is_endpoint = is_endpoint
         self.name = name
-        self.converters = _route_to_regex(str(route), is_endpoint)[1]

     def match(self, path):
         match = self.regex.search(path)
@@ -356,9 +371,6 @@ def _check_pattern_unmatched_angle_brackets(self):
             warnings.append(Warning(msg % (self.describe(), "<"), id="urls.W010"))
         return warnings

-    def _compile(self, route):
-        return re.compile(_route_to_regex(route, self._is_endpoint)[0])
-
     def __str__(self):
         return str(self._route)

diff --git a/tests/i18n/patterns/locale/en/LC_MESSAGES/django.mo b/tests/i18n/patterns/locale/en/LC_MESSAGES/django.mo
index ec7644b504c3..b1f63b103106 100644
Binary files a/tests/i18n/patterns/locale/en/LC_MESSAGES/django.mo and b/tests/i18n/patterns/locale/en/LC_MESSAGES/django.mo differ
diff --git a/tests/i18n/patterns/locale/en/LC_MESSAGES/django.po b/tests/i18n/patterns/locale/en/LC_MESSAGES/django.po
index 9a14a80ceb8f..ac98eb5f08aa 100644
--- a/tests/i18n/patterns/locale/en/LC_MESSAGES/django.po
+++ b/tests/i18n/patterns/locale/en/LC_MESSAGES/django.po
@@ -7,31 +7,53 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2011-06-15 11:33+0200\n"
+"POT-Creation-Date: 2024-03-01 21:18+0000\n"
 "PO-Revision-Date: 2011-06-14 16:16+0100\n"
 "Last-Translator: Jannis Leidel <jannis@leidel.info>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Language: \n"

 #: urls/default.py:11
-msgid "^translated/$"
+#, fuzzy
+#| msgid "^translated/$"
+msgid "translated/"
 msgstr "^translated/$"

 #: urls/default.py:12
+#, fuzzy
+#| msgid "^translated/$"
+msgid "^translated-regex/$"
+msgstr "^translated/$"
+
+#: urls/default.py:14
 msgid "^translated/(?P<slug>[\\w-]+)/$"
 msgstr "^translated/(?P<slug>[\\w-]+)/$"

-#: urls/default.py:17
+#: urls/default.py:25
+msgid "^with-arguments/(?P<argument>[\\w-]+)/(?:(?P<optional>[\\w-]+).html)?$"
+msgstr ""
+
+#: urls/default.py:29
 msgid "^users/$"
 msgstr "^users/$"

-#: urls/default.py:18 urls/wrong.py:7
+#: urls/default.py:31 urls/wrong.py:7
 msgid "^account/"
 msgstr "^account/"

 #: urls/namespace.py:9 urls/wrong_namespace.py:10
 msgid "^register/$"
 msgstr "^register/$"
+
+#: urls/namespace.py:10
+msgid "^register-without-slash$"
+msgstr ""
+
+#: urls/namespace.py:11
+#, fuzzy
+#| msgid "^register/$"
+msgid "register-as-path/"
+msgstr "^register/$"
diff --git a/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.mo b/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.mo
index 5eac50466cb7..544bfdbfc664 100644
Binary files a/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.mo and b/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.mo differ
diff --git a/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.po b/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.po
index a938e3371dea..aa58c506d562 100644
--- a/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.po
+++ b/tests/i18n/patterns/locale/nl/LC_MESSAGES/django.po
@@ -7,29 +7,37 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2011-06-15 11:33+0200\n"
+"POT-Creation-Date: 2024-03-01 21:18+0000\n"
 "PO-Revision-Date: 2011-06-14 16:16+0100\n"
 "Last-Translator: Jannis Leidel <jannis@leidel.info>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Language: \n"
 "Plural-Forms: nplurals=2; plural=(n != 1)\n"

 #: urls/default.py:11
-msgid "^translated/$"
-msgstr "^vertaald/$"
+msgid "translated/"
+msgstr "vertaald/"

 #: urls/default.py:12
+msgid "^translated-regex/$"
+msgstr "^vertaald-regex/$"
+
+#: urls/default.py:14
 msgid "^translated/(?P<slug>[\\w-]+)/$"
 msgstr "^vertaald/(?P<slug>[\\w-]+)/$"

-#: urls/default.py:17
+#: urls/default.py:25
+msgid "^with-arguments/(?P<argument>[\\w-]+)/(?:(?P<optional>[\\w-]+).html)?$"
+msgstr ""
+
+#: urls/default.py:29
 msgid "^users/$"
 msgstr "^gebruikers/$"

-#: urls/default.py:18 urls/wrong.py:7
+#: urls/default.py:31 urls/wrong.py:7
 msgid "^account/"
 msgstr "^profiel/"

@@ -37,6 +45,10 @@ msgstr "^profiel/"
 msgid "^register/$"
 msgstr "^registreren/$"

-#: urls/namespace.py:12
+#: urls/namespace.py:10
+msgid "^register-without-slash$"
+msgstr ""
+
+#: urls/namespace.py:11
 msgid "register-as-path/"
 msgstr "registreren-als-pad/"
diff --git a/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.mo b/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.mo
index 1d7b346c278c..8e36cb206409 100644
Binary files a/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.mo and b/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.mo differ
diff --git a/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.po b/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.po
index fd3388e4b013..464d14bc1f75 100644
--- a/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.po
+++ b/tests/i18n/patterns/locale/pt_BR/LC_MESSAGES/django.po
@@ -7,32 +7,50 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2011-06-15 11:34+0200\n"
+"POT-Creation-Date: 2024-03-01 21:18+0000\n"
 "PO-Revision-Date: 2011-06-14 16:17+0100\n"
 "Last-Translator: Jannis Leidel <jannis@leidel.info>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Language: \n"
 "Plural-Forms: nplurals=2; plural=(n > 1)\n"

 #: urls/default.py:11
-msgid "^translated/$"
-msgstr "^traduzidos/$"
+msgid "translated/"
+msgstr "traduzidos/"

 #: urls/default.py:12
+msgid "^translated-regex/$"
+msgstr "^traduzidos-regex/$"
+
+#: urls/default.py:14
 msgid "^translated/(?P<slug>[\\w-]+)/$"
 msgstr "^traduzidos/(?P<slug>[\\w-]+)/$"

-#: urls/default.py:17
+#: urls/default.py:25
+msgid "^with-arguments/(?P<argument>[\\w-]+)/(?:(?P<optional>[\\w-]+).html)?$"
+msgstr ""
+
+#: urls/default.py:29
 msgid "^users/$"
 msgstr "^usuarios/$"

-#: urls/default.py:18 urls/wrong.py:7
+#: urls/default.py:31 urls/wrong.py:7
 msgid "^account/"
 msgstr "^conta/"

 #: urls/namespace.py:9 urls/wrong_namespace.py:10
 msgid "^register/$"
 msgstr "^registre-se/$"
+
+#: urls/namespace.py:10
+msgid "^register-without-slash$"
+msgstr ""
+
+#: urls/namespace.py:11
+#, fuzzy
+#| msgid "^register/$"
+msgid "register-as-path/"
+msgstr "^registre-se/$"
diff --git a/tests/i18n/patterns/tests.py b/tests/i18n/patterns/tests.py
index e2fee904b149..bd329e69f8e7 100644
--- a/tests/i18n/patterns/tests.py
+++ b/tests/i18n/patterns/tests.py
@@ -134,6 +134,9 @@ class URLTranslationTests(URLTestCaseBase):
     def test_no_prefix_translated(self):
         with translation.override("en"):
             self.assertEqual(reverse("no-prefix-translated"), "/translated/")
+            self.assertEqual(
+                reverse("no-prefix-translated-regex"), "/translated-regex/"
+            )
             self.assertEqual(
                 reverse("no-prefix-translated-slug", kwargs={"slug": "yeah"}),
                 "/translated/yeah/",
@@ -141,6 +144,7 @@ def test_no_prefix_translated(self):

         with translation.override("nl"):
             self.assertEqual(reverse("no-prefix-translated"), "/vertaald/")
+            self.assertEqual(reverse("no-prefix-translated-regex"), "/vertaald-regex/")
             self.assertEqual(
                 reverse("no-prefix-translated-slug", kwargs={"slug": "yeah"}),
                 "/vertaald/yeah/",
@@ -148,6 +152,9 @@ def test_no_prefix_translated(self):

         with translation.override("pt-br"):
             self.assertEqual(reverse("no-prefix-translated"), "/traduzidos/")
+            self.assertEqual(
+                reverse("no-prefix-translated-regex"), "/traduzidos-regex/"
+            )
             self.assertEqual(
                 reverse("no-prefix-translated-slug", kwargs={"slug": "yeah"}),
                 "/traduzidos/yeah/",
@@ -180,7 +187,7 @@ def test_translate_url_utility(self):
                 "/nl/profiel/registreren-als-pad/",
             )
             self.assertEqual(translation.get_language(), "en")
-            # URL with parameters.
+            # re_path() URL with parameters.
             self.assertEqual(
                 translate_url("/en/with-arguments/regular-argument/", "nl"),
                 "/nl/with-arguments/regular-argument/",
@@ -191,6 +198,11 @@ def test_translate_url_utility(self):
                 ),
                 "/nl/with-arguments/regular-argument/optional.html",
             )
+            # path() URL with parameter.
+            self.assertEqual(
+                translate_url("/en/path-with-arguments/regular-argument/", "nl"),
+                "/nl/path-with-arguments/regular-argument/",
+            )

         with translation.override("nl"):
             self.assertEqual(translate_url("/nl/gebruikers/", "en"), "/en/users/")
diff --git a/tests/i18n/patterns/urls/default.py b/tests/i18n/patterns/urls/default.py
index 418e9f568568..090b92eeca66 100644
--- a/tests/i18n/patterns/urls/default.py
+++ b/tests/i18n/patterns/urls/default.py
@@ -8,7 +8,8 @@
 urlpatterns = [
     path("not-prefixed/", view, name="not-prefixed"),
     path("not-prefixed-include/", include("i18n.patterns.urls.included")),
-    re_path(_(r"^translated/$"), view, name="no-prefix-translated"),
+    path(_("translated/"), view, name="no-prefix-translated"),
+    re_path(_(r"^translated-regex/$"), view, name="no-prefix-translated-regex"),
     re_path(
         _(r"^translated/(?P<slug>[\w-]+)/$"),
         view,
@@ -25,6 +26,11 @@
         view,
         name="with-arguments",
     ),
+    path(
+        _("path-with-arguments/<str:argument>/"),
+        view,
+        name="path-with-arguments",
+    ),
     re_path(_(r"^users/$"), view, name="users"),
     re_path(
         _(r"^account/"), include("i18n.patterns.urls.namespace", namespace="account")

Stop URL system checks from compiling regular expressions
Description

(last modified by Adam Johnson)
Continuing my project to optimize the system checks, I found some good optimizations under django.core.checks.urls.check_url_config(), which showed up as quite expensive in profiling.
Looking down the call tree, it seems the most expensive part of this process is compiling the each URL pattern’s regular expression. This is unnecessary work though, as the checks only need *uncompiled* regular expression patterns. Using the compiled versions “undoes” the lazy-compile optimization that LocaleRegexDescriptor was created for in #27453 / 6e222dae5636f875c19ec66f730a4241abe33faa, at least for any process that runs checks.
The checks were fetching the uncompiled pattern with self.regex.pattern, which makse LocaleRegexDescriptor compile the pattern only to then read the uncompiled pattern from its pattern attribute.
Additionally, RoutePattern was calling _route_to_regex() twice to fetch its two result variables in different places: once in __init__() and again in _compile() (in the non-translated case). This function has non-trivial cost so avoiding double execution is worth it.
Before optimization stats:
check_url_config took 67ms, or ~10% of the time for checks.
LocaleRegexDescriptor.__get__() showed 965 calls taking ~60ms, ~9% of the total runtime of checks.
re.compile() showed 741 calls for 94ms.
_route_to_regex() had 1900 calls taking 18ms (~2.6% of the total runtime).
After optimization:
check_url_config() took 5ms, ~0.9% of the new total runtime.
The calls to LocaleRegexDescriptor.__get__ are gone.
re.compile() drops to 212 calls from other sites, for a total of 51ms.
_route_to_regex() drops to the expected 950 calls, taking half the time at 9ms.
(I also tried benchmarking with django-asv but got inconclusive results where change was within the error margins.)


diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py
index 6d5c0708a322..78063a134d2a 100644
--- a/django/contrib/admin/options.py
+++ b/django/contrib/admin/options.py
@@ -41,6 +41,7 @@
 from django.core.paginator import Paginator
 from django.db import models, router, transaction
 from django.db.models.constants import LOOKUP_SEP
+from django.db.models.functions import Cast
 from django.forms.formsets import DELETION_FIELD_NAME, all_valid
 from django.forms.models import (
     BaseInlineFormSet,
@@ -1207,9 +1208,33 @@ def construct_search(field_name):
         may_have_duplicates = False
         search_fields = self.get_search_fields(request)
         if search_fields and search_term:
-            orm_lookups = [
-                construct_search(str(search_field)) for search_field in search_fields
-            ]
+            str_annotations = {}
+            orm_lookups = []
+            for field in search_fields:
+                if field.endswith("__exact"):
+                    field_name = field.rsplit("__exact", 1)[0]
+                    try:
+                        field_obj = queryset.model._meta.get_field(field_name)
+                    except FieldDoesNotExist:
+                        lookup = construct_search(field)
+                        orm_lookups.append(lookup)
+                        continue
+                    # Add string cast annotations for non-string exact lookups.
+                    if not isinstance(field_obj, (models.CharField, models.TextField)):
+                        str_annotations[f"{field_name}_str"] = Cast(
+                            field_name, output_field=models.CharField()
+                        )
+                        orm_lookups.append(f"{field_name}_str__exact")
+                    else:
+                        lookup = construct_search(field)
+                        orm_lookups.append(lookup)
+                else:
+                    lookup = construct_search(str(field))
+                    orm_lookups.append(lookup)
+
+            if str_annotations:
+                queryset = queryset.annotate(**str_annotations)
+
             term_queries = []
             for bit in smart_split(search_term):
                 if bit.startswith(('"', "'")) and bit[0] == bit[-1]:

diff --git a/tests/admin_changelist/admin.py b/tests/admin_changelist/admin.py
index 349ef7d465b6..d9dc498e8427 100644
--- a/tests/admin_changelist/admin.py
+++ b/tests/admin_changelist/admin.py
@@ -48,6 +48,7 @@ class ChildAdmin(admin.ModelAdmin):
     list_display = ["name", "parent"]
     list_per_page = 10
     list_filter = ["parent", "age"]
+    search_fields = ["age__exact", "name__exact"]

     def get_queryset(self, request):
         return super().get_queryset(request).select_related("parent")
diff --git a/tests/admin_changelist/tests.py b/tests/admin_changelist/tests.py
index d8055a809be2..a823a72f7d7f 100644
--- a/tests/admin_changelist/tests.py
+++ b/tests/admin_changelist/tests.py
@@ -860,6 +860,25 @@ def test_custom_lookup_with_pk_shortcut(self):
         cl = m.get_changelist_instance(request)
         self.assertCountEqual(cl.queryset, [abcd])

+    def test_search_with_exact_lookup_for_non_string_field(self):
+        child = Child.objects.create(name="Asher", age=11)
+        model_admin = ChildAdmin(Child, custom_site)
+
+        for search_term, expected_result in [
+            ("11", [child]),
+            ("Asher", [child]),
+            ("1", []),
+            ("A", []),
+            ("random", []),
+        ]:
+            request = self.factory.get("/", data={SEARCH_VAR: search_term})
+            request.user = self.superuser
+            with self.subTest(search_term=search_term):
+                # 1 query for filtered result, 1 for filtered count, 1 for total count.
+                with self.assertNumQueries(3):
+                    cl = model_admin.get_changelist_instance(request)
+                self.assertCountEqual(cl.queryset, expected_result)
+
     def test_no_distinct_for_m2m_in_list_filter_without_params(self):
         """
         If a ManyToManyField is in list_filter but isn't in any lookup params,

Make ModelAdmin.search_fields raise data errors on __exact lookups for non-string fields.
Description

Currently, all queries are done as string lookups which gives something like this on PostgreSQL, for example: ("admin_views_pluggablesearchperson"."age"::text) = UPPER(20)). It would be more efficient if the admin cast the search value to an integer and used that for the query.

['PR: \u200bhttps://github.com/django/django/pull/6219', 1456649223.0]
['I think we should favor the approach suggested in #26184 instead.', 1456652736.0]
["I agree, let's revisit this if it doesn't.", 1457113813.0]
["As far as I tested, you can now use something like search_fields = ['votes__exact'] to do the appropriate query, however, if a non-integer search term is entered, the page crashes with ValueError invalid literal for int() with base 10: 'abc' from IntegerField.get_prep_value(). Probably that exception should be caught in ModelAdmin.get_search_results() and no results returned.", 1532099527.0]
['PR: \u200bhttps://github.com/django/django/pull/10339', 1535321698.0]
["Replying to Tim Graham: As far as I tested, you can now use something like search_fields = ['votes__exact'] to do the appropriate query, however, if a non-integer search term is entered, the page crashes with ValueError invalid literal for int() with base 10: 'abc' from IntegerField.get_prep_value(). Probably that exception should be caught in ModelAdmin.get_search_results() and no results returned. We have to consider that there might be multiple search fields of mixed types though, i.e. search_fields = ['votes__exact', 'title', 'text__contains'], just so the matches for other fields does not get discarded just because one of the fields threw an error. Instead of trying to catch these errors from .filter() as in the previous patch, we could run to_python() for each search field in order to test if that particular field would raise an exception later on when the query is run.", 1593498353.0]
['#34191 was a duplicate for DecimalField.', 1669781193.0]
['Replying to Mariusz Felisiak: #34191 was a duplicate for DecimalField. You can use iexact instead of exact for non-string fields, I think, it even makes sense, because the word exact means complete equality even with type. It works for me.', 1671817064.0]
['can i work on this?', 1706431804.0]
['PR: \u200bhttps://github.com/django/django/pull/17885', 1708402891.0]


diff --git a/django/db/models/options.py b/django/db/models/options.py
index 9b3106f67ef6..e63a81c2d8e7 100644
--- a/django/db/models/options.py
+++ b/django/db/models/options.py
@@ -395,9 +395,11 @@ def can_migrate(self, connection):
             )
         return True

-    @property
+    @cached_property
     def verbose_name_raw(self):
         """Return the untranslated verbose name."""
+        if isinstance(self.verbose_name, str):
+            return self.verbose_name
         with override(None):
             return str(self.verbose_name)

diff --git a/tests/model_meta/models.py b/tests/model_meta/models.py
index 6da62be2ac26..bc69d61a59cd 100644
--- a/tests/model_meta/models.py
+++ b/tests/model_meta/models.py
@@ -1,6 +1,7 @@
 from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
+from django.utils.translation import gettext_lazy as _

 class Relation(models.Model):
@@ -124,6 +125,9 @@ class Person(BasePerson):
     # GR fields
     generic_relation_concrete = GenericRelation(Relation)

+    class Meta:
+        verbose_name = _("Person")
+

 class ProxyPerson(Person):
     class Meta:
diff --git a/tests/model_meta/tests.py b/tests/model_meta/tests.py
index fe2f6e63da8d..7204a5e93a14 100644
--- a/tests/model_meta/tests.py
+++ b/tests/model_meta/tests.py
@@ -222,6 +222,17 @@ def test_get_fields_only_searches_forward_on_apps_not_ready(self):
             opts.apps.models_ready = True

+class VerboseNameRawTests(SimpleTestCase):
+    def test_string(self):
+        # Clear cached property.
+        Relation._meta.__dict__.pop("verbose_name_raw", None)
+        self.assertEqual(Relation._meta.verbose_name_raw, "relation")
+
+    def test_gettext(self):
+        Person._meta.__dict__.pop("verbose_name_raw", None)
+        self.assertEqual(Person._meta.verbose_name_raw, "Person")
+
+
 class RelationTreeTests(SimpleTestCase):
     all_models = (Relation, AbstractPerson, BasePerson, Person, ProxyPerson, Relating)

Cache Options.verbose_name_raw
Description

(last modified by Adam Johnson)
Another candidate for caching, like #35230, following the same system check profiling.
The Model._meta.verbose_name_raw property returns the stringified version of the verbose_name attribute whilst temporarily disabling translations. It is only used in django.contrib.auth for creating permission names.
I found this property was taking ~15% of the total runtime for system checks on a project with 118 models. calls. Turning it into a cached_property and adding a no-translation fast path saves nearly all this cost, with the below results.
Before: 520 calls taking 10ms
After: 105 calls taking ~0ms


diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py
index a2cbd87bd..0d5ec475e 100644
--- a/supervision/detection/utils.py
+++ b/supervision/detection/utils.py
@@ -720,25 +720,71 @@ def move_masks(
         masks (npt.NDArray[np.bool_]): A 3D array of binary masks corresponding to the
             predictions. Shape: `(N, H, W)`, where N is the number of predictions, and
             H, W are the dimensions of each mask.
-        offset (npt.NDArray[np.int32]): An array of shape `(2,)` containing non-negative
-            int values `[dx, dy]`.
+        offset (npt.NDArray[np.int32]): An array of shape `(2,)` containing int values
+            `[dx, dy]`. Supports both positive and negative values for bidirectional
+            movement.
         resolution_wh (Tuple[int, int]): The width and height of the desired mask
             resolution.

     Returns:
         (npt.NDArray[np.bool_]) repositioned masks, optionally padded to the specified
             shape.
-    """

-    if offset[0] < 0 or offset[1] < 0:
-        raise ValueError(f"Offset values must be non-negative integers. Got: {offset}")
+    Examples:
+        ```python
+        import numpy as np
+        import supervision as sv

+        mask = np.array([[[False, False, False, False],
+                         [False, True,  True,  False],
+                         [False, True,  True,  False],
+                         [False, False, False, False]]], dtype=bool)
+
+        offset = np.array([1, 1])
+        sv.move_masks(mask, offset, resolution_wh=(4, 4))
+        # array([[[False, False, False, False],
+        #         [False, False, False, False],
+        #         [False, False,  True,  True],
+        #         [False, False,  True,  True]]], dtype=bool)
+
+        offset = np.array([-2, 2])
+        sv.move_masks(mask, offset, resolution_wh=(4, 4))
+        # array([[[False, False, False, False],
+        #         [False, False, False, False],
+        #         [False, False, False, False],
+        #         [True,  False, False, False]]], dtype=bool)
+        ```
+    """
     mask_array = np.full((masks.shape[0], resolution_wh[1], resolution_wh[0]), False)
-    mask_array[
-        :,
-        offset[1] : masks.shape[1] + offset[1],
-        offset[0] : masks.shape[2] + offset[0],
-    ] = masks
+
+    if offset[0] < 0:
+        source_x_start = -offset[0]
+        source_x_end = min(masks.shape[2], resolution_wh[0] - offset[0])
+        destination_x_start = 0
+        destination_x_end = min(resolution_wh[0], masks.shape[2] + offset[0])
+    else:
+        source_x_start = 0
+        source_x_end = min(masks.shape[2], resolution_wh[0] - offset[0])
+        destination_x_start = offset[0]
+        destination_x_end = offset[0] + source_x_end - source_x_start
+
+    if offset[1] < 0:
+        source_y_start = -offset[1]
+        source_y_end = min(masks.shape[1], resolution_wh[1] - offset[1])
+        destination_y_start = 0
+        destination_y_end = min(resolution_wh[1], masks.shape[1] + offset[1])
+    else:
+        source_y_start = 0
+        source_y_end = min(masks.shape[1], resolution_wh[1] - offset[1])
+        destination_y_start = offset[1]
+        destination_y_end = offset[1] + source_y_end - source_y_start
+
+    if source_x_end > source_x_start and source_y_end > source_y_start:
+        mask_array[
+            :,
+            destination_y_start:destination_y_end,
+            destination_x_start:destination_x_end,
+        ] = masks[:, source_y_start:source_y_end, source_x_start:source_x_end]

     return mask_array

diff --git a/test/detection/test_utils.py b/test/detection/test_utils.py
index 87e50f6a4..d93c72c83 100644
--- a/test/detection/test_utils.py
+++ b/test/detection/test_utils.py
@@ -16,6 +16,7 @@
     merge_data,
     merge_metadata,
     move_boxes,
+    move_masks,
     process_roboflow_result,
     scale_boxes,
     xcycwh_to_xyxy,
@@ -442,6 +443,268 @@ def test_move_boxes(
         assert np.array_equal(result, expected_result)

+@pytest.mark.parametrize(
+    "masks, offset, resolution_wh, expected_result, exception",
+    [
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([0, 0]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([-1, -1]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [True, True, False, False],
+                        [True, True, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([-2, -2]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [True, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([-3, -3]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([-2, -1]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [True, False, False, False],
+                        [True, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([-1, -2]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [True, True, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([-2, 2]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [True, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([3, 3]),
+            (4, 4),
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+        (
+            np.array(
+                [
+                    [
+                        [False, False, False, False],
+                        [False, True, True, False],
+                        [False, True, True, False],
+                        [False, False, False, False],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            np.array([3, 3]),
+            (6, 6),
+            np.array(
+                [
+                    [
+                        [False, False, False, False, False, False],
+                        [False, False, False, False, False, False],
+                        [False, False, False, False, False, False],
+                        [False, False, False, False, False, False],
+                        [False, False, False, False, True, True],
+                        [False, False, False, False, True, True],
+                    ]
+                ],
+                dtype=bool,
+            ),
+            DoesNotRaise(),
+        ),
+    ],
+)
+def test_move_masks(
+    masks: np.ndarray,
+    offset: np.ndarray,
+    resolution_wh: Tuple[int, int],
+    expected_result: np.ndarray,
+    exception: Exception,
+) -> None:
+    with exception:
+        result = move_masks(masks=masks, offset=offset, resolution_wh=resolution_wh)
+        np.testing.assert_array_equal(result, expected_result)
+
+
 @pytest.mark.parametrize(
     "xyxy, factor, expected_result, exception",
     [

`move_masks` only supports movement in positive direction
If you compare the code of `move_masks`, `move_detections` and `move_oriented_boxes`, you'll find that only mask code restricts offset direction:

```python
    if offset[0] < 0 or offset[1] < 0:
        raise ValueError(f"Offset values must be non-negative integers. Got: {offset}")
```

It should be possible to move masks in either direction, even if it results in cropping.

To complete this:
- [ ] Change the code so masks can be moved with negative offset
- [ ] Create a unit test suite for `move_masks`

---

It would help us immensely and speed up the review process if you could create a [Colab](https://colab.research.google.com/) showcasing the changes, but for this task it is optional. You may use the [Starter Template](https://colab.research.google.com/drive/1rin7WrS-UvVIe-_Gfxmu-yVslGphOq89?usp=sharing).

Hi @LinasKo, I would like to contribute to this. Can you assign it to me?


diff --git a/supervision/detection/core.py b/supervision/detection/core.py
index 74c663a89..14f4c0ef8 100644
--- a/supervision/detection/core.py
+++ b/supervision/detection/core.py
@@ -1201,6 +1201,8 @@ def __getitem__(
         """
         if isinstance(index, str):
             return self.data.get(index)
+        if self.is_empty():
+            return Detections.empty()
         if isinstance(index, int):
             index = [index]
         return Detections(

diff --git a/test/detection/test_core.py b/test/detection/test_core.py
index 61796bef2..dfa784fcb 100644
--- a/test/detection/test_core.py
+++ b/test/detection/test_core.py
@@ -223,6 +223,12 @@
             None,
             pytest.raises(IndexError),
         ),
+        (
+            Detections.empty(),
+            np.isin(Detections.empty()["class_name"], ["cat", "dog"]),
+            Detections.empty(),
+            DoesNotRaise(),
+        ),  # Filter an empty detections by specific class names
     ],
 )
 def test_getitem(

Crash when filtering empty detections: xyxy shape (0, 0, 4).
Reproduction code:

```python
import supervision as sv
import numpy as np

CLASSES = [0, 1, 2]

prediction = sv.Detections.empty()
prediction = prediction[np.isin(prediction["class_name"], CLASSES)]
```

Error:

```
Traceback (most recent call last):
  File "/Users/linasko/.settler_workspace/pr/supervision-fresh/run_detections.py", line 7, in <module>
    prediction = prediction[np.isin(prediction["class_name"], CLASSES)]
  File "/Users/linasko/.settler_workspace/pr/supervision-fresh/supervision/detection/core.py", line 1206, in __getitem__
    return Detections(
  File "<string>", line 10, in __init__
  File "/Users/linasko/.settler_workspace/pr/supervision-fresh/supervision/detection/core.py", line 144, in __post_init__
    validate_detections_fields(
  File "/Users/linasko/.settler_workspace/pr/supervision-fresh/supervision/validators/__init__.py", line 120, in validate_detections_fields
    validate_xyxy(xyxy)
  File "/Users/linasko/.settler_workspace/pr/supervision-fresh/supervision/validators/__init__.py", line 11, in validate_xyxy
    raise ValueError(
ValueError: xyxy must be a 2D np.ndarray with shape (_, 4), but got shape (0, 0, 4)
```


diff --git a/sentence_transformers/cross_encoder/CrossEncoder.py b/sentence_transformers/cross_encoder/CrossEncoder.py
index c8d8d1d84..c4345017a 100644
--- a/sentence_transformers/cross_encoder/CrossEncoder.py
+++ b/sentence_transformers/cross_encoder/CrossEncoder.py
@@ -123,8 +123,7 @@ def __init__(
         if device is None:
             device = get_device_name()
             logger.info(f"Use pytorch device: {device}")
-
-        self._target_device = torch.device(device)
+        self.model.to(device)

         if default_activation_function is not None:
             self.default_activation_function = default_activation_function
@@ -154,11 +153,11 @@ def smart_batching_collate(self, batch: list[InputExample]) -> tuple[BatchEncodi
             *texts, padding=True, truncation="longest_first", return_tensors="pt", max_length=self.max_length
         )
         labels = torch.tensor(labels, dtype=torch.float if self.config.num_labels == 1 else torch.long).to(
-            self._target_device
+            self.model.device
         )

         for name in tokenized:
-            tokenized[name] = tokenized[name].to(self._target_device)
+            tokenized[name] = tokenized[name].to(self.model.device)

         return tokenized, labels

@@ -174,7 +173,7 @@ def smart_batching_collate_text_only(self, batch: list[InputExample]) -> BatchEn
         )

         for name in tokenized:
-            tokenized[name] = tokenized[name].to(self._target_device)
+            tokenized[name] = tokenized[name].to(self.model.device)

         return tokenized

@@ -232,7 +231,6 @@ def fit(
                 scaler = torch.npu.amp.GradScaler()
             else:
                 scaler = torch.cuda.amp.GradScaler()
-        self.model.to(self._target_device)

         if output_path is not None:
             os.makedirs(output_path, exist_ok=True)
@@ -272,7 +270,7 @@ def fit(
                 train_dataloader, desc="Iteration", smoothing=0.05, disable=not show_progress_bar
             ):
                 if use_amp:
-                    with torch.autocast(device_type=self._target_device.type):
+                    with torch.autocast(device_type=self.model.device.type):
                         model_predictions = self.model(**features, return_dict=True)
                         logits = activation_fct(model_predictions.logits)
                         if self.config.num_labels == 1:
@@ -438,7 +436,6 @@ def predict(

         pred_scores = []
         self.model.eval()
-        self.model.to(self._target_device)
         with torch.no_grad():
             for features in iterator:
                 model_predictions = self.model(**features, return_dict=True)
@@ -604,3 +601,21 @@ def push_to_hub(
             tags=tags,
             **kwargs,
         )
+
+    def to(self, device: int | str | torch.device | None = None) -> None:
+        return self.model.to(device)
+
+    @property
+    def _target_device(self) -> torch.device:
+        logger.warning(
+            "`CrossEncoder._target_device` has been removed, please use `CrossEncoder.device` instead.",
+        )
+        return self.device
+
+    @_target_device.setter
+    def _target_device(self, device: int | str | torch.device | None = None) -> None:
+        self.to(device)
+
+    @property
+    def device(self) -> torch.device:
+        return self.model.device

diff --git a/tests/test_cross_encoder.py b/tests/test_cross_encoder.py
index 9c17aa093..027751d1a 100644
--- a/tests/test_cross_encoder.py
+++ b/tests/test_cross_encoder.py
@@ -192,3 +192,35 @@ def test_bfloat16() -> None:

     ranking = model.rank("Hello there!", ["Hello, World!", "Heya!"])
     assert isinstance(ranking, list)
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA must be available to test moving devices effectively.")
+@pytest.mark.parametrize("device", ["cpu", "cuda"])
+def test_device_assignment(device):
+    model = CrossEncoder("cross-encoder/stsb-distilroberta-base", device=device)
+    assert model.device.type == device
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA must be available to test moving devices effectively.")
+def test_device_switching():
+    # test assignment using .to
+    model = CrossEncoder("cross-encoder/stsb-distilroberta-base", device="cpu")
+    assert model.device.type == "cpu"
+    assert model.model.device.type == "cpu"
+
+    model.to("cuda")
+    assert model.device.type == "cuda"
+    assert model.model.device.type == "cuda"
+
+    del model
+    torch.cuda.empty_cache()
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA must be available to test moving devices effectively.")
+def test_target_device_backwards_compat():
+    model = CrossEncoder("cross-encoder/stsb-distilroberta-base", device="cpu")
+    assert model.device.type == "cpu"
+
+    assert model._target_device.type == "cpu"
+    model._target_device = "cuda"
+    assert model.device.type == "cuda"

`CrossEncoder` is not pushed to cuda until predict is called, even if cuda is specified as device.
Hi, this is more like a question rather than a bug or issue.

When I specify the target device during initialization of any CrossEncoder, the model is not pushed to that device until the `predict` or the `fit` method is called, until then the model is kept in cpu.

```python
from sentence_transformers import CrossEncoder

model2 = CrossEncoder("mixedbread-ai/mxbai-rerank-large-v1", device="cuda:0")
print(model2.model.device)
# cpu
```

I mean I expect the model to be pushed to the specified device during initialization and until I am calling predict it is taking up my system Ram.
Is there any high level reason(s) why this is the case?

I’m not sure of the intention behind this implementation, but I think it’s because the following code within the fit function is where the data is first transferred to the GPU.

https://github.com/UKPLab/sentence-transformers/blob/df6a8e8278b49e7ca01401f46799610106a7b640/sentence_transformers/cross_encoder/CrossEncoder.py#L235

yes, but until I call fit or predict my model is kept in the cpu, which is inconvenient IMO and also takes up the ram.
Hello!

Apologies for the delay.
This was a design decision made by my predecessor, it was also the case for Sentence Transformer models, but it has been updated there (See #2351) as I believe it's better to immediately move the model to the desired device.

I'll fix this when I start updating cross-encoders soon, although I'm also open to a PR much like #2351 in the meantime.

- Tom Aarsen
Hello @tomaarsen!
thanks for the response!

I would like to create the PR to fix this, could you please assign this to me?
Gladly!


diff --git a/sentence_transformers/trainer.py b/sentence_transformers/trainer.py
index 3fd20eb1f..2a7907e3a 100644
--- a/sentence_transformers/trainer.py
+++ b/sentence_transformers/trainer.py
@@ -209,7 +209,7 @@ def __init__(
             "args": args,
             "data_collator": data_collator,
             "train_dataset": train_dataset,
-            "eval_dataset": eval_dataset,
+            "eval_dataset": eval_dataset if eval_dataset is not None or evaluator is None else "dummy",
             "model_init": model_init,
             "compute_metrics": compute_metrics,
             "callbacks": callbacks,
@@ -222,6 +222,12 @@ def __init__(
         else:
             super_kwargs["tokenizer"] = tokenizer
         super().__init__(**super_kwargs)
+        # Transformers v4.46.0 introduced a ValueError if `eval_dataset` is None while eval_strategy is not "no",
+        # but in Sentence Transformers you can also evaluate without an eval_dataset via an evaluator, so we set
+        # it to "dummy" in that case to avoid the ValueError
+        if self.eval_dataset == "dummy":
+            self.eval_dataset = None
+
         # Every Sentence Transformer model can always return a loss, so we set this to True
         # to avoid having to specify it in the data collator or model's forward
         self.can_return_loss = True

diff --git a/tests/test_trainer.py b/tests/test_trainer.py
index 64fbf827e..e7e9827d5 100644
--- a/tests/test_trainer.py
+++ b/tests/test_trainer.py
@@ -8,8 +8,10 @@

 import pytest
 import torch
+from datasets.dataset_dict import DatasetDict

 from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
+from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator
 from sentence_transformers.training_args import SentenceTransformerTrainingArguments
 from sentence_transformers.util import is_datasets_available, is_training_available
 from tests.utils import SafeTemporaryDirectory
@@ -230,3 +232,47 @@ def test_trainer(
         original_embeddings = original_model.encode("The cat is on the mat.", convert_to_tensor=True)
         new_embeddings = model.encode("The cat is on the the mat.", convert_to_tensor=True)
         assert not torch.equal(original_embeddings, new_embeddings)
+
+
+@pytest.mark.parametrize("use_eval_dataset", [True, False])
+@pytest.mark.parametrize("use_evaluator", [True, False])
+def test_trainer_no_eval_dataset_with_eval_strategy(
+    stsb_bert_tiny_model: SentenceTransformer,
+    stsb_dataset_dict: DatasetDict,
+    use_eval_dataset: bool,
+    use_evaluator: bool,
+    tmp_path: Path,
+) -> None:
+    # Expect a crash when `args.eval_strategy` is not "no" but neither `eval_dataset` or `evaluator` is provided
+    # Otherwise, the trainer should be created without any issues
+    model = stsb_bert_tiny_model
+    train_dataset = stsb_dataset_dict["train"].select(range(10))
+    eval_dataset = stsb_dataset_dict["validation"].select(range(10))
+    evaluator = EmbeddingSimilarityEvaluator(
+        sentences1=eval_dataset["sentence1"],
+        sentences2=eval_dataset["sentence2"],
+        scores=[score / 5 for score in eval_dataset["score"]],
+        name="stsb-validation",
+    )
+    loss = losses.CosineSimilarityLoss(model=model)
+    args = SentenceTransformerTrainingArguments(output_dir=tmp_path, eval_strategy="steps")
+
+    kwargs = {}
+    if use_eval_dataset:
+        kwargs["eval_dataset"] = eval_dataset
+    if use_evaluator:
+        kwargs["evaluator"] = evaluator
+
+    if not use_eval_dataset and not use_evaluator:
+        context = pytest.raises(ValueError, match=".*`args.eval_strategy`.*")
+    else:
+        context = nullcontext()
+
+    with context:
+        SentenceTransformerTrainer(
+            model=model,
+            args=args,
+            train_dataset=train_dataset,
+            loss=loss,
+            **kwargs,
+        )

`eval_dataset` or `evaluator`?
The docs say

```
You can use both an eval_dataset and an evaluator, one or the other, or neither. They evaluate based on the eval_strategy and eval_steps [Training Arguments](https://www.sbert.net/docs/sentence_transformer/training_overview.html#training-arguments).
```

But when I omit `eval_dataset` and only pass an `evaluator` into my args I get this from `SentenceTransformerTrainer`:

```
File ~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:419, in Trainer.__init__(self, model, args, data_collator, train_dataset, eval_dataset, processing_class, model_init, compute_loss_func, compute_metrics, callbacks, optimizers, preprocess_logits_for_metrics)
    [413](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:413)         raise ValueError(
    [414](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:414)             "When using `batch_eval_metrics`, your `compute_metrics` function must take a `compute_result`"
    [415](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:415)             " boolean argument which will be triggered after the last batch of the eval set to signal that the"
    [416](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:416)             " summary statistics should be returned by the function."
    [417](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:417)         )
    [418](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:418) if args.eval_strategy is not None and args.eval_strategy != "no" and eval_dataset is None:
--> [419](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:419)     raise ValueError(
    [420](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:420)         f"You have set `args.eval_strategy` to {args.eval_strategy} but you didn't pass an `eval_dataset` to `Trainer`. Either set `args.eval_strategy` to `no` or pass an `eval_dataset`. "
    [421](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:421)     )
    [422](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:422) self.args = args
    [423](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:423) self.compute_loss_func = compute_loss_func

ValueError: You have set `args.eval_strategy` to IntervalStrategy.STEPS but you didn't pass an `eval_dataset` to `Trainer`. Either set `args.eval_strategy` to `no` or pass an `eval_dataset`.
```

My versions are

```
    "sentence-transformers>=3.2.1",
    "transformers[torch]>=4.46.1",
```

Downgrading to `transformers==4.44.0` seems to have fixed
Hello!

Thanks for reporting - this is indeed likely an issue with the newest `transformers` version(s). I'll try and chase it down and include a fix in the next release. I'll keep you in the loop.

- Tom Aarsen
The original PR that introduced this problem: https://github.com/huggingface/transformers/pull/33743


diff --git a/pypdf/_writer.py b/pypdf/_writer.py
index 08f06f33e..a296caee4 100644
--- a/pypdf/_writer.py
+++ b/pypdf/_writer.py
@@ -3000,6 +3000,8 @@ def _insert_filtered_annotations(
                             outlist.append(self._add_object(anc))
             else:
                 d = cast("DictionaryObject", ano["/A"])["/D"]
+                if isinstance(d, NullObject):
+                    continue
                 if isinstance(d, str):
                     # it is a named dest
                     if str(d) in self.get_named_dest_root():

diff --git a/tests/test_writer.py b/tests/test_writer.py
index b6df2da05..c10782262 100644
--- a/tests/test_writer.py
+++ b/tests/test_writer.py
@@ -2484,6 +2484,16 @@ def test_append_pdf_with_dest_without_page(caplog):
     assert len(writer.named_destinations) == 3

+@pytest.mark.enable_socket
+def test_destination_is_nullobject():
+    """Tests for #2958"""
+    url = "https://github.com/user-attachments/files/17822279/C0.00.-.COVER.SHEET.pdf"
+    name = "iss2958.pdf"
+    source_data = BytesIO(get_data_from_url(url, name=name))
+    writer = PdfWriter()
+    writer.append(source_data)
+
+
 @pytest.mark.enable_socket
 def test_destination_page_is_none():
     """Tests for #2963"""

PdfWriter().append throwing 'NullObject' object is not subscriptable for a specific PDF file
I am using pypdf to merge PDF files. I haven't had an issue, until I hit a specific PDF file. Every other PDF file (even ones from the same "bundle" [plan set]). I just have no idea why this PDF file is having an issue with the Writer, or if there is any sort of workaround/front end adjustment I can make to the PDF file.

## Environment

```bash
$ python -m platform
Windows-11-10.0.22631-SP0

$ python -c "import pypdf;print(pypdf._debug_versions)"
pypdf==5.1.0, crypt_provider=('local_crypt_fallback', '0.0.0'), PIL=none
```

## Code + PDF

This is a minimal, complete example that shows the issue:

```python
import glob
import pypdf

fromp = '\\\\UNC\\Path\\*.pdf' #actual path exists, other PDF files merge fine

merger = pypdf.PdfWriter()

paths = glob.glob(fromp)
paths.sort()

for pdf in paths:
    merger.append(pdf) #Get error here on the problem PDF file
```

File link below:
[C0.00 - COVER SHEET.pdf](https://github.com/user-attachments/files/17822279/C0.00.-.COVER.SHEET.pdf)

## Traceback

This is the complete traceback I see:

```
Traceback (most recent call last):
  File "<stdin>", line 2, in <module>
  File "C:\Users\ethanwade\AppData\Roaming\Python\Python312\site-packages\pypdf\_writer.py", line 2638, in append
    self.merge(
  File "C:\Users\ethanwade\AppData\Roaming\Python\Python312\site-packages\pypdf\_writer.py", line 2796, in merge
    lst = self._insert_filtered_annotations(
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\ethanwade\AppData\Roaming\Python\Python312\site-packages\pypdf\_writer.py", line 2994, in _insert_filtered_annotations
    p = self._get_cloned_page(d[0], pages, reader)
                              ~^^^
TypeError: 'NullObject' object is not subscriptable
```

Thanks for the report. This specific PDF declares a destination of `null`, which does not look right:

```
58 0 obj
<<
/Border [0 0 0]
/A <<
/Type /Action
/S /GoTo
/D null
>>
/NM (AJNXWZZHGGPOHNQA)
/Rect [1995 2679 2007 2706]
/Subtype /Link
>>
endobj
```

Inserting

```python
                if isinstance(d, NullObject):
                    continue
```

after https://github.com/py-pdf/pypdf/blob/bd5b962e81c7cd9ad29aa7ff7dedb4197326ebbb/pypdf/_writer.py#L3002 seems to fix this.


diff --git a/pypdf/_cmap.py b/pypdf/_cmap.py
index e8e23f9ab..54c54436e 100644
--- a/pypdf/_cmap.py
+++ b/pypdf/_cmap.py
@@ -527,6 +527,8 @@ def _type1_alternative(
                         v = chr(int(words[2][4:], 16))
                     except ValueError:  # pragma: no cover
                         continue
+                else:
+                    continue
             map_dict[chr(i)] = v
             int_entry.append(i)
     return map_dict, int_entry

diff --git a/tests/test_cmap.py b/tests/test_cmap.py
index 2a83bc3b5..07a778520 100644
--- a/tests/test_cmap.py
+++ b/tests/test_cmap.py
@@ -259,3 +259,13 @@ def test_too_many_differences():
     name = "iss2836.pdf"
     reader = PdfReader(BytesIO(get_data_from_url(url, name=name)))
     assert reader.pages[0].extract_text() == ""
+
+
+@pytest.mark.enable_socket
+def test_iss2925():
+    url = (
+        "https://github.com/user-attachments/files/17621508/2305.09315.pdf"
+    )
+    name = "iss2925.pdf"
+    reader = PdfReader(BytesIO(get_data_from_url(url, name=name)))
+    assert "slicing on the PDG to extract the relevant contextual" in reader.pages[3].extract_text()

Undefined variable in text extraction with version 5.1.0
Our CI-pipelines run latest version of `pypdf` in some of our example notebooks.
These fail with the following exception with version `5.1.0`

## Environment

Github runner - `ubuntu-latest`

## Code + PDF

This is a minimal, complete example that shows the issue:

```python
import requests
from pdf2image import convert_from_path
from pypdf import PdfReader

def download_pdf(url):
    response = requests.get(url)
    if response.status_code == 200:
        return BytesIO(response.content)
    else:
        raise Exception(f"Failed to download PDF: Status code {response.status_code}")

def get_pdf_images(pdf_url):
    # Download the PDF
    pdf_file = download_pdf(pdf_url)
    # Save the PDF temporarily to disk (pdf2image requires a file path)
    temp_file = "temp.pdf"
    with open(temp_file, "wb") as f:
        f.write(pdf_file.read())
    reader = PdfReader(temp_file)
    page_texts = []
    for page_number in range(len(reader.pages)):
        page = reader.pages[page_number]
        text = page.extract_text()
        page_texts.append(text)
    images = convert_from_path(temp_file)
    assert len(images) == len(page_texts)
    return (images, page_texts)

sample_pdfs = [
    {
        "title": "ConocoPhillips Sustainability Highlights - Nature (24-0976)",
        "url": "https://static.conocophillips.com/files/resources/24-0976-sustainability-highlights_nature.pdf",
    },
    {
        "title": "ConocoPhillips Managing Climate Related Risks",
        "url": "https://static.conocophillips.com/files/resources/conocophillips-2023-managing-climate-related-risks.pdf",
    },
    {
        "title": "ConocoPhillips 2023 Sustainability Report",
        "url": "https://static.conocophillips.com/files/resources/conocophillips-2023-sustainability-report.pdf",
    },
]

for pdf in sample_pdfs:
    page_images, page_texts = get_pdf_images(pdf["url"])
    pdf["images"] = page_images
    pdf["texts"] = page_texts
```

Pdfs in links above, but also applies to several others.

## Traceback

This is the complete traceback I see:

```python
UnboundLocalError                         Traceback (most recent call last)
Cell In[8], line 2
      1 for pdf in sample_pdfs:
----> 2     page_images, page_texts = get_pdf_images(pdf["url"])
      3     pdf["images"] = page_images
      4     pdf["texts"] = page_texts

Cell In[6], line 24, in get_pdf_images(pdf_url)
     22 for page_number in range(len(reader.pages)):
     23     page = reader.pages[page_number]
---> 24     text = page.extract_text()
     25     page_texts.append(text)
     26 images = convert_from_path("temp.pdf")

File ~/work/pyvespa/pyvespa/.venv/lib/python3.10/site-packages/pypdf/_page.py:2393, in PageObject.extract_text(self, orientations, space_width, visitor_operand_before, visitor_operand_after, visitor_text, extraction_mode, *args, **kwargs)
   2390 if isinstance(orientations, int):
   2391     orientations = (orientations,)
-> 2393 return self._extract_text(
   2394     self,
   2395     self.pdf,
   2396     orientations,
   2397     space_width,
   2398     PG.CONTENTS,
   2399     visitor_operand_before,
   2400     visitor_operand_after,
   2401     visitor_text,
   2402 )

File ~/work/pyvespa/pyvespa/.venv/lib/python3.10/site-packages/pypdf/_page.py:1868, in PageObject._extract_text(self, obj, pdf, orientations, space_width, content_key, visitor_operand_before, visitor_operand_after, visitor_text)
   1866 if "/Font" in resources_dict:
   1867     for f in cast(DictionaryObject, resources_dict["/Font"]):
-> 1868         cmaps[f] = build_char_map(f, space_width, obj)
   1869 cmap: Tuple[
   1870     Union[str, Dict[int, str]], Dict[str, str], str, Optional[DictionaryObject]
   1871 ] = (
   (...)
   1875     None,
   1876 )  # (encoding,CMAP,font resource name,dictionary-object of font)
   1877 try:

File ~/work/pyvespa/pyvespa/.venv/lib/python3.10/site-packages/pypdf/_cmap.py:33, in build_char_map(font_name, space_width, obj)
     19 """
     20 Determine information about a font.
     21
   (...)
     30
     31 """
     32 ft: DictionaryObject = obj["/Resources"]["/Font"][font_name]  # type: ignore
---> 33 font_subtype, font_halfspace, font_encoding, font_map = build_char_map_from_dict(
     34     space_width, ft
     35 )
     36 return font_subtype, font_halfspace, font_encoding, font_map, ft

File ~/work/pyvespa/pyvespa/.venv/lib/python3.10/site-packages/pypdf/_cmap.py:56, in build_char_map_from_dict(space_width, ft)
     42 """
     43 Determine information about a font.
     44
   (...)
     53
     54 """
     55 font_type = cast(str, ft["/Subtype"].get_object())
---> 56 encoding, map_dict = get_encoding(ft)
     58 space_key_char = get_actual_str_key(" ", encoding, map_dict)
     59 font_width_map = build_font_width_map(ft, space_width * 2.0)

File ~/work/pyvespa/pyvespa/.venv/lib/python3.10/site-packages/pypdf/_cmap.py:129, in get_encoding(ft)
    125 def get_encoding(
    126     ft: DictionaryObject
    127 ) -> Tuple[Union[str, Dict[int, str]], Dict[Any, Any]]:
    128     encoding = _parse_encoding(ft)
--> 129     map_dict, int_entry = _parse_to_unicode(ft)
    131     # Apply rule from PDF ref 1.7 §5.9.1, 1st bullet:
    132     #   if cmap not empty encoding should be discarded
    133     #   (here transformed into identity for those characters)
    134     # If encoding is a string it is expected to be an identity translation.
    135     if isinstance(encoding, dict):

File ~/work/pyvespa/pyvespa/.venv/lib/python3.10/site-packages/pypdf/_cmap.py:212, in _parse_to_unicode(ft)
    210 if "/ToUnicode" not in ft:
    211     if ft.get("/Subtype", "") == "/Type1":
--> 212         return _type1_alternative(ft, map_dict, int_entry)
    213     else:
    214         return {}, []

File ~/work/pyvespa/pyvespa/.venv/lib/python3.10/site-packages/pypdf/_cmap.py:530, in _type1_alternative(ft, map_dict, int_entry)
    528                 except ValueError:  # pragma: no cover
    529                     continue
--> 530         map_dict[chr(i)] = v
    531         int_entry.append(i)
    532 return map_dict, int_entry

UnboundLocalError: local variable 'v' referenced before assignment
```

Thanks for the report. Apparently, https://github.com/py-pdf/pypdf/commit/96b46add0d61940f099f40a9676bb8fff300eaa6#diff-92b0af0cd341537aaf4208e79fac6fca9faca0e6021135693779ab5c82f5593dL535-L536 went missing when applying the latest text extraction changes and this case has never been covered by any tests.

Feel free to submit a corresponding PR including a test which would previously fail.