diff --git a/scapy/contrib/automotive/doip.py b/scapy/contrib/automotive/doip.py
index b9d279bcf6b..3fcbdc5dad4 100644
--- a/scapy/contrib/automotive/doip.py
+++ b/scapy/contrib/automotive/doip.py
@@ -239,10 +239,7 @@ def answers(self, other):

     def hashret(self):
         # type: () -> bytes
-        if self.payload_type in [0x8001, 0x8002, 0x8003]:
-            return bytes(self)[:2] + struct.pack(
-                "H", self.target_address ^ self.source_address)
-        return bytes(self)[:2]
+        return bytes(self)[:3]

     def post_build(self, pkt, pay):
         # type: (bytes, bytes) -> bytes

diff --git a/test/contrib/automotive/doip.uts b/test/contrib/automotive/doip.uts
index 9a7d61e3b7d..b5963a0d817 100644
--- a/test/contrib/automotive/doip.uts
+++ b/test/contrib/automotive/doip.uts
@@ -10,6 +10,8 @@

 = Load Contrib Layer

+from test.testsocket import TestSocket, cleanup_testsockets, UnstableSocket
+
 load_contrib("automotive.doip", globals_dict=globals())
 load_contrib("automotive.uds", globals_dict=globals())

@@ -406,6 +408,30 @@ assert pkts[0][DoIP].payload_length == 2
 assert pkts[0][DoIP:2].payload_length == 7
 assert pkts[1][DoIP].payload_length == 103

+= Doip logical addressing
+
+filename = scapy_path("/test/pcaps/doip_functional_request.pcap.gz")
+tx_sock = TestSocket(DoIP)
+rx_sock = TestSocket(DoIP)
+tx_sock.pair(rx_sock)
+
+for pkt in PcapReader(filename):
+    if pkt.haslayer(DoIP):
+        tx_sock.send(pkt[DoIP])
+
+ans, unans = rx_sock.sr(DoIP(bytes(DoIP(payload_type=0x8001, source_address=0xe80, target_address=0xe400) / UDS() / UDS_TP())), multi=True, timeout=0.1, verbose=False)
+
+cleanup_testsockets()
+
+ans.summary()
+if unans:
+    unans.summary()
+
+assert len(ans) == 8
+ans.summary()
+assert len(unans) == 0
+
+
 + DoIP Communication tests

 = Load libraries
diff --git a/test/pcaps/doip_functional_request.pcap.gz b/test/pcaps/doip_functional_request.pcap.gz
new file mode 100644
index 00000000000..c2b9e9cf35f
Binary files /dev/null and b/test/pcaps/doip_functional_request.pcap.gz differ

DoIP functional addressing responses not associated with preceding request
### Brief description

When sending a request over DoIP using functional addressing, the DoIP source addesses of the ECU responses will be different from the target address of the preceding request, as the ECUs respond from their actual address instead of the functional "broadcast address" used in the request. `DoIP.hashret` effectively filters out these "mismatched" packets, so when using `DoIPSocket.sr(...)` with a functional target address, no responses will ever be returned.

### Scapy version

2.6.0

### Python version

3.11

### Operating system

Linux 6.1.21-v8+

### Additional environment information

_No response_

### How to reproduce

```python3
from scapy.contrib.automotive.doip import *
from scapy.contrib.automotive.uds import *

ip="169.254.207.236"
sock = DoIPSocket(ip)

ans, unans = sock.sr(DoIP(payload_type=0x8001, source_address=0xe80, target_address=0xe400) / UDS() / UDS_TP(), multi=True, timeout=2)
```

### Actual result

```console
INFO: Routing activation successful! Target address set to: 0x3107
Begin emission:
Finished sending 1 packets.
.........
Received 9 packets, got 0 answers, remaining 1 packets
```

### Expected result

```console
INFO: Routing activation successful! Target address set to: 0x3107
Begin emission:
Finished sending 1 packets.
.********
Received 9 packets, got 8 answers, remaining 0 packets
```

This result was obtained by monkey-patching DoIP.hashret to ignore the addresses, based on a config setting, analogous to how scapy handles IP broadcast traffic with the `conf.checkIPaddr` setting (see below)

### Related resources

Just for reference, this is the monkey-patch I'm using at the moment to enable me to run functional addressing requests (by disabling the setting whenever I need it):

```python3
from scapy.config import conf
from scapy.contrib.automotive.doip import DoIP

def hashret_patched(self):
    if self.payload_type in [0x8001, 0x8002, 0x8003]:
        if conf.checkDoIPaddr:
            return bytes(self)[:2] + struct.pack(
                "H", self.target_address ^ self.source_address)
        return bytes(self)[:2]
    return bytes(self)[:2]

conf.checkDoIPaddr = True
DoIP.hashret = hashret_patched
```

But there is maybe/probably a better way.

Thanks for this issue.

Could you please reference a section in the DoIP Standard which describes this behaviour?
I couldn't find anything related to this.

Besides this, you can also pin the target_address in the DoIPSocket.__init__ function:
https://github.com/secdev/scapy/blob/c38a5de175be8e59742be473f4fb2dd6edef5503/scapy/contrib/automotive/doip.py#L440

Additionally, you can do the routing activation request yourself and eliminate the automatical setting of the target_address.

Could you please proved a pcap file for analysis?
Hey, thanks for taking a look.

I don't think there's anything in the spec that explicitly describes this behavior, it seems more like emergent behavior of the general principles where functionally addressed diagnostic requests are broadcast, and that responses sent always have the source address set according to the sending ECU. Otherwise (e.g. if the ECU responded with the source address set to the functional address the preceding request was addressed to), the recipient wouldn't be able to associate the individual diagnostic message responses with the ECU sending them, when they receive multiple responses for a single broadcast request, right?

I'm by no means an expert on this, I've just observed this behavior with the ECUs I'm working with, and it makes sense to me that this is how it works. Maybe I'm missing something? I have attached a pcap file that demonstrates the traffic I see when running the snippets I posted above: [doip_functional_request.pcap.zip](https://github.com/user-attachments/files/17330147/doip_functional_request.pcap.zip)

Hi, I’ve checked the DoIP standard regarding the described behaviour. I found this:
![httpswww autosar orgfileadminstandardsR20-11CPAUTOSAR_SWS_DiagnosticOverIP pdf#page40](https://github.com/user-attachments/assets/8f88f146-80a9-4266-88dc-51ca8f776609)

So you are doing a routing activation with SA 0x3107 but than your UDS packet is sent to Address 0xe400.

By strictly following the standard it should not even be possible to send a UDS packet to this TA.

To me, this behaviour looks like an out of specification behaviour. Neither the autosar standard nor the ISO13400 standard specify a broadcast mechanism as you show with your pcap file.

It would be great if you can provide any written reference for this feature.

I guess it must not have been clear from my previous comments, but the setup I have here is that my test device is connected to a gateway (which has address 0x3107). The gateway is connected to a number of ECUs on the vehicle network side.

ISO 13400-2:2019(E) Section 7.8 reads:
> [...] Functional logical addresses are used to address messages to groups of, or all of, the diagnostic application layer entities within a vehicle. [...] **For a DoIP Gateway the reception of a functionally addressed diagnostics message implies a multi- or broadcast on the connected in-vehicle sub-networks.** [...]

Thus, when sending functionally-addressed requests to the gateway, it will, in turn, broadcast them on the vehicle network, and route back any responses from the connected ECUs.


diff --git a/django/core/management/commands/inspectdb.py b/django/core/management/commands/inspectdb.py
index 77605b178f69..58594fb66f80 100644
--- a/django/core/management/commands/inspectdb.py
+++ b/django/core/management/commands/inspectdb.py
@@ -106,9 +106,12 @@ def handle_inspection(self, options):
                         connection.introspection.get_primary_key_columns(
                             cursor, table_name
                         )
+                        or []
                     )
                     primary_key_column = (
-                        primary_key_columns[0] if primary_key_columns else None
+                        primary_key_columns[0]
+                        if len(primary_key_columns) == 1
+                        else None
                     )
                     unique_columns = [
                         c["columns"][0]
@@ -128,6 +131,11 @@ def handle_inspection(self, options):
                 yield ""
                 yield "class %s(models.Model):" % model_name
                 known_models.append(model_name)
+
+                if len(primary_key_columns) > 1:
+                    fields = ", ".join([f"'{col}'" for col in primary_key_columns])
+                    yield f"    pk = models.CompositePrimaryKey({fields})"
+
                 used_column_names = []  # Holds column names used in the table so far
                 column_to_field_name = {}  # Maps column names to names of model fields
                 used_relations = set()  # Holds foreign relations used in the table.
@@ -151,12 +159,6 @@ def handle_inspection(self, options):
                     # Add primary_key and unique, if necessary.
                     if column_name == primary_key_column:
                         extra_params["primary_key"] = True
-                        if len(primary_key_columns) > 1:
-                            comment_notes.append(
-                                "The composite primary key (%s) found, that is not "
-                                "supported. The first column is selected."
-                                % ", ".join(primary_key_columns)
-                            )
                     elif column_name in unique_columns:
                         extra_params["unique"] = True

diff --git a/tests/inspectdb/tests.py b/tests/inspectdb/tests.py
index 1be4efc43051..131bd45ce89e 100644
--- a/tests/inspectdb/tests.py
+++ b/tests/inspectdb/tests.py
@@ -655,11 +655,10 @@ def test_composite_primary_key(self):
             call_command("inspectdb", table_name, stdout=out)
             output = out.getvalue()
             self.assertIn(
-                f"column_1 = models.{field_type}(primary_key=True)  # The composite "
-                f"primary key (column_1, column_2) found, that is not supported. The "
-                f"first column is selected.",
+                "pk = models.CompositePrimaryKey('column_1', 'column_2')",
                 output,
             )
+            self.assertIn(f"column_1 = models.{field_type}()", output)
             self.assertIn(
                 "column_2 = models.%s()"
                 % connection.features.introspected_field_types["IntegerField"],

Add support for CompositePrimaryKey in inspectdb
Description

We can now replace the half-measure created when introspecting a composite primary key with the actual field.
PR


diff --git a/django/utils/html.py b/django/utils/html.py
index bc336d88a66c..0d107a0da9fe 100644
--- a/django/utils/html.py
+++ b/django/utils/html.py
@@ -357,6 +357,8 @@ def handle_word(
                     domain = punycode(domain)
                 except UnicodeError:
                     return word
+                local = quote(local, safe="")
+                domain = quote(domain, safe="")
                 url = self.mailto_template.format(local=local, domain=domain)
                 nofollow_attr = ""
             # Make link.

diff --git a/tests/utils_tests/test_html.py b/tests/utils_tests/test_html.py
index dc3768e6fae0..0beaf98bff2b 100644
--- a/tests/utils_tests/test_html.py
+++ b/tests/utils_tests/test_html.py
@@ -376,6 +376,19 @@ def test_urlize(self):
                 + "한.글." * 15
                 + "aaa",
             ),
+            (
+                # RFC 6068 requires a mailto URI to percent-encode a number of
+                # characters that can appear in <addr-spec>.
+                "yes;this=is&a%valid!email@example.com",
+                'yes;this=is&a%valid!email@example.com",
+            ),
+            (
+                # Urlizer shouldn't urlize the "?org" part of this. But since
+                # it does, RFC 6068 requires percent encoding the "?".
+                "test@example.com?org",
+                'test@example.com?org',
+            ),
         )
         for value, output in tests:
             with self.subTest(value=value):

Urlize incorrectly handles punctuation in email addresses
Description

Several punctuation characters (%, &, +, !, etc.) can—and sometimes do—appear in the local part of an email address (before the @). The urlize template filter doesn't correctly encode them, which can result in broken mailto links.
Example (Django 5.1.4):
from django.template.defaultfilters import urlize
urlize("it%dept@example.org")
# 'it%dept@example.org'
# Expected:
# 'it%dept@example.org'
Clicking the resulting mailto link might work as expected, or do nothing, or could launch a mail composer with a missing or incorrect email address, depending on the specific address, browser and email app. Sequences that could also be percent-encoded characters (like "%de" in the example) are especially prone to unpredictable results.
The mailto URI spec RFC 6068 requires percent encoding most punctuation in this situation (section 2, item 1; also see section 5, Encoding).
Proposed fix: apply urllib.parse.quote() to local where the mailto link is constructed in django.utils.html.Urlizer. (Although not strictly necessary, it wouldn't hurt to also quote domain there.)


diff --git a/django/core/validators.py b/django/core/validators.py
index 8732ddf7adbf..c4e734c1d82a 100644
--- a/django/core/validators.py
+++ b/django/core/validators.py
@@ -2,7 +2,7 @@
 import math
 import re
 from pathlib import Path
-from urllib.parse import urlsplit, urlunsplit
+from urllib.parse import urlsplit

 from django.core.exceptions import ValidationError
 from django.utils.deconstruct import deconstructible
@@ -128,8 +128,6 @@ def __call__(self, value):

 @deconstructible
 class URLValidator(RegexValidator):
-    ul = "\u00a1-\uffff"  # Unicode letters range (must not be a raw string).
-
     # IP patterns
     ipv4_re = (
         r"(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)"
@@ -177,31 +175,17 @@ def __call__(self, value):
             splitted_url = urlsplit(value)
         except ValueError:
             raise ValidationError(self.message, code=self.code, params={"value": value})
-        try:
-            super().__call__(value)
-        except ValidationError as e:
-            # Trivial case failed. Try for possible IDN domain
-            if value:
-                scheme, netloc, path, query, fragment = splitted_url
-                try:
-                    netloc = punycode(netloc)  # IDN -> ACE
-                except UnicodeError:  # invalid domain part
-                    raise e
-                url = urlunsplit((scheme, netloc, path, query, fragment))
-                super().__call__(url)
-            else:
-                raise
-        else:
-            # Now verify IPv6 in the netloc part
-            host_match = re.search(r"^\[(.+)\](?::[0-9]{1,5})?$", splitted_url.netloc)
-            if host_match:
-                potential_ip = host_match[1]
-                try:
-                    validate_ipv6_address(potential_ip)
-                except ValidationError:
-                    raise ValidationError(
-                        self.message, code=self.code, params={"value": value}
-                    )
+        super().__call__(value)
+        # Now verify IPv6 in the netloc part
+        host_match = re.search(r"^\[(.+)\](?::[0-9]{1,5})?$", splitted_url.netloc)
+        if host_match:
+            potential_ip = host_match[1]
+            try:
+                validate_ipv6_address(potential_ip)
+            except ValidationError:
+                raise ValidationError(
+                    self.message, code=self.code, params={"value": value}
+                )

         # The maximum length of a full host name is 253 characters per RFC 1034
         # section 3.1. It's defined to be 255 bytes or less, but this includes

diff --git a/tests/validators/tests.py b/tests/validators/tests.py
index 4ae0f6413e5e..7455c93d407e 100644
--- a/tests/validators/tests.py
+++ b/tests/validators/tests.py
@@ -126,6 +126,7 @@
     "http://مثال.إختبار",
     "http://例子.测试",
     "http://उदाहरण.परीक्षा",
+    "https://މިހާރު.com",  # (valid in IDNA 2008 but not IDNA 2003)
     "http://-.~_!$&'()*+,;=%40:80%2f@example.com",
     "http://xn--7sbb4ac0ad0be6cf.xn--p1ai",
     "http://1337.net",

Dead code in URLValidator
Description

There's some dead code in django.core.validators.URLValidator which could be removed:
The entire Trivial case failed. Try for possible IDN domain section is no longer useful. This code attempts to re-validate a failed URL after encoding an international domain name using IDNA 2003 ("punycode"). But the URLValidator regular expressions have allowed all IDNs since Commit 2e65d56 (for #20003, in 2015), so the super call will never fail with a validation error that switching to IDNA 2003 would let pass.
The `ul` unicode letters property is no longer used. The regular expressions that had used it were moved into DomainNameValidator in Commit 4971a9a (for #18119, in Django 5.1).
For the first case, one way to verify the code is no longer in use is to run URLValidator on https://މިހާރު.com, which is a domain allowed by IDNA 2008 but prohibited by IDNA 2003. If the punycode() branch were coming into play, that URL would be rejected:
from django.core.validators import URLValidator
URLValidator()("https://މިހާރު.com")
# (No error)
from django.utils.encoding import punycode
punycode("މިހާރު.com")
# UnicodeError: Violation of BIDI requirement 3
# encoding with 'idna' codec failed


diff --git a/django/db/backends/postgresql/schema.py b/django/db/backends/postgresql/schema.py
index 75bf33147233..964009988cbb 100644
--- a/django/db/backends/postgresql/schema.py
+++ b/django/db/backends/postgresql/schema.py
@@ -120,6 +120,8 @@ def _create_like_index_sql(self, model, field):
         return None

     def _using_sql(self, new_field, old_field):
+        if new_field.generated:
+            return ""
         using_sql = " USING %(column)s::%(type)s"
         new_internal_type = new_field.get_internal_type()
         old_internal_type = old_field.get_internal_type()

diff --git a/tests/migrations/test_operations.py b/tests/migrations/test_operations.py
index 6312a7d4a2e1..5557426e274f 100644
--- a/tests/migrations/test_operations.py
+++ b/tests/migrations/test_operations.py
@@ -6210,6 +6210,37 @@ def _test_add_generated_field(self, db_persist):
             operation.database_backwards(app_label, editor, new_state, project_state)
         self.assertColumnNotExists(f"{app_label}_pony", "modified_pink")

+    @skipUnlessDBFeature("supports_stored_generated_columns")
+    def test_generated_field_changes_output_field(self):
+        app_label = "test_gfcof"
+        operation = migrations.AddField(
+            "Pony",
+            "modified_pink",
+            models.GeneratedField(
+                expression=F("pink") + F("pink"),
+                output_field=models.IntegerField(),
+                db_persist=True,
+            ),
+        )
+        from_state, to_state = self.make_test_state(app_label, operation)
+        # Add generated column.
+        with connection.schema_editor() as editor:
+            operation.database_forwards(app_label, editor, from_state, to_state)
+        # Update output_field used in the generated field.
+        operation = migrations.AlterField(
+            "Pony",
+            "modified_pink",
+            models.GeneratedField(
+                expression=F("pink") + F("pink"),
+                output_field=models.DecimalField(decimal_places=2, max_digits=16),
+                db_persist=True,
+            ),
+        )
+        from_state = to_state.clone()
+        to_state = self.apply_operations(app_label, from_state, [operation])
+        with connection.schema_editor() as editor:
+            operation.database_forwards(app_label, editor, from_state, to_state)
+
     @skipUnlessDBFeature("supports_stored_generated_columns")
     def test_add_generated_field_stored(self):
         self._test_add_generated_field(db_persist=True)

Changing output_field for GeneratedField leads to ProgrammingError with Postgres 16.5+
Description

Consider the following model and assume the initial migration has been applied:
class Order(models.Model):
	order_no = models.IntegerField()
	item_no = models.CharField(max_length=25)
	qty = models.IntegerField()
	cost = models.DecimalField(max_digits=10, decimal_places=2)
	total_cost = models.GeneratedField(
		expression=F("cost") * F("qty"),
		output_field=models.BigIntegerField(),
		db_persist=True,
	)
During a code review we determined the output field should be a Decimal field and the field was modified as follows:
total_cost = models.GeneratedField(
	expression=F("cost") * F("qty"),
	output_field=models.DecimalField(decimal_places=2, max_digits=16),
	db_persist=True,
)
And a new migration was generated:
migrations.AlterField(
	model_name='order',
	name='total_cost',
	field=models.GeneratedField(db_persist=True, expression=django.db.models.expressions.CombinedExpression(models.F('cost'), '*', models.F('qty')), output_field=models.DecimalField(decimal_places=2, max_digits=16)),
),
In Postgres 16.4 and earlier, this migration is applied without error. (I'm aware that the value of the total_cost field is not recomputed for existing records when this migration is applied.).
Starting with Postgres 16.5 and up, this migration fails with the following error:
psycopg2.errors.InvalidColumnDefinition: cannot specify USING when altering type of generated column
DETAIL: Column "total_cost" is a generated column.
...
django.db.utils.ProgrammingError: cannot specify USING when altering type of generated column
This appears to be a result of the following change in Postgres 16.5 (release notes):
Disallow a USING clause when altering the type of a generated column (Peter Eisentraut) [§](https://postgr.es/c/5867ee005)
A generated column already has an expression specifying the column contents, so including USING doesn't make sense.
The Django documentation for GeneratedField makes it clear that
There are many database-specific restrictions on generated fields that Django doesn’t validate and the database may raise an error
For this reason, I almost didn't open a ticket. However, there is logic in db/backends/base/schema.py that checks if the expression of a GeneratedField changed. Consider this migration (which changes the expression from multiplication to addition):
migrations.AlterField(
	model_name='order',
	name='total_cost',
	field=models.GeneratedField(db_persist=True, expression=django.db.models.expressions.CombinedExpression(models.F('cost'), '+', models.F('qty')), output_field=models.BigIntegerField()),
),
Attempting to apply this migration will raise the following error (even in Postgres 16.4):
ValueError: Modifying GeneratedFields is not supported - the field sales.Order.total_cost must be removed and re-added with the new definition.
This error is more helpful. It explains the problem better and even suggests a workaround.
Should we throw a similar error if the output_field of a GeneratedField is changed? Or add a tip to the documentation?
The above was tested with:
Django version 5.1.3
psycopg2 version 2.9.10
Postgres versions: 16.4, 16.5, 16.6, 17.0, and 17.2

['Thank you for the detailed report Replicated the error tests/migrations/test_operations.py diff --git a/tests/migrations/test_operations.py b/tests/migrations/test_operations.py index 6312a7d4a2..1b75c609b3 100644 a b class OperationTests(OperationTestBase): 61856185 with self.assertRaisesMessage(ValueError, msg): 61866186 self.apply_operations(app_label, project_state, operations) 61876187 6188 @skipUnlessDBFeature("supports_stored_generated_columns") 6189 def test_generated_field_changes_output_field(self): 6190 app_label = "test_gfcof" 6191 operation = migrations.AddField( 6192 "Pony", 6193 "modified_pink", 6194 models.GeneratedField( 6195 expression=F("pink") + F("pink"), 6196 output_field=models.IntegerField(), 6197 db_persist=True, 6198 ), 6199 ) 6200 project_state, new_state = self.make_test_state(app_label, operation) 6201 # Add generated column. 6202 with connection.schema_editor() as editor: 6203 operation.database_forwards(app_label, editor, project_state, new_state) 6204 # Update output_field used in the generated field. 6205 operations = [ 6206 migrations.AlterField( 6207 "Pony", 6208 "modified_pink", 6209 models.GeneratedField( 6210 expression=F("pink") + F("pink"), 6211 output_field=models.DecimalField(decimal_places=2, max_digits=16), 6212 db_persist=True, 6213 ), 6214 ), 6215 ] 6216 new_state = self.apply_operations(app_label, new_state, operations) 6217 with connection.schema_editor() as editor: 6218 operation.database_forwards(app_label, editor, project_state, new_state) 6219 61886220 def _test_add_generated_field(self, db_persist): 61896221 app_label = "test_agf" ValueError: Modifying GeneratedFields is not supported - the field sales.Order.total_cost must be removed and re-added with the new definition. This error is more helpful. It explains the problem better and even suggests a workaround. Having this error instead makes sense to me. CC-ed a couple of other folks who might have thoughts', 1733281732.0]
['Just to make sure before we commit to a solution. Does Posgres still allows for generated field alterations but disallow that USING is not used? If that\'s the case it seems like a better solution would be to have generated field alteration continue to be supported simply not specify USING (which is effectively redundant with the column type) instead of disabling the feature? django/db/backends/postgresql/schema.py diff --git a/django/db/backends/postgresql/schema.py b/django/db/backends/postgresql/schema.py index 75bf331472..964009988c 100644 a b def _create_like_index_sql(self, model, field): 120120 return None 121121 122122 def _using_sql(self, new_field, old_field): 123 if new_field.generated: 124 return "" 123125 using_sql = " USING %(column)s::%(type)s" 124126 new_internal_type = new_field.get_internal_type() 125127 old_internal_type = old_field.get_internal_type(', 1733308733.0]
["Replying to Simon Charette: Just to make sure before we commit to a solution. Does Posgres still allows for generated field alterations but disallow that USING is not used? If that's the case it seems like a better solution would be to have generated field alteration continue to be supported simply not specify USING (which is effectively redundant with the column type) instead of disabling the feature? I found that I can rename the GeneratedField without any issues. Renaming a field within the expression throws the ValueError: ValueError: Modifying GeneratedFields is not supported - the field sales.Order.total_cost must be removed and re-added with the new definition. This error appears to be raised by Django, so it's not even getting to Postgres.", 1733309553.0]
['It seems to be latter as the above patch works with adjusted tests provided by Sarah tests/migrations/test_operations.py diff --git a/tests/migrations/test_operations.py b/tests/migrations/test_operations.py index 3ac813b899..7aa3bfbc5d 100644 a b def _test_add_generated_field(self, db_persist): 61356135 operation.database_backwards(app_label, editor, new_state, project_state) 61366136 self.assertColumnNotExists(f"{app_label}_pony", "modified_pink") 61376137 6138 @skipUnlessDBFeature("supports_stored_generated_columns") 6139 def test_generated_field_changes_output_field(self): 6140 app_label = "test_gfcof" 6141 operation = migrations.AddField( 6142 "Pony", 6143 "modified_pink", 6144 models.GeneratedField( 6145 expression=F("pink") + F("pink"), 6146 output_field=models.IntegerField(), 6147 db_persist=True, 6148 ), 6149 ) 6150 from_state, to_state = self.make_test_state(app_label, operation) 6151 # Add generated column. 6152 with connection.schema_editor() as editor: 6153 operation.database_forwards(app_label, editor, from_state, to_state) 6154 # Update output_field used in the generated field. 6155 operation = migrations.AlterField( 6156 "Pony", 6157 "modified_pink", 6158 models.GeneratedField( 6159 expression=F("pink") + F("pink"), 6160 output_field=models.DecimalField(decimal_places=2, max_digits=16), 6161 db_persist=True, 6162 ), 6163 ) 6164 from_state = to_state.clone() 6165 to_state = self.apply_operations(app_label, from_state, [operation]) 6166 with connection.schema_editor() as editor: 6167 operation.database_forwards(app_label, editor, from_state, to_state) 6168 61386169 @skipUnlessDBFeature("supports_stored_generated_columns") 61396170 def test_add_generated_field_stored(self): 61406171 self._test_add_generated_field(db_persist=True)', 1733309845.0]
['In summary what Postgres 16.5 changed is that you longer can specify USING on ALTER COLUMN for generated columns ALTER TABLE "test_gfcof_pony" ALTER COLUMN "modified_pink" TYPE numeric(16, 2) USING "modified_pink"::numeric(16, 2); Which makes sense as USING is redundant with TYPE and could result in further ambiguity more if you change the generated expression at the same time. Well the solution appears to simply not specifying USING when dealing with GeneratedField alterations as suggested in comment:2 ALTER TABLE "test_gfcof_pony" ALTER COLUMN "modified_pink" TYPE numeric(16, 2);', 1733310855.0]


diff --git a/django/forms/forms.py b/django/forms/forms.py
index 549a3adf6fa6..614f99039585 100644
--- a/django/forms/forms.py
+++ b/django/forms/forms.py
@@ -316,7 +316,7 @@ def full_clean(self):
         """
         Clean all of self.data and populate self._errors and self.cleaned_data.
         """
-        self._errors = ErrorDict()
+        self._errors = ErrorDict(renderer=self.renderer)
         if not self.is_bound:  # Stop further processing.
             return
         self.cleaned_data = {}

diff --git a/tests/forms_tests/tests/test_forms.py b/tests/forms_tests/tests/test_forms.py
index cd909628cb03..d88ac33f24f0 100644
--- a/tests/forms_tests/tests/test_forms.py
+++ b/tests/forms_tests/tests/test_forms.py
@@ -5313,6 +5313,22 @@ class CommentForm(Form):
             "required>
",
         )

+    def test_custom_renderer_error_dict(self):
+        class CustomRenderer(DjangoTemplates):
+            def render(self, template_name, context, request=None):
+                if template_name == "django/forms/errors/dict/default.html":
+                    return "So many errors!"
+                return super().render(template_name, context, request)
+
+        form = Form({}, renderer=CustomRenderer())
+        form.full_clean()
+        form.add_error(None, "Test error")
+
+        self.assertHTMLEqual(
+            form.errors.render(),
+            "So many errors!",
+        )
+
     def test_cyclic_context_boundfield_render(self):
         class FirstNameForm(Form):
             first_name = CharField()

ErrorDict always uses default renderer
Description

When BaseForm.full_clean() instantiates ErrorDict, it doesn't pass it the renderer:
https://github.com/django/django/blob/1860a1afc9ac20750f932e8e0a94b32d096f2848/django/forms/forms.py#L319
Despite ErrorDict being ready to receive it and use it for rendering:
https://github.com/django/django/blob/1860a1afc9ac20750f932e8e0a94b32d096f2848/django/forms/utils.py#L124
Practically, this means customizations to the renderer are ignored when rendering the form errors using {{ errors }} in the template.
Building on top of the example and fix from #35987, I have a custom renderer that swaps out some templates:
from django import forms
from django.forms.renderers import TemplatesSetting
from django.forms.utils import ErrorList
from django.template.exceptions import TemplateDoesNotExist
class CustomRenderer(TemplatesSetting):
	def get_template(self, template_name):
		if template_name.startswith("django/forms/"):
			# Load our custom version from "custom/forms/" if it exists
			our_template = f"custom/{template_name.removeprefix('django/')}"
			try:
				return super().get_template(our_template)
			except TemplateDoesNotExist:
				pass
		return super().get_template(template_name)
class CustomErrorList(ErrorList):
	def copy(self):
		# Copying the fix from Django Ticket #35987
		copy = super().copy()
		copy.renderer = self.renderer
		return copy
class MyForm(forms.Form):
	default_renderer = CustomRenderer()
	def __init__(self, *args, error_class=CustomErrorList, **kwargs):
		super().__init__(*args, error_class=error_class, **kwargs)
The custom error list template uses some CSS utility classes from Tailwind, like text-red-600:
{% if errors %}
{% for field, error in errors %}
{{ field }}{{ error }}
{% endfor %}
{% endif %}
But creating a form with a non-field error and rendering the error dict uses the default renderer and its template:
In [1]: from example.forms import MyForm
 ...:
 ...: form = MyForm({})
 ...: form.full_clean()
 ...: form.add_error(None, "Test error")
In [2]: form.errors.render()
Out[2]: '

__all__

Test error

'
I need to override full_clean() to set the renderer:
class MyForm(forms.Form):
	...

	def full_clean(self):
		super().full_clean()
		# Fix a bug in Django where self._errors = ErrorDict is not passed the
		# renderer argument when initialized.
		self._errors.renderer = self.renderer
Then form errors use my custom template:
In [1]: from example.forms import MyForm
 ...:
 ...: form = MyForm({})
 ...: form.full_clean()
 ...: form.add_error(None, "Test error")
In [2]: form.errors.render()
Out[2]: '

__all__

Test error

'
I think this has probably been an issue ever since a custom renderer became possible in #31026. The argument was added to ErrorDict but missed in BaseForm.full_clean(), the only place where the class is instantiated.


diff --git a/django/forms/utils.py b/django/forms/utils.py
index d24711d1a0b5..27eabe57dc58 100644
--- a/django/forms/utils.py
+++ b/django/forms/utils.py
@@ -163,6 +163,7 @@ def as_data(self):
     def copy(self):
         copy = super().copy()
         copy.error_class = self.error_class
+        copy.renderer = self.renderer
         return copy

     def get_json_data(self, escape_html=False):

diff --git a/tests/forms_tests/tests/test_forms.py b/tests/forms_tests/tests/test_forms.py
index cd909628cb03..f93d9b41157f 100644
--- a/tests/forms_tests/tests/test_forms.py
+++ b/tests/forms_tests/tests/test_forms.py
@@ -4849,6 +4849,12 @@ class CustomForm(Form):
         form = CustomForm(renderer=custom)
         self.assertEqual(form.renderer, custom)

+    def test_get_context_errors(self):
+        custom = CustomRenderer()
+        form = Form(renderer=custom)
+        context = form.get_context()
+        self.assertEqual(context["errors"].renderer, custom)
+

 class TemplateTests(SimpleTestCase):
     def test_iterate_radios(self):
diff --git a/tests/forms_tests/tests/test_utils.py b/tests/forms_tests/tests/test_utils.py
index f9a5d4c82a6a..a50f86c934b3 100644
--- a/tests/forms_tests/tests/test_utils.py
+++ b/tests/forms_tests/tests/test_utils.py
@@ -2,6 +2,7 @@
 import json

 from django.core.exceptions import ValidationError
+from django.forms.renderers import DjangoTemplates
 from django.forms.utils import (
     ErrorDict,
     ErrorList,
@@ -161,6 +162,35 @@ def __str__(self):
             'example

',
         )

+    def test_error_list_copy(self):
+        e = ErrorList(
+            [
+                ValidationError(
+                    message="message %(i)s",
+                    params={"i": 1},
+                ),
+                ValidationError(
+                    message="message %(i)s",
+                    params={"i": 2},
+                ),
+            ]
+        )
+
+        e_copy = copy.copy(e)
+        self.assertEqual(e, e_copy)
+        self.assertEqual(e.as_data(), e_copy.as_data())
+
+    def test_error_list_copy_attributes(self):
+        class CustomRenderer(DjangoTemplates):
+            pass
+
+        renderer = CustomRenderer()
+        e = ErrorList(error_class="woopsies", renderer=renderer)
+
+        e_copy = e.copy()
+        self.assertEqual(e.error_class, e_copy.error_class)
+        self.assertEqual(e.renderer, e_copy.renderer)
+
     def test_error_dict_copy(self):
         e = ErrorDict()
         e["__all__"] = ErrorList(
@@ -183,6 +213,16 @@ def test_error_dict_copy(self):
         e_deepcopy = copy.deepcopy(e)
         self.assertEqual(e, e_deepcopy)

+    def test_error_dict_copy_attributes(self):
+        class CustomRenderer(DjangoTemplates):
+            pass
+
+        renderer = CustomRenderer()
+        e = ErrorDict(renderer=renderer)
+
+        e_copy = copy.copy(e)
+        self.assertEqual(e.renderer, e_copy.renderer)
+
     def test_error_dict_html_safe(self):
         e = ErrorDict()
         e["username"] = "Invalid username."

ErrorList.copy() reverts to default renderer
Description

When an ErrorList is copied, it loses track of any custom renderer, reverting to the default one. Practically, this means custom styles are not applied to non-field errors when rendering a whole form.
For example, I wrote a custom renderer that swaps some templates like so:
from django import forms
from django.forms.renderers import TemplatesSetting
from django.template.exceptions import TemplateDoesNotExist
class CustomRenderer(TemplatesSetting):
	def get_template(self, template_name):
		if template_name.startswith("django/forms/"):
			# Load our custom version from "custom/forms/" if it exists
			our_template = f"custom/{template_name.removeprefix('django/')}"
			try:
				return super().get_template(our_template)
			except TemplateDoesNotExist:
				pass
		return super().get_template(template_name)
class MyForm(forms.Form):
	default_renderer = CustomRenderer()
The custom error list template uses some CSS utility classes from Tailwind, like text-red-600:
{% if errors %}
{% for error in errors %}
{{ error }}
{% endfor %}
{% endif %}
Creating a form with a non-field error and rendering those errors uses the custom template:
In [1]: from example.forms import MyForm
 ...:
 ...: form = MyForm({})
 ...: form.full_clean()
 ...: form.add_error(None, "Test error")
In [2]: form.non_field_errors().render()
Out[2]: '

Test error

'
But rendering the whole form reverts to the default template, from the default renderer:
In [3]: form.render()
Out[3]: '

Test error

\n\n

'
This occurs because the ErrorList is copied in Form.get_context():
https://github.com/django/django/blob/1860a1afc9ac20750f932e8e0a94b32d096f2848/django/forms/forms.py#L225
The fix would be to also copy over renderer in ErrorList.copy():
https://github.com/django/django/blob/1860a1afc9ac20750f932e8e0a94b32d096f2848/django/forms/utils.py#L165
I think this has probably been an issue ever since a custom renderer became possible in #31026.


diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py
index 076667d41a8c..23ad424c5c31 100644
--- a/django/core/management/commands/makemessages.py
+++ b/django/core/management/commands/makemessages.py
@@ -40,7 +40,7 @@ def check_programs(*programs):

 def is_valid_locale(locale):
-    return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z].*$", locale)
+    return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z0-9].*$", locale)

 @total_ordering

diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py
index 7aa600c4c1e2..e4a6260c336a 100644
--- a/tests/i18n/test_extraction.py
+++ b/tests/i18n/test_extraction.py
@@ -179,6 +179,15 @@ def test_valid_locale_with_country(self):
         self.assertIn("processing locale en_GB", out.getvalue())
         self.assertIs(Path("locale/en_GB/LC_MESSAGES/django.po").exists(), True)

+    def test_valid_locale_with_numeric_region_code(self):
+        out = StringIO()
+        management.call_command(
+            "makemessages", locale=["ar_002"], stdout=out, verbosity=1
+        )
+        self.assertNotIn("invalid locale ar_002", out.getvalue())
+        self.assertIn("processing locale ar_002", out.getvalue())
+        self.assertIs(Path("locale/ar_002/LC_MESSAGES/django.po").exists(), True)
+
     def test_valid_locale_tachelhit_latin_morocco(self):
         out = StringIO()
         management.call_command(

makemessages: is_valid_locale regex fails to validate locales with numeric region codes
Description

(last modified by Juan Pablo Mallarino)
The is_valid_locale function in the makemessages management command uses a regular expression that doesn't account for locales with numeric region codes, such as es_419. The current regex only allows uppercase letters after the underscore. This excludes valid locale codes that use numbers in the region subtag.
This can cause issues when trying to generate message files for these locales. For example, running makemessages with es_419 will not generate the expected message files, potentially leading to missing translations.
Proposed Solution
Modify the regular expression in is_valid_locale to include numeric characters in the region subtag validation. The suggested change is:
From:
r"^[a-z]+_[A-Z].*$"
To:
r"^[a-z]+_[A-Z0-9].*$"
This change would allow the validation of locales while still maintaining the expected format for other locale codes. This simple modification would ensure broader compatibility and avoid unexpected behavior when working with valid locales containing numeric region codes.

['This is the PR that introduced the validating function: \u200bhttps://github.com/django/django/pull/15521 and the issue https://code.djangoproject.com/ticket/33565', 1733325193.0]
["Support for regions was added in #33078 (see also test case like de-1996) The validation added in #33565 doesn't match this", 1733370439.0]


diff --git a/django/contrib/postgres/fields/array.py b/django/contrib/postgres/fields/array.py
index 4171af82f9d2..a7e40703a3f8 100644
--- a/django/contrib/postgres/fields/array.py
+++ b/django/contrib/postgres/fields/array.py
@@ -169,7 +169,7 @@ def value_to_string(self, obj):
             else:
                 obj = AttributeSetter(base_field.attname, val)
                 values.append(base_field.value_to_string(obj))
-        return json.dumps(values)
+        return json.dumps(values, ensure_ascii=False)

     def get_transform(self, name):
         transform = super().get_transform(name)
diff --git a/django/contrib/postgres/fields/hstore.py b/django/contrib/postgres/fields/hstore.py
index cfc156ab596b..300458c0b118 100644
--- a/django/contrib/postgres/fields/hstore.py
+++ b/django/contrib/postgres/fields/hstore.py
@@ -43,7 +43,7 @@ def to_python(self, value):
         return value

     def value_to_string(self, obj):
-        return json.dumps(self.value_from_object(obj))
+        return json.dumps(self.value_from_object(obj), ensure_ascii=False)

     def formfield(self, **kwargs):
         return super().formfield(

diff --git a/tests/postgres_tests/test_array.py b/tests/postgres_tests/test_array.py
index ea7807687ea2..ba7151d4a2f5 100644
--- a/tests/postgres_tests/test_array.py
+++ b/tests/postgres_tests/test_array.py
@@ -1008,6 +1008,32 @@ def test_loading(self):
         self.assertEqual(instance.field, [1, 2, None])

+class TestStringSerialization(PostgreSQLSimpleTestCase):
+    field_values = [["Django", "Python", None], ["Джанго", "פייתון", None, "król"]]
+
+    @staticmethod
+    def create_json_data(array_field_value):
+        fields = {"field": json.dumps(array_field_value, ensure_ascii=False)}
+        return json.dumps(
+            [{"model": "postgres_tests.chararraymodel", "pk": None, "fields": fields}]
+        )
+
+    def test_encode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                instance = CharArrayModel(field=field_value)
+                data = serializers.serialize("json", [instance])
+                json_data = self.create_json_data(field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))
+
+    def test_decode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                json_data = self.create_json_data(field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+
+
 class TestValidation(PostgreSQLSimpleTestCase):
     def test_unbounded(self):
         field = ArrayField(models.IntegerField())
diff --git a/tests/postgres_tests/test_hstore.py b/tests/postgres_tests/test_hstore.py
index cac3eb742af0..2d19364736be 100644
--- a/tests/postgres_tests/test_hstore.py
+++ b/tests/postgres_tests/test_hstore.py
@@ -297,39 +297,53 @@ class MyModel(PostgreSQLModel):

 class TestSerialization(PostgreSQLSimpleTestCase):
-    test_data = json.dumps(
-        [
-            {
-                "model": "postgres_tests.hstoremodel",
-                "pk": None,
-                "fields": {
-                    "field": json.dumps({"a": "b"}),
-                    "array_field": json.dumps(
-                        [
-                            json.dumps({"a": "b"}),
-                            json.dumps({"b": "a"}),
-                        ]
-                    ),
-                },
-            }
-        ]
-    )
+    field_values = [
+        ({"a": "b"}, [{"a": "b"}, {"b": "a"}]),
+        (
+            {"все": "Трурль и Клапауций"},
+            [{"Трурль": "Клапауций"}, {"Клапауций": "Трурль"}],
+        ),
+    ]
+
+    @staticmethod
+    def create_json_data(field_value, array_field_value):
+        fields = {
+            "field": json.dumps(field_value, ensure_ascii=False),
+            "array_field": json.dumps(
+                [json.dumps(item, ensure_ascii=False) for item in array_field_value],
+                ensure_ascii=False,
+            ),
+        }
+        return json.dumps(
+            [{"model": "postgres_tests.hstoremodel", "pk": None, "fields": fields}]
+        )

     def test_dumping(self):
-        instance = HStoreModel(field={"a": "b"}, array_field=[{"a": "b"}, {"b": "a"}])
-        data = serializers.serialize("json", [instance])
-        self.assertEqual(json.loads(data), json.loads(self.test_data))
+        for field_value, array_field_value in self.field_values:
+            with self.subTest(field_value=field_value, array_value=array_field_value):
+                instance = HStoreModel(field=field_value, array_field=array_field_value)
+                data = serializers.serialize("json", [instance])
+                json_data = self.create_json_data(field_value, array_field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))

     def test_loading(self):
-        instance = list(serializers.deserialize("json", self.test_data))[0].object
-        self.assertEqual(instance.field, {"a": "b"})
-        self.assertEqual(instance.array_field, [{"a": "b"}, {"b": "a"}])
+        for field_value, array_field_value in self.field_values:
+            with self.subTest(field_value=field_value, array_value=array_field_value):
+                json_data = self.create_json_data(field_value, array_field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+                self.assertEqual(instance.array_field, array_field_value)

     def test_roundtrip_with_null(self):
-        instance = HStoreModel(field={"a": "b", "c": None})
-        data = serializers.serialize("json", [instance])
-        new_instance = list(serializers.deserialize("json", data))[0].object
-        self.assertEqual(instance.field, new_instance.field)
+        for field_value in [
+            {"a": "b", "c": None},
+            {"Енеїда": "Ти знаєш, він який суціга", "Зефір": None},
+        ]:
+            with self.subTest(field_value=field_value):
+                instance = HStoreModel(field=field_value)
+                data = serializers.serialize("json", [instance])
+                new_instance = list(serializers.deserialize("json", data))[0].object
+                self.assertEqual(instance.field, new_instance.field)

 class TestValidation(PostgreSQLSimpleTestCase):

Postgresql: ArrayField with Unicode characters gets serialized as string of "\u XXXX" characters
Description

In ArrayField.value_to_string(self, obj) the value is encoded using JSON.dumps(values), which produces escaped Unicode \u XXXX by default.
For example, an ArrayField with 3 elements ["один", "два", "три"] (1,2,3 in Russian) will produce
["\u043e\u0434\u0438\u043d", "\u0434\u0432\u0430", "\u0442\u0440\u0438"]
While this is not a bug per se, this becomes a nuisance when viewing on result of "dumpdata" management command:
The ArrayField fields will be encoded differently from other text fields.
Perhaps there should be an option to turn on/off the ensure_ascii parameter in JSON.dumps(values, ensure_ascii=option)) ?
The option can be enabled by default, as we do for 'hstore' field, or perhaps enabled conditionally:
in the field settings ArrayField(name='numbers', ascii_only=False)
in settings.py ( ARRAY_FIELD_ENSURE_ASCII )
I will be glad to submit a patch.

["Given we made the decision to have JSON serialization default to ensure_ascii=False when dealing with Unicode in #29249 (68fc21b3784aa34c7ba5515ab02ef0c7b6ee856d) I think we should use the same approach here and use ensure_ascii=False for any usage of json.dumps in Field.value_to_string for fields that might include text which includes ArrayField, and HStoreField. I don't think an additional field option to control this behavior and certainly not a setting is warranted here as it should be possible to subclass either field class to override value_to_string and ensure_ascii=False does constitute a more coherent default. Feel free to assign the issue to you and submit a PR with tests for ArrayField and HStoreField.", 1732638340.0]
['\u200bhttps://github.com/django/django/pull/18854', 1732680151.0]
['In ded48546: Fixed #35944 -- Handled serialization of Unicode values in ArrayField and HStoreField.', 1733387518.0]


diff --git a/django/template/loader_tags.py b/django/template/loader_tags.py
index 1874d8c52881..36703b47823a 100644
--- a/django/template/loader_tags.py
+++ b/django/template/loader_tags.py
@@ -242,7 +242,11 @@ def do_block(parser, token):
     return BlockNode(block_name, nodelist)

-def construct_relative_path(current_template_name, relative_name):
+def construct_relative_path(
+    current_template_name,
+    relative_name,
+    allow_recursion=False,
+):
     """
     Convert a relative path (starting with './' or '../') to the full template
     name based on the current_template_name.
@@ -264,7 +268,7 @@ def construct_relative_path(current_template_name, relative_name):
             "The relative path '%s' points outside the file hierarchy that "
             "template '%s' is in." % (relative_name, current_template_name)
         )
-    if current_template_name.lstrip("/") == new_name:
+    if not allow_recursion and current_template_name.lstrip("/") == new_name:
         raise TemplateSyntaxError(
             "The relative path '%s' was translated to template name '%s', the "
             "same template in which the tag appears."
@@ -346,7 +350,11 @@ def do_include(parser, token):
         options[option] = value
     isolated_context = options.get("only", False)
     namemap = options.get("with", {})
-    bits[1] = construct_relative_path(parser.origin.template_name, bits[1])
+    bits[1] = construct_relative_path(
+        parser.origin.template_name,
+        bits[1],
+        allow_recursion=True,
+    )
     return IncludeNode(
         parser.compile_filter(bits[1]),
         extra_context=namemap,

diff --git a/tests/template_tests/syntax_tests/test_include.py b/tests/template_tests/syntax_tests/test_include.py
index 3ee99b37981e..be0deee9260d 100644
--- a/tests/template_tests/syntax_tests/test_include.py
+++ b/tests/template_tests/syntax_tests/test_include.py
@@ -330,15 +330,43 @@ def test_include_recursive(self):
                 ],
             }
         ]
-        engine = Engine(app_dirs=True)
-        t = engine.get_template("recursive_include.html")
-        self.assertEqual(
-            "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
-            t.render(Context({"comments": comments}))
-            .replace(" ", "")
-            .replace("\n", " ")
-            .strip(),
-        )
+        with self.subTest(template="recursive_include.html"):
+            engine = Engine(app_dirs=True)
+            t = engine.get_template("recursive_include.html")
+            self.assertEqual(
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+                t.render(Context({"comments": comments}))
+                .replace(" ", "")
+                .replace("\n", " ")
+                .strip(),
+            )
+        with self.subTest(template="recursive_relative_include.html"):
+            engine = Engine(app_dirs=True)
+            t = engine.get_template("recursive_relative_include.html")
+            self.assertEqual(
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+                t.render(Context({"comments": comments}))
+                .replace(" ", "")
+                .replace("\n", " ")
+                .strip(),
+            )
+        with self.subTest(template="tmpl"):
+            engine = Engine()
+            template = """
+            Recursion!
+            {% for c in comments %}
+              {{ c.comment }}
+              {% if c.children %}{% include tmpl with comments=c.children %}{% endif %}
+            {% endfor %}
+            """
+            outer_tmpl = engine.from_string("{% include tmpl %}")
+            output = outer_tmpl.render(
+                Context({"tmpl": engine.from_string(template), "comments": comments})
+            )
+            self.assertEqual(
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+                output.replace(" ", "").replace("\n", " ").strip(),
+            )

     def test_include_cache(self):
         """
diff --git a/tests/template_tests/templates/recursive_relative_include.html b/tests/template_tests/templates/recursive_relative_include.html
new file mode 100644
index 000000000000..ae49cc0a4367
--- /dev/null
+++ b/tests/template_tests/templates/recursive_relative_include.html
@@ -0,0 +1,7 @@
+Recursion!
+{% for comment in comments %}
+    {{ comment.comment }}
+    {% if comment.children %}
+        {% include "./recursive_relative_include.html" with comments=comment.children %}
+    {% endif %}
+{% endfor %}

Allow `./` and `../` in paths when recursively including templates
Description

(last modified by Gabriel Nick Pivovarov)
Hi. Currently, when trying to recursively include a Django template within itself using the include tag with a path that contains ./ or ../, Django raises a TemplateSyntaxError. However, using a path that does not contain ./ or ../ does not raise the error. When the error is raised, the debug toolbar describes it like this:
TemplateSyntaxError at /
The relative path ‘“./ul.html”’ was translated to template name ‘app/ul.html’, the same template in which the tag appears.
Here is an example of a template in a Django app called app with the path app/templates/app/ul.html that would produce the error given above:

	{% for section in sections %}

{{ section.name }}

			{% if section.sections|length != 0 %}
				{% include "./ul.html" with sections=section.sections %}
			{% endif %}

	{% endfor %}

However, replacing the directory ./ul.html with the equivalent app/ul.html makes the error go away (assuming that the project's settings.py specifies APP_DIRS = True and the views and URLs are configured correctly). The actual paths are translated identically in both cases, and the behavior of the include tag should not depend simply on whether or not the path string uses ./ or ../ (or if it should, this is not documented in the Django documentation). Therefore, it seems that this is a bug. The expected behavior is that an error is only raised when recursively using the extends template, not when recursively using the include template.
Contrapositively, it appears that recursively extending a template using the extends tag with a path that does not contain ./ or ../ raises a TemplateDoesNotExist exception.
One possible fix is to modify the django/template/loader_tags.py file (https://github.com/django/django/blob/main/django/template/loader_tags.py) such that the error is raised when a template attempts to extend itself (not when a template attempts to include itself, which would otherwise be valid). The error handling logic in question starts on line 267 of that file within the construct_relative_path function; perhaps it should only be used when called from the do_extends function.
Here is a relevant discussion in the Django forums: https://forum.djangoproject.com/t/template-recursion-why-does-django-not-allow-and/31689

['Relative path support was added in #26402 and recursive include support was added in #3544. I think this was missed when working on #26402, replicated. Here is my test: tests/template_tests/syntax_tests/test_include.py diff --git a/tests/template_tests/syntax_tests/test_include.py b/tests/template_tests/syntax_tests/test_include.py index 3ee99b3798..6dafba5040 100644 a b class IncludeTests(SimpleTestCase): 339339 .replace("\\n", " ") 340340 .strip(), 341341 ) 342 t = engine.get_template("recursive_relative_include.html") 343 self.assertEqual( 344 "Recursion! A1 Recursion! B1 B2 B3 Recursion! C1", 345 t.render(Context({"comments": comments})) 346 .replace(" ", "") 347 .replace("\\n", " ") 348 .strip(), 349 ) 342350 343351 def test_include_cache(self): 344352 """ tests/template_tests/templates/recursive_relative_include.html diff --git a/tests/template_tests/templates/recursive_relative_include.html b/tests/template_tests/templates/recursive_relative_include.html index e69de29bb2..ae49cc0a43 100644 a b 1Recursion! 2{% for comment in comments %} 3 {{ comment.comment }} 4 {% if comment.children %} 5 {% include "./recursive_relative_include.html" with comments=comment.children %} 6 {% endif %} 7{% endfor %}', 1718185535.0]
['\u200bhttps://github.com/django/django/pull/18850', 1732629666.0]


diff --git a/django/core/management/commands/sqlmigrate.py b/django/core/management/commands/sqlmigrate.py
index 3e3151f0cf3c..076499b3e2cb 100644
--- a/django/core/management/commands/sqlmigrate.py
+++ b/django/core/management/commands/sqlmigrate.py
@@ -32,10 +32,9 @@ def add_arguments(self, parser):
         )

     def execute(self, *args, **options):
-        # sqlmigrate doesn't support coloring its output but we need to force
-        # no_color=True so that the BEGIN/COMMIT statements added by
-        # output_transaction don't get colored either.
-        options["no_color"] = True
+        # sqlmigrate doesn't support coloring its output, so make the
+        # BEGIN/COMMIT statements added by output_transaction colorless also.
+        self.style.SQL_KEYWORD = lambda noop: noop
         return super().execute(*args, **options)

     def handle(self, *args, **options):

diff --git a/tests/migrations/test_commands.py b/tests/migrations/test_commands.py
index 724c88a28fa8..acd7ef4c4e5e 100644
--- a/tests/migrations/test_commands.py
+++ b/tests/migrations/test_commands.py
@@ -9,6 +9,7 @@

 from django.apps import apps
 from django.core.management import CommandError, call_command
+from django.core.management.base import SystemCheckError
 from django.core.management.commands.makemigrations import (
     Command as MakeMigrationsCommand,
 )
@@ -859,7 +860,7 @@ def test_sqlmigrate_forwards(self):
         sqlmigrate outputs forward looking SQL.
         """
         out = io.StringIO()
-        call_command("sqlmigrate", "migrations", "0001", stdout=out)
+        call_command("sqlmigrate", "migrations", "0001", stdout=out, no_color=True)

         lines = out.getvalue().splitlines()

@@ -921,7 +922,14 @@ def test_sqlmigrate_backwards(self):
         call_command("migrate", "migrations", verbosity=0)

         out = io.StringIO()
-        call_command("sqlmigrate", "migrations", "0001", stdout=out, backwards=True)
+        call_command(
+            "sqlmigrate",
+            "migrations",
+            "0001",
+            stdout=out,
+            backwards=True,
+            no_color=True,
+        )

         lines = out.getvalue().splitlines()
         try:
@@ -1098,6 +1106,30 @@ def test_sqlmigrate_unrepresentable(self):
             ],
         )

+    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
+    def test_sqlmigrate_transaction_keywords_not_colorized(self):
+        out = io.StringIO()
+        with mock.patch(
+            "django.core.management.color.supports_color", lambda *args: True
+        ):
+            call_command("sqlmigrate", "migrations", "0001", stdout=out, no_color=False)
+        self.assertNotIn("\x1b", out.getvalue())
+
+    @override_settings(
+        MIGRATION_MODULES={"migrations": "migrations.test_migrations_no_operations"},
+        INSTALLED_APPS=["django.contrib.auth"],
+    )
+    def test_sqlmigrate_system_checks_colorized(self):
+        with (
+            mock.patch(
+                "django.core.management.color.supports_color", lambda *args: True
+            ),
+            self.assertRaisesMessage(SystemCheckError, "\x1b"),
+        ):
+            call_command(
+                "sqlmigrate", "migrations", "0001", skip_checks=False, no_color=False
+            )
+
     @override_settings(
         INSTALLED_APPS=[
             "migrations.migrations_test_apps.migrated_app",

sqlmigrate prevents normal colorization of system checks
Description

sqlmigrate forces no_color=True for the sake of not colorizing the sql keywords BEGIN; and COMMIT;, but this has the side effect of preventing system check output from being colorized.
(To reproduce, begin with a project that will emit a system check, and run sqlmigrate.)
Suggesting a small PR to preserve the non-colorization of BEGIN; and COMMIT; while still colorizing system checks.


diff --git a/django/forms/formsets.py b/django/forms/formsets.py
index c8e5893f19b6..c2663154d4a5 100644
--- a/django/forms/formsets.py
+++ b/django/forms/formsets.py
@@ -570,7 +570,12 @@ def formset_factory(
         "validate_max": validate_max,
         "renderer": renderer,
     }
-    return type(form.__name__ + "FormSet", (formset,), attrs)
+    form_name = form.__name__
+    if form_name.endswith("Form"):
+        formset_name = form_name + "Set"
+    else:
+        formset_name = form_name + "FormSet"
+    return type(formset_name, (formset,), attrs)

 def all_valid(formsets):

diff --git a/tests/forms_tests/tests/test_formsets.py b/tests/forms_tests/tests/test_formsets.py
index f80c1dc09e35..9f7012a11fd8 100644
--- a/tests/forms_tests/tests/test_formsets.py
+++ b/tests/forms_tests/tests/test_formsets.py
@@ -149,6 +149,12 @@ def test_basic_formset(self):
         self.assertFalse(formset.is_valid())
         self.assertFalse(formset.has_changed())

+    def test_formset_name(self):
+        ArticleFormSet = formset_factory(ArticleForm)
+        ChoiceFormSet = formset_factory(Choice)
+        self.assertEqual(ArticleFormSet.__name__, "ArticleFormSet")
+        self.assertEqual(ChoiceFormSet.__name__, "ChoiceFormSet")
+
     def test_form_kwargs_formset(self):
         """
         Custom kwargs set on the formset instance are passed to the

About the FormSet class name generated through the formset_factory.
Description

When you define Django Form, what format do you usually name it?
In my case, I usually name it "xxxForm".
The names of the forms used as examples in official documents related to Django Form are the same as my name define format.
# Working with forms docs.
class ContactForm(forms.Form):
	subject = forms.CharField(max_length=100)
	message = forms.CharField(widget=forms.Textarea)
	sender = forms.EmailField()
	cc_myself = forms.BooleanField(required=False)
# Working with forms docs.
class ArticleForm(forms.Form):
	title = forms.CharField()
	pub_date = forms.DateField()
# FormField docs.
class CommentForm(forms.Form):
	name = forms.CharField()
	url = forms.URLField()
	comment = forms.CharField()
The way django people name it may be based on the developers' tendencies or tastes, but I think most of them are built in the format as xxxForm.
The FormSet class I created by passing xxxForm name format to the formset_factory is named as follows.
>>> ArticleFormSet = formset_factory(ArticleForm)
>>> ArticleFormSet.__name__
'ArticleFormFormSet'
The name of the FormSet class created through the formset_factory in the form xxxForm name format is a little strange.
This is because when formset_factory creates a class through type, it adds a static "FormSet" to the form name.
def formset_factory(
	form,
	formset=BaseFormSet,
	extra=1,
	can_order=False,
	can_delete=False,
	max_num=None,
	validate_max=False,
	min_num=None,
	validate_min=False,
	absolute_max=None,
	can_delete_extra=True,
	renderer=None,
):
	...
	return type(form.__name__ + "FormSet", (formset,), attrs)
I wonder if the format of these names is intended :)

['I agree this is a bit odd. \u200bThe test for the Formset.__repr__() uses a form called Choice so the result is ChoiceFormSet as expected. As you pointed out, perhaps to match the documented convention of ending all form names with "Form" we could consider appending "Set" rather than "FormSet" if form.__name__ ends with Form. I can\'t think of any way this would break backward compatibility, and it might make debugging a little less confusing.', 1731782903.0]
['Thank you for your good opinion @Tim Graham ! It was not found because we tested it through Choice in the test. I agree to change "FormSet" to "Set".', 1731786303.0]
['\u200bPR', 1731789286.0]


diff --git a/django/contrib/admin/sites.py b/django/contrib/admin/sites.py
index dc67262afc50..3399bd87b85a 100644
--- a/django/contrib/admin/sites.py
+++ b/django/contrib/admin/sites.py
@@ -282,7 +282,7 @@ def wrapper(*args, **kwargs):
             path("autocomplete/", wrap(self.autocomplete_view), name="autocomplete"),
             path("jsi18n/", wrap(self.i18n_javascript, cacheable=True), name="jsi18n"),
             path(
-                "r/<int:content_type_id>/<path:object_id>/",
+                "r/<path:content_type_id>/<path:object_id>/",
                 wrap(contenttype_views.shortcut),
                 name="view_on_site",
             ),

diff --git a/tests/admin_views/tests.py b/tests/admin_views/tests.py
index c5d8b8f4f668..3f106f681418 100644
--- a/tests/admin_views/tests.py
+++ b/tests/admin_views/tests.py
@@ -8664,6 +8664,19 @@ def test_custom_admin_site(self):
             ),
         )

+    def test_view_on_site_url_non_integer_ids(self):
+        """The view_on_site URL accepts non-integer ids."""
+        self.assertEqual(
+            reverse(
+                "admin:view_on_site",
+                kwargs={
+                    "content_type_id": "37156b6a-8a82",
+                    "object_id": "37156b6a-8a83",
+                },
+            ),
+            "/test_admin/admin/r/37156b6a-8a82/37156b6a-8a83/",
+        )
+

 @override_settings(ROOT_URLCONF="admin_views.urls")
 class InlineAdminViewOnSiteTest(TestCase):

"view on site" URL doesn't accept non-integer ContentType pks
Description

Most admin URL paths use the path converter to allow any type of ID, however, the "view on site" uses the int converter for content_type_id.
This doesn't work on MongoDB which uses ObjectIdAutoField (bson.ObjectId) for all models since AutoField isn't supported.


diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py
index d1f31f021135..f9cafdb4bb40 100644
--- a/django/db/models/fields/__init__.py
+++ b/django/db/models/fields/__init__.py
@@ -392,7 +392,10 @@ def _check_db_default(self, databases=None, **kwargs):

         if (
             self.db_default is NOT_PROVIDED
-            or isinstance(self.db_default, Value)
+            or (
+                isinstance(self.db_default, Value)
+                or not hasattr(self.db_default, "resolve_expression")
+            )
             or databases is None
         ):
             return []

diff --git a/tests/invalid_models_tests/test_ordinary_fields.py b/tests/invalid_models_tests/test_ordinary_fields.py
index e30d41113809..1fcf3f708d47 100644
--- a/tests/invalid_models_tests/test_ordinary_fields.py
+++ b/tests/invalid_models_tests/test_ordinary_fields.py
@@ -1207,6 +1207,23 @@ class Model(models.Model):
         expected_error = Error(msg=msg, obj=field, id="fields.E012")
         self.assertEqual(errors, [expected_error])

+    def test_literals_not_treated_as_expressions(self):
+        """
+        DatabaseFeatures.supports_expression_defaults = False shouldn't
+        prevent non-expression literals (integer, float, boolean, etc.) from
+        being used as database defaults.
+        """
+
+        class Model(models.Model):
+            field = models.FloatField(db_default=1.0)
+
+        field = Model._meta.get_field("field")
+        with unittest.mock.patch.object(
+            connection.features, "supports_expression_defaults", False
+        ):
+            errors = field.check(databases=self.databases)
+        self.assertEqual(errors, [])
+

 @isolate_apps("invalid_models_tests")
 class GeneratedFieldTests(TestCase):

System check for default database values with expressions prohibits non-expressions
Description

Since its introduction in Django 5.0, the fields.E011 system check for database backends that have DatabaseFeatures.supports_expression_defaults = False requires literal defaults to be wrapped in Value.
There are a number of test models that have int, float and string db_defaults that will raise a system check error if DatabaseFeatures.supports_expression_defaults = False
since these models don't have required_db_features = {"supports_expression_defaults"}.
I'm working on MongoDB which doesn't support any database defaults, literal or expressions.

["I'm not sure if the appropriate solution is to add Value wrapping to all literal defaults in Django's test models or to modify the \u200bisinstance(db_default, Value) check to also accept int/float, str, etc.", 1728031068.0]
["I think either solution could work, but maybe there's a subtlety I don't remember. I think wrapping all literals in Value is most likely to just work.", 1728272037.0]
['The issue I see with requiring Value wrapping is that third-party apps may not know to do it and therefore run into compatibility issues with backends with supports_expression_defaults = False.\u200b For the other solution I suggested, I\'m not sure we can make the isinstance()check exhaustive for all possible non-expression types. Perhaps it\'s more appropriate to replace isinstance(self.db_default, Value) with not hasattr(self.db_default, "resolve_expression") to reject expressions.', 1728376052.0]


diff --git a/django/template/base.py b/django/template/base.py
index b974495c9c92..eaca428b10c4 100644
--- a/django/template/base.py
+++ b/django/template/base.py
@@ -57,7 +57,7 @@

 from django.template.context import BaseContext
 from django.utils.formats import localize
-from django.utils.html import conditional_escape, escape
+from django.utils.html import conditional_escape
 from django.utils.regex_helper import _lazy_re_compile
 from django.utils.safestring import SafeData, SafeString, mark_safe
 from django.utils.text import get_text_list, smart_split, unescape_string_literal
@@ -247,10 +247,10 @@ def get_exception_info(self, exception, token):
         for num, next in enumerate(linebreak_iter(self.source)):
             if start >= upto and end <= next:
                 line = num
-                before = escape(self.source[upto:start])
-                during = escape(self.source[start:end])
-                after = escape(self.source[end:next])
-            source_lines.append((num, escape(self.source[upto:next])))
+                before = self.source[upto:start]
+                during = self.source[start:end]
+                after = self.source[end:next]
+            source_lines.append((num, self.source[upto:next]))
             upto = next
         total = len(source_lines)

diff --git a/tests/template_tests/templates/test_extends_block_error.html b/tests/template_tests/templates/test_extends_block_error.html
index c4733747a237..8133c93ccd31 100644
--- a/tests/template_tests/templates/test_extends_block_error.html
+++ b/tests/template_tests/templates/test_extends_block_error.html
@@ -1,2 +1,2 @@
 {% extends "test_extends_block_error_parent.html" %}
-{% block content %}{% include "missing.html" %}{% endblock %}
+{% block content %}{% include "index.html" %}{% include "missing.html" %}{% include "index.html" %}{% endblock %}
diff --git a/tests/template_tests/tests.py b/tests/template_tests/tests.py
index 14df81669b12..7364c7ca6462 100644
--- a/tests/template_tests/tests.py
+++ b/tests/template_tests/tests.py
@@ -5,7 +5,6 @@
 from django.test import SimpleTestCase, override_settings
 from django.urls import NoReverseMatch
 from django.utils import translation
-from django.utils.html import escape

 class TemplateTestMixin:
@@ -157,10 +156,32 @@ def test_render_tag_error_in_extended_block(self):
         with self.assertRaises(TemplateDoesNotExist) as cm:
             template.render(context)
         if self.debug_engine:
+            self.assertEqual(
+                cm.exception.template_debug["before"],
+                '{% block content %}{% include "index.html" %}',
+            )
             self.assertEqual(
                 cm.exception.template_debug["during"],
-                escape('{% include "missing.html" %}'),
+                '{% include "missing.html" %}',
+            )
+            self.assertEqual(
+                cm.exception.template_debug["after"],
+                '{% include "index.html" %}{% endblock %}\n',
+            )
+            self.assertEqual(
+                cm.exception.template_debug["source_lines"][0],
+                (1, '{% extends "test_extends_block_error_parent.html" %}\n'),
+            )
+            self.assertEqual(
+                cm.exception.template_debug["source_lines"][1],
+                (
+                    2,
+                    '{% block content %}{% include "index.html" %}'
+                    '{% include "missing.html" %}'
+                    '{% include "index.html" %}{% endblock %}\n',
+                ),
             )
+            self.assertEqual(cm.exception.template_debug["source_lines"][2], (3, ""))

     def test_super_errors(self):
         """

Template system: escape() calls in get_exception_info() should be removed
Description

Here there are some calls to escape()
They shouldn't be there: escaping happens in templates for non-safe strings anyway, so there's no need.
And there _is_ a drawback: as an example, the Python Sentry SDK copies this info, but because it gets sent over the wire (as a JSON string) the information that this has already been escaped is lost, and on the receiving end it is escaped again.
This means that on the server-side the Error-tracking, in my case Bugsink will show doubly escaped html code snippets. This looks something like this:
<p class="relative text-slate-600 text-base md:text-xl mb-4 md:mb-5">
Removing the calls to escape simply solves this. Which makes sense: calling escape is simply not the responsibility of this piece of code, it should just stay marked as unsafe and be escape at the edges (on rendering).

['Given #33461 added force_escape to message, we might need to move that escaping into the template', 1731045354.0]


diff --git a/django/db/migrations/questioner.py b/django/db/migrations/questioner.py
index e1081ab70ac2..2e6119558188 100644
--- a/django/db/migrations/questioner.py
+++ b/django/db/migrations/questioner.py
@@ -160,8 +160,8 @@ def _ask_default(self, default=""):
             else:
                 try:
                     return eval(code, {}, {"datetime": datetime, "timezone": timezone})
-                except (SyntaxError, NameError) as e:
-                    self.prompt_output.write("Invalid input: %s" % e)
+                except Exception as e:
+                    self.prompt_output.write(f"{e.__class__.__name__}: {e}")

     def ask_not_null_addition(self, field_name, model_name):
         """Adding a NOT NULL field to a model."""

diff --git a/tests/migrations/test_questioner.py b/tests/migrations/test_questioner.py
index c1aebcb22491..ec1013923b06 100644
--- a/tests/migrations/test_questioner.py
+++ b/tests/migrations/test_questioner.py
@@ -61,10 +61,32 @@ def test_questioner_no_default_no_user_entry(self, mock_input):
         )

     @mock.patch("builtins.input", side_effect=["bad code", "exit"])
-    def test_questioner_no_default_bad_user_entry_code(self, mock_input):
+    def test_questioner_no_default_syntax_error(self, mock_input):
         with self.assertRaises(SystemExit):
             self.questioner._ask_default()
-        self.assertIn("Invalid input: ", self.prompt.getvalue())
+        self.assertIn("SyntaxError: invalid syntax", self.prompt.getvalue())
+
+    @mock.patch("builtins.input", side_effect=["datetim", "exit"])
+    def test_questioner_no_default_name_error(self, mock_input):
+        with self.assertRaises(SystemExit):
+            self.questioner._ask_default()
+        self.assertIn(
+            "NameError: name 'datetim' is not defined", self.prompt.getvalue()
+        )
+
+    @mock.patch("builtins.input", side_effect=["datetime.dat", "exit"])
+    def test_questioner_no_default_attribute_error(self, mock_input):
+        with self.assertRaises(SystemExit):
+            self.questioner._ask_default()
+        self.assertIn(
+            "AttributeError: module 'datetime' has no attribute 'dat'",
+            self.prompt.getvalue(),
+        )
+
+    @mock.patch("builtins.input", side_effect=[KeyboardInterrupt()])
+    def test_questioner_no_default_keyboard_interrupt(self, mock_input):
+        with self.assertRaises(KeyboardInterrupt):
+            self.questioner._ask_default()

     @mock.patch("builtins.input", side_effect=["", "n"])
     def test_questioner_no_default_no_user_entry_boolean(self, mock_input):

Loop on all errors in `InteractiveMigrationQuestioner._ask_default()`
Description

When changing some fields, the migration questioner can ask for new field default values entered through a Python prompt.
Yesterday, I made a typo which made the makemigrations process exit. This was a little frustrating as there were several fields to fix, and my work answering for the first fields was lost.
Currently, the questioner loops on SyntaxError and NameError (source). My mistake lead to an AttributeError, hence the crash and lost work. I propose we extend this clause to Exception - there's little harm in displaying the error and looping.
Here's a reproducing example. Say you drop null=True from two model fields:
example/models.py
diff --git example/models.py example/models.py
index 816fe37..9743877 100644

22
33
44class Book(models.Model):
5     title = models.CharField(max_length=100, null=True)
6     published_date = models.DateField(null=True)
 5    title = models.CharField(max_length=100)
 6    published_date = models.DateField()
makemigrations asks you about the first one:
$ ./manage.py makemigrations example
It is impossible to change a nullable field 'title' on book to non-nullable without providing a default. This is because the database needs something to populate existing rows.
Please select a fix:
 1) Provide a one-off default now (will be set on all existing rows with a null value for this column)
 2) Ignore for now. Existing rows that contain NULL values will have to be handled manually, for example with a RunPython or RunSQL operation.
 3) Quit and manually define a default value in models.py.
Select an option: 1
Please enter the default value as valid Python.
The datetime and django.utils.timezone modules are available, so it is possible to provide e.g. timezone.now as a value.
Type 'exit' to exit this prompt
>>> "Unknown book"
Immediately after this, it asks about the second field. Making a typo like .dat() instead of .date() crashes the process:
It is impossible to change a nullable field 'published_date' on book to non-nullable without providing a default. This is because the database needs something to populate existing rows.
Please select a fix:
 1) Provide a one-off default now (will be set on all existing rows with a null value for this column)
 2) Ignore for now. Existing rows that contain NULL values will have to be handled manually, for example with a RunPython or RunSQL operation.
 3) Quit and manually define a default value in models.py.
Select an option: 1
Please enter the default value as valid Python.
The datetime and django.utils.timezone modules are available, so it is possible to provide e.g. timezone.now as a value.
Type 'exit' to exit this prompt
>>> datetime.dat(1970, 1, 1)
Traceback (most recent call last):
 File "/..././manage.py", line 21, in <module>
	main()
 ...
 File "/.../.venv/.../django/db/migrations/questioner.py", line 162, in _ask_default
	return eval(code, {}, {"datetime": datetime, "timezone": timezone})
		 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "<string>", line 1, in <module>
AttributeError: module 'datetime' has no attribute 'dat'. Did you mean: 'date'?
But if you instead made a typo in the datetime name, it would loop:
>>> datetme.date(1970,1,1)
Invalid input: name 'datetme' is not defined
>>>

['As long as KeyboardInterrupt(BaseException) is allowed to bubble up (which catching for Exception allows) I agree that not trying to catch a predefined set of exceptions is a better default.', 1730539235.0]
['In 3434fab7: Refs #35882 -- Added test for migration questioner KeyboardInterrupt.', 1731917749.0]
['In e035db1: Fixed #35882 -- Made migration questioner loop on all errors.', 1731917750.0]


diff --git a/django/template/base.py b/django/template/base.py
index ee2e145c041a..b974495c9c92 100644
--- a/django/template/base.py
+++ b/django/template/base.py
@@ -533,9 +533,13 @@ def skip_past(self, endtag):
     def extend_nodelist(self, nodelist, node, token):
         # Check that non-text nodes don't appear before an extends tag.
         if node.must_be_first and nodelist.contains_nontext:
+            if self.origin.template_name:
+                origin = repr(self.origin.template_name)
+            else:
+                origin = "the template"
             raise self.error(
                 token,
-                "%r must be the first tag in the template." % node,
+                "{%% %s %%} must be the first tag in %s." % (token.contents, origin),
             )
         if not isinstance(node, TextNode):
             nodelist.contains_nontext = True

diff --git a/tests/template_tests/test_extends.py b/tests/template_tests/test_extends.py
index ce1838654bac..0d2a93468ce3 100644
--- a/tests/template_tests/test_extends.py
+++ b/tests/template_tests/test_extends.py
@@ -1,9 +1,9 @@
 import os

-from django.template import Context, Engine, TemplateDoesNotExist
+from django.template import Context, Engine, TemplateDoesNotExist, TemplateSyntaxError
 from django.test import SimpleTestCase

-from .utils import ROOT
+from .utils import ROOT, setup

 RECURSIVE = os.path.join(ROOT, "recursive_templates")

@@ -181,3 +181,17 @@ def test_block_override_in_extended_included_template(self):
         )
         template = engine.get_template("base.html")
         self.assertEqual(template.render(Context({})), "12AB")
+
+    @setup(
+        {"index.html": "{% block content %}B{% endblock %}{% extends 'base.html' %}"}
+    )
+    def test_extends_not_first_tag_in_extended_template(self):
+        msg = "{% extends 'base.html' %} must be the first tag in 'index.html'."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.get_template("index.html")
+
+    def test_extends_not_first_tag_in_extended_template_from_string(self):
+        template_string = "{% block content %}B{% endblock %}{% extends 'base.html' %}"
+        msg = "{% extends 'base.html' %} must be the first tag in the template."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            Engine().from_string(template_string)

Improve non-first {% extends %} error message
Description

Currently if you put {% extends %} after another tag, the error looks like:
TemplateSyntaxError: <ExtendsNode: extends 'base.html'> must be the first tag in the template.
ExtendsNode is a leaked internal detail. Showing its repr() is a bit confusing, especially for a beginner programmer.
The message would be clearer if it showed the actual text of the tag:
TemplateSyntaxError: {% extends 'base.html' %} must be the first tag in the template.

['So After looking at the code I am a bit stuck on what should be done, the error message is constructed using the __repr__ method of ExtendNode class # The way the error is constructed if node.must_be_first and nodelist.contains_nontext: raise self.error( token, "%r must be the first tag in the template." % node, ) And this is the __repr__ function # __repr__ function for ExtendsNode class def __repr__(self): return "<%s: extends %s>" % (self.__class__.__name__, self.parent_name.token) I don\'t think changing the repr method or hard coding the error message is any better. So what should be done ?', 1727505575.0]
['The repr should not be changed, it’s still used in other contexts like debugging. Change the error message to show/build the extends tag source.', 1727511576.0]
["Just sent a PR to github to add a clear error message. \u200bPR Edit: I didn't notice the first PR, sorry about that.", 1728200226.0]


diff --git a/django/db/migrations/operations/models.py b/django/db/migrations/operations/models.py
index 9aad9c809ee7..c8f7a2627a91 100644
--- a/django/db/migrations/operations/models.py
+++ b/django/db/migrations/operations/models.py
@@ -184,6 +184,38 @@ def reduce(self, operation, app_label):
                     managers=operation.managers,
                 ),
             ]
+        elif (
+            isinstance(operation, AlterModelTable)
+            and self.name_lower == operation.name_lower
+        ):
+            return [
+                CreateModel(
+                    self.name,
+                    fields=self.fields,
+                    options={
+                        **self.options,
+                        "db_table": operation.table,
+                    },
+                    bases=self.bases,
+                    managers=self.managers,
+                ),
+            ]
+        elif (
+            isinstance(operation, AlterModelTableComment)
+            and self.name_lower == operation.name_lower
+        ):
+            return [
+                CreateModel(
+                    self.name,
+                    fields=self.fields,
+                    options={
+                        **self.options,
+                        "db_table_comment": operation.table_comment,
+                    },
+                    bases=self.bases,
+                    managers=self.managers,
+                ),
+            ]
         elif (
             isinstance(operation, AlterTogetherOptionOperation)
             and self.name_lower == operation.name_lower

diff --git a/tests/migrations/test_optimizer.py b/tests/migrations/test_optimizer.py
index 2acbc7f09f74..3ed30102bf15 100644
--- a/tests/migrations/test_optimizer.py
+++ b/tests/migrations/test_optimizer.py
@@ -154,6 +154,46 @@ def test_create_alter_model_managers(self):
             ],
         )

+    def test_create_alter_model_table(self):
+        self.assertOptimizesTo(
+            [
+                migrations.CreateModel("Foo", fields=[]),
+                migrations.AlterModelTable(
+                    name="foo",
+                    table="foo",
+                ),
+            ],
+            [
+                migrations.CreateModel(
+                    "Foo",
+                    fields=[],
+                    options={
+                        "db_table": "foo",
+                    },
+                ),
+            ],
+        )
+
+    def test_create_alter_model_table_comment(self):
+        self.assertOptimizesTo(
+            [
+                migrations.CreateModel("Foo", fields=[]),
+                migrations.AlterModelTableComment(
+                    name="foo",
+                    table_comment="A lovely table.",
+                ),
+            ],
+            [
+                migrations.CreateModel(
+                    "Foo",
+                    fields=[],
+                    options={
+                        "db_table_comment": "A lovely table.",
+                    },
+                ),
+            ],
+        )
+
     def test_create_model_and_remove_model_options(self):
         self.assertOptimizesTo(
             [

Reduce CreateModel + AlterModelTable to CreateModel
Description

Like #33572, migration optimization can reduce CreateModel and AlterModelTable for the same model down to just CreateModel with db_table in options.


diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py
index 04a5edbd3086..dfc3137f76fd 100644
--- a/django/contrib/staticfiles/storage.py
+++ b/django/contrib/staticfiles/storage.py
@@ -308,22 +308,23 @@ def post_process(self, paths, dry_run=False, **options):
                 processed_adjustable_paths[name] = (name, hashed_name, processed)

         paths = {path: paths[path] for path in adjustable_paths}
-        substitutions = False
-
+        unresolved_paths = []
         for i in range(self.max_post_process_passes):
-            substitutions = False
+            unresolved_paths = []
             for name, hashed_name, processed, subst in self._post_process(
                 paths, adjustable_paths, hashed_files
             ):
                 # Overwrite since hashed_name may be newer.
                 processed_adjustable_paths[name] = (name, hashed_name, processed)
-                substitutions = substitutions or subst
+                if subst:
+                    unresolved_paths.append(name)

-            if not substitutions:
+            if not unresolved_paths:
                 break

-        if substitutions:
-            yield "All", None, RuntimeError("Max post-process passes exceeded.")
+        if unresolved_paths:
+            problem_paths = ", ".join(sorted(unresolved_paths))
+            yield problem_paths, None, RuntimeError("Max post-process passes exceeded.")

         # Store the processed paths
         self.hashed_files.update(hashed_files)

diff --git a/tests/staticfiles_tests/project/loop/baz.css b/tests/staticfiles_tests/project/loop/baz.css
new file mode 100644
index 000000000000..4021a1b1e6ca
--- /dev/null
+++ b/tests/staticfiles_tests/project/loop/baz.css
@@ -0,0 +1,3 @@
+body {
+    background-color: #fafafa;
+}
diff --git a/tests/staticfiles_tests/test_storage.py b/tests/staticfiles_tests/test_storage.py
index d6ea03b7446a..9ca4d6255329 100644
--- a/tests/staticfiles_tests/test_storage.py
+++ b/tests/staticfiles_tests/test_storage.py
@@ -186,7 +186,9 @@ def test_import_loop(self):
         err = StringIO()
         with self.assertRaisesMessage(RuntimeError, "Max post-process passes exceeded"):
             call_command("collectstatic", interactive=False, verbosity=0, stderr=err)
-        self.assertEqual("Post-processing 'All' failed!\n\n", err.getvalue())
+        self.assertEqual(
+            "Post-processing 'bar.css, foo.css' failed!\n\n", err.getvalue()
+        )
         self.assertPostCondition()

     def test_post_processing(self):

Improve `RuntimeError: Max post-process passes exceeded.` error
Description

(last modified by Michael)
Having just spend 3 hours trying to debug a collect static issue, to help future travellers, I recommend printing the files that caused max depth to be exceeded, often when a file references itself it causes recursion, but the only clue it currently prints is 'All' which does not narrow down the problem. We can surely do better than that! The proposed change prints the problem files only that keep chaning and can't be resolved:
So instead of getting:
Post-processing 'All' failed!
We get the new and improved:
Post-processing 'jsapp/jsapp/notify.min.js' failed!
Or if more than one file:
Post-processing 'jsapp/jsapp/notify.min.js, jsapp/jsapp/somethingelse.min.js' failed!
I recommend changing from:
# contrib/staticfiles/storage.py line 313: in def post_process(self, paths, dry_run=False, **options):
		unresolved_paths = []
		for i in range(self.max_post_process_passes):
			substitutions = False
			for name, hashed_name, processed, subst in self._post_process(
				paths, adjustable_paths, hashed_files
			):
				# Overwrite since hashed_name may be newer.
				processed_adjustable_paths[name] = (name, hashed_name, processed)
				if subst and i == self.max_post_process_passes - 1:
					unresolved_paths.append(name)
				substitutions = substitutions or subst
			if not substitutions:
				break
		if substitutions:
			problem_paths_str = ", ".join(unresolved_paths) if unresolved_paths else "All"
			yield problem_paths_str, None, RuntimeError("Max post-process passes exceeded.")
I recommend changing to (see the comments on the right of the proposed changed lines 1/5 to 5/5):
# contrib/staticfiles/storage.py line 313: in def post_process(self, paths, dry_run=False, **options):
		unresolved_paths = []											# < -- add this line 1/5
		for i in range(self.max_post_process_passes):
			substitutions = False
			for name, hashed_name, processed, subst in self._post_process(
				paths, adjustable_paths, hashed_files
			):
				# Overwrite since hashed_name may be newer.
				processed_adjustable_paths[name] = (name, hashed_name, processed)
				if subst and i == self.max_post_process_passes - 1:	 # < -- add this line 2/5
					unresolved_paths.append(name)						# < -- add this line 3/5
				substitutions = substitutions or subst
			if not substitutions:
				break
		if substitutions:
			problem_paths_str = ", ".join(unresolved_paths) if unresolved_paths else "All"	 # < -- add this line 4/5
			yield problem_paths_str, None, RuntimeError("Max post-process passes exceeded.") # < -- change this line 5/5

['Adding the suggestion as a diff django/contrib/staticfiles/storage.py diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py index 04a5edbd30..5c41fd6828 100644 a b class HashedFilesMixin: 309309 310310 paths = {path: paths[path] for path in adjustable_paths} 311311 substitutions = False 312 312 unresolved_paths = [] 313313 for i in range(self.max_post_process_passes): 314314 substitutions = False 315315 for name, hashed_name, processed, subst in self._post_process( … … class HashedFilesMixin: 318318 # Overwrite since hashed_name may be newer. 319319 processed_adjustable_paths[name] = (name, hashed_name, processed) 320320 substitutions = substitutions or subst 321 if subst and i == self.max_post_process_passes - 1: 322 unresolved_paths.append(name) 321323 322324 if not substitutions: 323325 break 324326 325327 if substitutions: 326 yield "All", None, RuntimeError("Max post-process passes exceeded.") 328 problem_paths_str = ", ".join(unresolved_paths) if unresolved_paths else "All" 329 yield problem_paths_str, None, RuntimeError("Max post-process passes exceeded.") 327330 328331 # Store the processed paths 329332 self.hashed_files.update(hashed_files) Appreciate clear error messages so sounds good to me', 1723434306.0]