diff --git a/strawberry_django/mutations/resolvers.py b/strawberry_django/mutations/resolvers.py
index 50c76677..53a58cbe 100644
--- a/strawberry_django/mutations/resolvers.py
+++ b/strawberry_django/mutations/resolvers.py
@@ -15,6 +15,7 @@
 import strawberry
 from django.db import models, transaction
 from django.db.models.base import Model
+from django.db.models.fields import Field
 from django.db.models.fields.related import ManyToManyField
 from django.db.models.fields.reverse_related import (
     ForeignObjectRel,
@@ -44,7 +45,11 @@
 )

 if TYPE_CHECKING:
-    from django.db.models.manager import ManyToManyRelatedManager, RelatedManager
+    from django.db.models.manager import (
+        BaseManager,
+        ManyToManyRelatedManager,
+        RelatedManager,
+    )
     from strawberry.types.info import Info

@@ -88,6 +93,7 @@ def _parse_data(
     value: Any,
     *,
     key_attr: str | None = None,
+    full_clean: bool | FullCleanOptions = True,
 ):
     obj, data = _parse_pk(value, model, key_attr=key_attr)
     parsed_data = {}
@@ -97,10 +103,21 @@ def _parse_data(
                 continue

             if isinstance(v, ParsedObject):
-                if v.pk is None:
-                    v = create(info, model, v.data or {})  # noqa: PLW2901
+                if v.pk in {None, UNSET}:
+                    related_field = cast("Field", get_model_fields(model).get(k))
+                    related_model = related_field.related_model
+                    v = create(  # noqa: PLW2901
+                        info,
+                        cast("type[Model]", related_model),
+                        v.data or {},
+                        key_attr=key_attr,
+                        full_clean=full_clean,
+                        exclude_m2m=[related_field.name],
+                    )
                 elif isinstance(v.pk, models.Model) and v.data:
-                    v = update(info, v.pk, v.data, key_attr=key_attr)  # noqa: PLW2901
+                    v = update(  # noqa: PLW2901
+                        info, v.pk, v.data, key_attr=key_attr, full_clean=full_clean
+                    )
                 else:
                     v = v.pk  # noqa: PLW2901

@@ -222,6 +239,7 @@ def prepare_create_update(
     data: dict[str, Any],
     key_attr: str | None = None,
     full_clean: bool | FullCleanOptions = True,
+    exclude_m2m: list[str] | None = None,
 ) -> tuple[
     Model,
     dict[str, object],
@@ -237,6 +255,7 @@ def prepare_create_update(
     fields = get_model_fields(model)
     m2m: list[tuple[ManyToManyField | ForeignObjectRel, Any]] = []
     direct_field_values: dict[str, object] = {}
+    exclude_m2m = exclude_m2m or []

     if dataclasses.is_dataclass(data):
         data = vars(data)
@@ -256,6 +275,8 @@ def prepare_create_update(
                 # (but only if the instance is already saved and we are updating it)
                 value = False  # noqa: PLW2901
         elif isinstance(field, (ManyToManyField, ForeignObjectRel)):
+            if name in exclude_m2m:
+                continue
             # m2m will be processed later
             m2m.append((field, value))
             direct_field_value = False
@@ -269,14 +290,19 @@ def prepare_create_update(
                 cast("type[Model]", field.related_model),
                 value,
                 key_attr=key_attr,
+                full_clean=full_clean,
             )
             if value is None and not value_data:
                 value = None  # noqa: PLW2901

             # If foreign object is not found, then create it
-            elif value is None:
-                value = field.related_model._default_manager.create(  # noqa: PLW2901
-                    **value_data,
+            elif value in {None, UNSET}:
+                value = create(  # noqa: PLW2901
+                    info,
+                    field.related_model,
+                    value_data,
+                    key_attr=key_attr,
+                    full_clean=full_clean,
                 )

             # If foreign object does not need updating, then skip it
@@ -309,6 +335,7 @@ def create(
     key_attr: str | None = None,
     full_clean: bool | FullCleanOptions = True,
     pre_save_hook: Callable[[_M], None] | None = None,
+    exclude_m2m: list[str] | None = None,
 ) -> _M: ...

@@ -321,10 +348,10 @@ def create(
     key_attr: str | None = None,
     full_clean: bool | FullCleanOptions = True,
     pre_save_hook: Callable[[_M], None] | None = None,
+    exclude_m2m: list[str] | None = None,
 ) -> list[_M]: ...

-@transaction.atomic
 def create(
     info: Info,
     model: type[_M],
@@ -333,12 +360,43 @@ def create(
     key_attr: str | None = None,
     full_clean: bool | FullCleanOptions = True,
     pre_save_hook: Callable[[_M], None] | None = None,
+    exclude_m2m: list[str] | None = None,
+) -> list[_M] | _M:
+    return _create(
+        info,
+        model._default_manager,
+        data,
+        key_attr=key_attr,
+        full_clean=full_clean,
+        pre_save_hook=pre_save_hook,
+        exclude_m2m=exclude_m2m,
+    )
+
+
+@transaction.atomic
+def _create(
+    info: Info,
+    manager: BaseManager,
+    data: dict[str, Any] | list[dict[str, Any]],
+    *,
+    key_attr: str | None = None,
+    full_clean: bool | FullCleanOptions = True,
+    pre_save_hook: Callable[[_M], None] | None = None,
+    exclude_m2m: list[str] | None = None,
 ) -> list[_M] | _M:
+    model = manager.model
     # Before creating your instance, verify this is not a bulk create
     # if so, add them one by one. Otherwise, get to work.
     if isinstance(data, list):
         return [
-            create(info, model, d, key_attr=key_attr, full_clean=full_clean)
+            create(
+                info,
+                model,
+                d,
+                key_attr=key_attr,
+                full_clean=full_clean,
+                exclude_m2m=exclude_m2m,
+            )
             for d in data
         ]

@@ -365,6 +423,7 @@ def create(
         data=data,
         full_clean=full_clean,
         key_attr=key_attr,
+        exclude_m2m=exclude_m2m,
     )

     # Creating the instance directly via create() without full-clean will
@@ -376,7 +435,7 @@ def create(

     # Create the instance using the manager create method to respect
     # manager create overrides. This also ensures support for proxy-models.
-    instance = model._default_manager.create(**create_kwargs)
+    instance = manager.create(**create_kwargs)

     for field, value in m2m:
         update_m2m(info, instance, field, value, key_attr)
@@ -393,6 +452,7 @@ def update(
     key_attr: str | None = None,
     full_clean: bool | FullCleanOptions = True,
     pre_save_hook: Callable[[_M], None] | None = None,
+    exclude_m2m: list[str] | None = None,
 ) -> _M: ...

@@ -405,6 +465,7 @@ def update(
     key_attr: str | None = None,
     full_clean: bool | FullCleanOptions = True,
     pre_save_hook: Callable[[_M], None] | None = None,
+    exclude_m2m: list[str] | None = None,
 ) -> list[_M]: ...

@@ -417,6 +478,7 @@ def update(
     key_attr: str | None = None,
     full_clean: bool | FullCleanOptions = True,
     pre_save_hook: Callable[[_M], None] | None = None,
+    exclude_m2m: list[str] | None = None,
 ) -> _M | list[_M]:
     # Unwrap lazy objects since they have a proxy __iter__ method that will make
     # them iterables even if the wrapped object isn't
@@ -433,6 +495,7 @@ def update(
                 key_attr=key_attr,
                 full_clean=full_clean,
                 pre_save_hook=pre_save_hook,
+                exclude_m2m=exclude_m2m,
             )
             for instance in instances
         ]
@@ -443,6 +506,7 @@ def update(
         data=data,
         key_attr=key_attr,
         full_clean=full_clean,
+        exclude_m2m=exclude_m2m,
     )

     if pre_save_hook is not None:
@@ -554,15 +618,22 @@ def update_m2m(
     use_remove = True
     if isinstance(field, ManyToManyField):
         manager = cast("RelatedManager", getattr(instance, field.attname))
+        reverse_field_name = field.remote_field.related_name  # type: ignore
     else:
         assert isinstance(field, (ManyToManyRel, ManyToOneRel))
         accessor_name = field.get_accessor_name()
+        reverse_field_name = field.field.name
         assert accessor_name
         manager = cast("RelatedManager", getattr(instance, accessor_name))
         if field.one_to_many:
             # remove if field is nullable, otherwise delete
             use_remove = field.remote_field.null is True

+    # Create a data dict containing the reference to the instance and exclude it from
+    # nested m2m creation (to break circular references)
+    ref_instance_data = {reverse_field_name: instance}
+    exclude_m2m = [reverse_field_name]
+
     to_add = []
     to_remove = []
     to_delete = []
@@ -581,7 +652,11 @@ def update_m2m(
         need_remove_cache = need_remove_cache or bool(values)
         for v in values:
             obj, data = _parse_data(
-                info, cast("type[Model]", manager.model), v, key_attr=key_attr
+                info,
+                cast("type[Model]", manager.model),
+                v,
+                key_attr=key_attr,
+                full_clean=full_clean,
             )
             if obj:
                 data.pop(key_attr, None)
@@ -621,14 +696,17 @@ def update_m2m(

                 existing.discard(obj)
             else:
-                if key_attr not in data:  # we have a Input Type
-                    obj, _ = manager.get_or_create(**data)
-                else:
-                    data.pop(key_attr)
-                    obj = manager.create(**data)
-
-                if full_clean:
-                    obj.full_clean(**full_clean_options)
+                # If we've reached here, the key_attr should be UNSET or missing. So
+                # let's remove it if it is there.
+                data.pop(key_attr, None)
+                obj = _create(
+                    info,
+                    manager,
+                    data | ref_instance_data,
+                    key_attr=key_attr,
+                    full_clean=full_clean,
+                    exclude_m2m=exclude_m2m,
+                )
                 existing.discard(obj)

         for remaining in existing:
@@ -645,6 +723,7 @@ def update_m2m(
                 cast("type[Model]", manager.model),
                 v,
                 key_attr=key_attr,
+                full_clean=full_clean,
             )
             if obj and data:
                 data.pop(key_attr, None)
@@ -656,18 +735,28 @@ def update_m2m(
                 data.pop(key_attr, None)
                 to_add.append(obj)
             elif data:
-                if key_attr not in data:
-                    manager.get_or_create(**data)
-                else:
-                    data.pop(key_attr)
-                    manager.create(**data)
+                # If we've reached here, the key_attr should be UNSET or missing. So
+                # let's remove it if it is there.
+                data.pop(key_attr, None)
+                _create(
+                    info,
+                    manager,
+                    data | ref_instance_data,
+                    key_attr=key_attr,
+                    full_clean=full_clean,
+                    exclude_m2m=exclude_m2m,
+                )
             else:
                 raise AssertionError

         need_remove_cache = need_remove_cache or bool(value.remove)
         for v in value.remove or []:
             obj, data = _parse_data(
-                info, cast("type[Model]", manager.model), v, key_attr=key_attr
+                info,
+                cast("type[Model]", manager.model),
+                v,
+                key_attr=key_attr,
+                full_clean=full_clean,
             )
             data.pop(key_attr, None)
             assert not data

diff --git a/tests/projects/schema.py b/tests/projects/schema.py
index 669ab286..f61ae7c8 100644
--- a/tests/projects/schema.py
+++ b/tests/projects/schema.py
@@ -335,6 +335,12 @@ class MilestoneIssueInput:
     name: strawberry.auto

+@strawberry_django.partial(Issue)
+class MilestoneIssueInputPartial:
+    name: strawberry.auto
+    tags: Optional[list[TagInputPartial]]
+
+
 @strawberry_django.partial(Project)
 class ProjectInputPartial(NodeInputPartial):
     name: strawberry.auto
@@ -351,6 +357,8 @@ class MilestoneInput:
 @strawberry_django.partial(Milestone)
 class MilestoneInputPartial(NodeInputPartial):
     name: strawberry.auto
+    issues: Optional[list[MilestoneIssueInputPartial]]
+    project: Optional[ProjectInputPartial]

 @strawberry.type
@@ -519,6 +527,11 @@ class Mutation:
         argument_name="input",
         key_attr="name",
     )
+    create_project_with_milestones: ProjectType = mutations.create(
+        ProjectInputPartial,
+        handle_django_errors=True,
+        argument_name="input",
+    )
     update_project: ProjectType = mutations.update(
         ProjectInputPartial,
         handle_django_errors=True,
diff --git a/tests/projects/snapshots/schema.gql b/tests/projects/snapshots/schema.gql
index b5f31d33..777b1c27 100644
--- a/tests/projects/snapshots/schema.gql
+++ b/tests/projects/snapshots/schema.gql
@@ -95,6 +95,8 @@ input CreateProjectInput {

 union CreateProjectPayload = ProjectType | OperationInfo

+union CreateProjectWithMilestonesPayload = ProjectType | OperationInfo
+
 input CreateQuizInput {
   title: String!
   fullCleanOptions: Boolean! = false
@@ -365,12 +367,19 @@ input MilestoneInput {
 input MilestoneInputPartial {
   id: GlobalID
   name: String
+  issues: [MilestoneIssueInputPartial!]
+  project: ProjectInputPartial
 }

 input MilestoneIssueInput {
   name: String!
 }

+input MilestoneIssueInputPartial {
+  name: String
+  tags: [TagInputPartial!]
+}
+
 input MilestoneOrder {
   name: Ordering
   project: ProjectOrder
@@ -431,6 +440,7 @@ type Mutation {
   updateIssueWithKeyAttr(input: IssueInputPartialWithoutId!): UpdateIssueWithKeyAttrPayload!
   deleteIssue(input: NodeInput!): DeleteIssuePayload!
   deleteIssueWithKeyAttr(input: MilestoneIssueInput!): DeleteIssueWithKeyAttrPayload!
+  createProjectWithMilestones(input: ProjectInputPartial!): CreateProjectWithMilestonesPayload!
   updateProject(input: ProjectInputPartial!): UpdateProjectPayload!
   createMilestone(input: MilestoneInput!): CreateMilestonePayload!
   createProject(
diff --git a/tests/projects/snapshots/schema_with_inheritance.gql b/tests/projects/snapshots/schema_with_inheritance.gql
index 352c8e1b..a99f50e4 100644
--- a/tests/projects/snapshots/schema_with_inheritance.gql
+++ b/tests/projects/snapshots/schema_with_inheritance.gql
@@ -159,6 +159,13 @@ input MilestoneFilter {
 input MilestoneInputPartial {
   id: GlobalID
   name: String
+  issues: [MilestoneIssueInputPartial!]
+  project: ProjectInputPartial
+}
+
+input MilestoneIssueInputPartial {
+  name: String
+  tags: [TagInputPartial!]
 }

 input MilestoneOrder {
@@ -304,6 +311,12 @@ type PageInfo {
   endCursor: String
 }

+input ProjectInputPartial {
+  id: GlobalID
+  name: String
+  milestones: [MilestoneInputPartial!]
+}
+
 input ProjectOrder {
   id: Ordering
   name: Ordering
@@ -401,6 +414,11 @@ input StrFilterLookup {
   iRegex: String
 }

+input TagInputPartial {
+  id: GlobalID
+  name: String
+}
+
 type TagType implements Node & Named {
   """The Globally Unique ID of this object"""
   id: GlobalID!
diff --git a/tests/test_input_mutations.py b/tests/test_input_mutations.py
index f48d54e3..ce405202 100644
--- a/tests/test_input_mutations.py
+++ b/tests/test_input_mutations.py
@@ -1,4 +1,7 @@
+from unittest.mock import patch
+
 import pytest
+from django.core.exceptions import ValidationError
 from strawberry.relay import from_base64, to_base64

 from tests.utils import GraphQLTestClient, assert_num_queries
@@ -10,7 +13,7 @@
     TagFactory,
     UserFactory,
 )
-from .projects.models import Issue, Milestone, Project
+from .projects.models import Issue, Milestone, Project, Tag

 @pytest.mark.django_db(transaction=True)
@@ -245,8 +248,8 @@ def test_input_create_mutation(db, gql_client: GraphQLTestClient):
 @pytest.mark.django_db(transaction=True)
 def test_input_create_mutation_nested_creation(db, gql_client: GraphQLTestClient):
     query = """
-    mutation CreateMilestone ($input: MilestoneInput!) {
-      createMilestone (input: $input) {
+    mutation CreateIssue ($input: IssueInput!) {
+      createIssue (input: $input) {
         __typename
         ... on OperationInfo {
           messages {
@@ -255,45 +258,70 @@ def test_input_create_mutation_nested_creation(db, gql_client: GraphQLTestClient
             message
           }
         }
-        ... on MilestoneType {
+        ... on IssueType {
           id
           name
-          project {
+          milestone {
             id
             name
+            project {
+              id
+              name
+            }
           }
         }
       }
     }
     """
     assert not Project.objects.filter(name="New Project").exists()
+    assert not Milestone.objects.filter(name="New Milestone").exists()
+    assert not Issue.objects.filter(name="New Issue").exists()

     res = gql_client.query(
         query,
         {
             "input": {
-                "name": "Some Milestone",
-                "project": {
-                    "name": "New Project",
+                "name": "New Issue",
+                "milestone": {
+                    "name": "New Milestone",
+                    "project": {
+                        "name": "New Project",
+                    },
                 },
             },
         },
     )
+
     assert res.data
-    assert isinstance(res.data["createMilestone"], dict)
+    assert isinstance(res.data["createIssue"], dict)

-    typename, _pk = from_base64(res.data["createMilestone"].pop("id"))
-    assert typename == "MilestoneType"
+    typename, pk = from_base64(res.data["createIssue"].get("id"))
+
+    assert typename == "IssueType"
+    issue = Issue.objects.get(pk=pk)
+    assert issue.name == "New Issue"
+
+    milestone = Milestone.objects.get(name="New Milestone")
+    assert milestone.name == "New Milestone"

     project = Project.objects.get(name="New Project")
+    assert project.name == "New Project"
+
+    assert milestone.project_id == project.pk
+    assert issue.milestone_id == milestone.pk

     assert res.data == {
-        "createMilestone": {
-            "__typename": "MilestoneType",
-            "name": "Some Milestone",
-            "project": {
-                "id": to_base64("ProjectType", project.pk),
-                "name": project.name,
+        "createIssue": {
+            "__typename": "IssueType",
+            "id": to_base64("IssueType", issue.pk),
+            "name": "New Issue",
+            "milestone": {
+                "id": to_base64("MilestoneType", milestone.pk),
+                "name": "New Milestone",
+                "project": {
+                    "id": to_base64("ProjectType", project.pk),
+                    "name": "New Project",
+                },
             },
         },
     }
@@ -377,6 +405,479 @@ def test_input_create_with_m2m_mutation(db, gql_client: GraphQLTestClient):
     }

+@pytest.mark.django_db(transaction=True)
+def test_input_create_mutation_with_multiple_level_nested_creation(
+    db, gql_client: GraphQLTestClient
+):
+    query = """
+    mutation createProjectWithMilestones ($input: ProjectInputPartial!) {
+      createProjectWithMilestones (input: $input) {
+        __typename
+        ... on OperationInfo {
+          messages {
+            kind
+            field
+            message
+          }
+        }
+        ... on ProjectType {
+          id
+          name
+          milestones {
+            id
+            name
+            issues {
+              id
+              name
+              tags {
+                name
+              }
+            }
+          }
+        }
+      }
+    }
+    """
+
+    shared_tag = TagFactory.create(name="Shared Tag")
+    shared_tag_id = to_base64("TagType", shared_tag.pk)
+
+    res = gql_client.query(
+        query,
+        {
+            "input": {
+                "name": "Some Project",
+                "milestones": [
+                    {
+                        "name": "Some Milestone",
+                        "issues": [
+                            {
+                                "name": "Some Issue",
+                                "tags": [
+                                    {"name": "Tag 1"},
+                                    {"name": "Tag 2"},
+                                    {"name": "Tag 3"},
+                                    {"id": shared_tag_id},
+                                ],
+                            }
+                        ],
+                    },
+                    {
+                        "name": "Another Milestone",
+                        "issues": [
+                            {
+                                "name": "Some Issue",
+                                "tags": [
+                                    {"name": "Tag 4"},
+                                    {"id": shared_tag_id},
+                                ],
+                            },
+                            {
+                                "name": "Another Issue",
+                                "tags": [
+                                    {"name": "Tag 5"},
+                                    {"id": shared_tag_id},
+                                ],
+                            },
+                            {
+                                "name": "Third issue",
+                                "tags": [
+                                    {"name": "Tag 6"},
+                                    {"id": shared_tag_id},
+                                ],
+                            },
+                        ],
+                    },
+                ],
+            },
+        },
+    )
+
+    assert res.data
+    assert isinstance(res.data["createProjectWithMilestones"], dict)
+
+    projects = Project.objects.all()
+    project_typename, project_pk = from_base64(
+        res.data["createProjectWithMilestones"].pop("id")
+    )
+    assert project_typename == "ProjectType"
+    assert projects[0] == Project.objects.get(pk=project_pk)
+
+    milestones = Milestone.objects.all()
+    assert len(milestones) == 2
+    assert len(res.data["createProjectWithMilestones"]["milestones"]) == 2
+
+    some_milestone = res.data["createProjectWithMilestones"]["milestones"][0]
+    milestone_typename, milestone_pk = from_base64(some_milestone.pop("id"))
+    assert milestone_typename == "MilestoneType"
+    assert milestones[0] == Milestone.objects.get(pk=milestone_pk)
+
+    another_milestone = res.data["createProjectWithMilestones"]["milestones"][1]
+    milestone_typename, milestone_pk = from_base64(another_milestone.pop("id"))
+    assert milestone_typename == "MilestoneType"
+    assert milestones[1] == Milestone.objects.get(pk=milestone_pk)
+
+    issues = Issue.objects.all()
+    assert len(issues) == 4
+    assert len(some_milestone["issues"]) == 1
+    assert len(another_milestone["issues"]) == 3
+
+    # Issues for first milestone
+    fetched_issue = some_milestone["issues"][0]
+    issue_typename, issue_pk = from_base64(fetched_issue.pop("id"))
+    assert issue_typename == "IssueType"
+    assert issues[0] == Issue.objects.get(pk=issue_pk)
+    # Issues for second milestone
+    for i in range(3):
+        fetched_issue = another_milestone["issues"][i]
+        issue_typename, issue_pk = from_base64(fetched_issue.pop("id"))
+        assert issue_typename == "IssueType"
+        assert issues[i + 1] == Issue.objects.get(pk=issue_pk)
+
+    tags = Tag.objects.all()
+    assert len(tags) == 7
+    assert len(issues[0].tags.all()) == 4  # 3 new tags + shared tag
+    assert len(issues[1].tags.all()) == 2  # 1 new tag + shared tag
+    assert len(issues[2].tags.all()) == 2  # 1 new tag + shared tag
+    assert len(issues[3].tags.all()) == 2  # 1 new tag + shared tag
+
+    assert res.data == {
+        "createProjectWithMilestones": {
+            "__typename": "ProjectType",
+            "name": "Some Project",
+            "milestones": [
+                {
+                    "name": "Some Milestone",
+                    "issues": [
+                        {
+                            "name": "Some Issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 1"},
+                                {"name": "Tag 2"},
+                                {"name": "Tag 3"},
+                            ],
+                        }
+                    ],
+                },
+                {
+                    "name": "Another Milestone",
+                    "issues": [
+                        {
+                            "name": "Some Issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 4"},
+                            ],
+                        },
+                        {
+                            "name": "Another Issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 5"},
+                            ],
+                        },
+                        {
+                            "name": "Third issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 6"},
+                            ],
+                        },
+                    ],
+                },
+            ],
+        },
+    }
+
+
+@pytest.mark.django_db(transaction=True)
+def test_input_update_mutation_with_multiple_level_nested_creation(
+    db, gql_client: GraphQLTestClient
+):
+    query = """
+    mutation UpdateProject ($input: ProjectInputPartial!) {
+      updateProject (input: $input) {
+        __typename
+        ... on OperationInfo {
+          messages {
+            kind
+            field
+            message
+          }
+        }
+        ... on ProjectType {
+          id
+          name
+          milestones {
+            id
+            name
+            issues {
+              id
+              name
+              tags {
+                name
+              }
+            }
+          }
+        }
+      }
+    }
+    """
+
+    project = ProjectFactory.create(name="Some Project")
+
+    shared_tag = TagFactory.create(name="Shared Tag")
+    shared_tag_id = to_base64("TagType", shared_tag.pk)
+
+    res = gql_client.query(
+        query,
+        {
+            "input": {
+                "id": to_base64("ProjectType", project.pk),
+                "milestones": [
+                    {
+                        "name": "Some Milestone",
+                        "issues": [
+                            {
+                                "name": "Some Issue",
+                                "tags": [
+                                    {"name": "Tag 1"},
+                                    {"name": "Tag 2"},
+                                    {"name": "Tag 3"},
+                                    {"id": shared_tag_id},
+                                ],
+                            }
+                        ],
+                    },
+                    {
+                        "name": "Another Milestone",
+                        "issues": [
+                            {
+                                "name": "Some Issue",
+                                "tags": [
+                                    {"name": "Tag 4"},
+                                    {"id": shared_tag_id},
+                                ],
+                            },
+                            {
+                                "name": "Another Issue",
+                                "tags": [
+                                    {"name": "Tag 5"},
+                                    {"id": shared_tag_id},
+                                ],
+                            },
+                            {
+                                "name": "Third issue",
+                                "tags": [
+                                    {"name": "Tag 6"},
+                                    {"id": shared_tag_id},
+                                ],
+                            },
+                        ],
+                    },
+                ],
+            },
+        },
+    )
+
+    assert res.data
+    assert isinstance(res.data["updateProject"], dict)
+
+    project_typename, project_pk = from_base64(res.data["updateProject"].pop("id"))
+    assert project_typename == "ProjectType"
+    assert project.pk == int(project_pk)
+
+    milestones = Milestone.objects.all()
+    assert len(milestones) == 2
+    assert len(res.data["updateProject"]["milestones"]) == 2
+
+    some_milestone = res.data["updateProject"]["milestones"][0]
+    milestone_typename, milestone_pk = from_base64(some_milestone.pop("id"))
+    assert milestone_typename == "MilestoneType"
+    assert milestones[0] == Milestone.objects.get(pk=milestone_pk)
+
+    another_milestone = res.data["updateProject"]["milestones"][1]
+    milestone_typename, milestone_pk = from_base64(another_milestone.pop("id"))
+    assert milestone_typename == "MilestoneType"
+    assert milestones[1] == Milestone.objects.get(pk=milestone_pk)
+
+    issues = Issue.objects.all()
+    assert len(issues) == 4
+    assert len(some_milestone["issues"]) == 1
+    assert len(another_milestone["issues"]) == 3
+
+    # Issues for first milestone
+    fetched_issue = some_milestone["issues"][0]
+    issue_typename, issue_pk = from_base64(fetched_issue.pop("id"))
+    assert issue_typename == "IssueType"
+    assert issues[0] == Issue.objects.get(pk=issue_pk)
+    # Issues for second milestone
+    for i in range(3):
+        fetched_issue = another_milestone["issues"][i]
+        issue_typename, issue_pk = from_base64(fetched_issue.pop("id"))
+        assert issue_typename == "IssueType"
+        assert issues[i + 1] == Issue.objects.get(pk=issue_pk)
+
+    tags = Tag.objects.all()
+    assert len(tags) == 7
+    assert len(issues[0].tags.all()) == 4  # 3 new tags + shared tag
+    assert len(issues[1].tags.all()) == 2  # 1 new tag + shared tag
+    assert len(issues[2].tags.all()) == 2  # 1 new tag + shared tag
+    assert len(issues[3].tags.all()) == 2  # 1 new tag + shared tag
+
+    assert res.data == {
+        "updateProject": {
+            "__typename": "ProjectType",
+            "name": "Some Project",
+            "milestones": [
+                {
+                    "name": "Some Milestone",
+                    "issues": [
+                        {
+                            "name": "Some Issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 1"},
+                                {"name": "Tag 2"},
+                                {"name": "Tag 3"},
+                            ],
+                        }
+                    ],
+                },
+                {
+                    "name": "Another Milestone",
+                    "issues": [
+                        {
+                            "name": "Some Issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 4"},
+                            ],
+                        },
+                        {
+                            "name": "Another Issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 5"},
+                            ],
+                        },
+                        {
+                            "name": "Third issue",
+                            "tags": [
+                                {"name": "Shared Tag"},
+                                {"name": "Tag 6"},
+                            ],
+                        },
+                    ],
+                },
+            ],
+        },
+    }
+
+
+@pytest.mark.parametrize("mock_model", ["Milestone", "Issue", "Tag"])
+@pytest.mark.django_db(transaction=True)
+def test_input_create_mutation_with_nested_calls_nested_full_clean(
+    db, gql_client: GraphQLTestClient, mock_model: str
+):
+    query = """
+    mutation createProjectWithMilestones ($input: ProjectInputPartial!) {
+      createProjectWithMilestones (input: $input) {
+        __typename
+        ... on OperationInfo {
+          messages {
+            kind
+            field
+            message
+          }
+        }
+        ... on ProjectType {
+          id
+          name
+          milestones {
+            id
+            name
+            issues {
+              id
+              name
+              tags {
+                name
+              }
+            }
+          }
+        }
+      }
+    }
+    """
+
+    shared_tag = TagFactory.create(name="Shared Tag")
+    shared_tag_id = to_base64("TagType", shared_tag.pk)
+
+    with patch(
+        f"tests.projects.models.{mock_model}.clean",
+        side_effect=ValidationError({"name": ValidationError("Invalid name")}),
+    ) as mocked_full_clean:
+        res = gql_client.query(
+            query,
+            {
+                "input": {
+                    "name": "Some Project",
+                    "milestones": [
+                        {
+                            "name": "Some Milestone",
+                            "issues": [
+                                {
+                                    "name": "Some Issue",
+                                    "tags": [
+                                        {"name": "Tag 1"},
+                                        {"name": "Tag 2"},
+                                        {"name": "Tag 3"},
+                                        {"id": shared_tag_id},
+                                    ],
+                                }
+                            ],
+                        },
+                        {
+                            "name": "Another Milestone",
+                            "issues": [
+                                {
+                                    "name": "Some Issue",
+                                    "tags": [
+                                        {"name": "Tag 4"},
+                                        {"id": shared_tag_id},
+                                    ],
+                                },
+                                {
+                                    "name": "Another Issue",
+                                    "tags": [
+                                        {"name": "Tag 5"},
+                                        {"id": shared_tag_id},
+                                    ],
+                                },
+                                {
+                                    "name": "Third issue",
+                                    "tags": [
+                                        {"name": "Tag 6"},
+                                        {"id": shared_tag_id},
+                                    ],
+                                },
+                            ],
+                        },
+                    ],
+                },
+            },
+        )
+
+    assert res.data
+    assert isinstance(res.data["createProjectWithMilestones"], dict)
+    assert res.data["createProjectWithMilestones"]["__typename"] == "OperationInfo"
+    assert mocked_full_clean.call_count == 1
+    assert res.data["createProjectWithMilestones"]["messages"] == [
+        {"field": "name", "kind": "VALIDATION", "message": "Invalid name"}
+    ]
+
+
 @pytest.mark.django_db(transaction=True)
 def test_input_update_mutation(db, gql_client: GraphQLTestClient):
     query = """

Perform validation check on updating nested fields

## Summary
If a nested field is updated via the parent field, a validation on the nested field should be performed, if some validators are provided on the django field.

## Feature Request Type

- [ ] Core functionality
- [x] Alteration (enhancement/optimization) of existing feature(s)
- [ ] New behavior

## Description

Assuming we have a Parent model with some nested fields:

```python
class Parent(models.Model);
   foo_field = models.CharField()

class Child(models.Model):
   url = models.URLField(validators=[validators.validate_url])
   parent_field = models.ForeignKey(to=Parent, related_name="children")
```

We create some types and a create mutation:

```python
@strawberry.type(Child, exclude=["parent_field"])
class Child:
    id: auto
    url: auto

@strawberry.input(Child)
class ChildInput(Child):
   pass

@strawberry.type(Parent)
class Parent:
   id: auto
   foo_field: auto
   children: List[Child]

@strawberry.input(Parent)
class ParentInput(Parent):
   pass

@strawberry.type
class Mutation:
    parent: Parent = mutations.create(types.ParentInput)
```

No matter which value is passed to the URL Field of the `Child`, the validator is not executed.

## Solution

Reason is that the `update_m2m` method in **resolvers.py** does not execute a `full_clean` on the respective object.

## Upvote & Fund

- We're using [Polar.sh](https://polar.sh/strawberry-graphql) so you can upvote and help fund this issue.
- We receive the funding once the issue is completed & confirmed by you.
- Thank you in advance for helping prioritize & fund our backlog.

<picture>
  <source media="(prefers-color-scheme: dark)" srcset="https://polar.sh/api/github/strawberry-graphql/strawberry-graphql-django/issues/398/pledge.svg?darkmode=1">

</picture>

Hey @tokr-bit ,

Thanks for reporting this!

I'm wondering if https://github.com/strawberry-graphql/strawberry-graphql-django/pull/394 will also fix this issue?

cc @sdobbelaere
Hey,
#394 won't fix the issue since the `update_m2m` method is not changed. If you and @sdobbelaere support the feature request, I can supply a PR.

@tokr-bit I would appreciate this fix.  I feel this would be something I'd run into myself as well during my project.
I'm not sure if this is why the issue was left open (since #405 was meant to fix it?) but there is an additional issue even with the fix where integrity errors raised by the creation happen before the call to `full_clean()`, which can lead to messages like this that leak model constraint and field names

```
{
    "data": null,
    "errors": [
        {
            "message": "duplicate key value violates unique constraint \"project_tags_unique_label_per_issue\"\nDETAIL:  Key (label, issue_id)=(L1, 55) already exists.\n",
            "locations": [
                {
                    "line": 2,
                    "column": 3
                }
            ],
            "path": [
                "createIssueWithTags"
            ]
        }
    ],
}
```


diff --git a/msal/application.py b/msal/application.py
index 5d1406af..a06df303 100644
--- a/msal/application.py
+++ b/msal/application.py
@@ -11,8 +11,6 @@
 from threading import Lock
 import os

-import requests
-
 from .oauth2cli import Client, JwtAssertionCreator
 from .oauth2cli.oidc import decode_part
 from .authority import Authority
@@ -425,6 +423,8 @@ def __init__(
         if http_client:
             self.http_client = http_client
         else:
+            import requests  # Lazy load
+
             self.http_client = requests.Session()
             self.http_client.verify = verify
             self.http_client.proxies = proxies
diff --git a/msal/authority.py b/msal/authority.py
index 145ce3d9..ecf6b777 100644
--- a/msal/authority.py
+++ b/msal/authority.py
@@ -5,11 +5,6 @@
     from urlparse import urlparse
 import logging

-# Historically some customers patched this module-wide requests instance.
-# We keep it here for now. They will be removed in next major release.
-import requests
-import requests as _requests
-
 from .exceptions import MsalServiceError

@@ -59,9 +54,10 @@ class Authority(object):
     _domains_without_user_realm_discovery = set([])

     @property
-    def http_client(self):  # Obsolete. We will remove this in next major release.
-        # A workaround: if module-wide requests is patched, we honor it.
-        return self._http_client if requests is _requests else requests
+    def http_client(self):  # Obsolete. We will remove this eventually
+        warnings.warn(
+            "authority.http_client might be removed in MSAL Python 1.21+", DeprecationWarning)
+        return self._http_client

     def __init__(self, authority_url, http_client, validate_authority=True):
         """Creates an authority instance, and also validates it.
@@ -84,7 +80,7 @@ def __init__(self, authority_url, http_client, validate_authority=True):
             payload = instance_discovery(
                 "https://{}{}/oauth2/v2.0/authorize".format(
                     self.instance, authority.path),
-                self.http_client)
+                self._http_client)
             if payload.get("error") == "invalid_instance":
                 raise ValueError(
                     "invalid_instance: "
@@ -104,7 +100,7 @@ def __init__(self, authority_url, http_client, validate_authority=True):
         try:
             openid_config = tenant_discovery(
                 tenant_discovery_endpoint,
-                self.http_client)
+                self._http_client)
         except ValueError:
             raise ValueError(
                 "Unable to get authority configuration for {}. "
@@ -124,7 +120,7 @@ def user_realm_discovery(self, username, correlation_id=None, response=None):
         # "federation_protocol", "cloud_audience_urn",
         # "federation_metadata_url", "federation_active_auth_url", etc.
         if self.instance not in self.__class__._domains_without_user_realm_discovery:
-            resp = response or self.http_client.get(
+            resp = response or self._http_client.get(
                 "https://{netloc}/common/userrealm/{username}?api-version=1.0".format(
                     netloc=self.instance, username=username),
                 headers={'Accept': 'application/json',
diff --git a/msal/oauth2cli/assertion.py b/msal/oauth2cli/assertion.py
index 0cf58799..855bd16b 100644
--- a/msal/oauth2cli/assertion.py
+++ b/msal/oauth2cli/assertion.py
@@ -4,8 +4,6 @@
 import uuid
 import logging

-import jwt
-

 logger = logging.getLogger(__name__)

@@ -99,6 +97,7 @@ def create_normal_assertion(
         Parameters are defined in https://tools.ietf.org/html/rfc7523#section-3
         Key-value pairs in additional_claims will be added into payload as-is.
         """
+        import jwt  # Lazy loading
         now = time.time()
         payload = {
             'aud': audience,
diff --git a/msal/oauth2cli/oauth2.py b/msal/oauth2cli/oauth2.py
index e092b3dd..54708004 100644
--- a/msal/oauth2cli/oauth2.py
+++ b/msal/oauth2cli/oauth2.py
@@ -17,8 +17,6 @@
 import string
 import hashlib

-import requests
-
 from .authcode import AuthCodeReceiver as _AuthCodeReceiver

 try:
@@ -159,6 +157,8 @@ def __init__(
                     "when http_client is in use")
             self._http_client = http_client
         else:
+            import requests  # Lazy loading
+
             self._http_client = requests.Session()
             self._http_client.verify = True if verify is None else verify
             self._http_client.proxies = proxies

diff --git a/tests/test_authority_patch.py b/tests/test_authority_patch.py
deleted file mode 100644
index 1feca62d..00000000
--- a/tests/test_authority_patch.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import unittest
-
-import msal
-from tests.http_client import MinimalHttpClient
-
-
-class DummyHttpClient(object):
-    def get(self, url, **kwargs):
-        raise RuntimeError("just for testing purpose")
-
-
-class TestAuthorityHonorsPatchedRequests(unittest.TestCase):
-    """This is only a workaround for an undocumented behavior."""
-    def test_authority_honors_a_patched_requests(self):
-        # First, we test that the original, unmodified authority is working
-        a = msal.authority.Authority(
-            "https://login.microsoftonline.com/common", MinimalHttpClient())
-        self.assertEqual(
-            a.authorization_endpoint,
-            'https://login.microsoftonline.com/common/oauth2/v2.0/authorize')
-
-        original = msal.authority.requests
-        try:
-            # Now we mimic a (discouraged) practice of patching authority.requests
-            msal.authority.requests = DummyHttpClient()
-            # msal.authority is expected to honor that patch.
-            with self.assertRaises(RuntimeError):
-                a = msal.authority.Authority(
-                    "https://login.microsoftonline.com/common", MinimalHttpClient())
-        finally:  # Tricky:
-            # Unpatch is necessary otherwise other test cases would be affected
-            msal.authority.requests = original

MSAL can consider using lazy import for `request`, `jwt`
**Describe the bug**

Importing MSAL is can cost ~300ms on Windows due to some heavy libraries like `request` and `jwt`:

```
python -X importtime -c "import msal"  2>perf.log; tuna .\perf.log
```

![image](https://user-images.githubusercontent.com/4003950/137872368-a85b9753-f46b-4ffe-acdd-323815486c1b.png)

MSAL can consider using lazy import for these libraries so that importing MSAL can be lightweight. (Azure CLI lazily imports MSAL: https://github.com/Azure/azure-cli/pull/19898)


diff --git a/python/ray/autoscaler/_private/gcp/config.py b/python/ray/autoscaler/_private/gcp/config.py
index fb6fdf2608dd6..4a96c08ed52a3 100644
--- a/python/ray/autoscaler/_private/gcp/config.py
+++ b/python/ray/autoscaler/_private/gcp/config.py
@@ -6,6 +6,9 @@
 import time
 from functools import partial, reduce

+import google_auth_httplib2
+import googleapiclient
+import httplib2
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -329,21 +332,40 @@ def _is_head_node_a_tpu(config: dict) -> bool:
     return get_node_type(node_configs[config["head_node_type"]]) == GCPNodeType.TPU

+def build_request(http, *args, **kwargs):
+    new_http = google_auth_httplib2.AuthorizedHttp(
+        http.credentials, http=httplib2.Http()
+    )
+    return googleapiclient.http.HttpRequest(new_http, *args, **kwargs)
+
+
 def _create_crm(gcp_credentials=None):
     return discovery.build(
-        "cloudresourcemanager", "v1", credentials=gcp_credentials, cache_discovery=False
+        "cloudresourcemanager",
+        "v1",
+        credentials=gcp_credentials,
+        requestBuilder=build_request,
+        cache_discovery=False,
     )

 def _create_iam(gcp_credentials=None):
     return discovery.build(
-        "iam", "v1", credentials=gcp_credentials, cache_discovery=False
+        "iam",
+        "v1",
+        credentials=gcp_credentials,
+        requestBuilder=build_request,
+        cache_discovery=False,
     )

 def _create_compute(gcp_credentials=None):
     return discovery.build(
-        "compute", "v1", credentials=gcp_credentials, cache_discovery=False
+        "compute",
+        "v1",
+        credentials=gcp_credentials,
+        requestBuilder=build_request,
+        cache_discovery=False,
     )

@@ -352,6 +374,7 @@ def _create_tpu(gcp_credentials=None):
         "tpu",
         TPU_VERSION,
         credentials=gcp_credentials,
+        requestBuilder=build_request,
         cache_discovery=False,
         discoveryServiceUrl="https://tpu.googleapis.com/$discovery/rest",
     )
diff --git a/python/ray/autoscaler/_private/gcp/node.py b/python/ray/autoscaler/_private/gcp/node.py
index 9fb2aa7833eb6..56a756a587ce8 100644
--- a/python/ray/autoscaler/_private/gcp/node.py
+++ b/python/ray/autoscaler/_private/gcp/node.py
@@ -34,6 +34,7 @@ class inheriting from ``GCPNode``. Those classes are essentially dicts
 from typing import Any, Dict, List, Optional, Tuple, Union
 from uuid import uuid4

+import httplib2
 from google_auth_httplib2 import AuthorizedHttp
 from googleapiclient.discovery import Resource
 from googleapiclient.errors import HttpError
@@ -335,7 +336,7 @@ class GCPCompute(GCPResource):

     def get_new_authorized_http(self, http: AuthorizedHttp) -> AuthorizedHttp:
         """Generate a new AuthorizedHttp object with the given credentials."""
-        new_http = AuthorizedHttp(http.credentials)
+        new_http = AuthorizedHttp(http.credentials, http=httplib2.Http())
         return new_http

     def wait_for_operation(
@@ -637,7 +638,7 @@ def path(self):

     def get_new_authorized_http(self, http: AuthorizedHttp) -> AuthorizedHttp:
         """Generate a new AuthorizedHttp object with the given credentials."""
-        new_http = AuthorizedHttp(http.credentials)
+        new_http = AuthorizedHttp(http.credentials, http=httplib2.Http())
         return new_http

     def wait_for_operation(

Setup Commands not running with Ray Cluster on GCP
### What happened + What you expected to happen

I have been creating Ray Clusters on cloud VM's in Google Cloud. I've been having issues with the setup_commands in the ray cluster YAML file. These are supposed to run when new nodes are made.

The commands always run correctly on the head node. However, sometimes when new workers are created by the autoscaler, one or both of the worker nodes is not setup correctly. No errors appear in logs, but the worker is not set up correctly. It appears to work / stop working randomly.

The YAML file below is the configuration file I've been using. You'll need to change the <project-id> in 3 places for your specific cloud project. I've been creating the clusters using ray up <yaml file name> on the google cloud shell, then SSH'ing into the head node to run scripts. The error first started appearing when the autoscaler added more than one worker.

### Versions / Dependencies

Ray most recent version.
The cluster is created from the google cloud shell.

### Reproduction script
```
# An unique identifier for the head node and workers of this cluster.
cluster_name: gpu-cluster

# Cloud-provider specific configuration.
provider:
    type: gcp
    region: us-east1
    availability_zone: "us-east1-c"
    project_id: <the project ID>  # Globally unique project id
# The maximum number of workers nodes to launch in addition to the head
# node.
max_workers: 2

# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 2.0

# This executes all commands on all nodes in the docker container,
# and opens all the necessary ports to support the Ray cluster.
# Empty string means disabled.
# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 20
# How Ray will authenticate with newly launched nodes.
auth:
    ssh_user: ubuntu

# Tell the autoscaler the allowed node types and the resources they provide.
available_node_types:
    ray_head:
        # The resources provided by this node type.
        resources: {"CPU": 16}
        # Provider-specific config for the head node, e.g. instance type.
        node_config:
            machineType: n1-standard-16
            serviceAccounts:
              - email: "ray-autoscaler-sa-v1@<project name>.iam.gserviceaccount.com"
                scopes:
                 - "https://www.googleapis.com/auth/cloud-platform"
            disks:
              - boot: true
                autoDelete: true
                type: PERSISTENT
                initializeParams:
                  diskSizeGb: 100
                  # See https://cloud.google.com/compute/docs/images for more images
                  sourceImage: projects/ml-images/global/images/c0-deeplearning-common-cu121-v20231209-debian-11
            scheduling:
              - onHostMaintenance: TERMINATE

    ray_worker_gpu:
        # The minimum number of nodes of this type to launch.
        # This number should be >= 0.
        min_workers: 0
        # The maximum number of workers nodes of this type to launch.
        # This takes precedence over min_workers.
        max_workers: 2
        # The resources provided by this node type.
        resources: {"CPU": 8, "GPU": 1}
        # Provider-specific config for the head node, e.g. instance type. By default
        # Ray will auto-configure unspecified fields such as subnets and ssh-keys.
        # For more documentation on available fields, see:
        # https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert
        node_config:
            machineType: n1-standard-8
            serviceAccounts:
              - email: "ray-autoscaler-sa-v1@<project-id>.iam.gserviceaccount.com"
                scopes:
                 - "https://www.googleapis.com/auth/cloud-platform"
            disks:
              - boot: true
                autoDelete: true
                type: PERSISTENT
                initializeParams:
                  diskSizeGb: 100
                  # See https://cloud.google.com/compute/docs/images for more images
                  sourceImage: projects/ml-images/global/images/c0-deeplearning-common-cu121-v20231209-debian-11
            # Make sure to set scheduling->onHostMaintenance to TERMINATE when GPUs are present
            # Run workers on preemtible instance by default.
            # Comment this out to use on-demand
            guestAccelerators:
              - acceleratorType: nvidia-tesla-t4
                acceleratorCount: 1
            metadata:
              items:
                - key: install-nvidia-driver
                  value: "True"
            scheduling:
              - preemptible: true
              - onHostMaintenance: TERMINATE
# Specify the node type of the head node (as configured above).
head_node_type: ray_head

# Files or directories to copy to the head and worker nodes. The format is a
# dictionary from REMOTE_PATH: LOCAL_PATH, e.g.
file_mounts: {}
#    "/path1/on/remote/machine": "/path1/on/local/machine",
#    "/path2/on/remote/machine": "/path2/on/local/machine",

# run before setup commands (also outside of any docker containers)
initialization_commands:
 - 'echo "Setup Commands Started" >> /home/ubuntu/logs.txt 2>&1'

# List of shell commands to run to set up nodes.
setup_commands:
  - 'echo "Setup Commands Started" >> /home/ubuntu/logs.txt 2>&1'
  - "pip3 install torch >> /home/ubuntu/logs.txt 2>&1"
  - "pip3 install torchvision >> /home/ubuntu/logs.txt 2>&1"
  - "pip3 install Pillow >> /home/ubuntu/logs.txt 2>&1"
  - "pip3 install requests >> /home/ubuntu/logs.txt 2>&1"
  - "pip3 install Flask >> /home/ubuntu/logs.txt 2>&1"

# Custom commands that will be run on the head node after common setup.
head_setup_commands:
  - 'echo "Head Commands Started" >> /home/ubuntu/logs.txt 2>&1'
  - "pip install google-api-python-client==1.7.8"

# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands:
  - 'echo "Worker command Started" >> /home/ubuntu/logs.txt 2>&1'

# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
    - ray stop
    - >-
      ulimit -n 65536;
      ray start
      --head
      --port=6379
      --object-manager-port=8076
      --autoscaling-config=~/ray_bootstrap_config.yaml

# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
    - ray stop
    - >-
      ulimit -n 65536;
      ray start
      --address=$RAY_HEAD_IP:6379
      --object-manager-port=8076
```

### Issue Severity

High: It blocks me from completing my task.

@arandhaw when you say worker nodes is not setup correctly, what's they symptoms?
I am experiencing the same issue.
When I start 1 worker node at a time, it works fine.
If the autoscaler tries to start multiple nodes it will fail to run the setup commands. The result is that ray is not installed.

> 2024-07-09 11:57:41,321 INFO updater.py:452 -- [5/7] Initializing command runner
> 2024-07-09 11:57:41,321 INFO updater.py:498 -- [6/7] No setup commands to run.
> 2024-07-09 11:57:41,322 INFO updater.py:503 -- [7/7] Starting the Ray runtime
> 2024-07-09 11:57:41,322 VINFO command_runner.py:371 -- Running `export RAY_OVERRIDE_RESOURCES='{"CPU":1}';export RAY_HEAD_IP=10.128.0.62; ray stop`
> 2024-07-09 11:57:41,322 VVINFO command_runner.py:373 -- Full command is `ssh -tt -i ~/ray_bootstrap_key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_24bf68e341/c022e6b155/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@10.128.15.193 bash --login -c -i 'source ~/.bashrc; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (export RAY_OVERRIDE_RESOURCES='"'"'{"CPU":1}'"'"';export RAY_HEAD_IP=10.128.0.62; ray stop)'`
>
> ==> /tmp/ray/session_latest/logs/monitor.log <==
> 2024-07-09 11:57:41,488 INFO discovery.py:873 -- URL being requested: POST https://compute.googleapis.com/compute/v1/projects/animated-bit-413114/zones/us-central1-a/instances/ray-gcp-99498f66b357d8db-worker-5b1c305c-compute/setLabels?alt=json
> 2024-07-09 11:57:41,791 INFO node.py:348 -- wait_for_compute_zone_operation: Waiting for operation operation-1720526261534-61ccf3ca540ad-b590f7ab-89ab5f7d to finish...
> 2024-07-09 11:57:41,792 INFO discovery.py:873 -- URL being requested: GET https://compute.googleapis.com/compute/v1/projects/animated-bit-413114/zones/us-central1-a/operations/operation-1720526261534-61ccf3ca540ad-b590f7ab-89ab5f7d?alt=json
> 2024-07-09 11:57:46,930 INFO discovery.py:873 -- URL being requested: GET https://compute.googleapis.com/compute/v1/projects/animated-bit-413114/zones/us-central1-a/operations/operation-1720526261534-61ccf3ca540ad-b590f7ab-89ab5f7d?alt=json
> 2024-07-09 11:57:47,058 INFO node.py:367 -- wait_for_compute_zone_operation: Operation operation-1720526261534-61ccf3ca540ad-b590f7ab-89ab5f7d finished.
>
> ==> /tmp/ray/session_latest/logs/monitor.out <==
> 2024-07-09 11:57:47,058 ERR updater.py:171 -- New status: update-failed
> 2024-07-09 11:57:47,064 ERR updater.py:173 -- !!!
> 2024-07-09 11:57:47,064 VERR updater.py:183 -- Exception details: {'message': 'SSH command failed.'}
> 2024-07-09 11:57:47,065 ERR updater.py:185 -- Full traceback: Traceback (most recent call last):
>   File "/home/ubuntu/.local/share/pipx/venvs/ray/lib/python3.12/site-packages/ray/autoscaler/_private/updater.py", line 166, in run
>     self.do_update()
>   File "/home/ubuntu/.local/share/pipx/venvs/ray/lib/python3.12/site-packages/ray/autoscaler/_private/updater.py", line 531, in do_update
>     self.cmd_runner.run(
>   File "/home/ubuntu/.local/share/pipx/venvs/ray/lib/python3.12/site-packages/ray/autoscaler/_private/command_runner.py", line 379, in run
>     return self._run_helper(
>            ^^^^^^^^^^^^^^^^^
>   File "/home/ubuntu/.local/share/pipx/venvs/ray/lib/python3.12/site-packages/ray/autoscaler/_private/command_runner.py", line 298, in _run_helper
>     raise click.ClickException(fail_msg) from None
> click.exceptions.ClickException: SSH command failed.

My `setup_commands` block is not empty, and it was successful in installing ray on the head node.
I think the setup commands are only run on the first worked that is being created.

I'm saying that because using `ray monitor` revealed that my setup command was being run on exactly on host, and all the others failed. Eventually, all the worker nodes have been configured.
@jjyao let me clarify exactly what seems to occur.
The problem occurs when the autoscaler creates new worker nodes.
What is supposed to happen is that during startup, the setup commands are run on the worker nodes.
Instead, what sometimes happens is that on some of the nodes, none of the setup commands are run.

Since I install dependencies in the setup commands (e.g., "pip install torch"), my ray jobs fail since none of the required libraries have been installed.

To be clear, the problem is not that the commands are failing and raising error messages. They are not being run at all.
I may have solved mt problem. It turns out the version of ray installed on the head and worker nodes was 2.8.1, whereas the version on the cluster launcher was 2.32.0. I just assumed that ray would install itself on the head/worker nodes, but I think it was using an older version part of the VM image. By adding "pip install -U ray[all]" to the setup commands, it seems to have fixed the problem.

It would be nice if the documentation was clearer (or if a meaningful error message was given by ray).
I'm having the same issue.
the new nodes are missing the setup commands.

Also using Ray Cluster on GCP VMs.
> pip install -U ray[all]

did not work for me.
I'm having same issue here.
The last clue I found is that this doesn't happen until version 2.6.3, and it happens from 2.7.0 onwards.
What I'm doing to handle that for now, is use the "startup-script", a field from GCP
https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert

In the **node_config**

```yaml
node_config:
  machineType: custom-24-61440
  metadata:
        items:
          - key: startup-script
            value: |
              #!/bin/bash
              sudo mkdir -p /fiftyone/datasets
              <more commands here>
```
It's working as expected this way.
@Gabriellgpc Thank you for sharing! That's a good workaround, but I need the "file_mounts" option to sync files across the cluster and it seems hard to achieve that with startup scripts.
I've been struggling with this issue since version 2.7.0, but no fix has appeared yet.
@astron8t-voyagerx I'm using the startup command to do something similar actually, because I want to share large folder between nodes, and the file mount do not handle that, I'm using the GCP buckets to store my large files and mounting it on all the nodes using the gsfuse command.
So my node_config looks like that,
```
            metadata:
              items:
                - key: startup-script
                  value: |
                    #!/bin/bash
                    sudo mkdir -p /fiftyone/datasets
                    sudo gcsfuse --implicit-dirs -o allow_other -file-mode=777 -dir-mode=777 --only-dir datasets fiftyone /fiftyone/datasets
```
I'm observing something very similar to this. My setup is the following:

`cluster-config.yml`

```yml
[...]

provider:
  type: gcp
  region: us-east4
  availability_zone: us-east4-a
  project_id: [project-id]

[...]

cluster_synced_files: []
file_mounts_sync_continuously: false
rsync_exclude: []
rsync_filter:
  - ".gitignore"

initialization_commands: []

setup_commands:
  - cd ~/code && bazelisk run //:create_venv  # Use `rules_uv` to create `.venv` with pip packages.
  - # download ~200MB file
  - echo 'source ~/code/.venv/bin/activate' >> ~/.bashrc
  - python -m ensurepip && python -m pip install -U google-api-python-client

head_setup_commands: []
worker_setup_commands: []

head_start_ray_commands:
  - ray stop
  - >-
    ray start
    --head
    --port=6379
    --object-manager-port=8076
    --autoscaling-config=~/ray_bootstrap_config.yaml
    --disable-usage-stats

# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
  - ray stop
  - >-
    ray start
    --address=$RAY_HEAD_IP:6379
    --object-manager-port=8076
    --disable-usage-stats
```

 Snippet 1/3 of `/tmp/ray/session_latest/logs/monitor*`. Command 'ray' not found.

```log
[...]
To run a command as administrator (user "root"), use "sudo <command>".
See "man sudo_root" for details.

Command 'ray' not found, did you mean:
  command 'say' from deb gnustep-gui-runtime (0.30.0-3build3)
  command 'rar' from deb rar (2:6.23-1)
  command 'ra6' from deb ipv6toolkit (2.0+ds.1-2)
  command 'ra' from deb argus-client (1:3.0.8.2-6.2ubuntu1)
Try: sudo apt install <deb name>
2024-11-13 15:16:58,640 INFO log_timer.py:25 -- NodeUpdater: ray-cluster-tuomas-worker-9edc73c3-compute: Ray start commands failed [LogTimer=617ms]
2024-11-13 15:16:58,640 INFO log_timer.py:25 -- NodeUpdater: ray-cluster-tuomas-worker-9edc73c3-compute: Applied config 09ddcf9108642756fa08d76c833ac1f21e014382  [LogTimer=45322ms]
To run a command as administrator (user "root"), use "sudo <command>".
See "man sudo_root" for details.
[...]
```

 Snippet 2/3 of `/tmp/ray/session_latest/logs/monitor*`. Seems weird that there's "no initialization commands to run" and "no setup commands to run".

```
2024-11-13 15:14:49,324 VINFO command_runner.py:371 -- Running `uptime`                                                                         15:54:04 [351/1849]
2024-11-13 15:14:49,324 VVINFO command_runner.py:373 -- Full command is `ssh -tt -i ~/ray_bootstrap_key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/
null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_24bf68e3
41/aaa3e6b268/%C -o ControlPersist=10s -o ConnectTimeout=10s ubuntu@10.150.0.74 bash --login -c -i 'source ~/.bashrc; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignor
e && (uptime)'`
To run a command as administrator (user "root"), use "sudo <command>".
See "man sudo_root" for details.

 15:14:50 up 7 min,  1 user,  load average: 0.00, 0.00, 0.00
2024-11-13 15:14:50,402 SUCC updater.py:295 -- Success.
2024-11-13 15:14:50,402 INFO log_timer.py:25 -- NodeUpdater: ray-cluster-test-worker-60b525fc-compute: Got remote shell  [LogTimer=6672ms]
2024-11-13 15:14:54,984 INFO updater.py:339 -- New status: waiting-for-ssh
2024-11-13 15:14:54,984 INFO updater.py:277 -- [1/7] Waiting for SSH to become available
2024-11-13 15:14:54,984 INFO updater.py:281 -- Running `uptime` as a test.
2024-11-13 15:14:54,984 INFO updater.py:389 -- Updating cluster configuration. [hash=09ddcf9108642756fa08d76c833ac1f21e014382]
2024-11-13 15:14:54,985 INFO command_runner.py:204 -- Fetched IP: 10.150.0.83
2024-11-13 15:14:54,990 INFO log_timer.py:25 -- NodeUpdater: ray-cluster-test-worker-61eab44a-compute: Got IP  [LogTimer=6ms]
2024-11-13 15:14:54,990 VINFO command_runner.py:371 -- Running `uptime`
2024-11-13 15:14:54,991 VVINFO command_runner.py:373 -- Full command is `ssh -tt -i ~/ray_bootstrap_key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/
null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_24bf68e3
41/aaa3e6b268/%C -o ControlPersist=10s -o ConnectTimeout=10s ubuntu@10.150.0.83 bash --login -c -i 'source ~/.bashrc; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignor
e && (uptime)'`
To run a command as administrator (user "root"), use "sudo <command>".
See "man sudo_root" for details.

 15:14:56 up 7 min,  1 user,  load average: 0.16, 0.07, 0.03
2024-11-13 15:14:56,050 SUCC updater.py:295 -- Success.
2024-11-13 15:14:56,050 INFO log_timer.py:25 -- NodeUpdater: ray-cluster-test-worker-61eab44a-compute: Got remote shell  [LogTimer=1066ms]
2024-11-13 15:15:00,523 INFO updater.py:396 -- New status: syncing-files
2024-11-13 15:15:00,523 INFO updater.py:389 -- Updating cluster configuration. [hash=09ddcf9108642756fa08d76c833ac1f21e014382]2024-11-13 15:15:00,523   INFO update
r.py:254 -- [2/7] Processing file mounts

2024-11-13 15:15:00,528 INFO updater.py:271 -- [3/7] No worker file mounts to sync
2024-11-13 15:15:06,116 INFO updater.py:396 -- New status: syncing-files
2024-11-13 15:15:06,121 INFO updater.py:254 -- [2/7] Processing file mounts
2024-11-13 15:15:06,122 INFO updater.py:271 -- [3/7] No worker file mounts to sync
2024-11-13 15:15:11,733 INFO updater.py:407 -- New status: setting-up
2024-11-13 15:15:11,740 INFO updater.py:448 -- [4/7] No initialization commands to run.
2024-11-13 15:15:11,740 INFO updater.py:452 -- [5/7] Initializing command runner
2024-11-13 15:15:11,740 INFO updater.py:498 -- [6/7] No setup commands to run.
2024-11-13 15:15:11,740 INFO updater.py:503 -- [7/7] Starting the Ray runtime
```

 Snippet 3/3 of `/tmp/ray/session_latest/logs/monitor*`. Messy error log ending in SSL erorr.

```
[... more logs ...]

Warning: Permanently added '10.150.0.48' (ED25519) to the list of known hosts.
Warning: Permanently added '10.150.0.65' (ED25519) to the list of known hosts.
Shared connection to 10.150.0.48 closed.
Shared connection to 10.150.0.65 closed.
Warning: Permanently added '10.150.0.65' (ED25519) to the list of known hosts.
Shared connection to 10.150.0.65 closed.
Exception in thread Exception in thread Thread-207 (spawn_updater):
Thread-204 (spawn_updater)Traceback (most recent call last):
:
  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12/
threading.py", line 1075, in _bootstrap_inner
Traceback (most recent call last):
  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12/
threading.py", line 1075, in _bootstrap_inner
    self.run()
  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12/
threading.py", line 1012, in run
    self.run()    self._target(*self._args, **self._kwargs)

  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12/
threading.py", line 1012, in run
  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/autoscaler.py", line 1343, in spawn_updater
    updater = NodeUpdaterThread(
       self._target(*self._args, **self._kwargs)
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/autoscaler.py", line 1343, in spawn_updater
        updater = NodeUpdaterThread(
           ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^
^^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/updater.py", line 569, in __init__
^^    ^NodeUpdater.__init__(self, *args, **kwargs)^
^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/updater.py", line 103, in __init__
^
  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/updater.py", line 569, in __init__
    self.cmd_runner = provider.get_command_runner(
         NodeUpdater.__init__(self, *args, **kwargs)
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/updater.py", line 103, in __init__
          self.cmd_runner = provider.get_command_runner(
             ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^^^
^^^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/gcp/node_provider.py", line 334, in get_command_runner
^^    ^instance = resource.get_instance(node_id)^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^
    File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/gcp/node_provider.py", line 334, in get_command_runner
     instance = resource.get_instance(node_id)
  ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/gcp/node.py", line 454, in get_instance
^^^    .execute()
 ^ ^ ^  ^^^^^^^^^^^^^^
^^
  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/ray/autoscaler/_private/gcp/node.py", line 454, in get_instance
  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
        .execute()return wrapped(*args, **kwargs)

          ^ ^ ^ ^ ^ ^ ^^^^^^
^^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
^^^    ^return wrapped(*args, **kwargs)^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^
^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/http.py", line 923, in execute
^^    ^resp, content = _retry_request(^
^ ^ ^ ^ ^ ^ ^ ^   ^ ^ ^ ^ ^ ^ ^  ^
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/http.py", line 923, in execute
^^^    ^resp, content = _retry_request(^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/http.py", line 222, in _retry_request
          raise exception
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/http.py", line 191, in _retry_request
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/http.py", line 191, in _retry_request
 ^^^    ^resp, content = http.request(uri, method, *args, **kwargs)^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^
     File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/http.py", line 222, in _retry_request
       raise exception
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/googleapiclient/http.py", line 191, in _retry_request
  ^    ^resp, content = http.request(uri, method, *args, **kwargs)^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/google_auth_httplib2.py", line 218, in request
^^    ^response, content = self.http.request(^
^  ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/google_auth_httplib2.py", line 218, in request
 ^^    ^response, content = self.http.request(^
^ ^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/httplib2/__init__.py", line 1724, in request
         (response, content) = self._request(
         ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/httplib2/__init__.py", line 1724, in request
  ^^    ^(response, content) = self._request(^
^  ^ ^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/httplib2/__init__.py", line 1444, in _request
         (response, content) = self._conn_request(conn, request_uri, method, body, headers)
                ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/httplib2/__init__.py", line 1444, in _request
   ^^^^^    ^(response, content) = self._conn_request(conn, request_uri, method, body, headers)^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/httplib2/__init__.py", line 1396, in _conn_request
^^^    ^response = conn.getresponse()^
^ ^^   ^^   ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^^^
^  File "/home/ubuntu/code/.venv/lib/python3.12/site-packages/httplib2/__init__.py", line 1396, in _conn_request
^^^^^^
  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12/http/client.py", line 1428, in getresponse
    response = conn.getresponse()
        response.begin()
   File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/http/client.py", line 331, in begin
            version, status, reason = self._read_status()
^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/http/client.py", line 1428, in getresponse
      response.begin()
   File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12/http/client.py", line 331, in begin
         version, status, reason = self._read_status()
  ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
      File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3
.12/http/client.py", line 292, in _read_status
        line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^
^  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/http/client.py", line 292, in _read_status
^^^^    ^line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^
^  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/socket.py", line 720, in readinto
^^^^^^^^^^^    ^^^^^^return self._sock.recv_into(b)
^^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/socket.py", line 720, in readinto
  ^^^^^    ^return self._sock.recv_into(b)^
^ ^^^ ^^ ^ ^ ^ ^ ^ ^^^^ ^
   File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/ssl.py", line 1251, in recv_into
^^^^    ^return self.read(nbytes, buffer)^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^
^  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/ssl.py", line 1251, in recv_into
^^^    ^return self.read(nbytes, buffer)^
         ^ ^^ ^^^^^^^^^^^^^^^^^^
^  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/ssl.py", line 1103, in read
^^^^^^    ^return self._sslobj.read(len, buffer)^
^ ^ ^ ^ ^ ^ ^ ^
   File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/da272e0d09e1d329650b5ce1f38bb920/external/rules_python~~python~python_3_12_x86_64-unknown-linux-gnu/lib/python3.12
/ssl.py", line 1103, in read
  ^    ^return self._sslobj.read(len, buffer)^
^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^ssl^^^.^SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2570)^
^^^^^^^^
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2570)
Warning: Permanently added '10.150.0.69' (ED25519) to the list of known hosts.
Shared connection to 10.150.0.69 closed.
Shared connection to 10.150.0.69 closed.
Shared connection to 10.150.0.69 closed.
Shared connection to 10.150.0.69 closed.
Shared connection to 10.150.0.69 closed.

[...more logs...]
```

Which makes me suspect that this is perhaps some sort of threading issue?

Note that some of my workers run exactly as expected. This seems to be happening more often the more nodes I have and about a quarter to half of my nodes setup fine when I run with 32 nodes. It feels like this is happening much less frequently when running with fewer nodes.
@jjyao maybe we could do some investigation here? seems odd to have this reliability issue
same issue here, any updates?
I am currently trying to handle this issue. I'll put simpler reproduction steps here as a note for myself.

```yaml
cluster_name: ray-46451

provider:
    type: gcp
    region: us-west1
    availability_zone: us-west1-a
    project_id: <project_id>

max_workers: 12

upscaling_speed: 100.0

idle_timeout_minutes: 60
auth:
    ssh_user: ubuntu

available_node_types:
    ray_head_default:
        resources: {"CPU": 0}
        node_config:
            machineType: n1-standard-2
            disks:
              - boot: true
                autoDelete: true
                type: PERSISTENT
                initializeParams:
                  diskSizeGb: 50
                  sourceImage: projects/deeplearning-platform-release/global/images/common-cpu-v20240922

    ray_worker_small:
        min_workers: 0
        max_workers: 10
        resources: {"CPU": 2}
        node_config:
            machineType: n1-standard-2
            disks:
              - boot: true
                autoDelete: true
                type: PERSISTENT
                initializeParams:
                  diskSizeGb: 50
                  sourceImage: projects/deeplearning-platform-release/global/images/common-cpu-v20240922
            scheduling:
              - preemptible: true

head_node_type: ray_head_default

file_mounts: {}
cluster_synced_files: []
file_mounts_sync_continuously: false
rsync_exclude: []
rsync_filter:
  - ".gitignore"

initialization_commands:
 - 'echo "Run initialization_commands" >> /tmp/logs.txt 2>&1'

# List of shell commands to run to set up nodes.
setup_commands:
  - 'echo "Run setup_commands" >> /tmp/logs.txt 2>&1'
  - pip install emoji

# Custom commands that will be run on the head node after common setup.
head_setup_commands:
  - 'echo "Run head_setup_commands" >> /tmp/logs.txt 2>&1'
  - "pip install google-api-python-client==1.7.8"

# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands:
  - 'echo "Run worker_setup_commands" >> /tmp/logs.txt 2>&1'

# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
    - ray stop
    - >-
      ray start
      --head
      --port=6379
      --object-manager-port=8076
      --autoscaling-config=~/ray_bootstrap_config.yaml

# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
    - ray stop
    - >-
      ray start
      --address=$RAY_HEAD_IP:6379
      --object-manager-port=8076
```

Run `ray up` and `ray attach`. In head node run `ray job submit -- python task.py`

`task.py`

```python
import time
import ray

ray.init()

@ray.remote(num_cpus=2)
def f():
  time.sleep(10)
  try:
    import emoji
    return True
  except ModuleNotFoundError:
    return False

futures = [f.remote() for i in range(10)]
print(ray.get(futures))
```

Expected: All `True`.
Actual: Some are `False`.
Any luck with this? It seems like the issue can now be pretty reliably reproduced. Does anyone happen to have any workaround for this? I wonder if adding some delays/sleeps somewhere could help the issue.
@hartikainen Did you try Ray version 2.38.0? I followed the steps I mentioned above and experimented by adding `ray[default]==2.37.0` and `ray[default]==2.38.0` in the `setup_commands`. It seems that the issue doesn't occur with version 2.38.0. You can try updating to 2.38.0 first, and if it works without issues, I'll go ahead and close this issue.


diff --git a/backend/chainlit/server.py b/backend/chainlit/server.py
index b4bf0dd989..7c4a824b68 100644
--- a/backend/chainlit/server.py
+++ b/backend/chainlit/server.py
@@ -41,6 +41,7 @@
     APIRouter,
     Depends,
     FastAPI,
+    File,
     Form,
     HTTPException,
     Query,
@@ -839,11 +840,9 @@ async def delete_thread(

 @router.post("/project/file")
 async def upload_file(
+    current_user: Annotated[Union[User, PersistedUser], Depends(get_current_user)],
     session_id: str,
     file: UploadFile,
-    current_user: Annotated[
-        Union[None, User, PersistedUser], Depends(get_current_user)
-    ],
 ):
     """Upload a file to the session files directory."""

@@ -868,17 +867,21 @@ async def upload_file(

     content = await file.read()

+    assert file.filename, "No filename for uploaded file"
+    assert file.content_type, "No content type for uploaded file"
+
     file_response = await session.persist_file(
         name=file.filename, content=content, mime=file.content_type
     )

-    return JSONResponse(file_response)
+    return JSONResponse(content=file_response)

 @router.get("/project/file/{file_id}")
 async def get_file(
     file_id: str,
-    session_id: Optional[str] = None,
+    session_id: str,
+    current_user: Annotated[Union[User, PersistedUser], Depends(get_current_user)],
 ):
     """Get a file from the session files directory."""

@@ -888,10 +891,17 @@ async def get_file(

     if not session:
         raise HTTPException(
-            status_code=404,
-            detail="Session not found",
+            status_code=401,
+            detail="Unauthorized",
         )

+    if current_user:
+        if not session.user or session.user.identifier != current_user.identifier:
+            raise HTTPException(
+                status_code=401,
+                detail="You are not authorized to download files from this session",
+            )
+
     if file_id in session.files:
         file = session.files[file_id]
         return FileResponse(file["path"], media_type=file["type"])

diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 94ae4a596a..7cc4266371 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -1,5 +1,6 @@
 import datetime
 from contextlib import asynccontextmanager
+from typing import Callable
 from unittest.mock import AsyncMock, Mock

 import pytest
@@ -20,20 +21,30 @@ def persisted_test_user():

 @pytest.fixture
-def mock_session():
-    mock = Mock(spec=WebsocketSession)
-    mock.id = "test_session_id"
-    mock.user_env = {"test_env": "value"}
-    mock.chat_settings = {}
-    mock.chat_profile = None
-    mock.http_referer = None
-    mock.client_type = "webapp"
-    mock.languages = ["en"]
-    mock.thread_id = "test_thread_id"
-    mock.emit = AsyncMock()
-    mock.has_first_interaction = True
-
-    return mock
+def mock_session_factory(persisted_test_user: PersistedUser) -> Callable[..., Mock]:
+    def create_mock_session(**kwargs) -> Mock:
+        mock = Mock(spec=WebsocketSession)
+        mock.user = kwargs.get("user", persisted_test_user)
+        mock.id = kwargs.get("id", "test_session_id")
+        mock.user_env = kwargs.get("user_env", {"test_env": "value"})
+        mock.chat_settings = kwargs.get("chat_settings", {})
+        mock.chat_profile = kwargs.get("chat_profile", None)
+        mock.http_referer = kwargs.get("http_referer", None)
+        mock.client_type = kwargs.get("client_type", "webapp")
+        mock.languages = kwargs.get("languages", ["en"])
+        mock.thread_id = kwargs.get("thread_id", "test_thread_id")
+        mock.emit = AsyncMock()
+        mock.has_first_interaction = kwargs.get("has_first_interaction", True)
+        mock.files = kwargs.get("files", {})
+
+        return mock
+
+    return create_mock_session
+
+
+@pytest.fixture
+def mock_session(mock_session_factory) -> Mock:
+    return mock_session_factory()

 @asynccontextmanager
diff --git a/backend/tests/test_server.py b/backend/tests/test_server.py
index 6aa9c16d5a..36c65124d6 100644
--- a/backend/tests/test_server.py
+++ b/backend/tests/test_server.py
@@ -1,12 +1,19 @@
 import os
 from pathlib import Path
-from unittest.mock import Mock, create_autospec, mock_open
+import pathlib
+from typing import Callable
+from unittest.mock import AsyncMock, Mock, create_autospec, mock_open
+import datetime  # Added import for datetime

 import pytest
+import tempfile
+from chainlit.session import WebsocketSession
 from chainlit.auth import get_current_user
 from chainlit.config import APP_ROOT, ChainlitConfig, load_config
 from chainlit.server import app
 from fastapi.testclient import TestClient
+from chainlit.types import FileReference
+from chainlit.user import PersistedUser  # Added import for PersistedUser

 @pytest.fixture
@@ -219,7 +226,7 @@ def test_get_avatar_non_existent_favicon(

 def test_avatar_path_traversal(
-    test_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path
+    test_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path
 ):
     """Test to prevent potential path traversal in avatar route on Windows."""

@@ -240,6 +247,268 @@ def test_avatar_path_traversal(
     assert response.status_code == 400

+@pytest.fixture
+def mock_session_get_by_id_patched(mock_session: Mock, monkeypatch: pytest.MonkeyPatch):
+    test_session_id = "test_session_id"
+
+    # Mock the WebsocketSession.get_by_id method to return the mock session
+    monkeypatch.setattr(
+        "chainlit.session.WebsocketSession.get_by_id",
+        lambda session_id: mock_session if session_id == test_session_id else None,
+    )
+
+    return mock_session
+
+
+def test_get_file_success(
+    test_client: TestClient,
+    mock_session_get_by_id_patched: Mock,
+    tmp_path: pathlib.Path,
+    mock_get_current_user: Mock,
+):
+    """
+    Test successful retrieval of a file from a session.
+    """
+    # Set current_user to match session.user
+    mock_get_current_user.return_value = mock_session_get_by_id_patched.user
+
+    # Create test data
+    test_content = b"Test file content"
+    test_file_id = "test_file_id"
+
+    # Create a temporary file with the test content
+    test_file = tmp_path / "test_file"
+    test_file.write_bytes(test_content)
+
+    mock_session_get_by_id_patched.files = {
+        test_file_id: {
+            "id": test_file_id,
+            "path": test_file,
+            "name": "test.txt",
+            "type": "text/plain",
+            "size": len(test_content),
+        }
+    }
+
+    # Make the GET request to retrieve the file
+    response = test_client.get(
+        f"/project/file/{test_file_id}?session_id={mock_session_get_by_id_patched.id}"
+    )
+
+    # Verify the response
+    assert response.status_code == 200
+    assert response.content == test_content
+    assert response.headers["content-type"].startswith("text/plain")
+
+
+def test_get_file_not_existent_file(
+    test_client: TestClient,
+    mock_session_get_by_id_patched: Mock,
+    mock_get_current_user: Mock,
+):
+    """
+    Test retrieval of a non-existing file from a session.
+    """
+    # Set current_user to match session.user
+    mock_get_current_user.return_value = mock_session_get_by_id_patched.user
+
+    # Make the GET request to retrieve the file
+    response = test_client.get("/project/file/test_file_id?session_id=test_session_id")
+
+    # Verify the response
+    assert response.status_code == 404
+
+
+def test_get_file_non_existing_session(
+    test_client: TestClient,
+    tmp_path: pathlib.Path,
+    mock_session_get_by_id_patched: Mock,
+    mock_session: Mock,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """
+    Test that an unauthenticated user cannot retrieve a file uploaded by an authenticated user.
+    """
+
+    # Attempt to access the file without authentication by providing an invalid session_id
+    response = test_client.get(
+        f"/project/file/nonexistent?session_id=unauthenticated_session_id"
+    )
+
+    # Verify the response
+    assert response.status_code == 401  # Unauthorized
+
+
+def test_upload_file_success(
+    test_client: TestClient,
+    test_config: ChainlitConfig,
+    mock_session_get_by_id_patched: Mock,
+):
+    """Test successful file upload."""
+
+    # Prepare the files to upload
+    file_content = b"Sample file content"
+    files = {
+        "file": ("test_upload.txt", file_content, "text/plain"),
+    }
+
+    # Mock the persist_file method to return a known value
+    expected_file_id = "mocked_file_id"
+    mock_session_get_by_id_patched.persist_file = AsyncMock(
+        return_value={
+            "id": expected_file_id,
+            "name": "test_upload.txt",
+            "type": "text/plain",
+            "size": len(file_content),
+        }
+    )
+
+    # Make the POST request to upload the file
+    response = test_client.post(
+        "/project/file",
+        files=files,
+        params={"session_id": mock_session_get_by_id_patched.id},
+    )
+
+    # Verify the response
+    assert response.status_code == 200
+    response_data = response.json()
+    assert "id" in response_data
+    assert response_data["id"] == expected_file_id
+    assert response_data["name"] == "test_upload.txt"
+    assert response_data["type"] == "text/plain"
+    assert response_data["size"] == len(file_content)
+
+    # Verify that persist_file was called with the correct arguments
+    mock_session_get_by_id_patched.persist_file.assert_called_once_with(
+        name="test_upload.txt", content=file_content, mime="text/plain"
+    )
+
+
+def test_file_access_by_different_user(
+    test_client: TestClient,
+    mock_session_get_by_id_patched: Mock,
+    persisted_test_user: PersistedUser,
+    tmp_path: pathlib.Path,
+    mock_session_factory: Callable[..., Mock],
+):
+    """Test that a file uploaded by one user cannot be accessed by another user."""
+
+    # Prepare the files to upload
+    file_content = b"Sample file content"
+    files = {
+        "file": ("test_upload.txt", file_content, "text/plain"),
+    }
+
+    # Mock the persist_file method to return a known value
+    expected_file_id = "mocked_file_id"
+    mock_session_get_by_id_patched.persist_file = AsyncMock(
+        return_value={
+            "id": expected_file_id,
+            "name": "test_upload.txt",
+            "type": "text/plain",
+            "size": len(file_content),
+        }
+    )
+
+    # Make the POST request to upload the file
+    response = test_client.post(
+        "/project/file",
+        files=files,
+        params={"session_id": mock_session_get_by_id_patched.id},
+    )
+
+    # Verify the response
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "id" in response_data
+    file_id = response_data["id"]
+
+    # Create a second session with a different user
+    second_session = mock_session_factory(
+        id="another_session_id",
+        user=PersistedUser(
+            id="another_user_id",
+            createdAt=datetime.datetime.now().isoformat(),
+            identifier="another_user_identifier",
+        ),
+    )
+
+    # Attempt to access the uploaded file using the second user's session
+    response = test_client.get(
+        f"/project/file/{file_id}?session_id={second_session.id}"
+    )
+
+    # Verify that the access attempt fails
+    assert response.status_code == 401  # Unauthorized
+
+
+def test_upload_file_missing_file(
+    test_client: TestClient,
+    mock_session: Mock,
+):
+    """Test file upload with missing file in the request."""
+
+    # Make the POST request without a file
+    response = test_client.post(
+        "/project/file",
+        data={"session_id": mock_session.id},
+    )
+
+    # Verify the response
+    assert response.status_code == 422  # Unprocessable Entity
+    assert "detail" in response.json()
+
+
+def test_upload_file_invalid_session(
+    test_client: TestClient,
+):
+    """Test file upload with an invalid session."""
+
+    # Prepare the files to upload
+    file_content = b"Sample file content"
+    files = {
+        "file": ("test_upload.txt", file_content, "text/plain"),
+    }
+
+    # Make the POST request with an invalid session_id
+    response = test_client.post(
+        "/project/file",
+        files=files,
+        data={"session_id": "invalid_session_id"},
+    )
+
+    # Verify the response
+    assert response.status_code == 422
+
+
+def test_upload_file_unauthorized(
+    test_client: TestClient,
+    test_config: ChainlitConfig,
+    mock_session_get_by_id_patched: Mock,
+):
+    """Test file upload without proper authorization."""
+
+    # Mock the upload_file_session to have no user
+    mock_session_get_by_id_patched.user = None
+
+    # Prepare the files to upload
+    file_content = b"Sample file content"
+    files = {
+        "file": ("test_upload.txt", file_content, "text/plain"),
+    }
+
+    # Make the POST request to upload the file
+    response = test_client.post(
+        "/project/file",
+        files=files,
+        data={"session_id": mock_session_get_by_id_patched.id},
+    )
+
+    assert response.status_code == 422
+
+
 def test_project_translations_file_path_traversal(
     test_client: TestClient, monkeypatch: pytest.MonkeyPatch
 ):

fix(security): add auth to /project/file get endpoint
fixes https://github.com/Chainlit/chainlit/issues/1101

Thanks @qvalentin for the report & fix! We'd like to take this along in the next release.

Any chance you could add a regression unittest demonstrating the issue and it's resolution?