diff --git a/arkouda/dataframe.py b/arkouda/dataframe.py
index 10c98b1c69..6291b22ff8 100644
--- a/arkouda/dataframe.py
+++ b/arkouda/dataframe.py
@@ -1457,7 +1457,7 @@ def _prep_data(self, index=False, columns=None):
             data = {c: self.data[c] for c in columns}

         if index:
-            data["Index"] = self.index
+            data["Index"] = self.index.values
         return data

     def to_hdf(self, path, index=False, columns=None, file_type="distribute"):
diff --git a/arkouda/io.py b/arkouda/io.py
index f386ac3a77..cdfbd6b3b2 100644
--- a/arkouda/io.py
+++ b/arkouda/io.py
@@ -1,7 +1,6 @@
 import glob
 import json
 import os
-import warnings
 from typing import Dict, List, Mapping, Optional, Union, cast
 from warnings import warn

@@ -334,7 +333,7 @@ def _parse_errors(rep_msg, allow_errors: bool = False):
     file_errors = rep_msg["file_errors"] if "file_errors" in rep_msg else []
     if allow_errors and file_errors:
         file_error_count = rep_msg["file_error_count"] if "file_error_count" in rep_msg else -1
-        warnings.warn(
+        warn(
             f"There were {file_error_count} errors reading files on the server. "
             + f"Sample error messages {file_errors}",
             RuntimeWarning,
@@ -822,10 +821,8 @@ def to_parquet(
     mode : {'truncate' | 'append'}
         By default, truncate (overwrite) the output files if they exist.
         If 'append', attempt to create new dataset in existing files.
-    file_type : str ("single" | "distribute")
-            Default: distribute
-            Single writes the dataset to a single file
-            Distribute writes the dataset to a file per locale
+        'append' is deprecated, please use the multi-column write
+

     Returns
     -------
@@ -865,12 +862,32 @@ def to_parquet(
     if mode.lower() not in ["append", "truncate"]:
         raise ValueError("Allowed modes are 'truncate' and 'append'")

-    datasetNames, pdarrays = _bulk_write_prep(columns, names)
+    if mode.lower() == "append":
+        warn(
+            "Append has been deprecated when writing Parquet files. "
+            "Please write all columns to the file at once.",
+            DeprecationWarning,
+        )

-    for arr, name in zip(pdarrays, cast(List[str], datasetNames)):
-        arr.to_parquet(prefix_path=prefix_path, dataset=name, mode=mode, compressed=compressed)
-        if mode.lower() == "truncate":
-            mode = "append"
+    datasetNames, pdarrays = _bulk_write_prep(columns, names)
+    # append or single column use the old logic
+    if mode.lower() == "append" or len(pdarrays) == 1:
+        for arr, name in zip(pdarrays, cast(List[str], datasetNames)):
+            arr.to_parquet(prefix_path=prefix_path, dataset=name, mode=mode, compressed=compressed)
+    else:
+        print(cast(
+            str,
+            generic_msg(
+                cmd="toParquet_multi",
+                args={
+                    "columns": pdarrays,
+                    "col_names": datasetNames,
+                    "filename": prefix_path,
+                    "num_cols": len(pdarrays),
+                    "compressed": compressed,
+                },
+            )
+        ))

 def to_hdf(
@@ -1234,7 +1251,7 @@ def read(
     elif file_format == "parquet":
         cmd = "readAllParquet"
     else:
-        warnings.warn(f"Unrecognized file format string: {file_format}. Inferring file type")
+        warn(f"Unrecognized file format string: {file_format}. Inferring file type")
         cmd = "readany"
     if iterative:  # iterative calls to server readhdf
         return {
@@ -1266,7 +1283,7 @@ def read(
         file_errors = rep["file_errors"] if "file_errors" in rep else []
         if allow_errors and file_errors:
             file_error_count = rep["file_error_count"] if "file_error_count" in rep else -1
-            warnings.warn(
+            warn(
                 f"There were {file_error_count} errors reading files on the server. "
                 + f"Sample error messages {file_errors}",
                 RuntimeWarning,
diff --git a/src/ArrowFunctions.cpp b/src/ArrowFunctions.cpp
index cb6adfc0ec..74e9799be5 100644
--- a/src/ArrowFunctions.cpp
+++ b/src/ArrowFunctions.cpp
@@ -366,6 +366,128 @@ int cpp_readColumnByName(const char* filename, void* chpl_arr, const char* colna
   }
 }

+// configure the schema for a multicolumn file
+std::shared_ptr<parquet::schema::GroupNode> SetupSchema(void* column_names, void* datatypes, int64_t colnum) {
+  parquet::schema::NodeVector fields;
+  auto cname_ptr = (char**)column_names;
+  auto dtypes_ptr = (int64_t*) datatypes;
+  for (int64_t i = 0; i < colnum; i++){
+    if(dtypes_ptr[i] == ARROWINT64)
+      fields.push_back(parquet::schema::PrimitiveNode::Make(cname_ptr[i], parquet::Repetition::REQUIRED, parquet::Type::INT64, parquet::ConvertedType::NONE));
+    else if(dtypes_ptr[i] == ARROWUINT64)
+      fields.push_back(parquet::schema::PrimitiveNode::Make(cname_ptr[i], parquet::Repetition::REQUIRED, parquet::Type::INT64, parquet::ConvertedType::UINT_64));
+    else if(dtypes_ptr[i] == ARROWBOOLEAN)
+      fields.push_back(parquet::schema::PrimitiveNode::Make(cname_ptr[i], parquet::Repetition::REQUIRED, parquet::Type::BOOLEAN, parquet::ConvertedType::NONE));
+    else if(dtypes_ptr[i] == ARROWDOUBLE)
+      fields.push_back(parquet::schema::PrimitiveNode::Make(cname_ptr[i], parquet::Repetition::REQUIRED, parquet::Type::DOUBLE, parquet::ConvertedType::NONE));
+    else if(dtypes_ptr[i] == ARROWSTRING)
+      fields.push_back(parquet::schema::PrimitiveNode::Make(cname_ptr[i], parquet::Repetition::REQUIRED, parquet::Type::BYTE_ARRAY, parquet::ConvertedType::NONE));
+  }
+  return std::static_pointer_cast<parquet::schema::GroupNode>(
+      parquet::schema::GroupNode::Make("schema", parquet::Repetition::REQUIRED, fields));
+}
+
+int cpp_writeMultiColToParquet(const char* filename, void* column_names,
+                                void** ptr_arr, void* datatypes,
+                                int64_t colnum, int64_t numelems, int64_t rowGroupSize,
+                                bool compressed, char** errMsg) {
+  try {
+    // initialize the file to write to
+    using FileClass = ::arrow::io::FileOutputStream;
+    std::shared_ptr<FileClass> out_file;
+    ARROWRESULT_OK(FileClass::Open(filename), out_file);
+
+    // Setup the parquet schema
+    std::shared_ptr<parquet::schema::GroupNode> schema = SetupSchema(column_names, datatypes, colnum);
+
+    parquet::WriterProperties::Builder builder;
+    if(compressed) {
+      builder.compression(parquet::Compression::SNAPPY);
+      builder.encoding(parquet::Encoding::RLE);
+    }
+    std::shared_ptr<parquet::WriterProperties> props = builder.build();
+
+    std::shared_ptr<parquet::ParquetFileWriter> file_writer =
+      parquet::ParquetFileWriter::Open(out_file, schema, props);
+
+    std::queue<int64_t> idxQueue; // queue used to track string byteIdx
+
+    auto dtypes_ptr = (int64_t*) datatypes;
+    int64_t numLeft = numelems; // number of elements remaining to write (rows)
+    int64_t x = 0;  // index to start writing batch from
+    while (numLeft > 0) {
+      // Append a RowGroup with a specific number of rows.
+      parquet::RowGroupWriter* rg_writer = file_writer->AppendRowGroup();
+      int64_t batchSize = rowGroupSize;
+      if(numLeft < rowGroupSize)
+        batchSize = numLeft;
+
+      // loop the columns and write the row groups
+      for(int64_t i = 0; i < colnum; i++){
+        int64_t dtype = dtypes_ptr[i];
+        if (dtype == ARROWINT64 || dtype == ARROWUINT64) {
+          auto data_ptr = (int64_t*)ptr_arr[i];
+          parquet::Int64Writer* int64_writer =
+              static_cast<parquet::Int64Writer*>(rg_writer->NextColumn());
+          int64_writer->WriteBatch(batchSize, nullptr, nullptr, &data_ptr[x]);
+        } else if(dtype == ARROWBOOLEAN) {
+          auto data_ptr = (bool*)ptr_arr[i];
+          parquet::BoolWriter* bool_writer =
+            static_cast<parquet::BoolWriter*>(rg_writer->NextColumn());
+          bool_writer->WriteBatch(batchSize, nullptr, nullptr, &data_ptr[x]);
+        } else if(dtype == ARROWDOUBLE) {
+          auto data_ptr = (double*)ptr_arr[i];
+          parquet::DoubleWriter* dbl_writer =
+            static_cast<parquet::DoubleWriter*>(rg_writer->NextColumn());
+          dbl_writer->WriteBatch(batchSize, nullptr, nullptr, &data_ptr[x]);
+        } else if(dtype == ARROWSTRING) {
+          auto data_ptr = (uint8_t*)ptr_arr[i];
+          parquet::ByteArrayWriter* ba_writer =
+          static_cast<parquet::ByteArrayWriter*>(rg_writer->NextColumn());
+          int64_t count = 0;
+          int64_t byteIdx = 0;
+
+          // identify the starting byte index
+          if (x > 0){
+            byteIdx = idxQueue.front();
+            idxQueue.pop();
+          }
+
+          while(count < batchSize) {
+            parquet::ByteArray value;
+            int16_t definition_level = 1;
+            value.ptr = reinterpret_cast<const uint8_t*>(&data_ptr[byteIdx]);
+            int64_t nextIdx = byteIdx;
+            while (data_ptr[nextIdx] != 0x00){
+              nextIdx++;
+            }
+            // subtract 1 since we have the null terminator
+            value.len = nextIdx - byteIdx;
+            ba_writer->WriteBatch(1, &definition_level, nullptr, &value);
+            count++;
+            byteIdx = nextIdx + 1;
+          }
+          if (numLeft - count > 0) {
+            idxQueue.push(byteIdx);
+          }
+        } else {
+          return ARROWERROR;
+        }
+      }
+      numLeft -= batchSize;
+      x += batchSize;
+    }
+
+    file_writer->Close();
+    ARROWSTATUS_OK(out_file->Close());
+
+    return 0;
+   } catch (const std::exception& e) {
+    *errMsg = strdup(e.what());
+    return ARROWERROR;
+  }
+}
+
 int cpp_writeColumnToParquet(const char* filename, void* chpl_arr,
                              int64_t colnum, const char* dsetname, int64_t numelems,
                              int64_t rowGroupSize, int64_t dtype, bool compressed,
@@ -783,4 +905,11 @@ extern "C" {
   void c_free_string(void* ptr) {
     cpp_free_string(ptr);
   }
+
+  int c_writeMultiColToParquet(const char* filename, void* column_names,
+                                void** ptr_arr, void* datatypes,
+                                int64_t colnum, int64_t numelems, int64_t rowGroupSize,
+                                bool compressed, char** errMsg){
+    return cpp_writeMultiColToParquet(filename, column_names, ptr_arr, datatypes, colnum, numelems, rowGroupSize, compressed, errMsg);
+  }
 }
diff --git a/src/ArrowFunctions.h b/src/ArrowFunctions.h
index 511e6fab11..e2902fd9f3 100644
--- a/src/ArrowFunctions.h
+++ b/src/ArrowFunctions.h
@@ -10,6 +10,7 @@
 #include <parquet/arrow/writer.h>
 #include <parquet/column_reader.h>
 #include <parquet/api/writer.h>
+#include <queue>
 extern "C" {
 #endif

@@ -79,6 +80,16 @@ extern "C" {
                                 const char* dsetname, int64_t numelems,
                                 int64_t dtype, bool compressed,
                                 char** errMsg);
+
+  int c_writeMultiColToParquet(const char* filename, void* column_names,
+                                void** ptr_arr, void* datatypes,
+                                int64_t colnum, int64_t numelems, int64_t rowGroupSize,
+                                bool compressed, char** errMsg);
+
+  int cpp_writeMultiColToParquet(const char* filename, void* column_names,
+                                  void** ptr_arr, void* datatypes,
+                                  int64_t colnum, int64_t numelems, int64_t rowGroupSize,
+                                  bool compressed, char** errMsg);

   const char* c_getVersionInfo(void);
   const char* cpp_getVersionInfo(void);
diff --git a/src/ParquetMsg.chpl b/src/ParquetMsg.chpl
index f9521a8cf0..0730cb0677 100644
--- a/src/ParquetMsg.chpl
+++ b/src/ParquetMsg.chpl
@@ -12,6 +12,7 @@ module ParquetMsg {
   use NumPyDType;
   use Sort;
   use CommAggregation;
+  use AryUtil;

   use SegmentedString;

@@ -769,6 +770,245 @@ module ParquetMsg {
     }
   }

+  proc writeMultiColParquet(filename: string, col_names: [] string,
+                              ncols: int, sym_names: [] string, targetLocales: [] locale,
+                              compressed: bool, st: borrowed SymTab): bool throws {
+
+    extern proc c_writeMultiColToParquet(filename, column_names, ptr_arr,
+                                      datatypes, colnum, numelems, rowGroupSize, compressed, errMsg): int;
+
+    var prefix: string;
+    var extension: string;
+    (prefix, extension) = getFileMetadata(filename);
+
+    // Generate the filenames based upon the number of targetLocales.
+    var filenames = generateFilenames(prefix, extension, targetLocales.size);
+
+    //Generate a list of matching filenames to test against.
+    var matchingFilenames = getMatchingFilenames(prefix, extension);
+
+    // TODO when APPEND is fully deprecated update this to not need the mode.
+    var filesExist = processParquetFilenames(filenames, matchingFilenames, TRUNCATE); // set to truncate. We will not be supporting appending.
+
+    coforall (loc, idx) in zip(targetLocales, filenames.domain) do on loc {
+      var pqErr = new parquetErrorMsg();
+      const fname = filenames[idx];
+
+      var ptrList: [0..#ncols] c_void_ptr;
+      var datatypes: [0..#ncols] int;
+      var sizeList: [0..#ncols] int;
+
+      var my_column_names = col_names;
+      var c_names: [0..#ncols] c_string;
+
+      var locSize: int = 0;
+      var sections_sizes: [0..#ncols] int; // only fill in sizes for str columns
+      forall (i, column) in zip(0..#ncols, sym_names) with (+ reduce locSize) {
+        var entry = st.lookup(column);
+        // need to calculate the total size of Strings on this local
+        if (entry.isAssignableTo(SymbolEntryType.SegStringSymEntry)) {
+          var e: SegStringSymEntry = toSegStringSymEntry(entry);
+          var segStr = new SegString("", e);
+          ref ss = segStr;
+          var lens = ss.getLengths();
+          const locDom = ss.offsets.a.localSubdomain();
+          var x: int;
+          for i in locDom do x += lens[i];
+          sections_sizes[i] = x;
+          locSize += x;
+        }
+      }
+
+      var str_vals: [0..#locSize] uint(8);
+      var str_idx = (+ scan sections_sizes) - sections_sizes;
+      forall (i, column, si) in zip(0..#ncols, sym_names, str_idx) {
+        // generate the local c string list of column names
+        c_names[i] = my_column_names[i].localize().c_str();
+
+        var entry = st.lookup(column);
+
+        // access the dtype of each
+        var entryDtype = DType.UNDEF;
+        if (entry.isAssignableTo(SymbolEntryType.TypedArraySymEntry)) {
+          entryDtype = (entry: borrowed GenSymEntry).dtype;
+        } else if (entry.isAssignableTo(SymbolEntryType.SegStringSymEntry)) {
+          entryDtype = (entry: borrowed SegStringSymEntry).dtype;
+        } else {
+          throw getErrorWithContext(
+              msg="Unknown SymEntry Type",
+              lineNumber=getLineNumber(),
+              routineName=getRoutineName(),
+              moduleName=getModuleName(),
+              errorClass='ValueError'
+          );
+        }
+
+        select entryDtype {
+          when DType.Int64 {
+            var e = toSymEntry(toGenSymEntry(entry), int);
+            var locDom = e.a.localSubdomain();
+            // set the pointer to the entry array in the list of Pointers
+            ptrList[i] = c_ptrTo(e.a[locDom]): c_void_ptr;
+            datatypes[i] = ARROWINT64;
+            sizeList[i] = locDom.size;
+          }
+          when DType.UInt64 {
+            var e = toSymEntry(toGenSymEntry(entry), uint);
+            var locDom = e.a.localSubdomain();
+            // set the pointer to the entry array in the list of Pointers
+            ptrList[i] = c_ptrTo(e.a[locDom]): c_void_ptr;
+            datatypes[i] = ARROWUINT64;
+            sizeList[i] = locDom.size;
+          }
+          when DType.Bool {
+            var e = toSymEntry(toGenSymEntry(entry), bool);
+            var locDom = e.a.localSubdomain();
+            // set the pointer to the entry array in the list of Pointers
+            ptrList[i] = c_ptrTo(e.a[locDom]): c_void_ptr;
+            datatypes[i] = ARROWBOOLEAN;
+            sizeList[i] = locDom.size;
+          } when DType.Float64 {
+            var e = toSymEntry(toGenSymEntry(entry), real);
+            var locDom = e.a.localSubdomain();
+            // set the pointer to the entry array in the list of Pointers
+            ptrList[i] = c_ptrTo(e.a[locDom]): c_void_ptr;
+            datatypes[i] = ARROWDOUBLE;
+            sizeList[i] = locDom.size;
+          } when DType.Strings {
+            var e: SegStringSymEntry = toSegStringSymEntry(entry);
+            var segStr = new SegString("", e);
+            ref ss = segStr;
+            var A = ss.offsets.a;
+            const lastOffset = A[A.domain.high];
+            const lastValIdx = ss.values.a.domain.high;
+            const locDom = ss.offsets.a.localSubdomain();
+
+            var localOffsets = A[locDom];
+            var startValIdx = localOffsets[locDom.low];
+            var endValIdx = if (lastOffset == localOffsets[locDom.high]) then lastValIdx else A[locDom.high + 1] - 1;
+            var valIdxRange = startValIdx..endValIdx;
+            ref olda = ss.values.a;
+            str_vals[si..#valIdxRange.size] = olda[valIdxRange];
+            ptrList[i] = c_ptrTo(str_vals[si]): c_void_ptr;
+            datatypes[i] = ARROWSTRING;
+            sizeList[i] = locDom.size;
+          } otherwise {
+            throw getErrorWithContext(
+                              msg="Writing Parquet files (multi-column) does not support columns of type %s".format(entryDtype),
+                              lineNumber=getLineNumber(),
+                              routineName=getRoutineName(),
+                              moduleName=getModuleName(),
+                              errorClass='DataTypeError'
+            );
+          }
+        }
+      }
+
+      // validate all elements same size
+      var numelems: int = sizeList[0];
+      if !(&& reduce (sizeList==numelems)) {
+        throw getErrorWithContext(
+              msg="Parquet columns must be the same size",
+              lineNumber=getLineNumber(),
+              routineName=getRoutineName(),
+              moduleName=getModuleName(),
+              errorClass='WriteModeError'
+        );
+      }
+      var result: int = c_writeMultiColToParquet(fname.localize().c_str(), c_ptrTo(c_names), c_ptrTo(ptrList), c_ptrTo(datatypes), ncols, numelems, ROWGROUPS, compressed, c_ptrTo(pqErr.errMsg));
+    }
+    return filesExist;
+  }
+
+  proc toParquetMultiColMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws {
+    const filename: string = msgArgs.getValueOf("filename");
+    const ncols: int = msgArgs.get("num_cols").getIntValue();
+
+    // get list of the names for the columns
+    var col_names: [0..#ncols] string = msgArgs.get("col_names").getList(ncols);
+
+    // get list of sym entry names holding column data
+    var sym_names: [0..#ncols] string = msgArgs.get("columns").getList(ncols);
+
+    var compressed = msgArgs.get("compressed").getBoolValue();
+
+    // Assuming all columns have same distribution, access the first to get target locales
+    var entry = st.lookup(sym_names[0]);
+
+    // access the dtype to create symentry from abstract
+    var entryDtype = DType.UNDEF;
+    if (entry.isAssignableTo(SymbolEntryType.TypedArraySymEntry)) {
+      entryDtype = (entry: borrowed GenSymEntry).dtype;
+    } else if (entry.isAssignableTo(SymbolEntryType.SegStringSymEntry)) {
+      entryDtype = (entry: borrowed SegStringSymEntry).dtype;
+    } else {
+      throw getErrorWithContext(
+          msg="Unknown SymEntry Type",
+          lineNumber=getLineNumber(),
+          routineName=getRoutineName(),
+          moduleName=getModuleName(),
+          errorClass='ValueError'
+      );
+    }
+
+    var targetLocales;
+    select entryDtype {
+      when DType.Int64 {
+        var e = toSymEntry(toGenSymEntry(entry), int);
+        targetLocales = e.a.targetLocales();
+      }
+      when DType.UInt64 {
+        var e = toSymEntry(toGenSymEntry(entry), uint);
+        targetLocales = e.a.targetLocales();
+      }
+      when DType.Bool {
+        var e = toSymEntry(toGenSymEntry(entry), bool);
+        targetLocales = e.a.targetLocales();
+      } when DType.Float64 {
+        var e = toSymEntry(toGenSymEntry(entry), real);
+        targetLocales = e.a.targetLocales();
+      } when DType.Strings {
+        var e: SegStringSymEntry = toSegStringSymEntry(entry);
+        var segStr = new SegString("", e);
+        targetLocales = segStr.offsets.a.targetLocales();
+      } otherwise {
+        throw getErrorWithContext(
+                          msg="Writing Parquet files (multi-column) does not support columns of type %s".format(entryDtype),
+                          lineNumber=getLineNumber(),
+                          routineName=getRoutineName(),
+                          moduleName=getModuleName(),
+                          errorClass='DataTypeError'
+        );
+      }
+    }
+
+    var warnFlag: bool;
+    try {
+      warnFlag = writeMultiColParquet(filename, col_names, ncols, sym_names, targetLocales, compressed, st);
+    } catch e: FileNotFoundError {
+      var errorMsg = "Unable to open %s for writing: %s".format(filename,e.message());
+      pqLogger.error(getModuleName(),getRoutineName(),getLineNumber(),errorMsg);
+      return new MsgTuple(errorMsg, MsgType.ERROR);
+    } catch e: WriteModeError {
+      var errorMsg = "Write mode error %s".format(e.message());
+      pqLogger.error(getModuleName(),getRoutineName(),getLineNumber(),errorMsg);
+      return new MsgTuple(errorMsg, MsgType.ERROR);
+    } catch e: Error {
+      var errorMsg = "problem writing to file %s".format(e.message());
+      pqLogger.error(getModuleName(),getRoutineName(),getLineNumber(),errorMsg);
+      return new MsgTuple(errorMsg, MsgType.ERROR);
+    }
+
+    if warnFlag {
+      var warnMsg = "Warning: possibly overwriting existing files matching filename pattern";
+      return new MsgTuple(warnMsg, MsgType.WARNING);
+    } else {
+      var repMsg = "wrote array to file";
+      pqLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
+      return new MsgTuple(repMsg, MsgType.NORMAL);
+    }
+  }
+
   proc lspqMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws {
     // reqMsg: "lshdf [<json_filename>]"
     var repMsg: string;
@@ -938,6 +1178,7 @@ module ParquetMsg {

   use CommandMap;
   registerFunction("readAllParquet", readAllParquetMsg, getModuleName());
+  registerFunction("toParquet_multi", toParquetMultiColMsg, getModuleName());
   registerFunction("writeParquet", toparquetMsg, getModuleName());
   registerFunction("lspq", lspqMsg, getModuleName());
   registerFunction("getnullparquet", nullIndicesMsg, getModuleName());

diff --git a/tests/dataframe_test.py b/tests/dataframe_test.py
index 06c9f83e8c..23ad865b98 100644
--- a/tests/dataframe_test.py
+++ b/tests/dataframe_test.py
@@ -528,7 +528,7 @@ def test_save(self):
             akdf.to_parquet(f"{tmp_dirname}/testName")

             ak_loaded = ak.DataFrame.load(f"{tmp_dirname}/testName")
-            self.assertTrue(validation_df.equals(ak_loaded.to_pandas()))
+            self.assertTrue(validation_df.equals(ak_loaded[akdf.columns].to_pandas()))

             # test save with index true
             akdf.to_parquet(f"{tmp_dirname}/testName_with_index.pq", index=True)
diff --git a/tests/import_export_test.py b/tests/import_export_test.py
index a6aad7039b..db8de2d28c 100644
--- a/tests/import_export_test.py
+++ b/tests/import_export_test.py
@@ -100,11 +100,10 @@ def test_export_parquet(self):
         akdf = self.build_arkouda_dataframe()
         with tempfile.TemporaryDirectory(dir=ImportExportTest.ie_test_base_tmp) as tmp_dirname:
             akdf.to_parquet(f"{tmp_dirname}/ak_write")
-            print(akdf.__repr__())

             pddf = ak.export(f"{tmp_dirname}/ak_write", write_file=f"{tmp_dirname}/pd_from_ak.parquet", index=True)
             self.assertEqual(len(glob.glob(f"{tmp_dirname}/pd_from_ak.parquet")), 1)
-            self.assertTrue(pddf.equals(akdf.to_pandas()))
+            self.assertTrue(pddf[akdf.columns].equals(akdf.to_pandas()))

             with self.assertRaises(RuntimeError):
                 pddf = ak.export(f"{tmp_dirname}/foo.h5", write_file=f"{tmp_dirname}/pd_from_ak.h5", index=True)

Multi-column Parquet write inefficiencies
Passing along a comment from another developer. " I've noticed that Parquet output from Arkouoda is currently very slow. The Arkouda client library saves multi-column dataframes by sending a separate writeParquet request to the server for each column, adding them to the file one-by-one. The ccp_appendColumnToParquet function doesn't add the column to the file in-place, though; instead it loads the whole existing file into memory, adds the new column to the table object in memory, and overwrites the whole file with the new version. That means it's loading and re-saving the content of all the old columns each time it adds a new one, which adds a lot of overhead since this happens for every column."
Multi-column Parquet write inefficiencies
Passing along a comment from another developer. " I've noticed that Parquet output from Arkouoda is currently very slow. The Arkouda client library saves multi-column dataframes by sending a separate writeParquet request to the server for each column, adding them to the file one-by-one. The ccp_appendColumnToParquet function doesn't add the column to the file in-place, though; instead it loads the whole existing file into memory, adds the new column to the table object in memory, and overwrites the whole file with the new version. That means it's loading and re-saving the content of all the old columns each time it adds a new one, which adds a lot of overhead since this happens for every column."

Thanks for bringing this issue to our attention. When the Parquet work was implemented, it was before the dataframe functionality had been implemented, so that use case wasn't considered in the initial design. As you have seen, the Parquet append functionality is very sub-optimal and that is because Parquet does not support appending to files natively, so we had to work around that by reading in the whole file and then rewriting it.

I do not have a great deal of familiarity with the dataframe code as it exists today, but I suspect that we should be able to implement a much faster implementation that only writes to the file once. I hope to get to this in the near future.
Thank you!
Spoke with @bmcdonald3. To allow me to get more familiar with the Parquet code, I am going to take this one.
There are some challenges presented by this that I will need to look into the best way to handle. Currently, all our Parquet support is designed to operate on a single Chapel array. In this case, we will most likely have instances where we have multiple Chapel Arrays of varying types needing to be written. In HDF5, this is a bit easier because we operate on the Objects directly in Chapel without needing to format them. Additionally, because the Apache Arrow library is C++ only, we have to make calls from Chapel to C and from C to C++. This should not be an issue, but noting as an additional complexity.

Ideally, I would like to configure methods to interact with objects based on ids/pointers like we are able to with HDF5. I am looking into configuring this now.
Thanks for bringing this issue to our attention. When the Parquet work was implemented, it was before the dataframe functionality had been implemented, so that use case wasn't considered in the initial design. As you have seen, the Parquet append functionality is very sub-optimal and that is because Parquet does not support appending to files natively, so we had to work around that by reading in the whole file and then rewriting it.

I do not have a great deal of familiarity with the dataframe code as it exists today, but I suspect that we should be able to implement a much faster implementation that only writes to the file once. I hope to get to this in the near future.
Thank you!
Spoke with @bmcdonald3. To allow me to get more familiar with the Parquet code, I am going to take this one.
There are some challenges presented by this that I will need to look into the best way to handle. Currently, all our Parquet support is designed to operate on a single Chapel array. In this case, we will most likely have instances where we have multiple Chapel Arrays of varying types needing to be written. In HDF5, this is a bit easier because we operate on the Objects directly in Chapel without needing to format them. Additionally, because the Apache Arrow library is C++ only, we have to make calls from Chapel to C and from C to C++. This should not be an issue, but noting as an additional complexity.

Ideally, I would like to configure methods to interact with objects based on ids/pointers like we are able to with HDF5. I am looking into configuring this now.


diff --git a/django/core/management/commands/runserver.py b/django/core/management/commands/runserver.py
index 132ee4c0795a..3795809a1226 100644
--- a/django/core/management/commands/runserver.py
+++ b/django/core/management/commands/runserver.py
@@ -188,3 +188,12 @@ def on_bind(self, server_port):
             f"Quit the server with {quit_command}.",
             file=self.stdout,
         )
+        if os.environ.get("HIDE_PRODUCTION_WARNING") != "true":
+            self.stdout.write(
+                self.style.WARNING(
+                    "WARNING: This is a development server. Do not use it in a "
+                    "production setting. Use a production WSGI or ASGI server "
+                    "instead.\nFor more information on production servers see: "
+                    "https://docs.djangoproject.com/en/stable/howto/deployment/"
+                )
+            )
diff --git a/docs/intro/tutorial01.txt b/docs/intro/tutorial01.txt
index db59ca49df75..0536eca2f776 100644
--- a/docs/intro/tutorial01.txt
+++ b/docs/intro/tutorial01.txt
@@ -134,6 +134,9 @@ You'll see the following output on the command line:
     Starting development server at http://127.0.0.1:8000/
     Quit the server with CONTROL-C.

+    WARNING: This is a development server. Do not use it in a production setting. Use a production WSGI or ASGI server instead.
+    For more information on production servers see: https://docs.djangoproject.com/en/stable/howto/deployment/
+
 .. note::
     Ignore the warning about unapplied database migrations for now; we'll deal
     with the database shortly.
diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt
index 8173224e4ceb..52eaaa331b65 100644
--- a/docs/ref/django-admin.txt
+++ b/docs/ref/django-admin.txt
@@ -947,6 +947,20 @@ multithreaded by default.
 Uses IPv6 for the development server. This changes the default IP address from
 ``127.0.0.1`` to ``::1``.

+.. envvar:: HIDE_PRODUCTION_WARNING
+
+.. versionadded:: 5.2
+
+By default, a warning is printed to the console that ``runserver`` is not
+suitable for production:
+
+.. code-block:: text
+
+    WARNING: This is a development server. Do not use it in a production setting. Use a production WSGI or ASGI server instead.
+    For more information on production servers see: https://docs.djangoproject.com/en/stable/howto/deployment/
+
+Set this environment variable to ``"true"`` to hide this warning.
+
 Examples of using different ports and addresses
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/docs/releases/5.2.txt b/docs/releases/5.2.txt
index 92bb501d61cb..6b63bc9524fd 100644
--- a/docs/releases/5.2.txt
+++ b/docs/releases/5.2.txt
@@ -194,7 +194,10 @@ Logging
 Management Commands
 ~~~~~~~~~~~~~~~~~~~

-* ...
+* A new warning is printed to the console when running :djadmin:`runserver` that
+  ``runserver`` is unsuitable for production. This warning can be hidden by
+  setting the :envvar:`HIDE_PRODUCTION_WARNING` environment variable to
+  ``"true"``.

 Migrations
 ~~~~~~~~~~

diff --git a/tests/admin_scripts/tests.py b/tests/admin_scripts/tests.py
index 2e77f2c97a62..67362460a99d 100644
--- a/tests/admin_scripts/tests.py
+++ b/tests/admin_scripts/tests.py
@@ -1597,6 +1597,13 @@ def test_zero_ip_addr(self):
             "Starting development server at http://0.0.0.0:8000/",
             self.output.getvalue(),
         )
+        self.assertIn(
+            "WARNING: This is a development server. Do not use it in a "
+            "production setting. Use a production WSGI or ASGI server instead."
+            "\nFor more information on production servers see: "
+            "https://docs.djangoproject.com/en/stable/howto/deployment/",
+            self.output.getvalue(),
+        )

     def test_on_bind(self):
         self.cmd.addr = "127.0.0.1"
@@ -1606,6 +1613,30 @@ def test_on_bind(self):
             "Starting development server at http://127.0.0.1:14437/",
             self.output.getvalue(),
         )
+        self.assertIn(
+            "WARNING: This is a development server. Do not use it in a "
+            "production setting. Use a production WSGI or ASGI server instead."
+            "\nFor more information on production servers see: "
+            "https://docs.djangoproject.com/en/stable/howto/deployment/",
+            self.output.getvalue(),
+        )
+
+    @mock.patch.dict(os.environ, {"HIDE_PRODUCTION_WARNING": "true"})
+    def test_hide_production_warning_with_environment_variable(self):
+        self.cmd.addr = "0"
+        self.cmd._raw_ipv6 = False
+        self.cmd.on_bind("8000")
+        self.assertIn(
+            "Starting development server at http://0.0.0.0:8000/",
+            self.output.getvalue(),
+        )
+        self.assertNotIn(
+            "WARNING: This is a development server. Do not use it in a "
+            "production setting. Use a production WSGI or ASGI server instead."
+            "\nFor more information on production servers see: "
+            "https://docs.djangoproject.com/en/stable/howto/deployment/",
+            self.output.getvalue(),
+        )

     @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
     def test_runner_addrport_ipv6(self):

Add warning to runserver that it should not be used for production
Description

As per this discussion on the forum, I think adding a warning to the start of runserver would be valuable to those new to Django and a healthy reminder to those coming back to Django.
The wording of the warning is:
WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
For more information on production servers see the documentation: https://docs.djangoproject.com/en/5.0/howto/deployment/
Additionally a flag should be added to optionally hide the warning. --hide-unsuitable-for-production was a suggestion.
Finally the following 2 pages of the documentation should have add some wording to highlight that a deployed project should not use runserver.
https://docs.djangoproject.com/en/5.0/howto/deployment/ (Add a warning or paragraph that mentions switching)
https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ (Add a section to switch away from runserver)
Add warning to runserver that it should not be used for production
Description

As per this discussion on the forum, I think adding a warning to the start of runserver would be valuable to those new to Django and a healthy reminder to those coming back to Django.
The wording of the warning is:
WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
For more information on production servers see the documentation: https://docs.djangoproject.com/en/5.0/howto/deployment/
Additionally a flag should be added to optionally hide the warning. --hide-unsuitable-for-production was a suggestion.
Finally the following 2 pages of the documentation should have add some wording to highlight that a deployed project should not use runserver.
https://docs.djangoproject.com/en/5.0/howto/deployment/ (Add a warning or paragraph that mentions switching)
https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ (Add a section to switch away from runserver)

['I think it\'s worth highlighting that it does say "development server": Starting development server at http://127.0.0.1:8000/ Quit the server with CTRL-BREAK. We also have a warning in the \u200brunserver docs and a \u200bwarning in the tutorial (note that the tutorial runserver output would need to be updated if we make a change here). So I think this is not that we don\'t have this already but a request to make this information more prominent', 1720668622.0]
['This will be the new output by default. Warning can be disabled by using --hide-unsuitable-for-production flag Starting development server at http://127.0.0.1:8000/ Quit the server with CTRL-BREAK. WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. For more information on production servers see the documentation: https://docs.djangoproject.com/en/5.0/howto/deployment/', 1720675079.0]
['Task list: add flag to runserver command add new message with flag to hide it update output at \u200bhttps://docs.djangoproject.com/en/5.0/intro/tutorial01/#the-development-server update reference docs: \u200bhttps://docs.djangoproject.com/en/5.0/ref/django-admin/#runserver update deployment checklist update deployment docs with a warning add tests for new flag update existing tests to include extended output', 1722421918.0]
['\u200bPR', 1722425099.0]
['Updated the PR.', 1723026415.0]
['I think it\'s worth highlighting that it does say "development server": Starting development server at http://127.0.0.1:8000/ Quit the server with CTRL-BREAK. We also have a warning in the \u200brunserver docs and a \u200bwarning in the tutorial (note that the tutorial runserver output would need to be updated if we make a change here). So I think this is not that we don\'t have this already but a request to make this information more prominent', 1720668622.0]
['This will be the new output by default. Warning can be disabled by using --hide-unsuitable-for-production flag Starting development server at http://127.0.0.1:8000/ Quit the server with CTRL-BREAK. WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. For more information on production servers see the documentation: https://docs.djangoproject.com/en/5.0/howto/deployment/', 1720675079.0]
['Task list: add flag to runserver command add new message with flag to hide it update output at \u200bhttps://docs.djangoproject.com/en/5.0/intro/tutorial01/#the-development-server update reference docs: \u200bhttps://docs.djangoproject.com/en/5.0/ref/django-admin/#runserver update deployment checklist update deployment docs with a warning add tests for new flag update existing tests to include extended output', 1722421918.0]
['\u200bPR', 1722425099.0]
['Updated the PR.', 1723026415.0]


diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e8480e8d0e3..bfc033f0990 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -45,7 +45,6 @@ repos:
           [
             types-toml,
             types-chardet,
-            types-appdirs,
             types-colorama,
             types-pyyaml,
             types-regex,
@@ -59,6 +58,7 @@ repos:
             pathspec,
             pytest, # and by extension... pluggy
             click,
+            platformdirs
           ]
         files: ^src/sqlfluff/.*
         # The mypy pre-commit hook by default sets a few arguments that we don't normally
diff --git a/pyproject.toml b/pyproject.toml
index caeaffdc56b..3762076deb2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,7 +67,7 @@ keywords = [
 ]
 dependencies = [
     # Used for finding os-specific application config dirs
-    "appdirs",
+    "platformdirs",
     # To get the encoding of files.
     "chardet",
     "click",
diff --git a/src/sqlfluff/core/config/loader.py b/src/sqlfluff/core/config/loader.py
index 4b4fdd4ebf8..e0df90d6b59 100644
--- a/src/sqlfluff/core/config/loader.py
+++ b/src/sqlfluff/core/config/loader.py
@@ -17,12 +17,15 @@
 import logging
 import os
 import os.path
+import sys
 from pathlib import Path
 from typing import (
     Optional,
 )

-import appdirs
+import platformdirs
+import platformdirs.macos
+import platformdirs.unix

 from sqlfluff.core.config.file import (
     cache,
@@ -55,22 +58,50 @@
 )

-def _get_user_config_dir_path() -> str:
+def _get_user_config_dir_path(sys_platform: str) -> str:
+    """Get the user config dir for this system.
+
+    Args:
+        sys_platform (str): The result of ``sys.platform()``. Provided
+            as an argument here for ease of testing. In normal usage
+            it should only be  called with ``sys.platform()``. This
+            argument only applies to switching between linux and macos.
+            Win32 detection still uses the underlying ``sys.platform()``
+            methods.
+    """
     appname = "sqlfluff"
     appauthor = "sqlfluff"

-    # On Mac OSX follow Linux XDG base dirs
-    # https://github.com/sqlfluff/sqlfluff/issues/889
-    user_config_dir_path = os.path.expanduser("~/.config/sqlfluff")
-    if appdirs.system == "darwin":
-        appdirs.system = "linux2"
-        user_config_dir_path = appdirs.user_config_dir(appname, appauthor)
-        appdirs.system = "darwin"
-
-    if not os.path.exists(user_config_dir_path):
-        user_config_dir_path = appdirs.user_config_dir(appname, appauthor)
+    # First try the default SQLFluff specific cross-platform config path.
+    cross_platform_path = os.path.expanduser("~/.config/sqlfluff")
+    if os.path.exists(cross_platform_path):
+        return cross_platform_path

-    return user_config_dir_path
+    # Then try the platform specific paths, for MacOS, we check
+    # the unix variant first to preferentially use the XDG config path if set.
+    # https://github.com/sqlfluff/sqlfluff/issues/889
+    if sys_platform == "darwin":
+        unix_config_path = platformdirs.unix.Unix(
+            appname=appname, appauthor=appauthor
+        ).user_config_dir
+        if os.path.exists(os.path.expanduser(unix_config_path)):
+            return unix_config_path
+        # Technically we could just delegate to the generic `user_config_dir`
+        # method, but for testing it's convenient to explicitly call the macos
+        # methods here.
+        return platformdirs.macos.MacOS(
+            appname=appname, appauthor=appauthor
+        ).user_config_dir
+    # NOTE: We could delegate to the generic `user_config_dir` method here,
+    # but for testing it's convenient to explicitly call the linux methods.
+    elif sys_platform == "linux":
+        return platformdirs.unix.Unix(
+            appname=appname, appauthor=appauthor
+        ).user_config_dir
+    # Defer to the self-detecting paths.
+    # NOTE: On Windows this means that the `sys_platform` argument is not
+    # applied.
+    return platformdirs.user_config_dir(appname, appauthor)

 def load_config_file(
@@ -218,7 +249,7 @@ def load_config_at_path(path: str) -> ConfigMappingType:

 def _load_user_appdir_config() -> ConfigMappingType:
     """Load the config from the user's OS specific appdir config directory."""
-    user_config_dir_path = _get_user_config_dir_path()
+    user_config_dir_path = _get_user_config_dir_path(sys.platform)
     if os.path.exists(user_config_dir_path):
         return load_config_at_path(user_config_dir_path)
     else:
@@ -283,16 +314,19 @@ def load_config_up_to_path(
         config_paths = iter_intermediate_paths(Path(path).absolute(), Path.cwd())
         config_stack = [load_config_at_path(str(p.resolve())) for p in config_paths]

-    # 4) Extra config paths
-    if not extra_config_path:
-        extra_config = {}
-    else:
-        if not os.path.exists(extra_config_path):
+    # 4) Extra config paths.
+    # When calling `load_config_file_as_dict` we resolve the path first so that caching
+    # is more efficient.
+    extra_config = {}
+    if extra_config_path:
+        try:
+            extra_config = load_config_file_as_dict(
+                str(Path(extra_config_path).resolve())
+            )
+        except FileNotFoundError:
             raise SQLFluffUserError(
-                f"Extra config '{extra_config_path}' does not exist."
+                f"Extra config path '{extra_config_path}' does not exist."
             )
-        # Resolve the path so that the caching is accurate.
-        extra_config = load_config_file_as_dict(str(Path(extra_config_path).resolve()))

     return nested_combine(
         user_appdir_config,

diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py
index 498b2cd82c3..f9751c96ea1 100644
--- a/test/cli/commands_test.py
+++ b/test/cli/commands_test.py
@@ -228,8 +228,8 @@ def test__cli__command_extra_config_fail():
             ],
         ],
         assert_output_contains=(
-            "Extra config 'test/fixtures/cli/extra_configs/.sqlfluffsdfdfdfsfd' does "
-            "not exist."
+            "Extra config path 'test/fixtures/cli/extra_configs/.sqlfluffsdfdfdfsfd' "
+            "does not exist."
         ),
     )

diff --git a/test/core/config/loader_test.py b/test/core/config/loader_test.py
index 14023f8839f..e21df5d5f8f 100644
--- a/test/core/config/loader_test.py
+++ b/test/core/config/loader_test.py
@@ -5,7 +5,6 @@
 from contextlib import contextmanager
 from unittest.mock import call, patch

-import appdirs
 import pytest

 from sqlfluff.core import FluffConfig
@@ -19,6 +18,7 @@
     _get_user_config_dir_path,
     _load_user_appdir_config,
 )
+from sqlfluff.core.errors import SQLFluffUserError

 config_a = {
     "core": {"testing_val": "foobar", "testing_int": 4, "dialect": "mysql"},
@@ -59,21 +59,43 @@ def test__config__load_file_f():
     assert cfg == config_a

+def test__config__load_file_missing_extra():
+    """Test loading config from a file path if extra path is not found."""
+    with pytest.raises(SQLFluffUserError):
+        load_config_up_to_path(
+            os.path.join("test", "fixtures", "config", "inheritance_a", "testing.sql"),
+            extra_config_path="non/existent/path",
+        )
+
+
 def test__config__load_nested():
     """Test nested overwrite and order of precedence of config files."""
     cfg = load_config_up_to_path(
         os.path.join(
             "test", "fixtures", "config", "inheritance_a", "nested", "blah.sql"
-        )
+        ),
+        extra_config_path=os.path.join(
+            "test",
+            "fixtures",
+            "config",
+            "inheritance_a",
+            "extra",
+            "this_can_have_any_name.cfg",
+        ),
     )
     assert cfg == {
         "core": {
+            # Outer .sqlfluff defines dialect & testing_val and not overridden.
             "dialect": "mysql",
             "testing_val": "foobar",
+            # tesing_int is defined in many. Inner pyproject.toml takes precedence.
             "testing_int": 1,
+            # testing_bar is defined only in setup.cfg
             "testing_bar": 7.698,
         },
-        "bar": {"foo": "foobar"},
+        # bar is defined in a few, but the extra_config takes precedence.
+        "bar": {"foo": "foobarextra"},
+        # fnarr is defined in a few. Inner tox.ini takes precedence.
         "fnarr": {"fnarr": {"foo": "foobar"}},
     }

@@ -158,37 +180,107 @@ def test__config__load_placeholder_cfg():
 @patch("os.path.exists")
 @patch("os.listdir")
 @pytest.mark.skipif(sys.platform == "win32", reason="Not applicable on Windows")
-def test__config__load_user_appdir_config(
-    mock_listdir, mock_path_exists, mock_xdg_home
+@pytest.mark.parametrize(
+    "sys_platform,xdg_exists,default_exists,resolved_config_path,paths_checked",
+    [
+        # On linux, if the default path exists, it should be the only path we check
+        # and the chosen config path.
+        ("linux", True, True, "~/.config/sqlfluff", ["~/.config/sqlfluff"]),
+        # On linux, if the default path doesn't exist, then (because for this
+        # test case we set XDG_CONFIG_HOME) it will check the default path
+        # but then on finding it to not exist it will then try the XDG path.
+        # In this case, neither actually exist and so what matters is that both
+        # are either checked or used - rather than one in particular being the
+        # end result.
+        (
+            "linux",
+            False,
+            False,
+            "~/.config/my/special/path/sqlfluff",
+            ["~/.config/sqlfluff"],
+        ),
+        # On MacOS, if the default config path and the XDG path don't exist, then
+        # we should resolve config to the default MacOS config path.
+        (
+            "darwin",
+            False,
+            False,
+            "~/Library/Application Support/sqlfluff",
+            ["~/.config/sqlfluff", "~/.config/my/special/path/sqlfluff"],
+        ),
+        # However, if XDG_CONFIG_HOME is set, and the path exists then that should
+        # be resolved _ahead of_ the default MacOS config path (as demonstrated
+        # by us not checking the presence of that path in the process).
+        # https://github.com/sqlfluff/sqlfluff/issues/889
+        (
+            "darwin",
+            True,
+            False,
+            "~/.config/my/special/path/sqlfluff",
+            ["~/.config/sqlfluff", "~/.config/my/special/path/sqlfluff"],
+        ),
+    ],
+)
+def test__config__get_user_config_dir_path(
+    mock_listdir,
+    mock_path_exists,
+    mock_xdg_home,
+    sys_platform,
+    xdg_exists,
+    default_exists,
+    resolved_config_path,
+    paths_checked,
 ):
     """Test loading config from user appdir."""
     xdg_home = os.environ.get("XDG_CONFIG_HOME")
     assert xdg_home, "XDG HOME should be set by the mock. Something has gone wrong."
     xdg_config_path = xdg_home + "/sqlfluff"

-    def path_exists(x):
-        if x == os.path.expanduser("~/.config/sqlfluff"):
+    def path_exists(check_path):
+        """Patch for os.path.exists which depends on test parameters.
+
+        Returns:
+            True, unless `default_exists` is `False` and the path passed to
+            the function is the default config path, or unless `xdg_exists`
+            is `False` and the path passed is the XDG config path.
+        """
+        resolved_path = os.path.expanduser(check_path)
+        if (
+            resolved_path == os.path.expanduser("~/.config/sqlfluff")
+            and not default_exists
+        ):
             return False
-        if x == xdg_config_path:
+        if resolved_path == os.path.expanduser(xdg_config_path) and not xdg_exists:
             return False
-        else:
-            return True
+        return True

     mock_path_exists.side_effect = path_exists

-    with patch.object(appdirs, attribute="system", new="darwin"):
-        resolved_path = _get_user_config_dir_path()
-        _load_user_appdir_config()
-    assert resolved_path == os.path.expanduser("~/Library/Application Support/sqlfluff")
-
+    # Get the config path as though we are on macOS.
+    resolved_path = _get_user_config_dir_path(sys_platform)
+    assert os.path.expanduser(resolved_path) == os.path.expanduser(resolved_config_path)
     mock_path_exists.assert_has_calls(
-        [
-            call(xdg_config_path),
-            call(os.path.expanduser("~/Library/Application Support/sqlfluff")),
-        ]
+        [call(os.path.expanduser(path)) for path in paths_checked]
     )

+@patch("os.path.exists")
+@patch("sqlfluff.core.config.loader.load_config_at_path")
+def test__config__load_user_appdir_config(mock_load_config, mock_path_exists):
+    """Test _load_user_appdir_config.
+
+    NOTE: We mock `load_config_at_path()` so we can be really focussed with this test
+    and also not need to actually interact with local home directories.
+    """
+    mock_load_config.side_effect = lambda x: {}
+    mock_path_exists.side_effect = lambda x: True
+    _load_user_appdir_config()
+    # It will check that the default config path exists...
+    mock_path_exists.assert_has_calls([call(os.path.expanduser("~/.config/sqlfluff"))])
+    # ...and assuming it does, it will try and load config files at that path.
+    mock_load_config.assert_has_calls([call(os.path.expanduser("~/.config/sqlfluff"))])
+
+
 def test__config__toml_list_config():
     """Test Parsing TOML list of values."""
     loaded_config = load_config_file(
diff --git a/test/fixtures/config/inheritance_a/extra/this_can_have_any_name.cfg b/test/fixtures/config/inheritance_a/extra/this_can_have_any_name.cfg
new file mode 100644
index 00000000000..544d5d08eda
--- /dev/null
+++ b/test/fixtures/config/inheritance_a/extra/this_can_have_any_name.cfg
@@ -0,0 +1,2 @@
+[sqlfluff:bar]
+foo=foobarextra

replace deprecated appdirs dependency with platformdirs
### Search before asking

- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.

### Description

https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1068011

python3-appdirs is dead upstream[1] and its Debian maintainer has indicated
that it should not be included in trixie[2]. A recommended replacement is
python3-platformdirs[3], which is a fork of appdirs with a very similar API.

Please migrate from appdirs to platformdirs or some other replacement,
so that appdirs can be removed.

### Use case

n/a

### Dialect

none

### Are you willing to work on and submit a PR to address the issue?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)


diff --git a/src/aiortc/rtcsctptransport.py b/src/aiortc/rtcsctptransport.py
index de5d0968d..c9439fc2f 100644
--- a/src/aiortc/rtcsctptransport.py
+++ b/src/aiortc/rtcsctptransport.py
@@ -1324,8 +1324,7 @@ async def _send(
             self._outbound_stream_seq[stream_id] = uint16_add(stream_seq, 1)

         # transmit outbound data
-        if not self._t3_handle:
-            await self._transmit()
+        await self._transmit()

     async def _send_chunk(self, chunk: Chunk) -> None:
         """

Slow sending messages with data channels
When I try to send 10 messages per second in datachannel I observe that messages are sent much slower, at 5 message per second.

I did a docker example to reproduce: https://gist.github.com/le-chat/844272e8d0f91dcbb61ba98ca635cd6b

From logs:
```
08:32:51.808407 sent №100, buffered 1750, queues: _sent 1, _outbound 0, _data_channel 50; _flight_size 35
08:33:01.895772 sent №200, buffered 3600, queues: _sent 1, _outbound 1, _data_channel 100; _flight_size 35
08:33:01.982040, 100 received: №99 at 2022-09-28 08:32:51.808277 (buffered 0)
08:33:11.973638 sent №300, buffered 5436, queues: _sent 1, _outbound 0, _data_channel 151; _flight_size 36
08:33:22.056580 sent №400, buffered 7200, queues: _sent 1, _outbound 1, _data_channel 200; _flight_size 36
08:33:22.160382, 200 received: №199 at 2022-09-28 08:33:01.895633 (buffered 0)
08:33:32.146720 sent №500, buffered 9036, queues: _sent 1, _outbound 0, _data_channel 251; _flight_size 36
08:33:42.228501 sent №600, buffered 10836, queues: _sent 1, _outbound 1, _data_channel 301; _flight_size 36
08:33:42.523312, 300 received: №299 at 2022-09-28 08:33:11.973609 (buffered 0)
08:33:52.314587 sent №700, buffered 12672, queues: _sent 1, _outbound 0, _data_channel 352; _flight_size 36
```
we can see messages are stuck on `_data_channel_queue`, though flight size is low and both `_outbound_queue` and `_sent_queue` are almost empty.

It looks like the second check in https://github.com/aiortc/aiortc/blob/1.3.2/src/aiortc/rtcsctptransport.py#L1651 erroneously breaks the loop in the very beginning and is redundant.

We have the same issue:
We try to send data for every frame which fails at 30fps.
For us there is an additional issue: we want the latency to be as low as possible; so the current code has an issue where even after applying your fix it will only send out batches of messages 10 times per second


diff --git a/db/install.py b/db/install.py
index 43440b398d..1372659c63 100644
--- a/db/install.py
+++ b/db/install.py
@@ -1,5 +1,6 @@
+from psycopg.errors import InsufficientPrivilege
 from sqlalchemy import text
-from sqlalchemy.exc import OperationalError
+from sqlalchemy.exc import OperationalError, ProgrammingError

 from db import engine
 from db.sql import install as sql_install
@@ -54,12 +55,20 @@ def _create_database(database_name, hostname, username, password, port, skip_con
             username, password, hostname, root_database, port,
             connect_args={"connect_timeout": 10}
         )
-        with root_db_engine.connect() as conn:
-            conn.execution_options(isolation_level="AUTOCOMMIT")
-            conn.execute(text(f'CREATE DATABASE "{database_name}"'))
-        root_db_engine.dispose()
-        print(f"Created DB is {database_name}.")
-        return True
+        try:
+            with root_db_engine.connect() as conn:
+                conn.execution_options(isolation_level="AUTOCOMMIT")
+                conn.execute(text(f'CREATE DATABASE "{database_name}"'))
+            root_db_engine.dispose()
+            print(f"Created DB is {database_name}.")
+            return True
+        except ProgrammingError as e:
+            if isinstance(e.orig, InsufficientPrivilege):
+                print(f"Database {database_name} could not be created due to Insufficient Privilege")
+                return False
+        except Exception:
+            print(f"Database {database_name} could not be created!")
+            return False
     else:
         print(f"Database {database_name} not created!")
         return False
diff --git a/docs/docs/installation/build-from-source/index.md b/docs/docs/installation/build-from-source/index.md
index c221d31d7e..4b403beee5 100644
--- a/docs/docs/installation/build-from-source/index.md
+++ b/docs/docs/installation/build-from-source/index.md
@@ -63,25 +63,25 @@ Then press Enter to customize this guide with your domain name.
     sudo -u postgres psql
     ```

-1. Mathesar needs a Postgres superuser to function correctly. Let's create a superuser.
+1. Let's create a Postgres user for Mathesar

     ```postgresql
-    CREATE USER mathesar WITH SUPERUSER ENCRYPTED PASSWORD '1234';
+    CREATE USER mathesar WITH ENCRYPTED PASSWORD '1234';
     ```

     !!! warning "Customize your password"
         Be sure to change the password `1234` in the command above to something more secure and private. Record your custom password somewhere safe. You will need to reference it later.

-1. Next, we have to create a database for storing Mathesar metadata.
+1. Next, we have to create a database for storing Mathesar metadata. Your PostgreSQL user will either need to be a `SUPERUSER` or `OWNER` of the database. In this guide, we will be setting the user to be `OWNER` of the database as it is slightly restrictive compared to a `SUPERUSER`.

     ```postgresql
-    CREATE DATABASE mathesar_django;
+    CREATE DATABASE mathesar_django OWNER mathesar;
     ```

 1. Now we let us create a database for storing your data.

     ```postgresql
-    CREATE DATABASE your_db_name;
+    CREATE DATABASE your_db_name OWNER mathesar;
     ```

 1. Press Ctrl+D to exit the `psql` shell.
diff --git a/docs/docs/installation/docker/index.md b/docs/docs/installation/docker/index.md
index 22362e6801..9c1c29c3b2 100644
--- a/docs/docs/installation/docker/index.md
+++ b/docs/docs/installation/docker/index.md
@@ -24,8 +24,11 @@ You'll need to install **[Docker](https://docs.docker.com/desktop/)** v23+
 #### Database for Mathesar's internal usage
 You'll need to:

-- Create a PostgreSQL database for Mathesar's internal usage.
-- Create a database user for Mathesar to use. The user should be a `SUPERUSER`, [see PostgreSQL docs for more information](https://www.postgresql.org/docs/13/sql-createrole.html).
+- Create or have a PostgreSQL user for Mathesar to use.
+- Create a PostgreSQL database for Mathesar's internal usage owned by that database user. (See the PostgreSQL [docs](https://www.postgresql.org/docs/13/ddl-priv.html) for more information.)
+
+    Alternatively, you can make the user a `SUPERUSER` which will give the user access to all the databases. (See the see PostgreSQL [docs](https://www.postgresql.org/docs/13/sql-createrole.html) for more information.)
+
 - Ensure that this database can accept network connections from the machine you're installing Mathesar on.
 - Have the following information for this database handy before installation:
     - Database hostname
@@ -40,11 +43,11 @@ Have the following information for all databases you'd like to connect to Mathes
 - Database hostname
 - Database port
 - Database name
-- Database username (should be a `SUPERUSER`, see above)
+- Database username (should be `SUPERUSER` or `OWNER` of the database, see above)
 - Database password

-!!! warning "Database creation"
-    Whenever the Docker container is started, we will attempt to create any databases in this list that don't already exist. So you don't need to ensure that they are created before installation.
+!!! info "Databases are automatically created"
+    You don't need to create these databases before installation. Whenever the Docker container is started, Mathesar will attempt to create any specified databases that don't already exist, so long as the user has [`CREATEDB` privilege](https://www.postgresql.org/docs/13/sql-createrole.html).

 ## Installation Steps

diff --git a/docs/docs/snippets/docker-compose-prerequisites.md b/docs/docs/snippets/docker-compose-prerequisites.md
index f9947c9289..fd8e061db0 100644
--- a/docs/docs/snippets/docker-compose-prerequisites.md
+++ b/docs/docs/snippets/docker-compose-prerequisites.md
@@ -24,7 +24,7 @@ You can create a new PostgreSQL database while setting up Mathesar or use our UI
 To connect Mathesar to an existing database:

 - The external database should be able to accept network connections from your Mathesar server.
-- You'll need to set up a database user for Mathesar to use. The user should be a `SUPERUSER`, [see PostgreSQL docs for more information](https://www.postgresql.org/docs/13/sql-createrole.html).
+- You'll need to set up a database user for Mathesar to use. The user should own that database, [see PostgreSQL docs for more information](https://www.postgresql.org/docs/13/ddl-priv.html)
 - Have the following information handy before installation:
     - Database hostname
     - Database port

The requirement of superuser postgresql access is problematic
## Problem
Mathesar needs a Postgres superuser to function correctly, from the docs at https://docs.mathesar.org/installation/build-from-source/

## Proposed solution
The mathesar user should not require superuser access.

## Additional context
The superuser is a global permission meaning that a user that has superuser permission will be able to access (and do stuff) not on the mathesar but *all* the databases of the RDBMS. Considering that many production systems have a single RDBMS hosting many application this is a major problem since the mathsar user won't have any access boundaries on the same RDBMS. The mathesar user access can be unlimited but *must* be bounded without the mathesar database.

Thanks for reporting this, @spapas. This is already a high-priority issue for us to resolve, requiring a superuser is one of the compromises we made to get our alpha version out of the door.
Hello friends, any news on this issue? This is very important for us and we can't actually use mathesar in procution until the supertuser requirement is dropped.

Would it be possible to clarify a bit why the mathesar usr needs global superuser access ? What would happen if we removed the RDBMS superuser access on the mathesar user and made him an owner of the corresponding databases (i.e mathesar_data and mathesar_django) ?

Thank you
Hi @spapas

We are working on this issue under the [Installations Improvement Project](https://wiki.mathesar.org/en/projects/installation-improvements-0_1_3). This will be part of Release v0.1.3 (coming next month)
Thank you @rajatvijay for answering me & giving a delivery estimate!

Kind regards,
Serafeim


diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index 9a6c29c27bdf..27faa252788d 100644
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -336,7 +336,7 @@ def shard_checkpoint(
     return shards, index

-def load_sharded_checkpoint(model, folder, strict=True):
+def load_sharded_checkpoint(model, folder, strict=True, prefer_safe=True):
     """
     This is the same as
     [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict)
@@ -350,6 +350,9 @@ def load_sharded_checkpoint(model, folder, strict=True):
         folder (`str` or `os.PathLike`): A path to a folder containing the sharded checkpoint.
         strict (`bool`, *optional`, defaults to `True`):
             Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint.
+        prefer_safe (`bool`, *optional*, defaults to `False`)
+            If both safetensors and PyTorch save files are present in checkpoint and `prefer_safe` is True, the
+            safetensors files will be loaded. Otherwise, PyTorch files are always loaded when possible.

     Returns:
         `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields
@@ -358,10 +361,32 @@ def load_sharded_checkpoint(model, folder, strict=True):
     """
     # Load the index
     index_file = os.path.join(folder, WEIGHTS_INDEX_NAME)
-    if not os.path.isfile(index_file):
-        raise ValueError(f"Can't find a checkpoint index ({WEIGHTS_INDEX_NAME}) in {folder}.")
+    safe_index_file = os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME)

-    with open(index_file, "r", encoding="utf-8") as f:
+    index_present = os.path.isfile(index_file)
+    safe_index_present = os.path.isfile(safe_index_file)
+
+    if not index_present and not (safe_index_present and is_safetensors_available()):
+        filenames = (
+            (WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_INDEX_NAME) if is_safetensors_available() else (WEIGHTS_INDEX_NAME,)
+        )
+        raise ValueError(f"Can't find a checkpoint index ({' or '.join(filenames)}) in {folder}.")
+
+    load_safe = False
+    if safe_index_present:
+        if prefer_safe:
+            if is_safetensors_available():
+                load_safe = True  # load safe due to preference
+            else:
+                logger.warning(
+                    f"Cannot load sharded checkpoint at {folder} safely since safetensors is not installed!"
+                )
+        elif not index_present:
+            load_safe = True  # load safe since we have no other choice
+
+    load_index = safe_index_file if load_safe else index_file
+
+    with open(load_index, "r", encoding="utf-8") as f:
         index = json.load(f)

     shard_files = list(set(index["weight_map"].values()))
@@ -381,11 +406,13 @@ def load_sharded_checkpoint(model, folder, strict=True):
             error_message += f"\nMissing key(s): {str_unexpected_keys}."
         raise RuntimeError(error_message)

+    loader = safe_load_file if load_safe else partial(torch.load, map_location="cpu")
+
     for shard_file in shard_files:
-        state_dict = torch.load(os.path.join(folder, shard_file), map_location="cpu")
+        state_dict = loader(os.path.join(folder, shard_file))
         model.load_state_dict(state_dict, strict=False)

-        # Make sure memory is fred before we load the next state dict.
+        # Make sure memory is freed before we load the next state dict.
         del state_dict
         gc.collect()

diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index a41d43edeb4f..d9fb7907585e 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -136,6 +136,8 @@
 from .training_args import OptimizerNames, ParallelMode, TrainingArguments
 from .utils import (
     CONFIG_NAME,
+    SAFE_WEIGHTS_INDEX_NAME,
+    SAFE_WEIGHTS_NAME,
     WEIGHTS_INDEX_NAME,
     WEIGHTS_NAME,
     can_return_loss,
@@ -146,6 +148,7 @@
     is_datasets_available,
     is_in_notebook,
     is_ipex_available,
+    is_safetensors_available,
     is_sagemaker_dp_enabled,
     is_sagemaker_mp_enabled,
     is_torch_compile_available,
@@ -198,6 +201,10 @@
     IS_SAGEMAKER_MP_POST_1_10 = False

+if is_safetensors_available():
+    import safetensors.torch
+
+
 skip_first_batches = None
 if is_accelerate_available():
     from accelerate import __version__ as accelerate_version
@@ -2078,15 +2085,22 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None):
         if model is None:
             model = self.model

-        if not os.path.isfile(os.path.join(resume_from_checkpoint, WEIGHTS_NAME)) and not os.path.isfile(
-            os.path.join(resume_from_checkpoint, WEIGHTS_INDEX_NAME)
+        config_file = os.path.join(resume_from_checkpoint, CONFIG_NAME)
+
+        weights_file = os.path.join(resume_from_checkpoint, WEIGHTS_NAME)
+        weights_index_file = os.path.join(resume_from_checkpoint, WEIGHTS_INDEX_NAME)
+        safe_weights_file = os.path.join(resume_from_checkpoint, SAFE_WEIGHTS_NAME)
+        safe_weights_index_file = os.path.join(resume_from_checkpoint, SAFE_WEIGHTS_INDEX_NAME)
+
+        if not any(
+            [os.path.isfile(f) for f in [weights_file, safe_weights_file, weights_index_file, safe_weights_index_file]]
         ):
             raise ValueError(f"Can't find a valid checkpoint at {resume_from_checkpoint}")

         logger.info(f"Loading model from {resume_from_checkpoint}.")

-        if os.path.isfile(os.path.join(resume_from_checkpoint, CONFIG_NAME)):
-            config = PretrainedConfig.from_json_file(os.path.join(resume_from_checkpoint, CONFIG_NAME))
+        if os.path.isfile(config_file):
+            config = PretrainedConfig.from_json_file(config_file)
             checkpoint_version = config.transformers_version
             if checkpoint_version is not None and checkpoint_version != __version__:
                 logger.warning(
@@ -2095,7 +2109,7 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None):
                     "yield to errors or unwanted behaviors."
                 )

-        if os.path.isfile(os.path.join(resume_from_checkpoint, WEIGHTS_NAME)):
+        if os.path.isfile(weights_file) or os.path.isfile(safe_weights_file):
             # If the model is on the GPU, it still works!
             if is_sagemaker_mp_enabled():
                 if os.path.isfile(os.path.join(resume_from_checkpoint, "user_content.pt")):
@@ -2111,7 +2125,7 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None):
                         logger.warning(
                             "Enabling FP16 and loading from smp < 1.10 checkpoint together is not suppported."
                         )
-                    state_dict = torch.load(os.path.join(resume_from_checkpoint, WEIGHTS_NAME), map_location="cpu")
+                    state_dict = torch.load(weights_file, map_location="cpu")
                     # Required for smp to not auto-translate state_dict from hf to smp (is already smp).
                     state_dict["_smp_is_partial"] = False
                     load_result = model.load_state_dict(state_dict, strict=True)
@@ -2119,7 +2133,11 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None):
                     del state_dict
             else:
                 # We load the model state dict on the CPU to avoid an OOM error.
-                state_dict = torch.load(os.path.join(resume_from_checkpoint, WEIGHTS_NAME), map_location="cpu")
+                if self.args.save_safetensors and os.path.isfile(safe_weights_file):
+                    state_dict = safetensors.torch.load_file(safe_weights_file, device="cpu")
+                else:
+                    state_dict = torch.load(weights_file, map_location="cpu")
+
                 # workaround for FSDP bug https://github.com/pytorch/pytorch/issues/82963
                 # which takes *args instead of **kwargs
                 load_result = model.load_state_dict(state_dict, False)
@@ -2128,15 +2146,18 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None):
                 self._issue_warnings_after_load(load_result)
         else:
             # We load the sharded checkpoint
-            load_result = load_sharded_checkpoint(model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled())
+            load_result = load_sharded_checkpoint(
+                model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled(), prefer_safe=self.args.save_safetensors
+            )
             if not is_sagemaker_mp_enabled():
                 self._issue_warnings_after_load(load_result)

     def _load_best_model(self):
         logger.info(f"Loading best model from {self.state.best_model_checkpoint} (score: {self.state.best_metric}).")
         best_model_path = os.path.join(self.state.best_model_checkpoint, WEIGHTS_NAME)
+        best_safe_model_path = os.path.join(self.state.best_model_checkpoint, SAFE_WEIGHTS_NAME)
         model = self.model_wrapped if is_sagemaker_mp_enabled() else self.model
-        if os.path.exists(best_model_path):
+        if os.path.exists(best_model_path) or os.path.exists(best_safe_model_path):
             if self.deepspeed:
                 if self.model_wrapped is not None:
                     # this removes the pre-hooks from the previous engine
@@ -2168,12 +2189,20 @@ def _load_best_model(self):
                     else:
                         # If the 'user_content.pt' file does NOT exist, load with the old smp api.
                         # Checkpoint must have been saved with the old smp api.
-                        state_dict = torch.load(best_model_path, map_location="cpu")
+                        if self.args.save_safetensors and os.path.isfile(best_safe_model_path):
+                            state_dict = safetensors.torch.load_file(best_safe_model_path, device="cpu")
+                        else:
+                            state_dict = torch.load(best_model_path, map_location="cpu")
+
                         state_dict["_smp_is_partial"] = False
                         load_result = model.load_state_dict(state_dict, strict=True)
                 else:
                     # We load the model state dict on the CPU to avoid an OOM error.
-                    state_dict = torch.load(best_model_path, map_location="cpu")
+                    if self.args.save_safetensors and os.path.isfile(best_safe_model_path):
+                        state_dict = safetensors.torch.load_file(best_safe_model_path, device="cpu")
+                    else:
+                        state_dict = torch.load(best_model_path, map_location="cpu")
+
                     # If the model is on the GPU, it still works!
                     # workaround for FSDP bug https://github.com/pytorch/pytorch/issues/82963
                     # which takes *args instead of **kwargs
@@ -2822,17 +2851,24 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
         # Save a trained model and configuration using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`
         if not isinstance(self.model, PreTrainedModel):
+            if state_dict is None:
+                state_dict = self.model.state_dict()
+
             if isinstance(unwrap_model(self.model), PreTrainedModel):
-                if state_dict is None:
-                    state_dict = self.model.state_dict()
-                unwrap_model(self.model).save_pretrained(output_dir, state_dict=state_dict)
+                unwrap_model(self.model).save_pretrained(
+                    output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
+                )
             else:
                 logger.info("Trainer.model is not a `PreTrainedModel`, only saving its state dict.")
-                if state_dict is None:
-                    state_dict = self.model.state_dict()
-                torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME))
+                if self.args.save_safetensors:
+                    safetensors.torch.save_file(state_dict, os.path.join(output_dir, SAFE_WEIGHTS_NAME))
+                else:
+                    torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME))
         else:
-            self.model.save_pretrained(output_dir, state_dict=state_dict)
+            self.model.save_pretrained(
+                output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
+            )
+
         if self.tokenizer is not None:
             self.tokenizer.save_pretrained(output_dir)

@@ -3531,7 +3567,7 @@ def _push_from_checkpoint(self, checkpoint_folder):

         output_dir = self.args.output_dir
         # To avoid a new synchronization of all model weights, we just copy the file from the checkpoint folder
-        modeling_files = [CONFIG_NAME, WEIGHTS_NAME]
+        modeling_files = [CONFIG_NAME, WEIGHTS_NAME, SAFE_WEIGHTS_NAME]
         for modeling_file in modeling_files:
             if os.path.isfile(os.path.join(checkpoint_folder, modeling_file)):
                 shutil.copy(os.path.join(checkpoint_folder, modeling_file), os.path.join(output_dir, modeling_file))
diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py
index 2a3c32673262..28387885de16 100644
--- a/src/transformers/training_args.py
+++ b/src/transformers/training_args.py
@@ -42,6 +42,7 @@
     get_full_repo_name,
     is_accelerate_available,
     is_psutil_available,
+    is_safetensors_available,
     is_sagemaker_dp_enabled,
     is_sagemaker_mp_enabled,
     is_torch_available,
@@ -261,6 +262,9 @@ class TrainingArguments:
         save_total_limit (`int`, *optional*):
             If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in
             `output_dir`.
+        save_safetensors (`bool`, *optional*, defaults to `False`):
+            Use [safetensors](https://huggingface.co/docs/safetensors) saving and loading for state dicts instead of
+            default `torch.load` and `torch.save`.
         save_on_each_node (`bool`, *optional*, defaults to `False`):
             When doing multi-node distributed training, whether to save models and checkpoints on each node, or only on
             the main one.
@@ -720,6 +724,12 @@ class TrainingArguments:
             )
         },
     )
+    save_safetensors: Optional[bool] = field(
+        default=False,
+        metadata={
+            "help": "Use safetensors saving and loading for state dicts instead of default torch.load and torch.save."
+        },
+    )
     save_on_each_node: bool = field(
         default=False,
         metadata={
@@ -1166,6 +1176,17 @@ def __post_init__(self):
                     f"steps, but found {self.save_steps}, which is not a round multiple of {self.eval_steps}."
                 )

+        safetensors_available = is_safetensors_available()
+        if self.save_safetensors and not safetensors_available:
+            raise ValueError(f"--save_safetensors={self.save_safetensors} requires safetensors to be installed!")
+        if not self.save_safetensors and safetensors_available:
+            logger.info(
+                f"Found safetensors installation, but --save_safetensors={self.save_safetensors}. "
+                f"Safetensors should be a preferred weights saving format due to security and performance reasons. "
+                f"If your model cannot be saved by safetensors please feel free to open an issue at "
+                f"https://github.com/huggingface/safetensors!"
+            )
+
         if self.load_best_model_at_end and self.metric_for_best_model is None:
             self.metric_for_best_model = "loss"
         if self.greater_is_better is None and self.metric_for_best_model is not None:

diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py
index 310842713bde..78b6afeacd4e 100644
--- a/tests/trainer/test_trainer.py
+++ b/tests/trainer/test_trainer.py
@@ -25,6 +25,7 @@
 import tempfile
 import time
 import unittest
+from itertools import product
 from pathlib import Path
 from unittest.mock import Mock, patch

@@ -54,6 +55,7 @@
     require_intel_extension_for_pytorch,
     require_optuna,
     require_ray,
+    require_safetensors,
     require_sentencepiece,
     require_sigopt,
     require_tokenizers,
@@ -73,10 +75,13 @@
 from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
 from transformers.training_args import OptimizerNames
 from transformers.utils import (
+    SAFE_WEIGHTS_INDEX_NAME,
+    SAFE_WEIGHTS_NAME,
     WEIGHTS_INDEX_NAME,
     WEIGHTS_NAME,
     is_apex_available,
     is_bitsandbytes_available,
+    is_safetensors_available,
     is_torchdistx_available,
 )
 from transformers.utils.hp_naming import TrialShortNamer
@@ -102,6 +107,9 @@
     )
     from transformers.modeling_utils import unwrap_model

+    if is_safetensors_available():
+        import safetensors.torch
+

 PATH_SAMPLE_TEXT = f"{get_tests_dir()}/fixtures/sample_text.txt"

@@ -345,8 +353,9 @@ def get_regression_trainer(a=0, b=0, double_output=False, train_len=64, eval_len

 class TrainerIntegrationCommon:
-    def check_saved_checkpoints(self, output_dir, freq, total, is_pretrained=True):
-        file_list = [WEIGHTS_NAME, "training_args.bin", "optimizer.pt", "scheduler.pt", "trainer_state.json"]
+    def check_saved_checkpoints(self, output_dir, freq, total, is_pretrained=True, safe_weights=False):
+        weights_file = WEIGHTS_NAME if not safe_weights else SAFE_WEIGHTS_NAME
+        file_list = [weights_file, "training_args.bin", "optimizer.pt", "scheduler.pt", "trainer_state.json"]
         if is_pretrained:
             file_list.append("config.json")
         for step in range(freq, total, freq):
@@ -356,7 +365,7 @@ def check_saved_checkpoints(self, output_dir, freq, total, is_pretrained=True):
                 self.assertTrue(os.path.isfile(os.path.join(checkpoint, filename)))

     def check_best_model_has_been_loaded(
-        self, output_dir, freq, total, trainer, metric, greater_is_better=False, is_pretrained=True
+        self, output_dir, freq, total, trainer, metric, greater_is_better=False, is_pretrained=True, safe_weights=False
     ):
         checkpoint = os.path.join(output_dir, f"checkpoint-{(total // freq) * freq}")
         log_history = TrainerState.load_from_json(os.path.join(checkpoint, "trainer_state.json")).log_history
@@ -370,7 +379,10 @@ def check_best_model_has_been_loaded(
             best_model.to(trainer.args.device)
         else:
             best_model = RegressionModel()
-            state_dict = torch.load(os.path.join(checkpoint, WEIGHTS_NAME))
+            if not safe_weights:
+                state_dict = torch.load(os.path.join(checkpoint, WEIGHTS_NAME))
+            else:
+                state_dict = safetensors.torch.load_file(os.path.join(checkpoint, SAFE_WEIGHTS_NAME))
             best_model.load_state_dict(state_dict)
             best_model.to(trainer.args.device)
         self.assertTrue(torch.allclose(best_model.a, trainer.model.a))
@@ -394,24 +406,43 @@ def check_trainer_state_are_the_same(self, trainer_state, trainer_state1):
                 _ = log1.pop(key, None)
             self.assertEqual(log, log1)

-    def convert_to_sharded_checkpoint(self, folder):
+    def convert_to_sharded_checkpoint(self, folder, save_safe=False, load_safe=False):
         # Converts a checkpoint of a regression model to a sharded checkpoint.
-        state_dict = torch.load(os.path.join(folder, WEIGHTS_NAME))
-        os.remove(os.path.join(folder, WEIGHTS_NAME))
+        if load_safe:
+            loader = safetensors.torch.load_file
+            weights_file = os.path.join(folder, SAFE_WEIGHTS_NAME)
+        else:
+            loader = torch.load
+            weights_file = os.path.join(folder, WEIGHTS_NAME)
+
+        if save_safe:
+            extension = "safetensors"
+            saver = safetensors.torch.save_file
+            index_file = os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME)
+            shard_name = SAFE_WEIGHTS_NAME
+        else:
+            extension = "bin"
+            saver = torch.save
+            index_file = os.path.join(folder, WEIGHTS_INDEX_NAME)
+            shard_name = WEIGHTS_NAME
+
+        state_dict = loader(weights_file)
+
+        os.remove(weights_file)
         keys = list(state_dict.keys())

         shard_files = [
-            WEIGHTS_NAME.replace(".bin", f"-{idx+1:05d}-of-{len(keys):05d}.bin") for idx in range(len(keys))
+            shard_name.replace(f".{extension}", f"-{idx+1:05d}-of-{len(keys):05d}.{extension}")
+            for idx in range(len(keys))
         ]
         index = {"metadata": {}, "weight_map": {key: shard_files[i] for i, key in enumerate(keys)}}

-        save_index_file = os.path.join(folder, WEIGHTS_INDEX_NAME)
-        with open(save_index_file, "w", encoding="utf-8") as f:
+        with open(index_file, "w", encoding="utf-8") as f:
             content = json.dumps(index, indent=2, sort_keys=True) + "\n"
             f.write(content)

         for param_name, shard_file in zip(keys, shard_files):
-            torch.save({param_name: state_dict[param_name]}, os.path.join(folder, shard_file))
+            saver({param_name: state_dict[param_name]}, os.path.join(folder, shard_file))

 @require_torch
@@ -1132,6 +1163,26 @@ def test_save_checkpoints(self):
             trainer.train()
             self.check_saved_checkpoints(tmpdir, 5, int(self.n_epochs * 64 / self.batch_size), False)

+    @require_safetensors
+    def test_safe_checkpoints(self):
+        for save_safetensors in [True, False]:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                trainer = get_regression_trainer(output_dir=tmpdir, save_steps=5, save_safetensors=save_safetensors)
+                trainer.train()
+                self.check_saved_checkpoints(
+                    tmpdir, 5, int(self.n_epochs * 64 / self.batch_size), safe_weights=save_safetensors
+                )
+
+            # With a regular model that is not a PreTrainedModel
+            with tempfile.TemporaryDirectory() as tmpdir:
+                trainer = get_regression_trainer(
+                    output_dir=tmpdir, save_steps=5, pretrained=False, save_safetensors=save_safetensors
+                )
+                trainer.train()
+                self.check_saved_checkpoints(
+                    tmpdir, 5, int(self.n_epochs * 64 / self.batch_size), False, safe_weights=save_safetensors
+                )
+
     @require_torch_multi_gpu
     def test_run_seq2seq_double_train_wrap_once(self):
         # test that we don't wrap the model more than once
@@ -1373,6 +1424,42 @@ def test_resume_training_with_shard_checkpoint(self):
             self.assertEqual(b, b1)
             self.check_trainer_state_are_the_same(state, state1)

+    @require_safetensors
+    @require_torch_up_to_2_gpus
+    def test_resume_training_with_safe_checkpoint(self):
+        # This test will fail for more than 2 GPUs since the batch size will get bigger and with the number of
+        # save_steps, the checkpoint will resume training at epoch 2 or more (so the data seen by the model
+        # won't be the same since the training dataloader is shuffled).
+
+        for initial_safe in [False, True]:
+            for loaded_safe in [False, True]:
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    trainer = get_regression_trainer(
+                        output_dir=tmpdir,
+                        train_len=128,
+                        save_steps=5,
+                        learning_rate=0.1,
+                        save_safetensors=initial_safe,
+                    )
+                    trainer.train()
+                    (a, b) = trainer.model.a.item(), trainer.model.b.item()
+                    state = dataclasses.asdict(trainer.state)
+
+                    checkpoint = os.path.join(tmpdir, "checkpoint-5")
+                    self.convert_to_sharded_checkpoint(checkpoint, load_safe=initial_safe, save_safe=loaded_safe)
+
+                    # Reinitialize trainer
+                    trainer = get_regression_trainer(
+                        output_dir=tmpdir, train_len=128, save_steps=5, learning_rate=0.1, save_safetensors=loaded_safe
+                    )
+
+                    trainer.train(resume_from_checkpoint=checkpoint)
+                    (a1, b1) = trainer.model.a.item(), trainer.model.b.item()
+                    state1 = dataclasses.asdict(trainer.state)
+                    self.assertEqual(a, a1)
+                    self.assertEqual(b, b1)
+                    self.check_trainer_state_are_the_same(state, state1)
+
     @require_torch_up_to_2_gpus
     def test_resume_training_with_gradient_accumulation(self):
         # This test will fail for more than 2 GPUs since the batch size will get bigger and with the number of
@@ -1522,6 +1609,30 @@ def test_load_best_model_at_end(self):
             self.check_saved_checkpoints(tmpdir, 5, total, is_pretrained=False)
             self.check_best_model_has_been_loaded(tmpdir, 5, total, trainer, "eval_loss", is_pretrained=False)

+    @require_safetensors
+    def test_load_best_model_from_safetensors(self):
+        total = int(self.n_epochs * 64 / self.batch_size)
+        for save_safetensors, pretrained in product([False, True], [False, True]):
+            with tempfile.TemporaryDirectory() as tmpdir:
+                trainer = get_regression_trainer(
+                    a=1.5,
+                    b=2.5,
+                    output_dir=tmpdir,
+                    learning_rate=0.1,
+                    eval_steps=5,
+                    evaluation_strategy="steps",
+                    save_steps=5,
+                    load_best_model_at_end=True,
+                    save_safetensors=save_safetensors,
+                    pretrained=pretrained,
+                )
+                self.assertFalse(trainer.args.greater_is_better)
+                trainer.train()
+                self.check_saved_checkpoints(tmpdir, 5, total, is_pretrained=pretrained, safe_weights=save_safetensors)
+                self.check_best_model_has_been_loaded(
+                    tmpdir, 5, total, trainer, "eval_loss", is_pretrained=pretrained, safe_weights=save_safetensors
+                )
+
     @slow
     def test_trainer_eval_mrpc(self):
         MODEL_ID = "bert-base-cased-finetuned-mrpc"

Implement safetensors checkpoint loading for Trainer
### Feature request

At the moment, Trainer loads models with `torch.load` method directly onto the cpu:

(`Trainer._load_from_checkpoint` method)
```python
                ...
                # We load the model state dict on the CPU to avoid an OOM error.
                state_dict = torch.load(os.path.join(resume_from_checkpoint, WEIGHTS_NAME), map_location="cpu")
                # workaround for FSDP bug https://github.com/pytorch/pytorch/issues/82963
                # which takes *args instead of **kwargs
                load_result = model.load_state_dict(state_dict, False)
                # release memory
                del state_dict
                self._issue_warnings_after_load(load_result)
                ...
```

Loading on cpu with safetensors is a lot faster, so this method should (?) be preffered if safetensors library is installed.

### Motivation

I care to speed up checkpointing process since I use it to store the best model checkpoint every time the metric is improved (via callback).

### Your contribution

The change should be straightforward, but I am not sure if safetensors checkpointing should be defaulted to if safetensors are installed, or configured manually like in `PreTrainedModel.from_pretrained` method.

The checkpoints are not saved in that format so there is no `model.safetensors` file to load from. We could add a training argument to use this format instead of the PyTorch format indeed.


diff --git a/validator/tests/test_repository_checks.py b/validator/tests/test_repository_checks.py
index aed318f..739a97d 100644
--- a/validator/tests/test_repository_checks.py
+++ b/validator/tests/test_repository_checks.py
@@ -11,6 +11,7 @@

 def test_check_repository() -> None:
     # Arrange
+    configuration = MagicMock()
     repository = MagicMock()
     repository.name = "test-repo"
     repository.full_name = "owner/test-repo"
@@ -18,7 +19,7 @@ def test_check_repository() -> None:
     repository.security_and_analysis.secret_scanning.status = "enabled"
     repository.security_and_analysis.dependabot_security_updates.status = "enabled"
     # Act
-    analysed_repository = check_repository(repository)
+    analysed_repository = check_repository(configuration, repository)
     # Assert
     assert analysed_repository.name == "test-repo"
     assert analysed_repository.full_name == "owner/test-repo"

Fix private vulnerability disclosure incorrect value