title: Out of memory when downloading 460mb file from S3 to AWS lambda using go aws sdk
body: <p>I'm running lambda using the aws-sdk-go-v2 but running into memory issues when downloading larger files. I've tried all sorts of combinations of partSize and concurrency but I either hit the timeout when setting concurrency to a small number or hit an out of memory issue. </p>
  
  <p>Does anyone know how to fix this or a better or other way of downloading files from S3 using go?</p>
  
  <pre><code>package main
  
  import (
      "context"
      "fmt"
      awsevents "github.com/aws/aws-lambda-go/events"
      "github.com/aws/aws-lambda-go/lambda"
      "github.com/aws/aws-sdk-go-v2/aws"
      "github.com/aws/aws-sdk-go-v2/aws/external"
      "github.com/aws/aws-sdk-go-v2/service/s3"
      "github.com/aws/aws-sdk-go-v2/service/s3/s3manager"
  )
  
  func handleDemoEvent(_ context.Context, s3Event awsevents.S3Event) error {
      for _, record := range s3Event.Records {
          return handleDemoRecord(record)
      }
  
      return nil
  }
  
  func handleDemoRecord(record awsevents.S3EventRecord) error {
      fmt.Println(record.ResponseElements)
      fmt.Printf("[%s - %s] Bucket = %s, Key = %s \n", record.EventSource, record.EventTime, record.S3.Bucket.Name, record.S3.Object.Key)
  
      bucket := record.S3.Bucket.Name
      item := record.S3.Object.Key
  
      config, err := external.LoadDefaultAWSConfig()
  
      if err != nil {
          return err
      }
  
      headObject := getHeadObject(bucket, item, config)
  
      fmt.Printf("%+v", headObject)
  
      downloader := s3manager.NewDownloader(config, func(d *s3manager.Downloader) {
          d.PartSize = 2 * 1024 * 1024
          d.Concurrency = 70
      })
  
      var buf []byte
      buffer := aws.NewWriteAtBuffer(buf)
  
      fmt.Println("Starting download of file")
      numBytesDownloaded, err := downloader.Download(buffer, &amp;s3.GetObjectInput{
          Bucket: aws.String(bucket),
          Key:    aws.String(item),
      })
  
      if err != nil {
          return err
      }
  
      fmt.Println("Downloaded ", numBytesDownloaded, " bytes")
      return nil
  }
  
  
  func getHeadObject(bucket string, key string, config aws.Config) *s3.HeadObjectResponse {
      client := s3.New(config)
      input := &amp;s3.HeadObjectInput{
          Bucket: aws.String(bucket),
          Key:    aws.String(key),
      }
  
      request := client.HeadObjectRequest(input)
  
      headObjectResponse, err := request.Send(context.TODO())
  
      if err != nil {
          panic(err)
      }
  
      fmt.Printf("Downloaded HeadObject: %v\n", request)
  
      return headObjectResponse
  }
  
  func main() {
      lambda.Start(handleDemoEvent)
  }
  </code></pre>
  
  <p>This is the error from the log stream, didn't include everything since it's really long.</p>
  
  <pre><code>START RequestId: 8f5bc332-b4ad-4749-97ac-ee7c65a4404a Version: $LATEST
  map[x-amz-request-id:AA3E4F41C6FCA9B9 x-amz-id-2:HEoJcflBdxLbSLkOOUD19kI7TboN8hDfkBf8vqQlFy1FQni2wUUi09cRwliZeDn6nFiNx73V2v9V5jreSlgOr1bpPDl9Lywe]
  [aws:s3 - 2020-01-24 08:25:11.271 +0000 UTC] Bucket = 5kstats-demos, Key = to-be-parsed/vitality-vs-mousesports-m1-inferno.dem 
  Downloaded HeadObject: {0xc000105b80 {
    Bucket: "5kstats-demos",
    Key: "to-be-parsed/vitality-vs-mousesports-m1-inferno.dem",
    RequestPayer: 
  } 0x908e80}
  {
    AcceptRanges: "bytes",
    ContentLength: 491103546,
    ContentType: "application/octet-stream",
    ETag: "\"f3870e5b519ea52dbd249f69f6072f45-30\"",
    LastModified: 2020-01-24 08:25:12 +0000 UTC,
    Metadata: {
      Event-Id: "4979",
      Event-Name: "EPICENTER 2019",
      Event-End-Sec: "1577012400000",
      Match-Id: "2338373",
      Event-Start-Sec: "1576580400000"
    },
    ObjectLockLegalHoldStatus: ,
    ObjectLockMode: ,
    ReplicationStatus: ,
    RequestCharged: ,
    ServerSideEncryption: ,
    StorageClass: 
  }Starting download of file
  fatal error: runtime: out of memory
  
  runtime stack:
  runtime.throw(0xac163b, 0x16)
      /usr/local/go/src/runtime/panic.go:608 +0x72
  runtime.sysMap(0xc0b8000000, 0x1c000000, 0x1112d18)
      /usr/local/go/src/runtime/mem_linux.go:156 +0xc7
  runtime.(*mheap).sysAlloc(0x10f96c0, 0x1c000000, 0x80, 0x0)
      /usr/local/go/src/runtime/malloc.go:619 +0x1c7
  runtime.(*mheap).grow(0x10f96c0, 0xd202, 0x0)
      /usr/local/go/src/runtime/mheap.go:920 +0x42
  runtime.(*mheap).allocSpanLocked(0x10f96c0, 0xd202, 0x1112d28, 0x432c55)
      /usr/local/go/src/runtime/mheap.go:848 +0x337
  runtime.(*mheap).alloc_m(0x10f96c0, 0xd202, 0xffffffffffff0101, 0x7ffe2e1d0780)
      /usr/local/go/src/runtime/mheap.go:692 +0x119
  runtime.(*mheap).alloc.func1()
      /usr/local/go/src/runtime/mheap.go:759 +0x4c
  runtime.(*mheap).alloc(0x10f96c0, 0xd202, 0x7ffe2e010101, 0x7ffe2e1d0820)
      /usr/local/go/src/runtime/mheap.go:758 +0x8a
  runtime.largeAlloc(0x1a403fe8, 0x101, 0xc00ba5bbb0)
      /usr/local/go/src/runtime/malloc.go:1019 +0x97
  runtime.mallocgc.func1()
      /usr/local/go/src/runtime/malloc.go:914 +0x46
  runtime.systemstack(0x456509)
      /usr/local/go/src/runtime/asm_amd64.s:351 +0x66
  runtime.mstart()
      /usr/local/go/src/runtime/proc.go:1229
  
  goroutine 10 [running]:
  runtime.systemstack_switch()
      /usr/local/go/src/runtime/asm_amd64.s:311 fp=0xc00ba5bb40 sp=0xc00ba5bb38 pc=0x456600
  runtime.mallocgc(0x1a403fe8, 0x9c9780, 0x41829684901, 0xc00ba5bc18)
      /usr/local/go/src/runtime/malloc.go:913 +0x896 fp=0xc00ba5bbe0 sp=0xc00ba5bb40 pc=0x40d646
  runtime.makeslice(0x9c9780, 0x1a403fe8, 0x1a403fe8, 0x8000, 0x3fe8, 0x0)
      /usr/local/go/src/runtime/slice.go:70 +0x77 fp=0xc00ba5bc10 sp=0xc00ba5bbe0 pc=0x441877
  5kparser/vendor/github.com/aws/aws-sdk-go-v2/aws.(*WriteAtBuffer).WriteAt(0xc0002c8210, 0xc001ade000, 0x3fe8, 0x8000, 0x1a400000, 0x0, 0x0, 0x0)
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/aws/types.go:190 +0xf9 fp=0xc00ba5bc78 sp=0xc00ba5bc10 pc=0x76bb69
  5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*dlchunk).Write(0xc000068740, 0xc001ade000, 0x3fe8, 0x8000, 0x3fe8, 0x0, 0x0)
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:590 +0x75 fp=0xc00ba5bcc8 sp=0xc00ba5bc78 pc=0x970c65
  io.copyBuffer(0xc20f20, 0xc000068740, 0xc215e0, 0xc0007da880, 0xc001ade000, 0x8000, 0x8000, 0xa162e0, 0xc000293600, 0xc215e0)
      /usr/local/go/src/io/io.go:404 +0x201 fp=0xc00ba5bd38 sp=0xc00ba5bcc8 pc=0x46f701
  io.Copy(0xc20f20, 0xc000068740, 0xc215e0, 0xc0007da880, 0xc0007da880, 0xc000419970, 0x0)
      /usr/local/go/src/io/io.go:364 +0x5a fp=0xc00ba5bd98 sp=0xc00ba5bd38 pc=0x46f4ba
  5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).tryDownloadChunk(0xc00008c750, 0xc0005cd0e0, 0xc20f20, 0xc000068740, 0x0, 0x0, 0x0)
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:470 +0x1e0 fp=0xc00ba5be28 sp=0xc00ba5bd98 pc=0x970430
  5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).downloadChunk(0xc00008c750, 0xc20280, 0xc0002c8210, 0x1a400000, 0x200000, 0x0, 0x0, 0x0, 0x0, 0x0)
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:429 +0x2a5 fp=0xc00ba5bf00 sp=0xc00ba5be28 pc=0x9700d5
  5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).downloadPart(0xc00008c750, 0xc00006c2a0)
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:378 +0x142 fp=0xc00ba5bfd0 sp=0xc00ba5bf00 pc=0x96fb42
  runtime.goexit()
      /usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00ba5bfd8 sp=0xc00ba5bfd0 pc=0x4586e1
  created by 5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).download
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:322 +0x1ed
  
  goroutine 1 [IO wait]:
  internal/poll.runtime_pollWait(0x7f018740cf00, 0x72, 0x0)
      /usr/local/go/src/runtime/netpoll.go:173 +0x66
  internal/poll.(*pollDesc).wait(0xc000120218, 0x72, 0xc00009a000, 0x0, 0x0)
      /usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
  internal/poll.(*pollDesc).waitRead(0xc000120218, 0xffffffffffffff00, 0x0, 0x0)
      /usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
  internal/poll.(*FD).Accept(0xc000120200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
      /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a0
  net.(*netFD).accept(0xc000120200, 0x7f0187415200, 0x0, 0x0)
      /usr/local/go/src/net/fd_unix.go:238 +0x42
  net.(*TCPListener).accept(0xc0000965a8, 0xc000171e30, 0xc000171e38, 0x18)
      /usr/local/go/src/net/tcpsock_posix.go:139 +0x2e
  net.(*TCPListener).Accept(0xc0000965a8, 0xada0f8, 0xc0000ba320, 0x7f0187415200, 0xc0000965f8)
      /usr/local/go/src/net/tcpsock.go:260 +0x47
  net/rpc.(*Server).Accept(0xc0000ba320, 0xc26c00, 0xc0000965a8)
      /usr/local/go/src/net/rpc/server.go:632 +0x69
  net/rpc.Accept(0xc26c00, 0xc0000965a8)
      /usr/local/go/src/net/rpc/server.go:692 +0x41
  5kparser/vendor/github.com/aws/aws-lambda-go/lambda.StartHandler(0xc21a20, 0xc00009bb80)
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-lambda-go/lambda/entry.go:60 +0x174
  5kparser/vendor/github.com/aws/aws-lambda-go/lambda.Start(0x9e9140, 0xad9dd8)
      /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-lambda-go/lambda/entry.go:41 +0x4d
  main.main()
      /home/softarn/go/src/5kparser/aws-lambda-runner.go:209 +0x39
  </code></pre>