title: how to download larger binary object from postgres using nodejs express?
body: <p>I have node js rest api script to query from Postgres tables. It's working fine, but When I try to fetch a large objects using &quot;lo_get&quot; of Postgres, I get</p>
  <blockquote>
  <p>JavaScript heap out of memory</p>
  </blockquote>
  <p>Below is the toy example.</p>
  <p>index.js</p>
  <pre><code>const express = require('express')
  const bodyParser = require('body-parser')
  const app = express()
  const port = 8000
  var controller= require('./src/controller');
  
  app.use(bodyParser.json())
  app.use(
    bodyParser.urlencoded({
      extended: true,
    })
  )
  
  app.use('/cont', controller);
  
  app.listen(port, () =&gt; {
    console.log(`App running on port ${port}.`)
  })
  </code></pre>
  <p>controller.js</p>
  <pre><code>var express = require('express');
  var router = express.Router();
  
  router.get('/fileID/:lo_oid/', function( req, res) {
      const lo_oid =  parseInt(req.params.lo_oid)
      model.exportLoSaint(lo_oid, function (err, object) {
          if (err) return res.send(err);
     #  Fixed the typo
     #     response.status(200).file(file);
          res.status(200).objects;
      });
  });
  
  module.exports = router;
  </code></pre>
  <p>Model.js</p>
  <pre><code>const Pool = require('pg').Pool
  var exportLoSaintQuery= &quot;SELECT lo_get($1)&quot;
  
  const exportLoSaint = (lo_oid, callback) =&gt; {
      pool.query( exportLoSaintQuery ,[lo_oid], (error, results) =&gt; {
                  if (error) {
                      callback(error);
                  }
                  else {
                      callback(results.rows)
                  }
              })
          }
  </code></pre>
  <p>on sending request .. <code>http://ipaddress:8000/cont/fileID/&lt;fileID&gt;/</code> getting below error.</p>
  <pre><code>
  &lt;--- JS stacktrace ---&gt;
  
  ==== JS stack trace =========================================
  
      0: ExitFrame [pc: 0x1b2cbd65be1d]
  Security context: 0x2bf3bbc08bd9 &lt;JSObject&gt;
      1: toJSON [0x259ded584f29] [buffer.js:~979] [pc=0x1b2cbd665145](this=0x306d2be17731 &lt;Uint8Array map = 0x9dd44699a69&gt;)
      2: arguments adaptor frame: 1-&gt;0
      3: InternalFrame [pc: 0x1b2cbd60ee75]
      4: EntryFrame [pc: 0x1b2cbd6092c1]
      5: builtin exit frame: stringify(this=0x0a6855c58f61 &lt;Object map = 0x428c84042a9&gt;,0x2160e26026f1 &lt;undefined&gt;,0x2160e...
  
  FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
   1: 0x56228da912e4 node::Abort() [node]
   2: 0x56228da91332  [node]
   3: 0x56228dc86d4a v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [node]
   4: 0x56228dc86fc5 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [node]
   5: 0x56228e0387d6  [node]
   6: 0x56228e04e018 v8::internal::Heap::AllocateRawWithRetryOrFail(int, v8::internal::AllocationSpace, v8::internal::AllocationAlignment) [node]
   7: 0x56228e01a63b v8::internal::Factory::AllocateRawArray(int, v8::internal::PretenureFlag) [node]
   8: 0x56228e01aad8 v8::internal::Factory::NewFixedArrayWithFiller(v8::internal::Heap::RootListIndex, int, v8::internal::Object*, v8::internal::PretenureFlag) [node]
   9: 0x56228dfe48fe  [node]
  10: 0x56228dfe4ad3  [node]
  11: 0x56228e143566 v8::internal::JSObject::AddDataElement(v8::internal::Handle&lt;v8::internal::JSObject&gt;, unsigned int, v8::internal::Handle&lt;v8::internal::Object&gt;, v8::internal::PropertyAttributes, v8::internal::ShouldThrow) [node]
  12: 0x56228e17897e v8::internal::Object::SetProperty(v8::internal::LookupIterator*, v8::internal::Handle&lt;v8::internal::Object&gt;, v8::internal::LanguageMode, v8::internal::Object::StoreFromKeyed) [node]
  13: 0x56228e2c1299 v8::internal::Runtime::SetObjectProperty(v8::internal::Isolate*, v8::internal::Handle&lt;v8::internal::Object&gt;, v8::internal::Handle&lt;v8::internal::Object&gt;, v8::internal::Handle&lt;v8::internal::Object&gt;, v8::internal::LanguageMode) [node]
  14: 0x56228e0a9b45 v8::internal::Runtime_KeyedStoreIC_Slow(int, v8::internal::Object**, v8::internal::Isolate*) [node]
  15: 0x1b2cbd65be1d 
  </code></pre>
  <p>The size of binary object in postgres is only 128 mb. So, I increased the size of node js to 1gb, it did not help. Other solution I found was not using express to download binary from postgres.</p>
  <p>******************* <strong>update 1</strong> *********** ************</p>
  <p>Model.js</p>
  <pre><code>const exportLoSaintForMaster = (oid, fileName, callback) =&gt; {
      var fileStream = require('fs').createWriteStream(fileName);
      var man = new LargeObjectManager({ pg: pool });
      pool.query('BEGIN', function (err, result) {
          if (err) {
              callback(err);
              pool.emit('error', err);
          }
          console.log(&quot;\nthe oid : %d\n&quot;, oid)
          var bufferSize = 16384;
          man.openAndReadableStream(oid, bufferSize, function (err, size, stream) {
              if (err) {
                  callback(err);
                  return console.error('Unable to read the given large object', err);
              }
  
              console.log('Streaming a large object with a total size of', size);
              stream.on('end', function () {
                  pool.query('COMMIT', callback);
              });
  
              // Store it as an image
  
              stream.pipe(fileStream);
          });
      });
      callback(fileStream)
  };
  </code></pre>
  <p><strike> The scripts waits for a long time and then error outs with the below message .. </strike></p>
  <pre><code>
          response.status(200).objects;
          ^
  
  ReferenceError: response is not defined
  
  
  
  </code></pre>
  <p>******************* <strong>Update 2</strong> *********************</p>
  <p>After fixing the typo above error is no more.
  yet I see some problems.. as the client-side is not able to download it. it shows below error on client side ..</p>
  <p>i tried to do get . I get below message</p>
  <pre><code>{&quot;_writableState&quot;:{&quot;objectMode&quot;:false,&quot;highWaterMark&quot;:16384,&quot;finalCalled&quot;:false,&quot;needDrain&quot;:false,&quot;ending&quot;:false,&quot;ended&quot;:false,&quot;finished&quot;:false,&quot;destroyed&quot;:false,&quot;decodeStrings&quot;:true,&quot;defaultEncoding&quot;:&quot;utf8&quot;,&quot;length&quot;:0,&quot;writing&quot;:false,&quot;corked&quot;:0,&quot;sync&quot;:true,&quot;bufferProcessing&quot;:false,&quot;writecb&quot;:null,&quot;writelen&quot;:0,&quot;bufferedRequest&quot;:null,&quot;lastBufferedRequest&quot;:null,&quot;pendingcb&quot;:0,&quot;prefinished&quot;:false,&quot;errorEmitted&quot;:false,&quot;emitClose&quot;:false,&quot;autoDestroy&quot;:false,&quot;bufferedRequestCount&quot;:0,&quot;corkedRequestsFree&quot;:{&quot;next&quot;:null,&quot;entry&quot;:null}},&quot;writable&quot;:true,&quot;_events&quot;:{},&quot;_eventsCount&quot;:0,&quot;path&quot;:&quot;myfile1&quot;,&quot;fd&quot;:null,&quot;flags&quot;:&quot;w&quot;,&quot;mode&quot;:438,&quot;autoClose&quot;:true,&quot;bytesWritten&quot;:0,&quot;closed&quot;:false}
  
  </code></pre>
  <p>I am using a postman to download it. I think the issue may be because of the way controller.js is invoking the script in model.js .</p>
  <p>Please suggest me</p>
  <p>******************* <strong>Update 3</strong> ****************</p>
  <p>controller.js</p>
  <pre><code>
  router.get('/fileID/:lo_oid/fileName/:fileName', function (req, res) {
      const oid = parseInt(req.params.lo_oid)
      const fileName = req.params.fileName
      model.exportLoSaintForMaster(oid,fileName, (err, stream) =&gt; stream.pipe(res));
  });
  
  </code></pre>
  <p>model.js</p>
  <pre><code>const exportLoSaintForMaster = (oid, fileName, callback) =&gt; {
      var fileStream = require('fs').createWriteStream(fileName);
      var man = new LargeObjectManager({ pg: pool });
      pool.query('BEGIN', function (err, result) {
          if (err) {
              callback(err);
              pool.emit('error', err);
          }
          console.log(&quot;\nthe oid : %d\n&quot;, oid)
          var bufferSize = 16384;
          man.openAndReadableStream(oid, bufferSize, function (err, size, stream) {
              if (err) {
                  callback(err);
                  return console.error('Unable to read the given large object', err);
              }
  
              console.log('Streaming a large object with a total size of', size);
              stream.on('end', function () {
                  pool.query('COMMIT', callback);
              });
  
              // Store it as an image
  
              stream.pipe(fileStream);
          });
      });
      callback(null, fileStream)
  }; 
  
  module.exports = {
  
      exportLoSaintForMaster
  }
  
  </code></pre>
  <p>getting below error</p>
  <pre><code>Error [ERR_STREAM_CANNOT_PIPE]: Cannot pipe, not readable
      at WriteStream.Writable.pipe (_stream_writable.js:243:24)
      ...
      ...
      
  </code></pre>