title: Using temporary files for working with large amount of data
body: <p>I am trying to implement interthread communication using a temporary serialized file (pointer table containing [no, offset, length] in data file and data file containing data). One thread should receive data (process it) and save it to memory. The second thread should read data from memory and display results. (The input thread is only appending data and the output thread is only reading.)</p>
  <p>I must compile it as 32-bit, so I try to solve the 2 GB limit by reading/writing a temporary file.</p>
  <p>I have implemented a simple example. But the problem is, that if I/O threads are working simultaneously, then the output thread does not read properly. If the input thread writes and closes the file, then the output thread reads and closes it works fine. I have done synchronization with shared_mutex and mutex with the same bad results.</p>
  <p>Thank you in advance for your responses.</p>
  <p><strong>UPDATE:</strong> Behavior gets better by resetting flags (stream.clear()) according to <a href="https://stackoverflow.com/questions/27807245/update-ifstream-object-after-data-is-written-to-its-file-using-ofstream">Update ifstream object after data is written to its file using ofstream</a> but still sometimes it fails and sometimes pass.</p>
  <p>Main:</p>
  <pre><code>int main() {        
        //Start input and output job
        std::thread input = std::thread(inputJob);
        std::thread output = std::thread(outputJob);
          
        //Wait here for end
        input.join();
        output.join();
      
      //HERE checking results
      
        return 0;
  }
  </code></pre>
  <p>Input job:</p>
  <pre><code>void inputJob() {
      is_on = true;
      //Loading input data
      for (int i = 1; i &lt; 10; i++) {
          student s(&quot;George&quot;, &quot;Patton&quot;, 100000 + i, (i % 2) &gt; 0, &quot;0A&quot;);
          for (int j = 1; j &lt; 4; j++) s.subjects.push_back(subject(rand(), &quot;MA&quot;, (j % 5)));
          s1.push_back(s);
      }
  
      //Save to binary file
      std::fstream data_stream, table_stream;
      table_stream.open(&quot;./table.data&quot;, std::fstream::out | std::fstream::trunc | std::fstream::binary);
      data_stream.open(&quot;./data.data&quot;, std::fstream::out | std::fstream::trunc | std::fstream::binary);
      size_t off = 0;
      if (table_stream.is_open() &amp;&amp; data_stream.is_open()) for (size_t i = 0; i &lt; s1.size(); i++) {
          std::string tmp = s1[i].toBinaryString();
          size_t sz = tmp.size();
          table_row t(i, off, sz);
          off += sz;
          {
              std::lock_guard lock(m);
              table_stream &lt;&lt; t.toBinaryString();
              data_stream &lt;&lt; tmp;
              cout &lt;&lt; &quot;Written&quot; &lt;&lt; endl;
          }       
      }
      table_stream.close();
      data_stream.close();
      is_on = false;
  }
  </code></pre>
  <p>output job:</p>
  <pre><code>    void outputJob() {
      //Load from binary file
      std::fstream data_stream, table_stream;
      table_stream.open(&quot;./table.data&quot;, std::fstream::in | std::fstream::binary);
      data_stream.open(&quot;./data.data&quot;, std::fstream::in | std::fstream::binary);
      if (table_stream.is_open() &amp;&amp; data_stream.is_open()) {
          size_t row_sz = sizeof(table_row);
          std::string line = &quot;&quot;;
          size_t index = 0;
  
              unsigned table_r = 0;
              bool was_empty = false;
              while (is_on || (was_empty == false)) {
                  {
                      std::lock_guard lock(m);
                      if (tryGetData(table_stream, line, row_sz, table_r) == 0 &amp;&amp; line.empty() == false) {
                          table_r += row_sz;
                          was_empty = false;
                          table_row row;
                          index = 0;
                          row.fromBinaryString(line, index, line.size());
  
                          if (tryGetData(data_stream, line, row.len, row.off) == 0 &amp;&amp; line.empty() == false) {
                              was_empty = false;
                              index = 0;
                              student tmp;
                              tmp.fromBinaryString(line, index);
                              if (VAL_CHECK == 1) s2.push_back(tmp);
                          }
                          else was_empty = true;
                      }
                      else was_empty = true;
                  }
              }
          
      }
      table_stream.close();
      data_stream.close();
  }
  </code></pre>
  <p>Try get data function:</p>
  <pre><code>    int tryGetData(std::fstream &amp;data_stream, std::string&amp; data, size_t data_sz, size_t offset) {
      int ret = 0;
      data = &quot;&quot;;
          if (data_stream.is_open()) {
              //Set ptr
              if (offset != UINT32_MAX) data_stream.seekp(offset);
  
              char c;
              while (data_sz &gt; 0 &amp;&amp; data_stream.get(c)) {
                  data.push_back(c);
                  data_sz--;
              }
              if (data_stream.eof()) ret = 1;
          }
      return ret;
  }
  </code></pre>