                      ["doc3" "Now we are engaged in a great civil war testing whether that nation or any nation so"]
                      ["doc4" "conceived and so dedicated can long endure We are met on a great battlefield of that war"]]
 
-          doc-data (-> (f/parallelize sc documents))
-
-          ;; total number of documents in corpus
-          num-docs (f/count doc-data)
+          doc-data (f/parallelize sc documents)
 
           ;; stopword filtered RDD of (document_id, term) tuples
           doc-term-seq (-> doc-data

          ]
      (-> tfidf-by-term
          f/collect
          clojure.pprint/pprint))))

;; Returns a seq of (doc_id, term) tuples
(f/defsparkfn gen-docid-term-tuples [x]
  (let [[doc-id content] x
        terms (clojure.string/split content #" ")]
    (map (fn [term] [doc-id term]) terms)))

;; Filters (doc_id, term) tuples if term is a stopword
