                 (conf/set conf)
                 (conf/set-executor-env env))
           sc (f/spark-context c)
+
           ;; sample docs and terms
           documents [["doc1" "a b c d"]
                      ["doc2" "a e f g"]
                      ["doc3" "a h i j"]
                      ["doc4" "a b b k l l l l"]]
+
           doc-data (-> (f/parallelize sc documents))
 
           ;; total number of documents in corpus

                                     [term [doc-id (double (/ term-feq terms-count))]]))
                        f/cache)

          ;; number of documents with given term in it
          doc-frequencies (-> doc-term-seq
                              (f/group-by (f/fn [[_ term]] term))
                              (f/map (f/fn [[term doc-seq]] [term (count doc-seq)]))

(ns flambo.tfidf
  (:require [flambo.api :as f]
            [flambo.conf :as conf]
            [flambo.utils :as u]
            [clojure.string :as s])
  (:gen-class))

(def master "local")
(def jars (f/jar-of-ns *ns*))
(def conf {})
(def env {})

(def stopwords #{"the" "is"}) ;; TODO add more stopwords

(defn -main [& args]
  (try
    (let [c (-> (conf/spark-conf)
                (conf/master master)
                (conf/app-name "tfidf")
                (conf/jars jars)
                (conf/set "spark.executor.memory" "2g")
                (conf/set "spark.kryoserializer.buffer.mb" "10")
                (conf/set "spark.default.parallelism" "288")
