   )
   )
 
+;; if the response is 403, capture the reset date and remove it from
+;; the available items
 
 (defn find-next-url
   "Figure out the next url to call

          {:success false :next-url nil :data nil :reason "Empty Response"}
          (do
            (let [data (parse-string-strict (:body response) true)
                  _ (log/sometimes 0.1 (log/info (:headers response) ))
                  next-url (find-next-url
                              (or (-> response :headers :link)
                                  (-> response :headers :Link) ;; json encoder causes this when executed remotely
                              ))]
              {:success true
                :next-url next-url
                :data data}))))
    (catch Throwable t
      (do
        (throw+ t)
        {:success false :reason (.getMessage t) :repos [] :next-url nil }))))


(defn pick-bee []
  (let [hive (wcar  {:pool {} :spec {:host "localhost" :port 6379}}
              (car/smembers "oss.worker.bees"))]
        (rand-nth hive)))

(defn assign-worker-bee
  [task param & {:keys [bee-picker] :or {bee-picker pick-bee}}]
  (let [bee (bee-picker)
        url (format "http://%s/%s/%s" bee task param )]
      (log/info url)
     (when-let [result (try
                        (dd/timed {} "worker.bee.call" {:service "oss"}
                          (-> (client/get url)

      next-url*
  ))

(defn to-int
  [v]
  (try
    (Integer/parseInt v)
    (catch Exception ex
      nil)))
(defn epoch-sec
  []
  (int (/ (hsm.utils/now->ep) 1000)))

(defn credit-report
  [bee report]
  (when report
    (when-let [reset (to-int (:X-RateLimit-Reset report))]
    (let [now (epoch-sec)
          credits-left (to-int (:X-RateLimit-Remaining report))
          report (merge report {:bee bee :rate (if (zero? credits-left) 0
                                                   (float (/ (- reset now) credits-left)))})]
    (wcar {:pool {} :spec {:host "localhost" :port 6379}}
      (car/hset "oss.worker.bee.state" bee (generate-string report))
      (car/zadd  "oss.worker.bee.tokens" credits-left bee )
      (car/zadd  "oss.worker.bee.credit" reset bee)
      (car/zadd  "oss.worker.bee.credit.rate" (str (:rate report))  bee )
      (car/publish "oss.worker.bee.credit.line"  (generate-string report))
      )))))
;; bee has time reset time already set, so don't use it
;; until the reset has been done.

(defn pick-bee-by-random []
  ; (log/info "Random bee pick")
  (let [[hive states] (wcar  {:pool {} :spec {:host "localhost" :port 6379}}
                        (car/smembers "oss.worker.bees")
                        (car/hgetall "oss.worker.bee.state"))
        states (filter (fn[x] (or (> (epoch-sec) (Integer/parseInt (:X-RateLimit-Reset x)))
                                  (> (Integer/parseInt (:X-RateLimit-Remaining x)) 0)))
                    (map (fn [[x y]] (parse-string y true)) (partition 2 states)))
        chosen (rand-nth states)]
        (if chosen
          (do
            (log/infof "Choses %s" chosen)
            (:bee chosen))
          (do
            (log/info "Rollback to random hive bee")
            (rand-nth hive)))))

(defn pick-bee-by-rank []
  (let [hive (mapv identity (partition 2 (wcar  {:pool {} :spec {:host "localhost" :port 6379}}
              (car/zrangebyscore "oss.worker.bee.tokens" 10 5000 "WITHSCORES" ))))
        selection (mapv identity (last hive))]
        (if (or (nil? selection) (empty? selection))
          (pick-bee-by-random)
          (do
            ; (log/infof "Lucky one is %s [%s]" selection hive)
            (first selection)))))

(defn pick-bee-by-rate []
  (let [hive (mapv identity (partition 2 (wcar  {:pool {} :spec {:host "localhost" :port 6379}}
              (car/zrangebyscore "oss.worker.bee.credit.rate" 0.01 5000 "WITHSCORES" ))))
        selection (mapv identity (last hive))]
        (if (or (nil? selection) (empty? selection))
          (pick-bee-by-random)
          (do
            ; (log/infof "Rate Lucky one is %s [%s]" selection hive)
            (first selection)))))

(defn pick-a-bee
  []
  (apply (rand-nth [pick-bee-by-rank
                    pick-bee-by-rank
                    pick-bee-by-random
                    pick-bee-by-rate
                    pick-bee-by-rate]) []))

;; if the chosen bee has the credit 0

(defn fetch-url
  "Picks up one of the worker bee as the delegate to fetch the URL and finds the
  next link. "
  [url & {:keys [bee-picker] :or {bee-picker pick-a-bee}}]

  (let [bee (bee-picker) ;
        fetcher (fn [url & args] (assign-worker-bee "get-url" bee (codec/url-encode url)
                                  ; :bee-picker (fn [] "localhost:10554")
                                   ))]
      (let [next-step (fetch-url-and-next url {:get-url-fn fetcher})
            next-url (:next-url next-step)
            _ (log/sometimes 0.9 (log/info bee (:credits next-step)))]
            (dd/increment {} "github.calls" 1 {:bee bee :result (not (nil? next-url))})
            (credit-report bee (:credits next-step))
        (if (nil? next-url)
          next-step
          (do
            (let [next-url* (clean-next-url next-url)
                  next-step* (assoc next-step :next-url next-url*)]
            (log/info (:next-url next-step*))
            next-step*
      ))))))


(defn user-data
