text:
from __future__ import division, print_function

import copy
import datetime
import re
import sys
import threading
import warnings

from collections import OrderedDict

import dateutil.tz
from dateutil.parser import parse as dateutil_parser

from .fetcher import Fetcher, REPO_CREATED_TAG_NAME
from .pygcgen_exceptions import ChangelogGeneratorError
from .reader import read_changelog

if sys.version_info.major == 3:
    # noinspection PyCompatibility
    from builtins import object, range, str

def timestring_to_datetime(timestring):
    """
    Convert an ISO formated date and time string to a datetime object.

    :param str timestring: String with date and time in ISO format.
    :rtype: datetime
    :return: datetime object
    """
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=UnicodeWarning)
        result = dateutil_parser(timestring)

    return result

# noinspection PyTypeChecker
class Generator(object):
    """
    A Generator responsible for all logic, related with
    change log generation from ready-to-parse issues.
    """

    def __init__(self, options):
        self.options = options
        self.tag_times_dict = {}
        self.issues = []
        self.issues2 = []
        self.pull_requests = []
        self.all_tags = []
        self.filtered_tags = []
        self.fetcher = Fetcher(options)

    def fetch_and_filter_issues_and_pr(self):
        issues, pull_requests = self.fetcher.fetch_closed_issues_and_pr()

        if self.options.verbose:
            print("Filtering issues and pull requests...")
        self.pull_requests = []
        if self.options.include_pull_request:
            self.pull_requests = self.get_filtered_pull_requests(pull_requests)

        self.issues = []
        if self.options.issues:
            self.issues = self.filter_by_labels(issues, "issues")

        self.fetch_events_for_issues_and_pr()
        self.issues = self.detect_actual_closed_dates(self.issues, "issues")
        self.pull_requests = self.detect_actual_closed_dates(
            self.pull_requests, "pull requests"
        )

    def fetch_events_for_issues_and_pr(self):
        """
        Fetch event for issues and pull requests

        @return [Array] array of fetched issues
        """

        # Async fetching events:
        self.fetcher.fetch_events_async(self.issues, "issues")
        self.fetcher.fetch_events_async(self.pull_requests, "pull requests")

    def fetch_tags_dates(self):
        """ Async fetching of all tags dates. """

        if self.options.verbose:
            print(
                "Fetching dates for {} tags...".format(len(self.filtered_tags))
            )

        def worker(tag):
            self.get_time_of_tag(tag)

        # Async fetching tags:
        threads = []
        max_threads = 50
        cnt = len(self.filtered_tags)
        for i in range(0, (cnt // max_threads) + 1):
            for j in range(max_threads):
                idx = i * 50 + j
                if idx == cnt:
                    break
                t = threading.Thread(target=worker,
                                     args=(self.filtered_tags[idx],))
                threads.append(t)
                t.start()
                if self.options.verbose > 2:
                    print(".", end="")
            for t in threads:
                t.join()
        if self.options.verbose > 2:
            print(".")
        if self.options.verbose > 1:
            print("Fetched dates for {} tags.".format(
                len(self.tag_times_dict))
            )

    def detect_actual_closed_dates(self, issues, kind):
        """
        Find correct closed dates, if issues was closed by commits.

        :param list issues: issues to check
        :param str kind: either "issues" or "pull requests"
        :rtype: list
        :return: issues with updated closed dates
        """

        if self.options.verbose:
            print("Fetching closed dates for {} {}...".format(
                len(issues), kind)
            )
        all_issues = copy.deepcopy(issues)
        for issue in all_issues:
            if self.options.verbose > 2:
                print(".", end="")
                if not issues.index(issue) % 30:
                    print("")
            self.find_closed_date_by_commit(issue)

            if not issue.get('actual_date', False):
                if issue.get('closed_at', False):
                    print("Skipping closed non-merged issue: #{0} {1}".format(
                        issue["number"], issue["title"]))

                all_issues.remove(issue)

        if self.options.verbose > 2:
            print(".")
        return all_issues

    def find_closed_date_by_commit(self, issue):
        """
        Fill "actual_date" parameter of specified issue by closed date of
        the commit, if it was closed by commit.

        :param dict issue: issue to edit
        """

        if not issue.get('events'):
            return
        # if it's PR -> then find "merged event", in case
        # of usual issue -> find closed date
        compare_string = "merged" if 'merged_at' in issue else "closed"
        # reverse! - to find latest closed event. (event goes in date order)
        # if it were reopened and closed again.
        issue['events'].reverse()
        found_date = False
        for event in issue['events']:
            if event["event"] == compare_string:
                self.set_date_from_event(event, issue)
                found_date = True
                break
        if not found_date:
            # TODO: assert issues, that remain without
            #       'actual_date' hash for some reason.
            print("\nWARNING: Issue without 'actual_date':"
                  " #{0} {1}".format(issue["number"], issue["title"]))

    def set_date_from_event(self, event, issue):
        """
        Set closed date from this issue.

        :param dict event: event data
        :param dict issue: issue data
        """

        if not event.get('commit_id', None):
            issue['actual_date'] = timestring_to_datetime(issue['closed_at'])
            return
        try:
            commit = self.fetcher.fetch_commit(event)
            issue['actual_date'] = timestring_to_datetime(
                commit['author']['date']
            )
        except ValueError:
            print("WARNING: Can't fetch commit {0}. "
                  "It is probably referenced from another repo.".
                  format(event['commit_id']))
            issue['actual_date'] = timestring_to_datetime(issue['closed_at'])

    @staticmethod
    def encapsulate_string(raw_string):
        """
        Encapsulate characters to make markdown look as expected.

        :param str raw_string: string to encapsulate
        :rtype: str
        :return: encapsulated input string
        """

        raw_string.replace('\\', '\\\\')
        enc_string = re.sub("([<>*_()\[\]#])", r"\\\1", raw_string)
        return enc_string

    def compound_changelog(self):
        """
        Main function to start change log generation

        :rtype: str
        :return: Generated change log file
        """

        self.fetch_and_filter_tags()
        tags_sorted = self.sort_tags_by_date(self.filtered_tags)
        self.filtered_tags = tags_sorted
        self.fetch_and_filter_issues_and_pr()

        log = str(self.options.frontmatter) \
            if self.options.frontmatter else u""
        log += u"{0}\n\n".format(self.options.header)

        if self.options.unreleased_only:
            log += self.generate_unreleased_section()
        else:
            log += self.generate_log_for_all_tags()

        try:
            with open(self.options.base) as fh:
                log += fh.read()
        except (TypeError, IOError):
            pass
        return log

    def generate_sub_section(self, issues, prefix):
        """
        Generate formated list of issues for changelog.

        :param list issues: Issues to put in sub-section.
        :param str prefix: Title of sub-section.
        :rtype: str
        :return: Generated ready-to-add sub-section.
        """

        log = ""
        if issues:
            if not self.options.simple_list:
                log += u"{0}\n\n".format(prefix)
            for issue in issues:
                merge_string = self.get_string_for_issue(issue)
                log += u"- {0}\n".format(merge_string)
            log += "\n"
        return log

    def generate_header(self, newer_tag_name, newer_tag_link,
                        newer_tag_time,
                        older_tag_link, project_url):
        """
        Generate a header for a tag section with specific parameters.

        :param str newer_tag_name: Name (title) of newer tag.
        :param str newer_tag_link: Tag name of newer tag, used for links.
                               Could be same as **newer_tag_name** or some
                               specific value, like `HEAD`.
        :param datetime newer_tag_time: Date and time when
                                        newer tag was created.
        :param str older_tag_link: Tag name of older tag, used for links.
        :param str project_url: URL for current project.
        :rtype: str
        :return: Generated ready-to-add tag section.
        """

        log = ""
        # Generate date string:
        # noinspection PyUnresolvedReferences
        time_string = newer_tag_time.strftime(self.options.date_format)

        # Generate tag name and link
        if self.options.release_url:
            release_url = self.options.release_url.format(newer_tag_link)
        else:
            release_url = u"{project_url}/tree/{newer_tag_link}".format(
                project_url=project_url, newer_tag_link=newer_tag_link)

        if not self.options.unreleased_with_date and \
                newer_tag_name == self.options.unreleased_label:
            log += u"## [{newer_tag_name}]({release_url})\n\n".format(
                newer_tag_name=newer_tag_name, release_url=release_url)
        else:
            log += u"## [{newer_tag_name}]({release_url}) " \
                   u"({time_string})\n".format(
                        newer_tag_name=newer_tag_name,
                        release_url=release_url,
                        time_string=time_string
                   )

        if self.options.compare_link \
            and older_tag_link != REPO_CREATED_TAG_NAME:
            # Generate compare link
            log += u"[Full Changelog]"
            log += u"({project_url}/compare/{older_tag_link}".format(
                project_url=project_url,
                older_tag_link=older_tag_link,
            )
            log += u"...{newer_tag_link})\n\n".format(
                newer_tag_link=newer_tag_link
            )
        return log

    def generate_log_between_tags(self, older_tag, newer_tag):
        """
        Generate log between 2 specified tags.

        :param dict older_tag: All issues before this tag's date will be
                               excluded. May be special value, if new tag is
                               the first tag. (Means **older_tag** is when
                               the repo was created.)
        :param dict newer_tag: All issues after this tag's date  will be
                               excluded. May be title of unreleased section.
        :rtype: str
        :return: Generated ready-to-add tag section for newer tag.
        """

        filtered_issues, filtered_pull_requests = \
            self.filter_issues_for_tags(newer_tag, older_tag)

        older_tag_name = older_tag["name"] if older_tag \
            else self.detect_since_tag()

        if not filtered_issues and not filtered_pull_requests:
            # do not generate an unreleased section if it would be empty
            return ""
        return self.generate_log_for_tag(
            filtered_pull_requests, filtered_issues,
            newer_tag, older_tag_name)

    def filter_issues_for_tags(self, newer_tag, older_tag):
        """
        Apply all filters to issues and pull requests.

        :param dict older_tag: All issues before this tag's date will be
                               excluded. May be special value, if new tag is
                               the first tag. (Means **older_tag** is when
                               the repo  was created.)
        :param dict newer_tag: All issues after this tag's date  will be
                               excluded. May be title of unreleased section.
        :rtype: list(dict), list(dict)
        :return: Filtered issues and pull requests.
        """

        filtered_pull_requests = self.delete_by_time(self.pull_requests,
                                                     older_tag, newer_tag)
        filtered_issues = self.delete_by_time(self.issues, older_tag,
                                              newer_tag)

        newer_tag_name = newer_tag["name"] if newer_tag else None

        if self.options.filter_issues_by_milestone:
            # delete excess irrelevant issues (according milestones).Issue #22.
            filtered_issues = self.filter_by_milestone(
                filtered_issues, newer_tag_name, self.issues
            )
            filtered_pull_requests = self.filter_by_milestone(
                filtered_pull_requests, newer_tag_name, self.pull_requests
            )
        return filtered_issues, filtered_pull_requests

    def generate_log_for_all_tags(self):
        """
        The full cycle of generation for whole project.

        :rtype: str
        :return: The complete change log for released tags.
        """

        if self.options.verbose:
            print("Generating log...")
        self.issues2 = copy.deepcopy(self.issues)

        log1 = ""
        if self.options.with_unreleased:
            log1 = self.generate_unreleased_section()

        log = ""
        for index in range(len(self.filtered_tags) - 1):
            log += self.do_generate_log_for_all_tags_part1(log, index)

        if self.options.tag_separator and log1:
            log = log1 + self.options.tag_separator + log
        else:
            log = log1 + log

        if len(self.filtered_tags) != 0:
            log += self.do_generate_log_for_all_tags_part2(log)

        return log

    def do_generate_log_for_all_tags_part1(self, log, index):
        if self.options.verbose > 1:
            print("\tgenerate log for {}".format(
                self.filtered_tags[index]["name"]))
        log2 = self.generate_log_between_tags(
            self.filtered_tags[index + 1], self.filtered_tags[index])
        if self.options.tag_separator and log and log2:
            return self.options.tag_separator + log2
        return log2

    def do_generate_log_for_all_tags_part2(self, log):
        older_tag = self.last_older_tag()
        if self.options.verbose > 1:
            print("\tgenerate log for {}".format(
                self.filtered_tags[-1]["name"]))
        log2 = self.generate_log_between_tags(
            older_tag, self.filtered_tags[-1])
        if self.options.tag_separator and log and log2:
            return self.options.tag_separator + log2
        return log2

    def last_older_tag(self):
        older_tag = {"name": self.get_temp_tag_for_repo_creation()}
        if self.options.between_tags or self.options.since_tag:
            older_tag_date = self.get_time_of_tag(older_tag)
            newer_tag_date = self.get_time_of_tag(self.filtered_tags[-1])
            for tag in self.all_tags:
                tag_date = self.get_time_of_tag(tag)
                if older_tag_date < tag_date < newer_tag_date:
                    older_tag = tag
                    older_tag_date = tag_date
        return older_tag

    def generate_unreleased_section(self):
        """
        Generate log for unreleased closed issues.

        :rtype: str
        :return: Generated ready-to-add unreleased section.
        """
        if not self.filtered_tags:
            return ""
        now = datetime.datetime.utcnow()
        now = now.replace(tzinfo=dateutil.tz.tzutc())
        head_tag = {"name": self.options.unreleased_label}
        self.tag_times_dict[head_tag["name"]] = now
        unreleased_log = self.generate_log_between_tags(
            self.filtered_tags[0], head_tag)
        return unreleased_log

    def get_string_for_issue(self, issue):
        """
        Parse issue and generate single line formatted issue line.

        Example output:
            - Add coveralls integration [\#223](https://github.com/skywinder/github-changelog-generator/pull/223) ([skywinder](https://github.com/skywinder))
            - Add coveralls integration [\#223](https://github.com/skywinder/github-changelog-generator/pull/223) (@skywinder)

        :param dict issue: Fetched issue from GitHub.
        :rtype: str
        :return: Markdown-formatted single issue.
        """

        encapsulated_title = self.encapsulate_string(issue['title'])
        try:
            title_with_number = u"{0} [\\#{1}]({2})".format(
                encapsulated_title, issue["number"], issue["html_url"]
            )
        except UnicodeEncodeError:
            # TODO: why did i add this? Is it needed?
            title_with_number = "ERROR ERROR ERROR: #{0} {1}".format(
                issue["number"], issue['title']
            )
            print(title_with_number, '\n', issue["html_url"])
        return self.issue_line_with_user(title_with_number, issue)

    def issue_line_with_user(self, line, issue):
        """
        If option author is enabled, a link to the profile of the author
        of the pull reqest will be added to the issue line.

        :param str line: String containing a markdown-formatted single issue.
        :param dict issue: Fetched issue from GitHub.
        :rtype: str
        :return: Issue line with added author link.
        """
        if not issue.get("pull_request") or not self.options.author:
            return line

        if not issue.get("user"):
            line += u" (Null user)"
        elif self.options.username_as_tag:
            line += u" (@{0})".format(
                issue["user"]["login"]
            )
        else:
            line += u" ([{0}]({1}))".format(
                issue["user"]["login"], issue["user"]["html_url"]
            )
        return line

    def generate_log_for_tag(self,
                             pull_requests,
                             issues,
                             newer_tag,
                             older_tag_name):
        """
        Generates log for tag section with header and body.

        :param list(dict) pull_requests: List of PR's in this tag section.
        :param list(dict) issues: List of issues in this tag section.
        :param dict newer_tag: Github data of tag for this section.
        :param str older_tag_name: Older tag, used for the links.
                                   May be special value, if **newer tag** is
                                   the first tag. (Means **older_tag** is when
                                   the repo was created.)
        :rtype: str
        :return: Ready-to-add and parsed tag section.
        """

        newer_tag_link, newer_tag_name, \
        newer_tag_time = self.detect_link_tag_time(newer_tag)

        github_site = "https://github.com" or self.options.github_endpoint
        project_url = "{0}/{1}/{2}".format(
            github_site, self.options.user, self.options.project)

        log = self.generate_header(newer_tag_name, newer_tag_link,
                                   newer_tag_time, older_tag_name, project_url)
        if self.options.issues:
            # Generate issues:
            log += self.issues_to_log(issues, pull_requests)
        if self.options.include_pull_request:
            # Generate pull requests:
            log += self.generate_sub_section(
                pull_requests, self.options.merge_prefix
            )
        return log

    def issues_to_log(self, issues, pull_requests):
        """
        Generate ready-to-paste log from list of issues and pull requests.

        :param list(dict) issues: List of issues in this tag section.
        :param list(dict) pull_requests: List of PR's in this tag section.
        :rtype: str
        :return: Generated log for issues and pull requests.
        """

        log = ""
        sections_a, issues_a = self.parse_by_sections(
            issues, pull_requests)

        for section, s_issues in sections_a.items():
            log += self.generate_sub_section(s_issues, section)
        log += self.generate_sub_section(issues_a, self.options.issue_prefix)
        return log

    def parse_by_sections(self, issues, pull_requests):
        """
        This method sort issues by types (bugs, features, etc. or
        just closed issues) by labels.

        :param list(dict) issues: List of issues in this tag section.
        :param list(dict) pull_requests: List of PR's in this tag section.
        :rtype: dict(list(dict)), list(dict)
        :return: Issues and PR's sorted into sections.
        """

        issues_a = []
        sections_a = OrderedDict()

        if not self.options.sections:
            return [sections_a, issues]
        for key in self.options.sections:
            sections_a.update({key: []})
        self.parse_by_sections_for_issues(issues, sections_a, issues_a)
        self.parse_by_sections_for_pr(pull_requests, sections_a)
        return [sections_a, issues_a]

    def parse_by_sections_for_issues(self, issues, sections_a, issues_a):
        for section, sect_labels in self.options.sections.items():
            added_issues = []
            for issue in issues:
                is_labels = issue.get('labels')
                if is_labels:
                    is_lbls = set(l["name"] for l in is_labels)
                    if is_lbls.intersection(set(sect_labels)):
                        sections_a[section].append(issue)
                        added_issues.append(issue)
                        continue
                issues_a.append(issue)
                added_issues.append(issue)
            for iss in added_issues:
                issues.remove(iss)

    def parse_by_sections_for_pr(self, pull_requests, sections_a):
        for section, sect_labels in self.options.sections.items():
            added_pull_requests = []
            for pr in pull_requests:
                pr_labels = pr.get('labels')
                if pr_labels:
                    pr_lbls = set(l["name"] for l in pr_labels)
                    if pr_lbls.intersection(set(sect_labels)):
                        sections_a[section].append(pr)
                        added_pull_requests.append(pr)
                        continue
            for pr in added_pull_requests:
                pull_requests.remove(pr)

    def exclude_issues_by_labels(self, issues):
        """
        Delete all issues with labels from exclude-labels option.

        :param list(dict) issues: All issues for tag.
        :rtype: list(dict)
        :return: Filtered issues.
        """
        if not self.options.exclude_labels:
            return copy.deepcopy(issues)

        remove_issues = set()
        exclude_labels = self.options.exclude_labels
        include_issues = []
        for issue in issues:
            for label in issue["labels"]:
                if label["name"] in exclude_labels:
                    remove_issues.add(issue["number"])
                    break
        for issue in issues:
            if issue["number"] not in remove_issues:
                include_issues.append(issue)
        return include_issues

    def filter_by_milestone(self, filtered_issues, tag_name, all_issues):
        """
        :param list(dict) filtered_issues: Filtered issues.
        :param str tag_name: Name (title) of tag.
        :param list(dict) all_issues: All issues.
        :rtype: list(dict)
        :return: Filtered issues according milestone.
        """

        filtered_issues = self.remove_issues_in_milestones(filtered_issues)
        if tag_name:
            # add missed issues (according milestones)
            issues_to_add = self.find_issues_to_add(all_issues, tag_name)
            filtered_issues.extend(issues_to_add)
        return filtered_issues

    @staticmethod
    def find_issues_to_add(all_issues, tag_name):
        """
        Add all issues, that should be in that tag, according to milestone.

        :param list(dict) all_issues: All issues.
        :param str tag_name: Name (title) of tag.
        :rtype: List[dict]
        :return: Issues filtered by milestone.
        """

        filtered = []
        for issue in all_issues:
            if issue.get("milestone"):
                if issue["milestone"]["title"] == tag_name:
                    iss = copy.deepcopy(issue)
                    filtered.append(iss)
        return filtered

    def remove_issues_in_milestones(self, filtered_issues):
        """
        :param list(dict) filtered_issues: Filtered issues.
        :rtype: list(dict)
        :return: List with removed issues, that contain milestones with
                 same name as a tag.
        """
        for issue in filtered_issues:
            # leave issues without milestones
            if issue["milestone"]:
                # check, that this milestone is in tag list:
                for tag in self.filtered_tags:
                    if tag["name"] == issue["milestone"]["title"]:
                        filtered_issues.remove(issue)
        return filtered_issues

    def delete_by_time(self, issues, older_tag, newer_tag):
        """
        Filter issues that belong to specified tag range.

        :param list(dict) issues: Issues to filter.
        :param dict older_tag: All issues before this tag's date will be
                               excluded. May be special value, if **newer_tag**
                               is the first tag. (Means **older_tag** is when
                               the repo was created.)
        :param dict newer_tag: All issues after this tag's date  will be
                               excluded. May be title of unreleased section.
        :rtype: list(dict)
        :return: Filtered issues.
        """

        if not older_tag and not newer_tag:
            # in case if no tags are specified - return unchanged array
            return copy.deepcopy(issues)

        newer_tag_time = self.get_time_of_tag(newer_tag)
        older_tag_time = self.get_time_of_tag(older_tag)
        filtered = []
        for issue in issues:
            if issue.get('actual_date'):
                rslt = older_tag_time < issue['actual_date'] <= newer_tag_time
                if rslt:
                    filtered.append(copy.deepcopy(issue))
        return filtered

    def include_issues_by_labels(self, all_issues):
        """
        Include issues with labels, specified in self.options.include_labels.

        :param list(dict) all_issues: All issues.
        :rtype: list(dict)
        :return: Filtered issues.
        """

        included_by_labels = self.filter_by_include_labels(all_issues)
        wo_labels = self.filter_wo_labels(all_issues)
        il = set([f["number"] for f in included_by_labels])
        wl = set([w["number"] for w in wo_labels])
        filtered_issues = []
        for issue in all_issues:
            if issue["number"] in il or issue["number"] in wl:
                filtered_issues.append(issue)
        return filtered_issues

    def filter_wo_labels(self, all_issues):
        """
        Filter all issues that don't have a label.

        :rtype: list(dict)
        :return: Issues without labels.
        """

        issues_wo_labels = []
        if not self.options.add_issues_wo_labels:
            for issue in all_issues:
                if not issue['labels']:
                    issues_wo_labels.append(issue)
        return issues_wo_labels

    def filter_by_include_labels(self, issues):
        """
        Filter issues to include only issues with labels
        specified in include_labels.

        :param list(dict) issues: Pre-filtered issues.
        :rtype: list(dict)
        :return: Filtered issues.
        """

        if not self.options.include_labels:
            return copy.deepcopy(issues)
        filtered_issues = []
        include_labels = set(self.options.include_labels)
        for issue in issues:
            labels = [label["name"] for label in issue["labels"]]
            if include_labels.intersection(labels):
                filtered_issues.append(issue)
        return filtered_issues

    def filter_by_labels(self, all_issues, kind):
        """
        Filter issues for include/exclude labels.

        :param list(dict) all_issues: All issues.
        :param str kind: Either "issues" or "pull requests".
        :rtype: list(dict)
        :return: Filtered issues.
        """

        filtered_issues = self.include_issues_by_labels(all_issues)
        filtered = self.exclude_issues_by_labels(filtered_issues)
        if self.options.verbose > 1:
            print("\tremaining {}: {}".format(kind, len(filtered)))
        return filtered

    def get_filtered_pull_requests(self, pull_requests):
        """
        This method fetches missing params for PR and filter them
        by specified options. It include add all PR's with labels
        from options.include_labels and exclude all from
        options.exclude_labels.

        :param list(dict) pull_requests: All pull requests.
        :rtype: list(dict)
        :return: Filtered pull requests.
        """

        pull_requests = self.filter_by_labels(pull_requests, "pull requests")
        pull_requests = self.filter_merged_pull_requests(pull_requests)
        if self.options.verbose > 1:
            print("\tremaining pull requests: {}".format(len(pull_requests)))
        return pull_requests

    def filter_merged_pull_requests(self, pull_requests):
        """
        This method filter only merged PR and fetch missing required
        attributes for pull requests. Using merged date is more correct
        than closed date.

        :param list(dict) pull_requests: Pre-filtered pull requests.
        :rtype: list(dict)
        :return:
        """

        if self.options.verbose:
            print("Fetching merge date for pull requests...")
        closed_pull_requests = self.fetcher.fetch_closed_pull_requests()

        if not pull_requests:
            return []
        pulls = copy.deepcopy(pull_requests)
        for pr in pulls:
            fetched_pr = None
            for fpr in closed_pull_requests:
                if fpr['number'] == pr['number']:
                    fetched_pr = fpr
            if fetched_pr:
                pr['merged_at'] = fetched_pr['merged_at']
                closed_pull_requests.remove(fetched_pr)

        for pr in pulls:
            if not pr.get('merged_at'):
                pulls.remove(pr)
        return pulls

    def fetch_and_filter_tags(self):
        """
        Fetch and filter tags, fetch dates and sort them in time order.
        """

        self.all_tags = self.fetcher.get_all_tags()
        self.filtered_tags = self.get_filtered_tags(self.all_tags)
        self.fetch_tags_dates()

    def sort_tags_by_date(self, tags):
        """
        Sort all tags by date.

        :param list(dict) tags: All tags.
        :rtype: list(dict)
        :return: Sorted list of tags.
        """

        if self.options.verbose:
            print("Sorting tags...")
        tags.sort(key=lambda x: self.get_time_of_tag(x))
        tags.reverse()
        return tags

    def get_time_of_tag(self, tag):
        """
        Get date and time for tag, fetching it if not already cached.

        :param dict tag: Tag to get the datetime for.
        :rtype: datetime
        :return: datetime for specified tag.
        """

        if not tag:
            raise ChangelogGeneratorError("tag is nil")

        name_of_tag = tag["name"]
        time_for_name = self.tag_times_dict.get(name_of_tag, None)
        if time_for_name:
            return time_for_name
        else:
            time_string = self.fetcher.fetch_date_of_tag(tag)
            try:
                self.tag_times_dict[name_of_tag] = \
                    timestring_to_datetime(time_string)
            except UnicodeWarning:
                print("ERROR ERROR:", tag)
                self.tag_times_dict[name_of_tag] = \
                    timestring_to_datetime(time_string)
            return self.tag_times_dict[name_of_tag]

    def detect_link_tag_time(self, tag):
        """
        Detect link, name and time for specified tag.

        :param dict tag: Tag data.
        :rtype: str, str, datetime
        :return: Link, name and time of the tag.
        """

        # if tag is nil - set current time
        newer_tag_time = self.get_time_of_tag(tag) if tag \
            else datetime.datetime.now()

        # if it's future release tag - set this value
        if tag["name"] == self.options.unreleased_label \
            and self.options.future_release:
            newer_tag_name = self.options.future_release
            newer_tag_link = self.options.future_release
        elif tag["name"] is not self.options.unreleased_label :
            # put unreleased label if there is no name for the tag
            newer_tag_name = tag["name"]
            newer_tag_link = newer_tag_name
        else:
            newer_tag_name = self.options.unreleased_label
            newer_tag_link = "HEAD"
        return [newer_tag_link, newer_tag_name, newer_tag_time]

    def detect_since_tag(self):
        """
        Try to find tag name to use as older tag for range of log creation.

        :rtype: str
        :return: Tag name to use as 'oldest' tag. May be special value,
                 indicating the creation of the repo.
        """
        return self.options.since_tag or self.version_of_first_item()

    def version_of_first_item(self):
        """
        Try to detect the newest tag from self.options.base, otherwise
        return a special value indicating the creation of the repo.

        :rtype: str
        :return: Tag name to use as 'oldest' tag. May be special value,
                 indicating the creation of the repo.
        """
        try:
            sections = read_changelog(self.options)
            return sections[0]["version"]
        except(IOError, TypeError):
            return self.get_temp_tag_for_repo_creation()

    def get_temp_tag_for_repo_creation(self):
        """
        If not already cached, fetch the creation date of the repo, cache it
        and return the special value indicating the creation of the repo.

        :rtype: str
        :return: value indicating the creation
        """
        tag_date = self.tag_times_dict.get(REPO_CREATED_TAG_NAME, None)
        if not tag_date:
            tag_name, tag_date = self.fetcher.fetch_repo_creation_date()
            self.tag_times_dict[tag_name] = timestring_to_datetime(tag_date)
        return REPO_CREATED_TAG_NAME

    def get_filtered_tags(self, all_tags):
        """
        Return tags after filtering tags in lists provided by
        option: --between-tags & --exclude-tags

        :param list(dict) all_tags: All tags.
        :rtype: list(dict)
        :return: Filtered tags.
        """

        filtered_tags = self.filter_since_tag(all_tags)
        if self.options.between_tags:
            filtered_tags = self.filter_between_tags(filtered_tags)
        if self.options.due_tag:
            filtered_tags = self.filter_due_tag(filtered_tags)
        return self.filter_excluded_tags(filtered_tags)

    def filter_since_tag(self, all_tags):
        """
        Filter tags according since_tag option.

        :param list(dict) all_tags: All tags.
        :rtype: list(dict)
        :return: Filtered tags.
        """

        tag = self.detect_since_tag()
        if not tag or tag == REPO_CREATED_TAG_NAME:
            return copy.deepcopy(all_tags)

        filtered_tags = []
        tag_names = [t["name"] for t in all_tags]
        try:
            idx = tag_names.index(tag)
        except ValueError:
            self.warn_if_tag_not_found(tag, "since-tag")
            return copy.deepcopy(all_tags)

        since_tag = all_tags[idx]
        since_date = self.get_time_of_tag(since_tag)
        for t in all_tags:
            tag_date = self.get_time_of_tag(t)
            if since_date <= tag_date:
                filtered_tags.append(t)
        return filtered_tags

    def filter_due_tag(self, all_tags):
        """
        Filter tags according due_tag option.

        :param list(dict) all_tags: Pre-filtered tags.
        :rtype: list(dict)
        :return: Filtered tags.
        """

        filtered_tags = []
        tag = self.options.due_tag
        tag_names = [t["name"] for t in all_tags]
        try:
            idx = tag_names.index(tag)
        except ValueError:
            self.warn_if_tag_not_found(tag, "due-tag")
            return copy.deepcopy(all_tags)

        due_tag = all_tags[idx]
        due_date = self.get_time_of_tag(due_tag)
        for t in all_tags:
            tag_date = self.get_time_of_tag(t)
            if tag_date <= due_date:
                filtered_tags.append(t)
        return filtered_tags

    def filter_between_tags(self, all_tags):
        """
        Filter tags according between_tags option.

        :param list(dict) all_tags: Pre-filtered tags.
        :rtype: list(dict)
        :return: Filtered tags.
        """

        tag_names = [t["name"] for t in all_tags]
        between_tags = []
        for tag in self.options.between_tags:
            try:
                idx = tag_names.index(tag)
            except ValueError:
                raise ChangelogGeneratorError(
                    "ERROR: can't find tag {0}, specified with "
                    "--between-tags option.".format(tag))
            between_tags.append(all_tags[idx])

        between_tags = self.sort_tags_by_date(between_tags)

        if len(between_tags) == 1:
            # if option --between-tags was only 1 tag given, duplicate it
            # to generate the changelog only for that one tag.
            between_tags.append(between_tags[0])

        older = self.get_time_of_tag(between_tags[1])
        newer = self.get_time_of_tag(between_tags[0])

        for tag in all_tags:
            if older < self.get_time_of_tag(tag) < newer:
                between_tags.append(tag)
        if older == newer:
            between_tags.pop(0)
        return between_tags

    def filter_excluded_tags(self, all_tags):
        """
        Filter tags according exclude_tags and exclude_tags_regex option.

        :param list(dict) all_tags: Pre-filtered tags.
        :rtype: list(dict)
        :return: Filtered tags.
        """
        filtered_tags = copy.deepcopy(all_tags)
        if self.options.exclude_tags:
            filtered_tags = self.apply_exclude_tags(filtered_tags)
        if self.options.exclude_tags_regex:
            filtered_tags = self.apply_exclude_tags_regex(filtered_tags)
        return filtered_tags

    def apply_exclude_tags_regex(self, all_tags):
        """
        Filter tags according exclude_tags_regex option.

        :param list(dict) all_tags: Pre-filtered tags.
        :rtype: list(dict)
        :return: Filtered tags.
        """
        filtered = []
        for tag in all_tags:
            if not re.match(self.options.exclude_tags_regex, tag["name"]):
                filtered.append(tag)
        if len(all_tags) == len(filtered):
            self.warn_if_nonmatching_regex()
        return filtered

    def apply_exclude_tags(self, all_tags):
        """
        Filter tags according exclude_tags option.

        :param list(dict) all_tags: Pre-filtered tags.
        :rtype: list(dict)
        :return: Filtered tags.
        """
        filtered = copy.deepcopy(all_tags)
        for tag in all_tags:
            if tag["name"] not in self.options.exclude_tags:
                self.warn_if_tag_not_found(tag, "exclude-tags")
            else:
                filtered.remove(tag)
        return filtered

    def warn_if_nonmatching_regex(self):
        if not self.options.quiet:
            print(
                "WARNING: unable to reject any tag, using regex "
                "'{0}' in --exclude-tags-regex option.".format(
                    self.options.exclude_tags_regex
                )
            )

    def warn_if_tag_not_found(self, tag, option):
        if not self.options.quiet:
            print(
                "WARNING: can't find tag '{0}' specified with "
                "--{1} option.".format(tag, option)
            )


text:
using Colors
using Images
using Boltzmann
using MNIST
using ImageView
using Base.Test

function run_mnist()
    X, y = testdata()  # test data is smaller, no need to downsample
    normalize_samples!(X)
    binarize!(X;threshold=0.01)

    TrainSet = X[:,1:9000]
    ValidSet = X[:,9001:end]
    HiddenUnits = 100;
    Epochs = 10;
    Gibbs = 1;
    LearnRate = 0.05
    MonitorEvery=1

    rbm = BernoulliRBM(28*28, HiddenUnits, (28,28); momentum=0.0)

    # Attempt with L2 weight decay
    finalrbm,monitor = fit(rbm, TrainSet;n_iter=Epochs,
                          weight_decay="l2",
                          decay_magnitude=0.001,
                          lr=LearnRate,
                          validation=ValidSet,
                          NormalizationApproxIter=Gibbs,
                          monitor_every=MonitorEvery,
                          monitor_vis=true)

    info("==========monitor==============")
    println(monitor)
    info("===============================")

    WriteMonitorChartPDF(finalrbm,monitor,X,"testmonitor.pdf")
    SaveMonitorHDF5(monitor,"testmonitor.h5")
end

run_mnist()

redpajama_set_name: RedPajamaGithub


text:
// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.remote;

import static com.google.common.base.Strings.isNullOrEmpty;

import build.bazel.remote.execution.v2.ActionCacheGrpc;
import build.bazel.remote.execution.v2.ActionCacheGrpc.ActionCacheBlockingStub;
import build.bazel.remote.execution.v2.ActionCacheGrpc.ActionCacheFutureStub;
import build.bazel.remote.execution.v2.ActionResult;
import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc;
import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc.ContentAddressableStorageFutureStub;
import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.FindMissingBlobsRequest;
import build.bazel.remote.execution.v2.FindMissingBlobsResponse;
import build.bazel.remote.execution.v2.GetActionResultRequest;
import build.bazel.remote.execution.v2.UpdateActionResultRequest;
import com.google.bytestream.ByteStreamGrpc;
import com.google.bytestream.ByteStreamGrpc.ByteStreamStub;
import com.google.bytestream.ByteStreamProto.ReadRequest;
import com.google.bytestream.ByteStreamProto.ReadResponse;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.hash.HashCode;
import com.google.common.hash.HashingOutputStream;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.remote.RemoteRetrier.ProgressiveBackoff;
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.MissingDigestsFinder;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.protobuf.ByteString;
import io.grpc.CallCredentials;
import io.grpc.Context;
import io.grpc.Status;
import io.grpc.Status.Code;
import io.grpc.StatusRuntimeException;
import io.grpc.stub.StreamObserver;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import javax.annotation.Nullable;

/** A RemoteActionCache implementation that uses gRPC calls to a remote cache server. */
@ThreadSafe
public class GrpcCacheClient implements RemoteCacheClient, MissingDigestsFinder {
  private final CallCredentials credentials;
  private final ReferenceCountedChannel channel;
  private final RemoteOptions options;
  private final DigestUtil digestUtil;
  private final RemoteRetrier retrier;
  private final ByteStreamUploader uploader;
  private final int maxMissingBlobsDigestsPerMessage;

  private AtomicBoolean closed = new AtomicBoolean();

  @VisibleForTesting
  public GrpcCacheClient(
      ReferenceCountedChannel channel,
      CallCredentials credentials,
      RemoteOptions options,
      RemoteRetrier retrier,
      DigestUtil digestUtil,
      ByteStreamUploader uploader) {
    this.credentials = credentials;
    this.channel = channel;
    this.options = options;
    this.digestUtil = digestUtil;
    this.retrier = retrier;
    this.uploader = uploader;
    maxMissingBlobsDigestsPerMessage = computeMaxMissingBlobsDigestsPerMessage();
    Preconditions.checkState(
        maxMissingBlobsDigestsPerMessage > 0, "Error: gRPC message size too small.");
  }

  private int computeMaxMissingBlobsDigestsPerMessage() {
    final int overhead =
        FindMissingBlobsRequest.newBuilder()
            .setInstanceName(options.remoteInstanceName)
            .build()
            .getSerializedSize();
    final int tagSize =
        FindMissingBlobsRequest.newBuilder()
                .addBlobDigests(Digest.getDefaultInstance())
                .build()
                .getSerializedSize()
            - FindMissingBlobsRequest.getDefaultInstance().getSerializedSize();
    // We assume all non-empty digests have the same size. This is true for fixed-length hashes.
    final int digestSize = digestUtil.compute(new byte[] {1}).getSerializedSize() + tagSize;
    return (options.maxOutboundMessageSize - overhead) / digestSize;
  }

  private ContentAddressableStorageFutureStub casFutureStub() {
    return ContentAddressableStorageGrpc.newFutureStub(channel)
        .withInterceptors(TracingMetadataUtils.attachMetadataFromContextInterceptor())
        .withCallCredentials(credentials)
        .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
  }

  private ByteStreamStub bsAsyncStub() {
    return ByteStreamGrpc.newStub(channel)
        .withInterceptors(TracingMetadataUtils.attachMetadataFromContextInterceptor())
        .withCallCredentials(credentials)
        .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
  }

  private ActionCacheBlockingStub acBlockingStub() {
    return ActionCacheGrpc.newBlockingStub(channel)
        .withInterceptors(TracingMetadataUtils.attachMetadataFromContextInterceptor())
        .withCallCredentials(credentials)
        .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
  }

  private ActionCacheFutureStub acFutureStub() {
    return ActionCacheGrpc.newFutureStub(channel)
        .withInterceptors(TracingMetadataUtils.attachMetadataFromContextInterceptor())
        .withCallCredentials(credentials)
        .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
  }

  @Override
  public void close() {
    if (closed.getAndSet(true)) {
      return;
    }
    uploader.release();
    channel.release();
  }

  /** Returns true if 'options.remoteCache' uses 'grpc' or an empty scheme */
  public static boolean isRemoteCacheOptions(RemoteOptions options) {
    if (isNullOrEmpty(options.remoteCache)) {
      return false;
    }
    // TODO(ishikhman): add proper URI validation/parsing for remote options
    return !(Ascii.toLowerCase(options.remoteCache).startsWith("http://")
        || Ascii.toLowerCase(options.remoteCache).startsWith("https://"));
  }

  @Override
  public ListenableFuture<ImmutableSet<Digest>> findMissingDigests(Iterable<Digest> digests) {
    if (Iterables.isEmpty(digests)) {
      return Futures.immediateFuture(ImmutableSet.of());
    }
    // Need to potentially split the digests into multiple requests.
    FindMissingBlobsRequest.Builder requestBuilder =
        FindMissingBlobsRequest.newBuilder().setInstanceName(options.remoteInstanceName);
    List<ListenableFuture<FindMissingBlobsResponse>> getMissingDigestCalls = new ArrayList<>();
    for (Digest digest : digests) {
      requestBuilder.addBlobDigests(digest);
      if (requestBuilder.getBlobDigestsCount() == maxMissingBlobsDigestsPerMessage) {
        getMissingDigestCalls.add(getMissingDigests(requestBuilder.build()));
        requestBuilder.clearBlobDigests();
      }
    }

    if (requestBuilder.getBlobDigestsCount() > 0) {
      getMissingDigestCalls.add(getMissingDigests(requestBuilder.build()));
    }

    ListenableFuture<ImmutableSet<Digest>> success =
        Futures.whenAllSucceed(getMissingDigestCalls)
            .call(
                () -> {
                  ImmutableSet.Builder<Digest> result = ImmutableSet.builder();
                  for (ListenableFuture<FindMissingBlobsResponse> callFuture :
                      getMissingDigestCalls) {
                    result.addAll(callFuture.get().getMissingBlobDigestsList());
                  }
                  return result.build();
                },
                MoreExecutors.directExecutor());
    return Futures.catchingAsync(
        success,
        RuntimeException.class,
        (e) ->
            Futures.immediateFailedFuture(
                new IOException(
                    String.format(
                        "findMissingBlobs(%d) for %s",
                        requestBuilder.getBlobDigestsCount(),
                        TracingMetadataUtils.fromCurrentContext().getActionId()),
                    e)),
        MoreExecutors.directExecutor());
  }

  private ListenableFuture<FindMissingBlobsResponse> getMissingDigests(
      FindMissingBlobsRequest request) {
    Context ctx = Context.current();
    return retrier.executeAsync(() -> ctx.call(() -> casFutureStub().findMissingBlobs(request)));
  }

  private ListenableFuture<ActionResult> handleStatus(ListenableFuture<ActionResult> download) {
    return Futures.catchingAsync(
        download,
        StatusRuntimeException.class,
        (sre) ->
            sre.getStatus().getCode() == Code.NOT_FOUND
                // Return null to indicate that it was a cache miss.
                ? Futures.immediateFuture(null)
                : Futures.immediateFailedFuture(new IOException(sre)),
        MoreExecutors.directExecutor());
  }

  @Override
  public ListenableFuture<ActionResult> downloadActionResult(
      ActionKey actionKey, boolean inlineOutErr) {
    GetActionResultRequest request =
        GetActionResultRequest.newBuilder()
            .setInstanceName(options.remoteInstanceName)
            .setActionDigest(actionKey.getDigest())
            .setInlineStderr(inlineOutErr)
            .setInlineStdout(inlineOutErr)
            .build();
    Context ctx = Context.current();
    return retrier.executeAsync(
        () -> ctx.call(() -> handleStatus(acFutureStub().getActionResult(request))));
  }

  @Override
  public void uploadActionResult(ActionKey actionKey, ActionResult actionResult)
      throws IOException, InterruptedException {
    try {
      retrier.execute(
          () ->
              acBlockingStub()
                  .updateActionResult(
                      UpdateActionResultRequest.newBuilder()
                          .setInstanceName(options.remoteInstanceName)
                          .setActionDigest(actionKey.getDigest())
                          .setActionResult(actionResult)
                          .build()));
    } catch (StatusRuntimeException e) {
      throw new IOException(e);
    }
  }

  @Override
  public ListenableFuture<Void> downloadBlob(Digest digest, OutputStream out) {
    if (digest.getSizeBytes() == 0) {
      return Futures.immediateFuture(null);
    }

    @Nullable Supplier<HashCode> hashSupplier = null;
    if (options.remoteVerifyDownloads) {
      HashingOutputStream hashOut = digestUtil.newHashingOutputStream(out);
      hashSupplier = hashOut::hash;
      out = hashOut;
    }

    return downloadBlob(digest, out, hashSupplier);
  }

  private ListenableFuture<Void> downloadBlob(
      Digest digest,
      OutputStream out,
      @Nullable Supplier<HashCode> hashSupplier) {
    Context ctx = Context.current();
    AtomicLong offset = new AtomicLong(0);
    ProgressiveBackoff progressiveBackoff = new ProgressiveBackoff(retrier::newBackoff);
    return Futures.catchingAsync(
        retrier.executeAsync(
            () ->
                ctx.call(() -> requestRead(offset, progressiveBackoff, digest, out, hashSupplier)),
            progressiveBackoff),
        StatusRuntimeException.class,
        (e) -> Futures.immediateFailedFuture(new IOException(e)),
        MoreExecutors.directExecutor());
  }

  public static String getResourceName(String instanceName, Digest digest) {
    String resourceName = "";
    if (!instanceName.isEmpty()) {
      resourceName += instanceName + "/";
    }
    return resourceName + "blobs/" + DigestUtil.toString(digest);
  }

  private ListenableFuture<Void> requestRead(
      AtomicLong offset,
      ProgressiveBackoff progressiveBackoff,
      Digest digest,
      OutputStream out,
      @Nullable Supplier<HashCode> hashSupplier) {
    String resourceName = getResourceName(options.remoteInstanceName, digest);
    SettableFuture<Void> future = SettableFuture.create();
    bsAsyncStub()
        .read(
            ReadRequest.newBuilder()
                .setResourceName(resourceName)
                .setReadOffset(offset.get())
                .build(),
            new StreamObserver<ReadResponse>() {
              @Override
              public void onNext(ReadResponse readResponse) {
                ByteString data = readResponse.getData();
                try {
                  data.writeTo(out);
                  offset.addAndGet(data.size());
                } catch (IOException e) {
                  future.setException(e);
                  // Cancel the call.
                  throw new RuntimeException(e);
                }
                // reset the stall backoff because we've made progress or been kept alive
                progressiveBackoff.reset();
              }

              @Override
              public void onError(Throwable t) {
                Status status = Status.fromThrowable(t);
                if (status.getCode() == Status.Code.NOT_FOUND) {
                  future.setException(new CacheNotFoundException(digest));
                } else {
                  future.setException(t);
                }
              }

              @Override
              public void onCompleted() {
                try {
                  if (hashSupplier != null) {
                    Utils.verifyBlobContents(
                        digest.getHash(), DigestUtil.hashCodeToString(hashSupplier.get()));
                  }
                  out.flush();
                  future.set(null);
                } catch (IOException e) {
                  future.setException(e);
                }
              }
            });
    return future;
  }

  @Override
  public ListenableFuture<Void> uploadFile(Digest digest, Path path) {
    return uploader.uploadBlobAsync(
        HashCode.fromString(digest.getHash()),
        Chunker.builder().setInput(digest.getSizeBytes(), path).build(),
        /* forceUpload= */ true);
  }

  @Override
  public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
    return uploader.uploadBlobAsync(
        HashCode.fromString(digest.getHash()),
        Chunker.builder().setInput(data.toByteArray()).build(),
        /* forceUpload= */ true);
  }
}


text:
from matplotlib import pyplot as plt
import numpy as np
import time

import jacobi1   # created from jacobi1.f90 using f2py

n = 51

# grid points:
x = np.linspace(0., 1., n)

# boundary conditions:
alpha = 20.
beta = 60.

# initial guess:
u = np.linspace(alpha, beta, n)

# right hand side:
f = 100. * np.exp(x)

niters = 0
plt.clf()
plt.plot(x, u, 'o-')
plt.title("After %s iterations" % niters)
plt.ylim([10., 80.])

ans = raw_input("Type return to start... ")

iters_per_plot = 200   # number of iterations between plots
nplots = 20            # number of plots to produce

for nn in range(nplots):
    u = jacobi1.iterate(u, iters_per_plot, f)
    plt.plot(x, u, 'o-')
    niters = niters + iters_per_plot
    plt.title("After %s iterations" % niters)
    plt.ylim([10., 80.])
    plt.draw()
    time.sleep(.5)


text:
import _plotly_utils.basevalidators

class RangeValidator(_plotly_utils.basevalidators.InfoArrayValidator):
    def __init__(self, plotly_name="range", parent_name="layout.yaxis", **kwargs):
        super(RangeValidator, self).__init__(
            plotly_name=plotly_name,
            parent_name=parent_name,
            anim=kwargs.pop("anim", True),
            edit_type=kwargs.pop("edit_type", "axrange"),
            implied_edits=kwargs.pop("implied_edits", {"autorange": False}),
            items=kwargs.pop(
                "items",
                [
                    {
                        "anim": True,
                        "editType": "axrange",
                        "impliedEdits": {"^autorange": False},
                        "valType": "any",
                    },
                    {
                        "anim": True,
                        "editType": "axrange",
                        "impliedEdits": {"^autorange": False},
                        "valType": "any",
                    },
                ],
            ),
            **kwargs,
        )


text:
namespace TraktApiSharp.Tests.Modules.TraktNetworksModule
{
    using FluentAssertions;
    using System;
    using System.Linq;
    using System.Net;
    using System.Threading.Tasks;
    using TestUtils;
    using Traits;
    using TraktApiSharp.Exceptions;
    using TraktApiSharp.Objects.Basic;
    using TraktApiSharp.Responses;
    using Xunit;

    [Category("Modules.Networks")]
    public partial class TraktNetworksModule_Tests
    {
        private const string GET_NETWORKS_URI = "networks";

        [Fact]
        public async Task Test_TraktNetworksModule_GetNetworks()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, GET_NETWORKS_JSON);
            TraktListResponse<ITraktNetwork> response = await client.Networks.GetNetworksAsync();

            response.Should().NotBeNull();
            response.IsSuccess.Should().BeTrue();
            response.Value.Should().NotBeNull();
            response.HasValue.Should().BeTrue();
            response.Exception.Should().BeNull();
            response.Value.Should().NotBeEmpty().And.HaveCount(2);

            ITraktNetwork[] networks = response.ToArray();

            networks[0].Should().NotBeNull();
            networks[0].Network.Should().Be("ABC(US)");

            networks[1].Should().NotBeNull();
            networks[1].Network.Should().Be("The CW");
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_NotFoundException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.NotFound);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktNotFoundException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_AuthorizationException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.Unauthorized);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktAuthorizationException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_BadRequestException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.BadRequest);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktBadRequestException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ForbiddenException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.Forbidden);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktForbiddenException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_MethodNotFoundException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.MethodNotAllowed);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktMethodNotFoundException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ConflictException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.Conflict);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktConflictException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ServerException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.InternalServerError);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktServerException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_BadGatewayException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, HttpStatusCode.BadGateway);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktBadGatewayException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_PreconditionFailedException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)412);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktPreconditionFailedException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ValidationException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)422);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktValidationException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_RateLimitException()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)429);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktRateLimitException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ServerUnavailableException_503()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)503);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktServerUnavailableException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ServerUnavailableException_504()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)504);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktServerUnavailableException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ServerUnavailableException_520()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)520);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktServerUnavailableException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ServerUnavailableException_521()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)521);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktServerUnavailableException>();
        }

        [Fact]
        public void Test_TraktNetworksModule_GetNetworks_Throws_ServerUnavailableException_522()
        {
            TraktClient client = TestUtility.GetMockClient(GET_NETWORKS_URI, (HttpStatusCode)522);
            Func<Task<TraktListResponse<ITraktNetwork>>> act = () => client.Networks.GetNetworksAsync();
            act.Should().Throw<TraktServerUnavailableException>();
        }
    }
}


text:
/* A test program doing nothing really, just linking to all the BSD MP
   functions that're supposed to exist. */

/*
Copyright (C) 2000 Free Software Foundation, Inc.

This file is part of the GNU MP Library.

The GNU MP Library is free software; you can redistribute it and/or modify
it under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.

The GNU MP Library is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
License for more details.

You should have received a copy of the GNU Library General Public License
along with the GNU MP Library; see the file COPYING.LIB.  If not, write to
the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
MA 02111-1307, USA.
*/

#include "mp.h"

int
main (argc, argv)
int argc;
char *argv[];
{
  MINT *a, *b, *c, *d;
  short  h;

  a = itom (123);
  b = xtom ("DEADBEEF");
  c = itom (0);
  d = itom (0);
  move (a, b);
  madd (a, b, c);
  msub (a, b, c);
  mult (a, b, c);
  mdiv (b, a, c, d);
  sdiv (b, 2, c, &h);
  msqrt (a, c, d);
  pow (b, a, a, c);
  rpow (a, 3, c);
  gcd (a, b, c);
  mcmp (a, b);
  if (argc > 1)
    min (c);
  mout (a);
  mtox (b);

  mfree(a);

  return 0;
}


text:
describe Api::V1::FeedVersionInfosController do
  context 'GET index' do
    it 'returns all FeedVersionInfos when no parameters provided' do
      feed_version_info = create_list(:feed_version_info, 2)
      get :index, total: true
      expect_json_types({ feed_version_infos: :array })
      expect_json({
        feed_version_infos: -> (feed_version_infos) {
          expect(feed_version_infos.length).to eq 2
        },
        meta: {
          total: 2
        }
      })
    end
  end
end


text:
/*
This package provides wrappers around the OpenCL API.

To use this package, ensure that the headers and your favourite OpenCL
distribution is installed.
On Linux, the headers are usually available in a
package like "opencl-headers", while the implementation is provided by the
graphics driver distribution.
On Windows, your graphics driver distribution should have all the necessary stuff.

There are some basic tests provided as examples, see cl_test.go.

*/
package cl

//#cgo LDFLAGS: -lOpenCL
//#include "include.h"
import "C"


text:
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" version="2.5">
  <servlet>
    <servlet-name>appServlet</servlet-name>
    <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
    <init-param>
      <param-name>contextConfigLocation</param-name>
      <param-value>WEB-INF/spring/servlet-context.xml</param-value>
    </init-param>
    <load-on-startup>1</load-on-startup>
  </servlet>
  <servlet-mapping>
    <servlet-name>appServlet</servlet-name>
    <url-pattern>/</url-pattern>
  </servlet-mapping>
  <listener>
    <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
  </listener>
  <context-param>
    <param-name>contextConfigLocation</param-name>
    <param-value>WEB-INF/spring/servlet-context.xml</param-value>
  </context-param>

</web-app>


text:
package net.kuujo.vertigo.cluster.data.impl;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

import net.kuujo.vertigo.cluster.data.AsyncMap;
import net.kuujo.vertigo.cluster.data.MapEvent;
import net.kuujo.vertigo.cluster.data.WatchableAsyncMap;
import net.kuujo.vertigo.cluster.data.MapEvent.Type;
import net.kuujo.vertigo.util.CountingCompletionHandler;

import org.vertx.java.core.AsyncResult;
import org.vertx.java.core.Handler;
import org.vertx.java.core.Vertx;
import org.vertx.java.core.eventbus.EventBus;
import org.vertx.java.core.eventbus.Message;
import org.vertx.java.core.impl.DefaultFutureResult;
import org.vertx.java.core.json.JsonObject;

/**
 * Wrapped watchable asynchronous map.
 *
 * @author Jordan Halterman
 *
 * @param <K> The map key type.
 * @param <V> The map value type.
 */
public class WrappedWatchableAsyncMap<K, V> implements WatchableAsyncMap<K, V> {
  private final AsyncMap<K, V> map;
  private final EventBus eventBus;
  private final Map<MapEvent.Type, Map<Handler<MapEvent<K, V>>, Handler<Message<JsonObject>>>> watchHandlers = new HashMap<>();

  public WrappedWatchableAsyncMap(AsyncMap<K, V> map, Vertx vertx) {
    this.map = map;
    this.eventBus = vertx.eventBus();
  }

  @Override
  public String name() {
    return map.name();
  }

  @Override
  public void put(K key, V value) {
    put(key, value, null);
  }

  @Override
  public void put(final K key, final V value, final Handler<AsyncResult<V>> doneHandler) {
    map.put(key, value, new Handler<AsyncResult<V>>() {
      @Override
      public void handle(AsyncResult<V> result) {
        eventBus.publish(String.format("%s.%s.%s", map.name(), key, MapEvent.Type.CHANGE.toString()), new JsonObject()
            .putString("type", MapEvent.Type.CHANGE.toString())
            .putValue("key", key)
            .putValue("value", value));
        String event = result.result() == null ? MapEvent.Type.CREATE.toString() : MapEvent.Type.UPDATE.toString();
        eventBus.publish(String.format("%s.%s.%s", map.name(), key, event), new JsonObject()
            .putString("type", event)
            .putValue("key", key)
            .putValue("value", value));
        if (doneHandler != null) {
          doneHandler.handle(result);
        }
      }
    });
  }

  @Override
  public void get(K key, Handler<AsyncResult<V>> resultHandler) {
    map.get(key, resultHandler);
  }

  @Override
  public void remove(K key) {
    remove(key, null);
  }

  @Override
  public void remove(final K key, final Handler<AsyncResult<V>> resultHandler) {
    map.remove(key, new Handler<AsyncResult<V>>() {
      @Override
      public void handle(AsyncResult<V> result) {
        eventBus.publish(String.format("%s.%s.%s", map.name(), key, MapEvent.Type.CHANGE.toString()), new JsonObject()
            .putString("type", MapEvent.Type.CHANGE.toString())
            .putValue("key", key)
            .putValue("value", result.result()));
        eventBus.publish(String.format("%s.%s.%s", map.name(), key, MapEvent.Type.DELETE.toString()), new JsonObject()
            .putString("type", MapEvent.Type.DELETE.toString())
            .putValue("key", key)
            .putValue("value", result.result()));
        if (resultHandler != null) {
          resultHandler.handle(result);
        }
      }
    });
  }

  @Override
  public void containsKey(K key, Handler<AsyncResult<Boolean>> resultHandler) {
    map.containsKey(key, resultHandler);
  }

  @Override
  public void keySet(Handler<AsyncResult<Set<K>>> resultHandler) {
    map.keySet(resultHandler);
  }

  @Override
  public void values(Handler<AsyncResult<Collection<V>>> resultHandler) {
    map.values(resultHandler);
  }

  @Override
  public void size(Handler<AsyncResult<Integer>> resultHandler) {
    map.size(resultHandler);
  }

  @Override
  public void isEmpty(Handler<AsyncResult<Boolean>> resultHandler) {
    map.isEmpty(resultHandler);
  }

  @Override
  public void clear() {
    map.clear();
  }

  @Override
  public void clear(Handler<AsyncResult<Void>> doneHandler) {
    map.clear(doneHandler);
  }

  @Override
  public void watch(K key, Handler<MapEvent<K, V>> handler) {
    watch(key, null, handler, null);
  }

  @Override
  public void watch(K key, Handler<MapEvent<K, V>> handler, Handler<AsyncResult<Void>> doneHandler) {
    watch(key, null, handler, doneHandler);
  }

  @Override
  public void watch(K key, Type event, Handler<MapEvent<K, V>> handler) {
    watch(key, event, handler, null);
  }

  @Override
  public void watch(final K key, final Type event, final Handler<MapEvent<K, V>> handler, Handler<AsyncResult<Void>> doneHandler) {
    if (event == null) {
      final CountingCompletionHandler<Void> counter = new CountingCompletionHandler<Void>(4).setHandler(doneHandler);
      addWatcher(key, MapEvent.Type.CHANGE, handler, counter);
      addWatcher(key, MapEvent.Type.CREATE, handler, counter);
      addWatcher(key, MapEvent.Type.UPDATE, handler, counter);
      addWatcher(key, MapEvent.Type.DELETE, handler, counter);
    } else {
      addWatcher(key, event, handler, doneHandler);
    }
  }

  private void addWatcher(K key, final MapEvent.Type event, final Handler<MapEvent<K, V>> handler, Handler<AsyncResult<Void>> doneHandler) {
    Map<Handler<MapEvent<K, V>>, Handler<Message<JsonObject>>> watchHandlers = this.watchHandlers.get(event);
    if (watchHandlers == null) {
      watchHandlers = new HashMap<>();
      this.watchHandlers.put(event, watchHandlers);
    }

    Handler<Message<JsonObject>> wrappedHandler;
    if (!watchHandlers.containsKey(handler)) {
      wrappedHandler = new Handler<Message<JsonObject>>() {
        @Override
        public void handle(Message<JsonObject> message) {
          K key = message.body().getValue("key");
          V value = message.body().getValue("value");
          handler.handle(new MapEvent<K, V>(event, key, value));
        }
      };
      watchHandlers.put(handler, wrappedHandler);
    } else {
      wrappedHandler = watchHandlers.get(handler);
    }
    eventBus.registerHandler(String.format("%s.%s.%s", map.name(), key, event.toString()), wrappedHandler, doneHandler);
  }

  @Override
  public void unwatch(K key, Handler<MapEvent<K, V>> handler) {
    unwatch(key, null, handler, null);
  }

  @Override
  public void unwatch(K key, Handler<MapEvent<K, V>> handler, Handler<AsyncResult<Void>> doneHandler) {
    unwatch(key, null, handler, doneHandler);
  }

  @Override
  public void unwatch(K key, Type event, Handler<MapEvent<K, V>> handler) {
    unwatch(key, event, handler, null);
  }

  @Override
  public void unwatch(K key, Type event, Handler<MapEvent<K, V>> handler, Handler<AsyncResult<Void>> doneHandler) {
    if (event == null) {
      final CountingCompletionHandler<Void> counter = new CountingCompletionHandler<Void>(4).setHandler(doneHandler);
      removeWatcher(key, MapEvent.Type.CHANGE, handler, counter);
      removeWatcher(key, MapEvent.Type.CREATE, handler, counter);
      removeWatcher(key, MapEvent.Type.UPDATE, handler, counter);
      removeWatcher(key, MapEvent.Type.DELETE, handler, counter);
    } else {
      removeWatcher(key, event, handler, doneHandler);
    }
  }

  private void removeWatcher(K key, MapEvent.Type event, Handler<MapEvent<K, V>> handler, Handler<AsyncResult<Void>> doneHandler) {
    Map<Handler<MapEvent<K, V>>, Handler<Message<JsonObject>>> watchHandlers = this.watchHandlers.get(event);
    if (watchHandlers == null || !watchHandlers.containsKey(handler)) {
      new DefaultFutureResult<Void>((Void) null).setHandler(doneHandler);
    } else {
      Handler<Message<JsonObject>> wrappedHandler = watchHandlers.remove(handler);
      eventBus.unregisterHandler(String.format("%s.%s.%s", map.name(), key, event.toString()), wrappedHandler, doneHandler);
    }
  }

}


text:
Audits

Filter by date

                from
                <input type="date" class="form-control" name="start" [(ngModel)]="fromDate" (ngModelChange)="onChangeDate($event)" required/>
                to
                <input type="date" class="form-control" name="end" [(ngModel)]="toDate" (ngModelChange)="onChangeDate($event)" required/>

                Date
                User
                State
                Extra data

                {{audit.timestamp| date:'medium'}}
                {{audit.principal}}
                {{audit.type}}

                    {{audit.data.message}}
                    Remote Address {{audit.data.remoteAddress}}

            <jhi-item-count [page]="page" [total]="totalItems" [itemsPerPage]="itemsPerPage"></jhi-item-count>

            <ngb-pagination [collectionSize]="totalItems" [(page)]="page" (pageChange)="loadPage(page)"></ngb-pagination>


text:
layout: post
title:  "搭建http/https代理"
date:   2016-05-23 21:58:55 +0800
categories: jekyll update
---
敬请期待...
{% highlight ruby %}
def print_hi(name)
  puts "Hi, #{name}"
end
print_hi('Tom')
#=> prints 'Hi, Tom' to STDOUT.
{% endhighlight %}

Check out the [Jekyll docs][jekyll-docs] for more info on how to get the most out of Jekyll. File all bugs/feature requests at [Jekyll's GitHub repo][jekyll-gh]. If you have questions, you can ask them on [Jekyll Talk][jekyll-talk].

[jekyll-docs]: http://jekyllrb.com/docs/home
[jekyll-gh]:   https://github.com/jekyll/jekyll
[jekyll-talk]: https://talk.jekyllrb.com/