text:
__author__ = 'saeedamen' # Saeed Amen

#
# Copyright 2016 Cuemacro
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and limitations under the License.
#

"""
LoaderBBG

Abstract class for download of Bloomberg daily, intraday data and reference data.

Implemented by
- LoaderBBGOpen (adapted version of new Bloomberg Open API for Python - recommended - although requires compilation)
- no longer support COM API

"""

from findatapy.market.datavendor import DataVendor

import abc
import datetime
import copy

class DataVendorBBG(DataVendor):

    def __init__(self):
        super(DataVendorBBG, self).__init__()
        self.logger = LoggerManager().getLogger(__name__)

    # implement method in abstract superclass
    def load_ticker(self, market_data_request):
        """
        load_ticker - Retrieves market data from external data source (in this case Bloomberg)

        Parameters
        ----------
        market_data_request : MarketDataRequest
            contains all the various parameters detailing time series start and finish, tickers etc

        Returns
        -------
        DataFrame
        """
        market_data_request_vendor = self.construct_vendor_market_data_request(market_data_request)

        data_frame = None
        self.logger.info("Request Bloomberg data")

        # do we need daily or intraday data?
        if (market_data_request.freq in ['daily', 'weekly', 'monthly', 'quarterly', 'yearly']):

            # for events times/dates separately needs ReferenceDataRequest (when specified)
            if 'release-date-time-full' in market_data_request.fields:

                # experimental!!
                # careful: make sure you copy the market data request object (when threading, altering that can
                # cause concurrency issues!)
                datetime_data_frame = self.get_reference_data(market_data_request_vendor, market_data_request)

                old_fields = copy.deepcopy(market_data_request.fields)
                old_vendor_fields = copy.deepcopy(market_data_request_vendor.fields)

                # remove fields 'release-date-time-full' from our request (and the associated field in the vendor)
                # if they are there
                try:
                    index = market_data_request.fields.index('release-date-time-full')

                    market_data_request.fields.pop(index)
                    market_data_request_vendor.fields.pop(index)
                except:
                    pass

                # download all the other event fields (uses HistoricalDataRequest to Bloomberg)
                # concatenate with date time fields
                if len(market_data_request_vendor.fields) > 0:
                    events_data_frame = self.get_daily_data(market_data_request, market_data_request_vendor)

                    col = events_data_frame.index.name
                    events_data_frame = events_data_frame.reset_index(drop = False)

                    data_frame = pandas.concat([events_data_frame, datetime_data_frame], axis = 1)
                    temp = data_frame[col]
                    del data_frame[col]
                    data_frame.index = temp
                else:
                    data_frame = datetime_data_frame

                market_data_request.fields = old_fields
                market_data_request_vendor.fields = old_vendor_fields

            # for all other daily/monthly/quarter data, we can use HistoricalDataRequest to Bloomberg
            else:
                data_frame = self.get_daily_data(market_data_request, market_data_request_vendor)

                try:
                    # convert fields with release-dt to dates (special case!)
                    for c in data_frame.columns:
                        if 'release-dt' in c:
                            data_frame[c] = (data_frame[c]).astype('int').astype(str).apply(
                                lambda x: pandas.to_datetime(x, format='%Y%m%d'))
                except:
                    pass

        # assume one ticker only
        # for intraday data we use IntradayDataRequest to Bloomberg
        if (market_data_request.freq in ['tick', 'intraday', 'second', 'minute', 'hourly']):
            market_data_request_vendor.tickers = market_data_request_vendor.tickers[0]

            if market_data_request.freq in ['tick', 'second']:
                data_frame = self.download_tick(market_data_request_vendor)
            else:
                data_frame = self.download_intraday(market_data_request_vendor)

            if data_frame is not None:
                if data_frame.empty:
                    try:
                        self.logger.info("No tickers returned for: " + market_data_request_vendor.tickers)
                    except:
                        pass

                    return None

                cols = data_frame.columns.values

                import pytz

                try:
                    data_frame = data_frame.tz_localize(pytz.utc)
                except:
                    data_frame = data_frame.tz_convert(pytz.utc)

                cols = market_data_request.tickers[0] + "." + cols
                data_frame.columns = cols

        self.logger.info("Completed request from Bloomberg.")

        return data_frame

    def get_daily_data(self, market_data_request, market_data_request_vendor):
        data_frame = self.download_daily(market_data_request_vendor)

        # convert from vendor to findatapy tickers/fields
        if data_frame is not None:
            if data_frame.empty:
                self.logger.info("No tickers returned for...")

                try:
                    self.logger.info(str(market_data_request_vendor.tickers))
                except: pass

                return None

            returned_fields = data_frame.columns.get_level_values(0)
            returned_tickers = data_frame.columns.get_level_values(1)

            # TODO if empty try downloading again a year later
            try:
                fields = self.translate_from_vendor_field(returned_fields, market_data_request)
            except:
                print('t')

            tickers = self.translate_from_vendor_ticker(returned_tickers, market_data_request)

            ticker_combined = []

            for i in range(0, len(fields)):
                ticker_combined.append(tickers[i] + "." + fields[i])

            data_frame.columns = ticker_combined
            data_frame.index.name = 'Date'

        return data_frame

    def get_reference_data(self, market_data_request_vendor, market_data_request):
        end = datetime.utcnow()

        from datetime import timedelta
        end = end + timedelta(days=365)# end.replace(year = end.year + 1)

        market_data_request_vendor.finish_date = end

        self.logger.debug("Requesting ref for " + market_data_request_vendor.tickers[0] + " etc.")

        data_frame = self.download_ref(market_data_request_vendor)

        self.logger.debug("Waiting for ref...")

        # convert from vendor to findatapy tickers/fields
        if data_frame is not None:
            returned_fields = data_frame.columns.get_level_values(0)
            returned_tickers = data_frame.columns.get_level_values(1)

        if data_frame is not None:
            # TODO if empty try downloading again a year later
            fields = self.translate_from_vendor_field(returned_fields, market_data_request)
            tickers = self.translate_from_vendor_ticker(returned_tickers, market_data_request)

            ticker_combined = []

            for i in range(0, len(fields)):
                ticker_combined.append(tickers[i] + "." + fields[i])

            data_frame.columns = ticker_combined

            # TODO coerce will be deprecated from pandas
            data_frame = data_frame.convert_objects(convert_dates = 'coerce', convert_numeric= 'coerce')

        return data_frame

    # implement method in abstract superclass
    @abc.abstractmethod
    def kill_session(self):
        return

    @abc.abstractmethod
    def download_tick(self, market_data_request):
        return

    @abc.abstractmethod
    def download_intraday(self, market_data_request):
        return

    @abc.abstractmethod
    def download_daily(self, market_data_request):
        return

    @abc.abstractmethod
    def download_ref(self, market_data_request):
        return

#######################################################################################################################

"""
LoaderBBGOpen

Calls the Bloomberg Open API to download market data: daily, intraday and reference data (needs blpapi).

"""

import abc
import copy
import collections
import datetime
import re

import pandas

try:
    import blpapi   # obtainable from Bloomberg website
except: pass

from findatapy.util.dataconstants import DataConstants
from findatapy.market.datavendorbbg import DataVendorBBG

from collections import defaultdict

class DataVendorBBGOpen(DataVendorBBG):
    def __init__(self):
        super(DataVendorBBGOpen, self).__init__()
        self.logger = LoggerManager().getLogger(__name__)

    def download_tick(self, market_data_request):
        # Bloomberg OpenAPI implementation
        low_level_loader = BBGLowLevelTick()

        # by default we download all available fields!
        data_frame = low_level_loader.load_time_series(market_data_request)

        # self.kill_session() # need to forcibly kill_session since can't always reopen

        return data_frame

    def download_intraday(self, market_data_request):
        # Bloomberg OpenAPI implementation
        low_level_loader = BBGLowLevelIntraday()

        # by default we download all available fields!
        data_frame = low_level_loader.load_time_series(market_data_request)

        # self.kill_session() # need to forcibly kill_session since can't always reopen

        return data_frame

    def download_daily(self, market_data_request):
        # Bloomberg Open API implementation
        low_level_loader = BBGLowLevelDaily()

        # by default we download all available fields!
        data_frame = low_level_loader.load_time_series(market_data_request)

        # self.kill_session() # need to forcibly kill_session since can't always reopen

        return data_frame

    def download_ref(self, market_data_request):

         # Bloomberg Open API implementation
        low_level_loader = BBGLowLevelRef()

        market_data_request_vendor_selective = copy.copy(market_data_request)
        market_data_request_vendor_selective.fields = ['ECO_FUTURE_RELEASE_DATE_LIST']

        data_frame =  low_level_loader.load_time_series(market_data_request_vendor_selective)

        # self.kill_session() # need to forcibly kill_session since can't always reopen

        return data_frame

    def kill_session(self):
        # TODO not really needed, because we automatically kill sessions
        BBGLowLevelDaily().kill_session(None)
        BBGLowLevelRef().kill_session(None)
        BBGLowLevelIntraday().kill_session(None)

########################################################################################################################
#### Lower level code to interact with Bloomberg Open API

class BBGLowLevelTemplate(object): # in order that the init function works in child classes

    _session = None

    def __init__(self):
        self._data_frame = None

        self.RESPONSE_ERROR = blpapi.Name("responseError")
        self.SESSION_TERMINATED = blpapi.Name("SessionTerminated")
        self.CATEGORY = blpapi.Name("category")
        self.MESSAGE = blpapi.Name("message")

        return

    def load_time_series(self, market_data_request):

        options = self.fill_options(market_data_request)

        #if(BBGLowLevelTemplate._session is None):
        session = self.start_bloomberg_session()
        #else:
        #    session = BBGLowLevelTemplate._session

        try:
            # if can't open the session, kill existing one
            # then try reopen (up to 5 times...)
            i = 0

            while i < 5:
                if session is not None:
                    if not session.openService("//blp/refdata"):
                        self.logger.info("Try reopening Bloomberg session... try " + str(i))
                        self.kill_session(session) # need to forcibly kill_session since can't always reopen
                        session = self.start_bloomberg_session()

                        if session is not None:
                            if session.openService("//blp/refdata"): i = 6
                else:
                    self.logger.info("Try opening Bloomberg session... try " + str(i))
                    session = self.start_bloomberg_session()

                i = i + 1

            # give error if still doesn't work after several tries..
            if not session.openService("//blp/refdata"):
                self.logger.error("Failed to open //blp/refdata")

                return

            self.logger.info("Creating request...")
            eventQueue = None # blpapi.EventQueue()

            # create a request
            self.send_bar_request(session, eventQueue)
            self.logger.info("Waiting for data to be returned...")

            # wait for events from session and collect the data
            self.event_loop(session, eventQueue)

        finally:
            # stop the session (will fail if NoneType)
            try:
                session.stop()
            except: pass

        return self._data_frame

    def event_loop(self, session, eventQueue):
        not_done = True

        data_frame = pandas.DataFrame()
        data_frame_slice = None

        while not_done:
            # nextEvent() method can be called with timeout to let
            # the program catch Ctrl-C between arrivals of new events
            event = session.nextEvent() # removed time out

            # Bloomberg will send us responses in chunks
            if event.eventType() == blpapi.Event.PARTIAL_RESPONSE:
                # self.logger.info("Processing Bloomberg Partial Response")
                data_frame_slice = self.process_response_event(event)
            elif event.eventType() == blpapi.Event.RESPONSE:
                # self.logger.info("Processing Bloomberg Full Response")
                data_frame_slice = self.process_response_event(event)
                not_done = False
            else:
                for msg in event:
                    if event.eventType() == blpapi.Event.SESSION_STATUS:
                        if msg.messageType() == self.SESSION_TERMINATED:
                            not_done = False

            # append DataFrame only if not empty
            if data_frame_slice is not None:
                if (data_frame.empty):
                    data_frame = data_frame_slice
                else:
                    # make sure we do not reattach a message we've already read
                    # sometimes Bloomberg can give us back the same message several times
                    # CAREFUL with this!
                    # compares the ticker names, and make sure they don't already exist in the time series
                    data_frame = self.combine_slices(data_frame, data_frame_slice)

        # make sure we do not have any duplicates in the time series
        if data_frame is not None:
            if data_frame.empty == False:
                data_frame.drop_duplicates(keep='last')

        self._data_frame = data_frame

    # process raw message returned by Bloomberg
    def process_response_event(self, event):
        data_frame = pandas.DataFrame()

        for msg in event:
            # generates a lot of output - so don't use unless for debugging purposes
            # self.logger.info(msg)

            if msg.hasElement(self.RESPONSE_ERROR):
                self.logger.error("REQUEST FAILED: " + str(msg.getElement(self.RESPONSE_ERROR)))
                continue

            data_frame_slice = self.process_message(msg)

            if (data_frame_slice is not None):
                if (data_frame.empty):
                    data_frame = data_frame_slice
                else:
                    data_frame = data_frame.append(data_frame_slice)
            else:
                data_frame = data_frame_slice

        return data_frame

    def get_previous_trading_date(self):
        tradedOn = datetime.date.today()

        while True:
            try:
                tradedOn -= datetime.timedelta(days=1)
            except OverflowError:
                return None

            if tradedOn.weekday() not in [5, 6]:
                return tradedOn

    # create a session for Bloomberg with appropriate server & port
    def start_bloomberg_session(self):
        tries = 0

        session = None

        # try up to 5 times to start a session
        while(tries < 5):
            try:
                # fill SessionOptions
                sessionOptions = blpapi.SessionOptions()
                sessionOptions.setServerHost(DataConstants().bbg_server)
                sessionOptions.setServerPort(DataConstants().bbg_server_port)

                self.logger.info("Starting Bloomberg session...")

                # create a Session
                session = blpapi.Session(sessionOptions)

                # start a Session
                if not session.start():
                    self.logger.error("Failed to start session.")
                    return

                self.logger.info("Returning session...")

                tries = 5
            except:
                tries = tries + 1

        # BBGLowLevelTemplate._session = session

        if session is None:
            self.logger.error("Failed to start session.")
            return

        return session

    def add_override(self, request, field, value):
        overrides = request.getElement("overrides")
        override1 = overrides.appendElement()
        override1.setElement("fieldId", field)
        override1.setElement("value", value)

    @abc.abstractmethod
    def process_message(self, msg):
        # to be implemented by subclass
        return

    # create request for data
    @abc.abstractmethod
    def send_bar_request(self, session, eventQueue):
        # to be implemented by subclass
        return

    # create request for data
    @abc.abstractmethod
    def combine_slices(self, data_frame, data_frame_slice):
        # to be implemented by subclass
        return

    def kill_session(self, session):
        if (session is not None):
            try:
                session.stop()

                self.logger.info("Stopping session...")
            finally:
                self.logger.info("Finally stopping session...")

            session = None

class BBGLowLevelDaily(BBGLowLevelTemplate):

    def __init__(self):
        super(BBGLowLevelDaily, self).__init__()

        self.logger = LoggerManager().getLogger(__name__)
        self._options = []

    def combine_slices(self, data_frame, data_frame_slice):
        if (data_frame_slice.columns.get_level_values(1).values[0]
            not in data_frame.columns.get_level_values(1).values):

            return data_frame.join(data_frame_slice, how="outer")

        return data_frame

    # populate options for Bloomberg request for asset daily request
    def fill_options(self, market_data_request):
        self._options = OptionsBBG()

        self._options.security = market_data_request.tickers
        self._options.startDateTime = market_data_request.start_date
        self._options.endDateTime = market_data_request.finish_date
        self._options.fields = market_data_request.fields

        return self._options

    def process_message(self, msg):
        # Process received events
        ticker = msg.getElement('securityData').getElement('security').getValue()
        fieldData = msg.getElement('securityData').getElement('fieldData')

        # SLOW loop (careful, not all the fields will be returned every time
        # hence need to include the field name in the tuple)
        data = defaultdict(dict)

        for i in range(fieldData.numValues()):
            for j in range(1, fieldData.getValue(i).numElements()):
                data[(str(fieldData.getValue(i).getElement(j).name()), ticker)][fieldData.getValue(i).getElement(0).getValue()] \
                    = fieldData.getValue(i).getElement(j).getValue()

        data_frame = pandas.DataFrame(data)

        # if obsolete ticker could return no values
        if (not(data_frame.empty)):
            # data_frame.columns = pandas.MultiIndex.from_tuples(data, names=['field', 'ticker'])
            data_frame.index = pandas.to_datetime(data_frame.index)
            self.logger.info("Read: " + ticker + ' ' + str(data_frame.index[0]) + ' - ' + str(data_frame.index[-1]))
        else:
            return None

        return data_frame

    # create request for data
    def send_bar_request(self, session, eventQueue):
        refDataService = session.getService("//blp/refdata")
        request = refDataService.createRequest("HistoricalDataRequest")

        request.set("startDate", self._options.startDateTime.strftime('%Y%m%d'))
        request.set("endDate", self._options.endDateTime.strftime('%Y%m%d'))

        # # only one security/eventType per request
        for field in self._options.fields:
            request.getElement("fields").appendValue(field)

        for security in self._options.security:
            request.getElement("securities").appendValue(security)

        self.logger.info("Sending Bloomberg Daily Request:" + str(request))
        session.sendRequest(request)

class BBGLowLevelRef(BBGLowLevelTemplate):

    def __init__(self):
        super(BBGLowLevelRef, self).__init__()

        self.logger = LoggerManager().getLogger(__name__)
        self._options = []

    # populate options for Bloomberg request for asset intraday request
    def fill_options(self, market_data_request):
        self._options = OptionsBBG()

        self._options.security = market_data_request.tickers
        self._options.startDateTime = market_data_request.start_date
        self._options.endDateTime = market_data_request.finish_date
        self._options.fields = market_data_request.fields

        return self._options

    def process_message(self, msg):
        data = collections.defaultdict(dict)

        # process received events
        securityDataArray = msg.getElement('securityData')

        index = 0

        for securityData in list(securityDataArray.values()):
            ticker = securityData.getElementAsString("security")
            fieldData = securityData.getElement("fieldData")

            for field in fieldData.elements():
                if not field.isValid():
                    field_name = "%s" % field.name()

                    self.logger.error(field_name + " is NULL")
                elif field.isArray():
                    # iterate over complex data returns.
                    field_name = "%s" % field.name()

                    for i, row in enumerate(field.values()):
                        data[(field_name, ticker)][index] = re.findall(r'"(.*?)"', "%s" % row)[0]

                        index = index + 1
                # else:
                    # vals.append(re.findall(r'"(.*?)"', "%s" % row)[0])
                    # print("%s = %s" % (field.name(), field.getValueAsString()))

            fieldExceptionArray = securityData.getElement("fieldExceptions")

            for fieldException in list(fieldExceptionArray.values()):
                errorInfo = fieldException.getElement("errorInfo")
                print(errorInfo.getElementAsString("category"), ":", \
                    fieldException.getElementAsString("fieldId"))

        data_frame = pandas.DataFrame(data)

        # if obsolete ticker could return no values
        if (not(data_frame.empty)):
            data_frame.columns = pandas.MultiIndex.from_tuples(data, names=['field', 'ticker'])
            self.logger.info("Reading: " + ticker + ' ' + str(data_frame.index[0]) + ' - ' + str(data_frame.index[-1]))
        else:
            return None

        return data_frame

    def combine_slices(self, data_frame, data_frame_slice):
        if (data_frame_slice.columns.get_level_values(1).values[0]
            not in data_frame.columns.get_level_values(1).values):

            return data_frame.join(data_frame_slice, how="outer")

        return data_frame

    # create request for data
    def send_bar_request(self, session, eventQueue):
        refDataService = session.getService("//blp/refdata")
        request = refDataService.createRequest('ReferenceDataRequest')

        self.add_override(request, 'TIME_ZONE_OVERRIDE', 23)    # force GMT time
        self.add_override(request, 'START_DT', self._options.startDateTime.strftime('%Y%m%d'))
        self.add_override(request, 'END_DT', self._options.endDateTime.strftime('%Y%m%d'))

        # only one security/eventType per request
        for field in self._options.fields:
            request.getElement("fields").appendValue(field)

        for security in self._options.security:
            request.getElement("securities").appendValue(security)

        self.logger.info("Sending Bloomberg Ref Request:" + str(request))
        session.sendRequest(request)

from operator import itemgetter

class BBGLowLevelIntraday(BBGLowLevelTemplate):

    def __init__(self):
        super(BBGLowLevelIntraday, self).__init__()

        self.logger = LoggerManager().getLogger(__name__)

        # constants
        self.BAR_DATA = blpapi.Name("barData")
        self.BAR_TICK_DATA = blpapi.Name("barTickData")
        self.OPEN = blpapi.Name("open")
        self.HIGH = blpapi.Name("high")
        self.LOW = blpapi.Name("low")
        self.CLOSE = blpapi.Name("close")
        self.VOLUME = blpapi.Name("volume")
        self.NUM_EVENTS = blpapi.Name("numEvents")
        self.TIME = blpapi.Name("time")

    def combine_slices(self, data_frame, data_frame_slice):
        return data_frame.append(data_frame_slice)

    # populate options for Bloomberg request for asset intraday request
    def fill_options(self, market_data_request):
        self._options = OptionsBBG()

        self._options.security = market_data_request.tickers[0]    # get 1st ticker only!
        self._options.event = market_data_request.trade_side.upper()
        self._options.barInterval = market_data_request.freq_mult
        self._options.startDateTime = market_data_request.start_date
        self._options.endDateTime = market_data_request.finish_date
        self._options.gapFillInitialBar = False

        if hasattr(self._options.startDateTime, 'microsecond'):
            self._options.startDateTime = self._options.startDateTime.replace(microsecond=0)

        if hasattr(self._options.endDateTime, 'microsecond'):
            self._options.endDateTime = self._options.endDateTime.replace(microsecond=0)

        return self._options

    # iterate through Bloomberg output creating a DataFrame output
    # implements abstract method
    def process_message(self, msg):
        data = msg.getElement(self.BAR_DATA).getElement(self.BAR_TICK_DATA)

        self.logger.info("Processing intraday data for " + str(self._options.security))

        data_vals = list(data.values())

        # data_matrix = numpy.zeros([len(data_vals), 6])
        # data_matrix.fill(numpy.nan)
        #
        # date_index = [None] * len(data_vals)
        #
        # for i in range(0, len(data_vals)):
        #     data_matrix[i][0] = data_vals[i].getElementAsFloat(self.OPEN)
        #     data_matrix[i][1] = data_vals[i].getElementAsFloat(self.HIGH)
        #     data_matrix[i][2] = data_vals[i].getElementAsFloat(self.LOW)
        #     data_matrix[i][3] = data_vals[i].getElementAsFloat(self.CLOSE)
        #     data_matrix[i][4] = data_vals[i].getElementAsInteger(self.VOLUME)
        #     data_matrix[i][5] = data_vals[i].getElementAsInteger(self.NUM_EVENTS)
        #
        #     date_index[i] = data_vals[i].getElementAsDatetime(self.TIME)
        #
        # self.logger.info("Dates between " + str(date_index[0]) + " - " + str(date_index[-1]))
        #
        # # create pandas dataframe with the Bloomberg output
        # return pandas.DataFrame(data = data_matrix, index = date_index,
        #                columns=['open', 'high', 'low', 'close', 'volume', 'events'])

        ## for loop method is touch slower
        # time_list = []
        # data_table = []

        # for bar in data_vals:
        #     data_table.append([bar.getElementAsFloat(self.OPEN),
        #                  bar.getElementAsFloat(self.HIGH),
        #                  bar.getElementAsFloat(self.LOW),
        #                  bar.getElementAsFloat(self.CLOSE),
        #                  bar.getElementAsInteger(self.VOLUME),
        #                  bar.getElementAsInteger(self.NUM_EVENTS)])
        #
        #     time_list.append(bar.getElementAsDatetime(self.TIME))

        # each price time point has multiple fields - marginally quicker
        tuple = [([bar.getElementAsFloat(self.OPEN),
                        bar.getElementAsFloat(self.HIGH),
                        bar.getElementAsFloat(self.LOW),
                        bar.getElementAsFloat(self.CLOSE),
                        bar.getElementAsInteger(self.VOLUME),
                        bar.getElementAsInteger(self.NUM_EVENTS)],
                        bar.getElementAsDatetime(self.TIME)) for bar in data_vals]

        data_table = list(map(itemgetter(0), tuple))
        time_list = list(map(itemgetter(1), tuple))

        try:
            self.logger.info("Dates between " + str(time_list[0]) + " - " + str(time_list[-1]))
        except:
            self.logger.info("No dates retrieved")
            return None

        # create pandas dataframe with the Bloomberg output
        return pandas.DataFrame(data = data_table, index = time_list,
                      columns=['open', 'high', 'low', 'close', 'volume', 'events'])

    # implement abstract method: create request for data
    def send_bar_request(self, session, eventQueue):
        refDataService = session.getService("//blp/refdata")
        request = refDataService.createRequest("IntradayBarRequest")

        # only one security/eventType per request
        request.set("security", self._options.security)
        request.set("eventType", self._options.event)
        request.set("interval", self._options.barInterval)

        # self.add_override(request, 'TIME_ZONE_OVERRIDE', 'GMT')

        if self._options.startDateTime and self._options.endDateTime:
            request.set("startDateTime", self._options.startDateTime)
            request.set("endDateTime", self._options.endDateTime)

        if self._options.gapFillInitialBar:
            request.append("gapFillInitialBar", True)

        self.logger.info("Sending Intraday Bloomberg Request...")

        session.sendRequest(request)

class BBGLowLevelTick(BBGLowLevelTemplate):

    def __init__(self):
        super(BBGLowLevelTick, self).__init__()

        self.logger = LoggerManager().getLogger(__name__)

        # constants
        self.TICK_DATA = blpapi.Name("tickData")
        self.COND_CODE = blpapi.Name("conditionCodes")
        self.TICK_SIZE = blpapi.Name("size")
        self.TIME = blpapi.Name("time")
        self.TYPE = blpapi.Name("type")
        self.VALUE = blpapi.Name("value")
        self.RESPONSE_ERROR = blpapi.Name("responseError")
        self.CATEGORY = blpapi.Name("category")
        self.MESSAGE = blpapi.Name("message")
        self.SESSION_TERMINATED = blpapi.Name("SessionTerminated")

    def combine_slices(self, data_frame, data_frame_slice):
        return data_frame.append(data_frame_slice)

    # populate options for Bloomberg request for asset intraday request
    def fill_options(self, market_data_request):
        self._options = OptionsBBG()

        self._options.security = market_data_request.tickers[0]    # get 1st ticker only!
        self._options.event = market_data_request.trade_side.upper()
        # self._options.barInterval = market_data_request.freq_mult
        self._options.startDateTime = market_data_request.start_date
        self._options.endDateTime = market_data_request.finish_date
        # self._options.gapFillInitialBar = False

        if hasattr(self._options.startDateTime, 'microsecond'):
            self._options.startDateTime = self._options.startDateTime.replace(microsecond=0)

        if hasattr(self._options.endDateTime, 'microsecond'):
            self._options.endDateTime = self._options.endDateTime.replace(microsecond=0)

        return self._options

    # iterate through Bloomberg output creating a DataFrame output
    # implements abstract method
    def process_message(self, msg):
        data = msg.getElement(self.TICK_DATA).getElement(self.TICK_DATA)

        self.logger.info("Processing tick data for " + str(self._options.security))
        tuple = []

        data_vals = data.values()

        # for item in list(data_vals):
        #     if item.hasElement(self.COND_CODE):
        #         cc = item.getElementAsString(self.COND_CODE)
        #     else:
        #         cc = ""
        #
        #     # each price time point has multiple fields - marginally quicker
        #     tuple.append(([item.getElementAsFloat(self.VALUE),
        #                     item.getElementAsInteger(self.TICK_SIZE)],
        #                     item.getElementAsDatetime(self.TIME)))

        # slightly faster this way (note, we are skipping trade & CC fields)
        tuple = [([item.getElementAsFloat(self.VALUE),
                             item.getElementAsInteger(self.TICK_SIZE)],
                             item.getElementAsDatetime(self.TIME)) for item in data_vals]

        data_table = list(map(itemgetter(0), tuple))
        time_list = list(map(itemgetter(1), tuple))

        try:
            self.logger.info("Dates between " + str(time_list[0]) + " - " + str(time_list[-1]))
        except:
            self.logger.info("No dates retrieved")
            return None

        # create pandas dataframe with the Bloomberg output
        return pandas.DataFrame(data = data_table, index = time_list,
                      columns=['close', 'ticksize'])

    # implement abstract method: create request for data
    def send_bar_request(self, session, eventQueue):
        refDataService = session.getService("//blp/refdata")
        request = refDataService.createRequest("IntradayTickRequest")

        # only one security/eventType per request
        request.set("security", self._options.security)
        request.getElement("eventTypes").appendValue("TRADE")
        # request.set("eventTypes", self._options.event)
        request.set("includeConditionCodes", True)

        # self.add_override(request, 'TIME_ZONE_OVERRIDE', 'GMT')

        if self._options.startDateTime and self._options.endDateTime:
            request.set("startDateTime", self._options.startDateTime)
            request.set("endDateTime", self._options.endDateTime)

        self.logger.info("Sending Tick Bloomberg Request...")

        session.sendRequest(request)

#######################################################################################################################

from findatapy.util.loggermanager import LoggerManager
from datetime import datetime

class OptionsBBG:

    # TODO create properties that cannot be manipulated
    def __init__(self):
        self.logger = LoggerManager().getLogger(__name__)


text:
from django.conf.urls import patterns, include, url
from django.conf.urls.static import static
from django.conf import settings
from django.views.generic import TemplateView

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    url(r'^$', include('quotes.urls')),

    # Examples:
    # url(r'^$', 'dialogos.views.home', name='home'),
    # url(r'^dialogos/', include('dialogos.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),
)

# Uncomment the next line to serve media files in dev.
# urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

if settings.DEBUG:
    import debug_toolbar
    urlpatterns += patterns('',
                            url(r'^__debug__/', include(debug_toolbar.urls)),
                            )

redpajama_set_name: RedPajamaGithub


text:
package org.casaca.gpx4j.core.data;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;

public class Waypoint extends CoordinatesObject {
	//POSITION INFO
	private Degrees magvar;
	private BigDecimal geoIdHeight;

	//DESCRIPTION INFO
	private String name;
	private String cmt;
	private String description;
	private String src;
	private List<Link> links;
	private String sym;
	private String type;

	//ACCURACY INFO
	private Fix fix;
	private BigInteger sat;
	private BigDecimal hdop;
	private BigDecimal vdop;
	private BigDecimal pdop;
	private BigDecimal ageOfDgpsData;
	private DgpsStation dGpsId;

	public Waypoint(BigDecimal latitude, BigDecimal longitude) throws IllegalArgumentException{
		super(latitude, longitude);
		init();
	}

	public Waypoint(BigDecimal latitude, BigDecimal longitude, BigDecimal elevation, Calendar time) {
		super(latitude, longitude, elevation, time);
		init();
	}

	public Waypoint(BigDecimal latitude, BigDecimal longitude, BigDecimal elevation) {
		super(latitude, longitude, elevation);
		init();
	}

	private void init(){
		this.magvar = null;
		this.geoIdHeight = null;

		this.name = null;
		this.cmt = null;
		this.description = null;
		this.src = null;
		this.links = new ArrayList<Link>();
		this.sym = null;
		this.type = null;

		this.fix = null;
		this.sat = null;
		this.hdop = null;
		this.vdop = null;
		this.pdop = null;
		this.ageOfDgpsData = null;
		this.dGpsId = null;
	}

	public Degrees getMagvar() {
		return magvar;
	}

	public void setMagvar(Degrees magvar) {
		this.magvar = magvar;
	}

	public BigDecimal getGeoIdHeight() {
		return geoIdHeight;
	}

	public void setGeoIdHeight(BigDecimal geoIdHeight) {
		this.geoIdHeight = geoIdHeight;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public String getCmt() {
		return cmt;
	}

	public void setCmt(String cmt) {
		this.cmt = cmt;
	}

	public String getDescription() {
		return description;
	}

	public void setDescription(String description) {
		this.description = description;
	}

	public String getSrc() {
		return src;
	}

	public void setSrc(String src) {
		this.src = src;
	}

	public List<Link> getLinks() {
		return links;
	}

	public String getSym() {
		return sym;
	}

	public void setSym(String sym) {
		this.sym = sym;
	}

	public String getType() {
		return type;
	}

	public void setType(String type) {
		this.type = type;
	}

	public Fix getFix() {
		return fix;
	}

	public void setFix(Fix fix) {
		this.fix = fix;
	}

	public BigInteger getSat() {
		return sat;
	}

	public void setSat(BigInteger sat) {
		this.sat = sat;
	}

	public BigDecimal getHdop() {
		return hdop;
	}

	public void setHdop(BigDecimal hdop) {
		this.hdop = hdop;
	}

	public BigDecimal getVdop() {
		return vdop;
	}

	public void setVdop(BigDecimal vdop) {
		this.vdop = vdop;
	}

	public BigDecimal getPdop() {
		return pdop;
	}

	public void setPdop(BigDecimal pdop) {
		this.pdop = pdop;
	}

	public BigDecimal getAgeOfDgpsData() {
		return ageOfDgpsData;
	}

	public void setAgeOfDgpsData(BigDecimal ageOfDgpsData) {
		this.ageOfDgpsData = ageOfDgpsData;
	}

	public DgpsStation getdGpsId() {
		return dGpsId;
	}

	public void setdGpsId(DgpsStation dGpsId) {
		this.dGpsId = dGpsId;
	}
}


text:
type $npm$dropzone$FilesToString = (files: File | Array<File>) => string;
type $npm$dropzone$DrawImageOptions = {
  srcX: number,
  srcY: number,
  srcWidth: number,
  srcHeight: number
};

type $npm$dropzone$DropzoneOptions = {
  url: string | $npm$dropzone$FilesToString,
  method?: "post" | "put" | $npm$dropzone$FilesToString,
  parallelUploads?: number,
  maxFilesize?: number,
  filesizeBase?: number,
  paramName?: string,
  uploadMultiple?: boolean,
  headers?: { [key: string]: string },
  addRemoveLinks?: boolean,
  previewsContainer?: HTMLElement | string,
  clickable?: boolean,
  createImageThumbnails?: boolean,
  maxThumbnailFilesize?: number,
  thumbnailWidth?: number,
  thumbnailHeight?: number,
  maxFiles?: number,
  resize?: (file: File) => $npm$dropzone$DrawImageOptions,
  init?: () => mixed,
  acceptedFiles?: string,
  autoProcessQueue?: boolean,
  previewTemplate?: string,
  forceFallback?: boolean,
  fallback?: () => mixed,
  dictDefaultMessage?: string,
  dictFallbackMessage?: string,
  dictFallbackText?: string,
  dictInvalidFileType?: string,
  dictFileTooBig?: string,
  dictResponseError?: string,
  dictCancelUpload?: string,
  dictCancelUploadConfirmation?: string,
  dictRemoveFile?: string,
  dictMaxFilesExceeded?: string
};

declare module "dropzone" {
  declare class Dropzone {
    files: Array<File>;

    static constructor(
      selector: string,
      options: $npm$dropzone$DropzoneOptions
    ): Dropzone;
    static autoDiscover: boolean;
    static confirm(
      question: string,
      accepted: () => mixed,
      rejected?: () => mixed
    ): void;
    destroy(): void;
    disable(): void;
    enable(): void;
    getAcceptedFiles(): Array<File>;
    getQueuedFiles(): Array<File>;
    getRejectedFiles(): Array<File>;
    getUploadingFiles(): Array<File>;
    off(event: string, fn: (...args: Array<any>) => mixed): void;
    on(event: string, fn: (...args: Array<any>) => mixed): void;
    processQueue(): void;
    removeAllFiles(cancel?: boolean): void;
    removeFile(file: File): void;
  }
  declare module.exports: Class<Dropzone>;
}


text:
var logger = require('pomelo-logger').getLogger(__filename);
var utils = require('../util/utils');
var worldPlayerDao  = require('../dao/worldPlayerDao');
var playerEquipDao  = require('../dao/playerEquipDao');
var playerFractionDao = require('../dao/playerFractionDao');
var playerParamDao  = require('../dao/playerParamDao');
var playerItemDao   = require('../dao/playerItemDao');
var playerPetDao    = require('../dao/playerPetDao');

var async = require('async');
//var petConf = require('../../../shared/petConf');
var gameInit = require('../../../shared/gameInit');
var commonUtils = require('../../../shared/util/commonUtils');
var dataApi = require('../util/dataApi');

var PetCompoTrans = module.exports;

/**
 * Compose fraction into hero
 * @param {Object} mysqlc mysql client reference
 * @param {Number} playerId
 * @param {Number} star fraction star, also pet star
 * @param {Number} playerPetId sequence id for id in player_pet
 * @param {Number} num how many fractions used for one compo
 * @param {Number} petId pet_id in pet_data
 * @returns {Boolean} true or false
 */
PetCompoTrans.exec = function(mysqlc, playerId, star, playerPetId, num, petId, cb) {
  var type = gameInit.BAG.PET.type; // type of fraction, here is "pet"
  var typeId = commonUtils.getInitID(gameInit.BAG, type);

  star = Number(star);

  mysqlc.query('BEGIN', function(err, rows) { // start TRANSACTION
    if (err) {
      utils.invokeCallback(cb, err, null);
      return;
    }
    async.auto({
      // delete fraction
      deleteFraction: function(callback) {
        playerFractionDao.update(mysqlc, playerId, typeId, star, num, callback);
      },
      setPlayerPet: function(callback) {
        playerPetDao.add(mysqlc, playerPetId, playerId, petId, callback);
      }
    }, function(err, res) {
      var q; // query cmd
      if (err || !res) {
        q = 'ROLLBACK';
      }
      else {
        q = 'COMMIT';
      }
      mysqlc.query(q, function(err1, res1) {
        //mysqlc.transRelease(); // TODO: test
        if (err1) {
          utils.invokeCallback(cb, err1, null);
          return;
        }
        else {
          if (err) {
            console.log('[PetCompo] transaction query failed ');
            utils.invokeCallback(cb, err, null);
            return;
          }
          else {
            console.log('[PetCompo] transaction query finished');
            if (!res.deleteFraction || !res.setPlayerPet) {
              utils.invokeCallback(cb, null, false);
            }
            else {
              utils.invokeCallback(cb, null, true);
            }
          }
        }
      });
    });
  });
};


text:
class AddHtmlToPosts < ActiveRecord::Migration
  def change
    add_column :posts, :html, :text
    add_column :posts, :blurb, :text
  end
end


text:
  <button class="close" aria-label="Close" type="button" ng-click="ctrl.cancel()">×</button>

 Descarga material técnicos

No. Bitácora:

      <input type="text" class="form-control" ng-model="ctrl.No_Bitacora" disabled>

Clasificación del material:

      <select
        class="form-control"
        name="material"
        ng-disabled="ctrl.Detalle"
        ng-model="ctrl.SlctMaterial"
        ng-options="M.Descripcion for M in ctrl.Material track by M.catTipoArticuloClave"
        ng-change="ctrl.BuscarNombreArticulo()">
        <option value="">---Selecciona----</option>
      </select>

Descripción de artículo:

      <select
        class="form-control"
        name="articulo"
        ng-disabled="ctrl.Detalle"
        ng-model="ctrl.SlctArticulo"
        ng-options="A.Nombre for A in ctrl.DescripcionArticulo track by A.IdArticulo"
        ng-change="ctrl.BuscarTipoArticulo()">
        <option value="">---Selecciona----</option>
      </select>

Cantidad

      <input type="number" class="form-control" ng-disabled="ctrl.Detalle" ng-model="ctrl.CantidadDescarga" ng-show="ctrl.MostrarCD">

Metraje:

      <select class="form-control" ng-model="ctrl.TipoMetraje" ng-change="ctrl.SlctTipoMetraje()" ng-show="ctrl.MostrarTM">
        <option value="I">Interior</option>
        <option value="E">Exterior</option>
      </select>

Inicial

      <input type="number" class="form-control" ng-model="ctrl.MetrajeII" ng-show="ctrl.MostrarMII" placeholder="II">
      <input type="number" class="form-control" ng-model="ctrl.MetrajeIE" ng-show="ctrl.MostrarMIE" placeholder="IE">

Final

      <input type="number" class="form-control" ng-model="ctrl.MetrajeFI" ng-show="ctrl.MostrarMFI" placeholder="FI">
      <input type="number" class="form-control" ng-model="ctrl.MetrajeFE" ng-show="ctrl.MostrarMFE" placeholder="FE">

      <button title="Agregar un artículo" class="btn btn-info btn-sm" ng-disabled="ctrl.Detalle" ng-click="ctrl.AgregarArticulo()">Agregar</button>

        Clave
        Descripción
        Cantidad
        Tecnico
        Metraje inicial int.
        Metraje final int.
        Metraje inicial ext.
        Metraje final ext.
        Opciones

        {{ D.Clave}}
         {{D.Descripcion}}
         {{D.Cantidad}}
         {{D.Tecnico}}
         {{D.metrajeInicio}}
         {{D.metrajeFin}}
         {{D.metrajeInicioExterior}}
         {{D.metrajeFinExterior}}
         <button title="Eliminar artículo" class="btn btn-xs btn-danger " ng-click="ctrl.EliminarArticulo(D.Clave)"> Eliminar</button>

  <button title="Cancelar cambios" class="btn btn-default-bright btn-sm" type="button" ng-click="ctrl.cancel()">Cancelar</button>
  <button title="Guardar cambios" class="btn btn-primary btn-sm" type="button" ng-disabled="ctrl.Detalle" ng-click="ctrl.ok()">Guardar</button>


text:
'use strict';

var router = require('express').Router();

var App   = require('../controllers/application_controller'),
    Tessels = require('../controllers/tessels_controller'),
    Remote = require('../controllers/remote_controller'),
    Config = require('../controllers/configuration_controller');

// API key settings routes
router.get('/api/user/apikey', Config.apiKey); // get existing API key
router.post('/api/user/apikey', Config.genApiKey); // create and get new API key

// Authenticate API routes with API key
router.all('/api/tessel*', App.authenticate);

// Tessel routes
router.post('/api/tessel', Tessels.create); // create new remote Tessel
router.get('/api/tessel/:device_id', Tessels.details); // list details for one Tessel
router.put('/api/tessel/:device_id', Tessels.update); // update Tessel nickname/ allowed users
router.delete('/api/tessel/:device_id', Tessels.delete); // delete remote Tessel
router.get('/api/tessel', Tessels.list); // list details for all of user's Tessels

// Remote control routes
router.post('/api/tessel/:device_id/code', Remote.code); // tessel run or push as decided by attributes
router.get('/api/tessel/:device_id/network', Remote.network); // get network connection details
router.get('/api/tessel/:device_id/log', Remote.log); // start listening

module.exports = router;


text:
class MakePositionNullableInCategories < ActiveRecord::Migration
  def up
    change_column :categories, :position, :integer, null: true
  end

  def down
    execute "update categories set position=0 where position is null"
    change_column :categories, :position, :integer, null: false
  end
end


text:
module HackyHAL
  module Util
    def self.object_from_hash(hash, context_module)
      hash = hash.dup
      type = hash.delete(:type)

      unless type
        raise ArgumentError, "Must specify type to build object from hash.  Given: #{hash}"
      end

      context_module.const_get(type).new(hash)
    end

    def self.symbolize_keys_deep(h)
      h.keys.each do |k|
        ks = k.respond_to?(:to_sym) ? k.to_sym : k
        h[ks] = h.delete(k)
        symbolize_keys_deep(h[ks]) if h[ks].kind_of?(Hash)
      end
      h
    end
  end
end


text:
from setuptools import setup
from pip.req import parse_requirements

REQS = [str(ir.req) for ir in parse_requirements('requirements.txt', session=False)]

setup(
    name='tabletopscanner',
    packages=['tabletopscanner'],
    include_package_data=True,
    install_requires=REQS,
)


text:
package api_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"strconv"
	"time"

	"gopkg.in/redis.v4"

	"github.com/labstack/echo/engine/standard"
	. "github.com/onsi/gomega"
	"github.com/topfreegames/santiago/api"

	"github.com/uber-go/zap"
)

//GetTestRedisConn returns a connection to the test redis server
func GetTestRedisConn() (*redis.Client, error) {
	redisPort := 57575
	redisPortEnv := os.Getenv("REDIS_PORT")
	if redisPortEnv != "" {
		res, err := strconv.ParseInt(redisPortEnv, 10, 32)
		if err != nil {
			return nil, err
		}
		redisPort = int(res)
	}
	client := redis.NewClient(&redis.Options{
		Addr:     fmt.Sprintf("localhost:%d", redisPort),
		Password: "", // no password set
		DB:       0,  // use default DB
	})
	return client, nil
}

// GetDefaultTestApp returns a new Santiago API Application bound to 0.0.0.0:8888 for test
func GetDefaultTestApp(logger zap.Logger) (*api.App, error) {
	options := api.DefaultOptions()
	options.ConfigFile = "../config/test.yaml"
	return api.New(options, logger, false)
}

//Get from server
func Get(app *api.App, url string) (int, string) {
	return doRequest(app, "GET", url, "")
}

//Post to server
func Post(app *api.App, url, body string) (int, string) {
	return doRequest(app, "POST", url, body)
}

//PostJSON to server
func PostJSON(app *api.App, url string, body interface{}) (int, string) {
	result, err := json.Marshal(body)
	if err != nil {
		return 510, "Failed to marshal specified body to JSON format"
	}
	return Post(app, url, string(result))
}

//Put to server
func Put(app *api.App, url, body string) (int, string) {
	return doRequest(app, "PUT", url, body)
}

//PutJSON to server
func PutJSON(app *api.App, url string, body interface{}) (int, string) {
	result, err := json.Marshal(body)
	if err != nil {
		return 510, "Failed to marshal specified body to JSON format"
	}
	return Put(app, url, string(result))
}

//Delete from server
func Delete(app *api.App, url string) (int, string) {
	return doRequest(app, "DELETE", url, "")
}

var client *http.Client
var transport *http.Transport

func initClient() {
	if client == nil {
		transport = &http.Transport{DisableKeepAlives: true}
		client = &http.Client{Transport: transport}
	}
}

func doRequest(app *api.App, method, url, body string) (int, string) {
	initClient()
	defer transport.CloseIdleConnections()
	app.Engine.SetHandler(app.WebApp)
	ts := httptest.NewServer(app.Engine.(*standard.Server))

	var bodyBuff io.Reader
	if body != "" {
		bodyBuff = bytes.NewBuffer([]byte(body))
	}
	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", ts.URL, url), bodyBuff)
	req.Header.Set("Connection", "close")
	req.Close = true
	Expect(err).NotTo(HaveOccurred())

	res, err := client.Do(req)
	ts.Close()
	//Wait for port of httptest to be reclaimed by OS
	time.Sleep(50 * time.Millisecond)
	Expect(err).NotTo(HaveOccurred())

	b, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	Expect(err).NotTo(HaveOccurred())

	return res.StatusCode, string(b)
}


text:
require "active_record"
require "activerecord-import"
require "table_utils/progress"

class DataSource
  attr_reader :table_name, :input_path

  def initialize table_name, input_path, csv_opts = {}, &block
    @table_name, @input_path = table_name, input_path
    @csv_opts = csv_opts ? csv_opts.dup : {}
    ImportDSL.new self, &block
  end

  def import progress_opts = {}
    batch = []
    create_table
    TableUtils::Progress.over csv, progress_opts do |row, bar|
      transform_record row
      common_columns = model.column_names & row.headers
      batch << common_columns.map{ |c| row[c] }
      if batch.count >= 1000 || bar.finished?
        model.import common_columns, batch, validate: false
        batch = []
      end
    end if input_path
    after_import
  end

  def init_table
    create_table
    after_import
  end

  def define_table *args
    raise "No table definition supplied for #{@table_name}"
  end

  def transform_record ignored_record

  end

  def after_import

  end

  def table_exists?
    conn.table_exists? table_name
  end

  def drop_table
    conn.drop_table table_name
  end

  private

  class ImportDSL
    attr_reader :source

    def initialize source, &block
      @source = source
      instance_eval &block
    end

    [:define_table, :transform_record, :after_import].each do |hook|
      define_method hook do |&block|
        source.define_singleton_method hook, block
      end
    end
  end

  attr_accessor :csv_opts

  def conn
    @conn ||= ActiveRecord::Base.connection
  end

  def create_table
    conn.create_table table_name do |t|
      define_table t
    end unless table_exists?
  end

  def model
    _table_name = table_name
    @model ||= Class.new(ActiveRecord::Base) { self.table_name = _table_name }
  end

  def csv
    @csv ||= begin
               csv_opts[:headers] ||= model.column_names - %w(id created_at updated_at)
               csv_opts[:return_headers] = true
               CSV.open(input_path, csv_opts).tap do |csv|
                 csv.shift
                 csv.shift if Array === csv_opts[:headers]
               end
             end
  end
end


text:
<!DOCTYPE html>

    <link href='http://fonts.googleapis.com/css?family=Limelight|Lobster|Cinzel' rel='stylesheet' type='text/css'>
    <link type="text/css" rel="stylesheet" href="../stylesheets/blog-stylesheets.css">

    <title>Welcome to Ali's site :]</title>

Ali Yeysides

About

Blog

Projects

< W  e  l  c  o  m  e >

Hello World! My name is Ali Yeysides and I'm currently a student at Dev Bootcamp. Growing up, I've been obsessed with breaking things down and trying to figure how they worked. To me, software is the perfect medium to express my ideas and continue to challenge myself. My aspirations are to build meaningful products and in the process become a full-stack programmer. On my free time I enjoy video games, food, and anything outside.

How and When to use Relative, Absolute, and Fixed Positioning

Why is JavaScript popular in web development? What are the great things about JavaScript? What are its limitations?

Enumerable#Map/Select and it's Practices

What are Ruby Classes?

Ruby Classes and Modules

SQL Injection Hacking

Test Driven Development

Websites to Keep Up in your Browser

Shereef's Fireside Chat

My Thinking Style and Learning

Issues in Tech: Net Neutrality

Pairing & Giving Feedback

Imposter Syndrome

My Values

Conflicts

Asking the right Questions

GPS 1.1

Responsive CSS

< M  y     P  r  o  j  e  c  t  s >


text:
import flask
import pytest

import main

# Create a fake "app" for generating test request contexts.
@pytest.fixture(scope="module")
def app():
    return flask.Flask(__name__)

def test_parse_xml(app):
    with app.test_request_context(method='GET', data="<baz>foo</baz>"):
        res = main.parse_xml(flask.request)
        assert res == "{\n  \"baz\": \"foo\"\n}"

def test_parse_multipart_data(app, capsys):
    with app.test_request_context(method='POST', data={'foo': 'bar'}):
        res = main.parse_multipart(flask.request)
        out, _ = capsys.readouterr()
        assert res == 'Done!'
        assert out == 'Processed field: foo\n'

def test_parse_multipart_files(app, capsys):
    with open(__file__, 'rb') as file:
        with app.test_request_context(method='POST', data={'test.py': file}):
            res = main.parse_multipart(flask.request)
            out, _ = capsys.readouterr()
            assert res == 'Done!'
            assert out == 'Processed file: test.py\n'

def test_cors_enabled_function_preflight(app):
    with app.test_request_context(method='OPTIONS'):
        res = main.cors_enabled_function(flask.request)
        assert res[2].get('Access-Control-Allow-Origin') == '*'
        assert res[2].get('Access-Control-Allow-Methods') == 'GET'
        assert res[2].get('Access-Control-Allow-Headers') == 'Content-Type'
        assert res[2].get('Access-Control-Max-Age') == '3600'

def test_cors_enabled_function_main(app):
    with app.test_request_context(method='GET'):
        res = main.cors_enabled_function(flask.request)
        assert res[2].get('Access-Control-Allow-Origin') == '*'

def test_cors_enabled_function_auth_preflight(app):
    with app.test_request_context(method='OPTIONS'):
        res = main.cors_enabled_function_auth(flask.request)
        assert res[2].get('Access-Control-Allow-Origin') == \
            'https://mydomain.com'
        assert res[2].get('Access-Control-Allow-Methods') == 'GET'
        assert res[2].get('Access-Control-Allow-Headers') == 'Authorization'
        assert res[2].get('Access-Control-Max-Age') == '3600'
        assert res[2].get('Access-Control-Allow-Credentials') == 'true'

def test_cors_enabled_function_auth_main(app):
    with app.test_request_context(method='GET'):
        res = main.cors_enabled_function_auth(flask.request)
        assert res[2].get('Access-Control-Allow-Origin') == \
            'https://mydomain.com'
        assert res[2].get('Access-Control-Allow-Credentials') == 'true'


text:
//
// Created by byter on 12/18/17.
//

#ifndef THREEPP_SHADOWMATERIAL_H
#define THREEPP_SHADOWMATERIAL_H

#include "Material.h"
#include <threepp/core/Color.h>

namespace three {

/**
 * @author mrdoob / http://mrdoob.com/
 *
 * parameters = {
 *  color: <THREE.Color>,
 *  opacity: <float>
 * }
 */
class ShadowMaterial : public MaterialT<material::Diffuse>
{
private:
  ShadowMaterial()
     : MaterialT(material::InfoT<ShadowMaterial>(), material::Typer(this))
  {
    this->color = 0x000000;
    this->opacity = 1.0f;
    this->lights = true;
  }

  ShadowMaterial(const ShadowMaterial &material)
     : MaterialT(material, material::InfoT<ShadowMaterial>(), material::Typer(this))
  {
  }

public:
  using Ptr = std::shared_ptr<ShadowMaterial>;

  static Ptr make() {
    return Ptr(new ShadowMaterial());
  }

  ShadowMaterial *cloned() const override {
    return new ShadowMaterial(*this);
  }

  bool transparent() const override {return opacity < 1.0f;}
};

}

#endif //THREEPP_SHADOWMATERIAL_H