<file_sep>"""Get historical rates from git."""

import datetime
import gspread
import requests
import time
import xmltodict

sheetkey = "<KEY>"

gc = gspread.service_account()
sh = gc.open_by_key(sheetkey)
ws = sh.worksheet('kurzy')

r = requests.get('https://ban.tipsport.cz/c/feed.php?feed=1101&pid=10661&sid=12302&tid=2161&bid=3284')

data = {}
main = xmltodict.parse(r.content)

date_data = datetime.datetime.strptime(main['odds']['date'], "%d.%m.%Y %H:%M").isoformat()
date_retrieved = datetime.datetime.fromtimestamp(int(main['odds']['timestamp'])/1000).isoformat()

for competition in main['odds']['competition']:
    for event in competition['match']['event']:
        try:
            for odd in event['odd']:
                data[event['@name'] + ": " + odd['@fullName']] = float(odd['@rate'])
        except:
            nothing = True

headers = ws.get('A1:ZZ1')[0]
column_pointer = len(headers)
row_pointer = ws.row_count


for k in data:
    if k not in headers:
        headers.append(k)

h2c = {}
i = 0
for h in headers:
    h2c[h] = i 
    i += 1

row = [None] * i
for k in data:
    row[h2c[k]] = data[k]

row[0] = date_data
row[1] = date_retrieved

ws.append_row(row)
ws.update('A1', [headers])
time.sleep(1.5)

# subprocess.run(["git", "checkout", 'master'])

# len(main['odds']['competition'])

# len(competition['match'])

# competition['match'].keys()

# event.keys()
# event['@name']
# odd['@fullName']<file_sep>"""Test connection"""

import datetime
import os
import pandas as pd
import requests
import sys

url_root = "https://m.tipsport.cz/"
path = "2022/"

# authentization
# the first part is local, the other takes the values from Github secrets
try:
  sys.path.append('2022')
  import v3.secret as secret
  os.environ['TIPSPORT_USER'] = secret.TIPSPORT_USER
  os.environ['TIPSPORT_PASSWORD'] = secret.TIPSPORT_PASSWORD
  os.environ['TIPSPORT_PRODUCTID'] = secret.TIPSPORT_PRODUCTID
except:
  pass

headers = {'Content-Type': 'application/x-www-form-urlencoded'}
credentials = {
  'username': os.environ['TIPSPORT_USER'], 
  'password': <PASSWORD>['<PASSWORD>'],
  'productId': os.environ['TIPSPORT_PRODUCTID']
}

r = requests.post(url_root + 'rest/common/v1/session', data=credentials, headers=headers)
auth = r.json()
cookies = r.cookies
token = auth['sessionToken']
headers = {'Authorization': "Bearer {}".format(token)}

# 'společenské sázky' - get matches
r1 = requests.get(url_root + 'rest/external/offer/v1/sports', headers=headers, cookies=cookies)
data = r1.json()

matches = []
for category in data['data']['children']:
  for supersport in category['children']:
    if supersport['title'] == 'Společenské sázky':
      for sport in supersport['children']:
        for match in sport['children']:
          item = {
            'sport_title': sport['title'],
            'match_title': match['title'],
            'match_id': match['id']
          }
          matches.append(item)

# get races
races = []
for match in matches:
  r2 = requests.get(url_root + 'rest/external/offer/v2/competitions/{}/matches'.format(match['match_id']), headers=headers, cookies=cookies)
  data2 = r2.json()
  for race in data2['matches']:
    item = match.copy()
    item['race_name'] = race['name']
    item['race_id'] = race['id']
    races.append(item)

# get details
events = []
now = datetime.datetime.now().isoformat()
for race in races:
  r3 = requests.get(url_root + 'rest/offer/v2/matches/{}?withOdds=true'.format(race['race_id']), headers=headers, cookies=cookies)
  # print(race['race_name'])
  data3 = r3.json()
  datetimeClosed = data3['match']['datetimeClosed']
  for table in data3['match']['eventTables']:
    for box in table['boxes']:
      item = race.copy()
      item['match_close'] = datetimeClosed
      item['event_name'] = table['name']
      item['box_id'] = box['id']
      item['box_name'] = box['name']
      item['cells'] = []
      for cell in box['cells']:
        it = {
          'id': cell['id'],
          'name': cell['name'],
          'odd': cell['odd'],
          'date': now

        }
        item['cells'].append(it)
      events.append(item)


# prepare tables
try:
  meta = pd.read_csv(path + 'meta.csv')
except:
  meta = pd.DataFrame()
for event in events:
  e = event.copy()
  del e['cells']
  if (len(meta.index) == 0) or (meta.loc[meta['box_id'] == e['box_id']].empty):
    meta = meta.append(e, ignore_index=True)
  
  try:
    table = pd.read_csv(path + 'data/' + str(event['box_id']) + '.csv')
  except:
    table = pd.DataFrame()
  for cell in event['cells']:
    table = table.append(cell, ignore_index=True)
  table['id'] = table['id'].astype(int)
  table.to_csv(path + 'data/' + str(event['box_id']) + '.csv', index=False)

ids = ['match_id', 'race_id']
for i in ids:
  meta[i] = meta[i].astype(int)

meta.to_csv(path + 'meta.csv', index=False)


  

meta.to_csv('meta.csv')
<file_sep># tipsport.cz
Scrapers for tipsport.cz

Use:

- https://mandaty.cz
- https://www.seznamzpravy.cz/clanek/fakta-sazky-na-prezidenta-ku-pocitame-sance-kdo-vyhraje-volby-2023-201469
<file_sep>"""Get historical rates from git."""

import datetime
import gspread
import subprocess
import time
import xmltodict

sheetkey = "<KEY>"

gc = gspread.service_account()
sh = gc.open_by_key(sheetkey)
ws = sh.worksheet('kurzy')

proc = subprocess.run(["git", "log", "--oneline"], stdout=subprocess.PIPE)

output = subprocess.check_output(["git", "log", "--oneline"], text=True)

raw_rows = output.split('\n')

hashes = []
for row in raw_rows:
    it = row.split(' ')
    if (len(it) > 1) and (it[1] == 'Latest'):
        hashes.append(it[0])

hashes.reverse()

kk = 1
for hash in hashes:
    subprocess.run(["git", "checkout", hash])
    with open("../social.xml") as fin:
        xmlb = fin.read()
        xml = (bytes(xmlb, encoding='utf8'))

    data = {}
    main = xmltodict.parse(xml)

    date_data = datetime.datetime.strptime(main['odds']['date'], "%d.%m.%Y %H:%M").isoformat()
    date_retrieved = datetime.datetime.fromtimestamp(int(main['odds']['timestamp'])/1000).isoformat()

    for competition in main['odds']['competition']:
        for event in competition['match']['event']:
            try:
                for odd in event['odd']:
                    data[event['@name'] + ": " + odd['@fullName']] = float(odd['@rate'])
            except:
                nothing = True
    
    headers = ws.get('A1:ZZ1')[0]
    column_pointer = len(headers)
    row_pointer = ws.row_count


    for k in data:
        if k not in headers:
            headers.append(k)
    
    h2c = {}
    i = 0
    for h in headers:
        h2c[h] = i 
        i += 1
    
    row = [None] * i
    for k in data:
        row[h2c[k]] = data[k]

    row[0] = date_data
    row[1] = date_retrieved

    ws.append_row(row)
    ws.update('A1', [headers])
    time.sleep(1.5)
    
    kk += 1
    # if kk == 3:
    #     break



subprocess.run(["git", "checkout", 'master'])

# len(main['odds']['competition'])

# len(competition['match'])

# competition['match'].keys()

# event.keys()
# event['@name']
# odd['@fullName']<file_sep># scrapes odds from tipsport.cz and updates github datapackages

import csv
import datapackage #v0.8.3
import datetime
import json
import git
import os
import requests

import tipsport_cz_scraper_utils as utils
import settings

data_path = "data/"  # from this script to data

# repo settings
repo = git.Repo(settings.git_dir)
git_ssh_identity_file = settings.ssh_file
o = repo.remotes.origin
git_ssh_cmd = 'ssh -i %s' % git_ssh_identity_file

total_groups = 0
for fdir in settings.tipsport_dirs:
    dat = utils.scrape_races(fdir)
    date = datetime.datetime.utcnow().isoformat()
    for row in dat:
        data = utils.scrape_race(row)
        # load or create datapackage
        try:
            # load datapackage
            datapackage_url = settings.project_url + data_path + row['matchId']  +"/datapackage.json"
            dp = datapackage.DataPackage(datapackage_url)
        except:
            # create datapackage
            dp = datapackage.DataPackage()
            urldp = settings.project_url + "datapackage_prepared.json"
            rdp = requests.get(urldp)
            prepared = rdp.json()
            dp.descriptor['identifier'] = row['matchId']
            dp.descriptor['name'] = "tisport_cz_" + row['matchId']
            dp.descriptor['title'] = row['title'] + " - odds from tipsport.cz"
            dp.descriptor['description'] = "Scraped odds from tipsport.cz for: " + row['title']
            for k in prepared:
                dp.descriptor[k] = prepared[k]
            if not os.path.exists(settings.git_dir + data_path + row['matchId']):
                os.makedirs(settings.git_dir + data_path + row['matchId'])
            with open(settings.git_dir + data_path + row['matchId'] +'/datapackage.json', 'w') as fout:
                fout.write(dp.to_json())
            repo.git.add(settings.git_dir + data_path + row['matchId']  +'/datapackage.json')
            with open(settings.git_dir + data_path + row['matchId'] +'/odds.csv',"w") as fout:
                header = []
                for resource in dp.resources:
                    if resource.descriptor['name'] == 'odds':
                        for field in resource.descriptor['schema']['fields']:
                            header.append(field['name'])
                dw = csv.DictWriter(fout,header)
                dw.writeheader()
            repo.git.add(settings.git_dir + data_path + row['matchId']  +'/odds.csv')

        with open(settings.git_dir + data_path + row['matchId']  +'/odds.csv',"a") as fout:
            header = []
            attributes = ['date','title','odds','date_bet','identifier']
            for resource in dp.resources:
                if resource.descriptor['name'] == 'odds':
                    for field in resource.descriptor['schema']['fields']:
                        header.append(field['name'])
            dw = csv.DictWriter(fout,header)
            for ro in data:
                if ro['identifier'] == row['matchId']:
                    for bet in ro['rows']:
                        item = {
                            'date': date,
                            'title': bet['title'],
                            'odds': bet['odds'],
                            'date_bet': bet['date_bet'],
                            'identifier': bet['identifier'],
                        }
                        dw.writerow(item)

        repo.git.add(settings.git_dir + data_path + row['matchId']  +'/odds.csv')

    total_groups += len(dat)

with repo.git.custom_environment(GIT_COMMITTER_NAME=settings.bot_name, GIT_COMMITTER_EMAIL=settings.bot_email):
    repo.git.commit(message="happily updating data %s groups of bets" % (str(total_groups)), author="%s <%s>" % (settings.bot_name, settings.bot_email))
with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd):
    o.push()
<file_sep>"""Create daily averages."""

import numpy as np
import pandas as pd

path = "2022/"

meta = pd.read_csv(path + 'meta.csv')

for i, row in meta.iterrows():
    df = pd.read_csv(path + 'data/' + row['box_id'] + '.csv')
    df['date'] = pd.to_datetime(df['date']).dt.date
    pt = pd.pivot_table(df, index='date', values='odd', columns=['name'], aggfunc=np.average).reset_index()
    pt.to_csv(path + 'daily/' + row['box_id'] + '.csv', index=False)<file_sep>from lxml import html, etree
import re
import requests

import settings

def scrape_race(match):
    url = settings.tipsport_url + settings.tipsport_endpoint
    r = requests.post(url,data={"matchId":match['matchId']})
    data = []
    if r.status_code == 200:
        group = {'identifier': match['matchId']}
        domtree = html.fromstring(r.text)
        trs = domtree.xpath('//tbody/tr')
        date_bet = trs[1].xpath('td/span')[0].text.strip('()')
        rows = []
        for trn in range(2,len(trs)):
            tr = trs[trn]
            item = {}
            try:
                item['title'] = tr.xpath('td')[1].text.strip()
                item['identifier'] = tr.xpath('td/span')[0].text.strip()
                item['odds'] =  tr.xpath('td/a')[0].text.strip()
                item['date_bet'] = date_bet
                rows.append(item)
            except:
                nothing = None
        group['rows'] = rows
        data.append(group)
    return data

def scrape_races(fdir):
    url = settings.tipsport_url2 + fdir
    r = requests.get(url)
    data = []
    if r.status_code == 200:
        domtree = html.fromstring(r.text)
        table = domtree.xpath('//table[@class="tblOdds"]')[0]
        trs = table.xpath('tr[@id="oddViewMain0"]')
        for tr in trs:
            item = {}
            item['title'] = tr.xpath('td/a/span')[0].text.strip().strip(':')
            item['matchId'] = tr.xpath('td/a/span/@data-m')[0]
            data.append(item)
    return data

if __name__ == "__main__":
    # test:
    dat = scrape_races('spolecenske-sazky-25')
    for row in dat:
        data = scrape_race(row)
        print(data)
<file_sep># settings for scrapers
tipsport_url = "https://www.tipsport.cz/"
tipsport_url2 = "https://www.tipsport.cz/kurzy/"
tipsport_endpoint = "EventsMatchExtendedOddsAjaxAction.do"
tipsport_dirs = ['spolecenske-sazky-25']
# settings for github
project_url = "https://raw.githubusercontent.com/michalskop/tipsport.cz/master/"
# settings for MakaBot
git_dir = "/home/user/project/tipsport.cz/"
ssh_file = "/home/user/.ssh/ExampleBot"
bot_name = "ExampleBot"
bot_email = "<EMAIL>"
<file_sep>"""Download current odds, v3."""

import datetime
import json
import os
import pandas as pd
import requests

url_root_p = "https://partners.tipsport.cz/"
path = "v3/"

# authentization
# the first part is local, the other takes the values from Github secrets
try:
  # sys.path.append('2022')
  import v3.secret as secret
  os.environ['TIPSPORT_USER'] = secret.TIPSPORT_USER
  os.environ['TIPSPORT_PASSWORD'] = secret.TIPSPORT_PASSWORD
  os.environ['TIPSPORT_PRODUCTID'] = secret.TIPSPORT_PRODUCTID
  os.environ['PROXY_SERVERS'] = secret.PROXY_SERVERS
except:
  pass

# proxy
proxy_servers = {
  'https': os.environ.get('PROXY_SERVERS')
}

# authentization
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
credentials = {
  'username': os.getenv('TIPSPORT_USER'), 
  'password': os.getenv('<PASSWORD>'),
  'productId': os.getenv('TIPSPORT_PRODUCTID')
}
headers = {'Content-Type': 'application/json'}
for i in range(10):
  r = requests.post(url_root_p + 'rest/external/common/v2/session', data=json.dumps(credentials), headers=headers, proxies=proxy_servers)
  cookies = r.cookies
  if r.status_code == 200:
    break
  else:
    r2 = requests.get(url_root_p + 'rest/external/offer/v1/matches', headers=headers, cookies=cookies, proxies=proxy_servers)
    print(r.status_code, r2.status_code)

if r.status_code != 200:
  raise Exception('Could not authenticate, status code {}.'.format(r.status_code))

auth = r.json()
cookies = r.cookies
token = auth['sessionToken']
headers = {'Authorization': "Bearer {}".format(token)}

# 'společenské sázky' - get matches
matches = []
r2 = requests.get(url_root_p + 'rest/external/offer/v1/matches', headers=headers, cookies=cookies, proxies=proxy_servers)
data2 = r2.json()
# json.dump(data2, open(path + 'data2test.json', 'w'))

for match in data2['matches']:
  if match['nameSuperSport'] == 'Společenské sázky':
    matches.append(match)
now = datetime.datetime.now()
# json.dump(matches, open(path + 'matchestest.json', 'w'))
"105903"

matches3 = []
for match in matches:
  params = {
    'idCompetition': match['idCompetition'],
    'allEvents': 'True',
  }
  r3 = requests.get(url_root_p + 'rest/external/offer/v1/matches/' + str(match['id']), params=params, headers=headers, cookies=cookies, proxies=proxy_servers)
  data3 = r3.json()
  try:
    matches3.append(data3['match'])
  except:
    pass
# json.dump(matches3, open(path + 'matches3test.json', 'w'))

# 'společenské sázky' - get odds, read / write
try:
  meta = pd.read_csv(path + 'meta.csv')
except:
  meta = pd.DataFrame()
for match in matches3:
  # break
  match_id = match['id']
  try:
    table = pd.read_csv(path + 'data/' + str(match_id) + '.csv')
  except:
    table = pd.DataFrame()
  
  for et in match['eventTables']:
    # odds
    for box in et['boxes']:
      for cell in box['cells']:
        row = {
          'date': now,
          'id': cell['id'],
          'name': cell['name'],
          'odd': cell['odd'],
          'supername': box['name'],
          'hypername': et['name'],
        }
        table = pd.concat([table, pd.DataFrame([row])])

    table = table.drop_duplicates(subset=['id', 'date', 'name'])
    if len(table) > 0:
      table.to_csv(path + 'data/' + str(match_id) + '.csv', index=False)

    # meta
    meta_row = {
      'date': now,
      'match_id': match_id,
      'match_name': match['name'],
      'match_url': match['matchUrl'],
      'competition_id': match['idCompetition'],
      'competition_name': match['nameCompetition'],
      'sport_id': match['idSport'],
      'sport_name': match['nameSport'],
      'date_closed': datetime.datetime.fromtimestamp(match['dateClosed'] / 1000).isoformat(),
      'event_id': et['id'],
      'event_name': et['name'],
    }
    meta = pd.concat([meta, pd.DataFrame([meta_row])])

meta = meta.drop_duplicates(subset=['match_id', 'event_id', 'date'])
meta.to_csv(path + 'meta.csv', index=False)
<file_sep># Betting odds from Tipsport.cz

Scraper and data of betting odds from tipsport.cz with direct update to Github.

Restriction: Works for single bets only (i.e. - one line, one bet) now.

Data are structured in tabular datapackage format (http://frictionlessdata.io/guides/tabular-data-package/).

Odds for bets on **politics** are updated daily and accessible in this project.

### Example - the data of odds for Czech presidential candidates 2017
http://data.okfn.org/tools/view?url=https%3A%2F%2Fraw.githubusercontent.com%2Fmichalskop%2Ftipsport.cz%2Fmaster%2Fdata%2F2518628%2Fdatapackage.json

### Custom installation
Requirements:
- Python 3
- Python packages: csv, datetime, datapackage (>=v0.8), git, lxml, re, requests, os

Copy example settings into settings and correct it for your Github account (e.g., your bot's account)

    cp settings-example.py settings.py

Note: The origin for the local git project must be 'ssh' address (not 'https' one) for bot to work.

### Automation
You can automate the data retrieval using cron. Example:

    14 3 * * * /usr/bin/python3 /home/project/tipsport.cz/scraper.py > /dev/null 2>&1
<file_sep>"""Create daily averages."""

import numpy as np
import pandas as pd

path = "v3/"

meta = pd.read_csv(path + 'meta.csv')

matches = meta['match_id'].unique()

for match_id in matches:
  try:
    df = pd.read_csv(path + 'data/' + str(match_id) + '.csv')
    df['date'] = pd.to_datetime(df['date']).dt.date
    # v3.1 vs 3.0
    if 'supername' in df.columns:
      df['new_name'] = df['hypername'].fillna('') + ' - ' + df['supername'].fillna('') + ': ' + df['name']
      df['new_supername'] = df['hypername'].fillna('') + ' - ' + df['supername'].fillna('')
      new_supernames = df['new_supername'].unique()
      if len(new_supernames) > 2: # v3.0 + v3.1
        pt = pd.pivot_table(df, index='date', values='odd', columns=['new_name'], aggfunc=np.average).reset_index()
      else:
        pt = pd.pivot_table(df, index='date', values='odd', columns=['name'], aggfunc=np.average).reset_index()
    # v3.0
    else:
      pt = pd.pivot_table(df, index='date', values='odd', columns=['name'], aggfunc=np.average).reset_index()
    pt.to_csv(path + 'daily/' + str(match_id) + '.csv', index=False)
  except:
    pass
