dima/flashscore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
4a7e71241a7a95f3fff3e8ca85c845d2ebdc48a7
flashscore/parser.go

1595 lines
36 KiB
Go
Raw Normal View History

1st
2026-07-21 03:49:47 +03:00
package flashscore
import (
"encoding/json"
"fmt"
"html"
"regexp"
"strconv"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
xhtml "golang.org/x/net/html"
"gordenko.dev/dima/flashscore/model"
"gordenko.dev/dima/pretty"
)
// Парсит кастомный формат Flashscore. На выходе список матчей,
// без группировок по турнирам, раундам
/*
func parseListOfScheduledMatches(content string, sportID int) (result []ScheduledMatch, err error) {
matches := strings.Split(content, SymbolTilda)
for _, match := range matches {
match = strings.TrimSpace(match)
if match == "" {
continue
}
pairs := strings.Split(match, SymbolNot)
sm := ScheduledMatch{
SportID: sportID,
}
for _, pair := range pairs {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
kv := strings.Split(pair, SymbolDivide)
if len(kv) != 2 {
err = fmt.Errorf("Wrong key/value: %s", pair)
return
}
key := strings.TrimSpace(kv[0])
value := strings.TrimSpace(kv[1])
switch key {
case "AA":
//
sm.MatchID = value
case "AD":
var timestamp int64
timestamp, err = strconv.ParseInt(value, 10, 64)
if err != nil {
err = fmt.Errorf("Parse start time: %s; unixtime=%s", err, value)
return
}
sm.StartTime = time.Unix(timestamp, 0)
}
}
if sm.MatchID != "" && !sm.StartTime.IsZero() {
// Помимо матчей встречаются чемпионаты, поэтому эти поля могут быть пустыми
result = append(result, sm)
}
}
return
}
*/
//ZA zone: champ name extended
//ZL "champID"
/*
AA: 2XhOe000
AD: 1595772000
ADE: 1595772000
AB: 1
CR: 1
AC: 1
CX: Southampton
ER: Round 38
RW: 0
AX: 0
BX: -1
WQ:
WM: SOU
AE: Southampton
JA: fu16YetP
WU: southampton
WN: SHU
AF: Sheffield Utd
JB: bDRnslpP
WV: sheffield-utd
AW: 1
AN: n
Расшифровка:
ZA - champ name
AA - ID матча, нужен чтобы получить статистику по матчу
AD / ADE - время начала матча в UTC
CX - home team
AF - away team
WU - home team url part (нужно ли ?)
WV - away team url part (нужно ли ?)
ER - раунд (можно не парсить, но сгруппировать)
*/
//var reRoundNumber = regexp.MustCompile(`(?i)Round\s+(\d+)`)
// parseEncodedFixtures - парсит строку (html unescaped)
//func parseEncodedFixtures(content string) (rounds []FootballRound, err error) {
func parseMatchesResults(trs *goquery.Selection) (results []model.TeamMatch, err error) {
for idx := range trs.Nodes {
var m model.TeamMatch
m, err = parseMatchResultTr(trs.Eq(idx))
if err != nil {
err = fmt.Errorf("parseTr: %s", err)
return
}
results = append(results, m)
}
return
}
var reMatchIDinMatchResultTr = regexp.MustCompile(`(?i)open\('g_\d+_([^']+)'`)
func parseMatchResultTr(trSelection *goquery.Selection) (m model.TeamMatch, err error) {
onclickContent, exists := trSelection.Attr("onclick")
if !exists {
err = fmt.Errorf("onclick attr not found in match result tr")
return
}
groups := reMatchIDinMatchResultTr.FindStringSubmatch(onclickContent)
if len(groups) != 2 {
err = fmt.Errorf("MatchID not found in onclick attr of match result tr")
return
}
m.MatchID = groups[1]
//fmt.Printf("onclick: %s, exists: %t\n", val, exists)
//spanSelection := trSelection.Find("span")
tdSelection := trSelection.Find("td")
// Достаем дату
spanList := tdSelection.Eq(0).Find("span")
if spanList == nil {
//
return
}
//timestampStr := spanList.Get(0).FirstChild.Data
//fmt.Printf("Date: %s\n", timestampStr)
/*
m.StartTime, err = strconv.ParseInt(timestampStr, 10, 64)
if err != nil {
err = fmt.Errorf("Parse timestamp string: %s; timestamp=%s", err, timestampStr)
return
}
*/
var score string
score, err = parseTdContent(tdSelection.Eq(4))
if err != nil {
err = fmt.Errorf("Get score: %s", err)
return
}
m.HomeGoals, m.AwayGoals, err = parseScore(score)
if err != nil {
err = fmt.Errorf("parseScore: %s; score=%s", err, score)
return
}
// ВАЖНО!
m.IsMatchScoreLoaded = true
return
}
func parseTdContent(tdSelection *goquery.Selection) (team string, err error) {
// Достаем home
spanList := tdSelection.Find("span")
if spanList.Length() == 0 {
var tagContent string
tagContent, err = getTagContent(tdSelection)
if err != nil {
err = fmt.Errorf("getTagContent: %s", err)
return
}
err = fmt.Errorf("Tag <span> not found in <td>: %s", tagContent)
return
}
strongList := spanList.Find("strong")
if strongList.Length() == 0 {
team = spanList.Get(0).FirstChild.Data
} else {
/*
firstStrong := strongList.Get(0) ///.FirstChild.Data
if firstStrong == nil {
panic("firstS strong = nil")
}
*/
team = strongList.Get(0).FirstChild.Data
}
return
}
func parsePeriodTeamGoals(doc *goquery.Document, period int, side string) (goals int, err error) {
spanTeamGoals := doc.Find(fmt.Sprintf("span.p%d_%s", period, side))
if spanTeamGoals.Length() == 0 {
errmsg := fmt.Sprintf(`%s team goals for period %d not found (tag <span class="p%d_%s">)`,
strings.ToUpper(side), period, period, side)
err = fmt.Errorf(errmsg)
return
}
textNode := spanTeamGoals.Get(0).FirstChild
if textNode == nil {
err = fmt.Errorf("Teams goals <span> doesn't have a child node")
return
}
goalsStr := strings.TrimSpace(textNode.Data)
goals, err = strconv.Atoi(goalsStr)
if err != nil {
err = fmt.Errorf("Parse team goals: %s; text=%q", err, goalsStr)
return
}
return
}
func getTagContent(tagSelection *goquery.Selection) (_ string, err error) {
txt, err := tagSelection.Html()
if err != nil {
return
}
return html.UnescapeString(txt), nil
}
func parseScore(score string) (homeGoals, awayGoals int, err error) {
list := strings.Split(score, ":")
if len(list) != 2 {
err = fmt.Errorf("Wrong score string")
return
}
homeGoals, err = strconv.Atoi(strings.TrimSpace(list[0]))
if err != nil {
return
}
awayGoals, err = strconv.Atoi(strings.TrimSpace(list[1]))
if err != nil {
return
}
return
}
var reDirtyTeamID = regexp.MustCompile(`'/team([^']+)'`)
func parseTeam(divList *goquery.Selection) (team model.Team, err error) {
if divList.Length() == 0 {
err = fmt.Errorf("Team div not found")
return
}
aList := divList.First().Find("a.participant-imglink")
if aList.Length() == 0 {
err = fmt.Errorf("a.participant-imglink not found")
return
}
a := aList.Get(0)
textNode := a.FirstChild
if textNode == nil {
err = fmt.Errorf("a.participant-imglink have not content")
return
}
if textNode.Type != xhtml.TextNode {
err = fmt.Errorf("First child of a.participant-imglink must be TextNode, not %s",
nodeTypes[textNode.Type])
return
}
team.CanonicalName = strings.TrimSpace(textNode.Data)
onclick, exists := aList.Eq(0).Attr("onclick")
if !exists {
err = fmt.Errorf("a.participant-imglink have not 'onclick' attr")
return
}
groups := reDirtyTeamID.FindStringSubmatch(onclick)
if len(groups) != 2 {
err = fmt.Errorf("TeamID in 'onclick' attr not found")
return
}
str := groups[1]
// На всякий случай обрезаем слэш в конце
str = strings.TrimSuffix(str, "/")
team.TeamID = str
//parts := strings.Split(str, "/")
// Последняя секция - это teamID
//team.TeamID = parts[len(parts)-1]
return
}
func parseZoneOnTeamPage(doc *goquery.Document, sportID int) (zone model.TeamZone, err error) {
h2List := doc.Find("h2.tournament")
if h2List.Length() == 0 {
err = fmt.Errorf("Tag h2.tournament not found")
return
}
aList := h2List.First().Find("a")
if aList.Length() == 0 {
err = fmt.Errorf("Tags <a> not found in h2.tournament tag")
return
}
for i := 0; i < aList.Length(); i++ {
a := aList.Eq(i)
href, exists := a.Attr("href")
if !exists {
continue
}
href = strings.TrimSpace(href)
prefix, ok := sportPrefixes[sportID]
if !ok {
panic(fmt.Sprintf("Bug: sportID %d hasn't sport prefix", sportID))
}
href = strings.TrimSuffix(href, "/")
if href == prefix {
continue
}
if !strings.HasPrefix(href, prefix) {
continue
}
if href == "" {
continue
}
// Мы нашли zoneID
// Достаем имя зоны
textNode := aList.Get(i).FirstChild
if textNode == nil {
err = fmt.Errorf("Zone name not found in <a> content")
return
}
if textNode.Type != xhtml.TextNode {
err = fmt.Errorf("FirstChild of tag <a> must be TextNode, not %s",
nodeTypes[textNode.Type])
return
}
zone = model.TeamZone{
ZoneID: href,
Name: strings.TrimSpace(textNode.Data),
SportID: sportID,
}
return
}
//err = fmt.Errorf("ZoneID not found in <a> list")
return
}
// Футбол, парсим HTML код
func parseFootballTeamScoreByPeriods(doc *goquery.Document) (m teamScoreByPeriods, hasScoreByPeriods bool, err error) {
// Счет 1го тайма
m.P1HomeGoals, err = parsePeriodTeamGoals(doc, 1, "home")
if err != nil {
return
}
m.P1AwayGoals, err = parsePeriodTeamGoals(doc, 1, "away")
if err != nil {
return
}
// Счет 2го тайма
m.P2HomeGoals, err = parsePeriodTeamGoals(doc, 2, "home")
if err != nil {
return
}
m.P2AwayGoals, err = parsePeriodTeamGoals(doc, 2, "away")
if err != nil {
return
}
hasScoreByPeriods = true
return
}
// ODDS
// Футбол, парсим HTML код
func parseOddsTotal(doc *goquery.Document) (totals []Total, err error) {
// 2.5 => *Total
totalMap := make(map[string]*Total)
divTotal := doc.Find("#block-under-over-ft")
if divTotal.Length() == 0 {
err = fmt.Errorf("#block-under-over-ft not found")
return
}
tdList := divTotal.First().Find("td.bookmaker")
if tdList.Length() == 0 {
err = fmt.Errorf("List of td.bookmaker in #block-under-over-ft not found")
return
}
for idx := 0; idx < tdList.Length(); idx++ {
//tdList.Each(func(idx int, tdSelection *goquery.Selection) {
var offer TotalOffer
//.Get(0).Data //.Html()
//fmt.Printf("%s\n", txt)
//fmt.Printf("%v\n", tdSelection)
//tdSelection.Get(0)
//tdSelection.Next()
tdSelection := tdList.Eq(idx)
offer.Bookmaker, err = getBookmaker(tdSelection)
if err != nil {
return
}
priceTd := tdSelection.Next()
var totalValue string
totalValue, err = getTotalValue(priceTd)
if err != nil {
err = fmt.Errorf("get total value: %s", err)
return
}
priceTd = priceTd.Next()
var isRemovedByBookmaker bool
isRemovedByBookmaker, offer.Over, err = getTotalPrice(priceTd)
if err != nil {
err = fmt.Errorf("get over price: %s", err)
return
}
if isRemovedByBookmaker {
continue
}
priceTd = priceTd.Next()
isRemovedByBookmaker, offer.Under, err = getTotalPrice(priceTd)
if err != nil {
err = fmt.Errorf("get under price: %s", err)
return
}
if isRemovedByBookmaker {
continue
}
total, ok := totalMap[totalValue]
if !ok {
total = &Total{
Total: totalValue,
}
totalMap[totalValue] = total
}
total.Offers = append(total.Offers, offer)
}
for _, ptotal := range totalMap {
totals = append(totals, *ptotal)
}
return
}
func getTotalValue(tdSelection *goquery.Selection) (total string, err error) {
if tdSelection == nil {
err = fmt.Errorf("<td> total tag selection is nil")
return
}
tdNode := tdSelection.Get(0)
textNode := tdNode.FirstChild
if textNode == nil {
err = fmt.Errorf("Empty total's <td>")
return
}
if textNode.Type != xhtml.TextNode {
err = fmt.Errorf("First child of total's <td> must be TextNode, not %s",
nodeTypes[textNode.Type])
return
}
total = strings.TrimSpace(textNode.Data)
return
}
// Цена может быть перечеркнута, значит ее нужно выкинуть из результата
func getTotalPrice(tdSelection *goquery.Selection) (isRemovedByBookmaker bool, price float64, err error) {
if tdSelection == nil {
err = fmt.Errorf("<td> total price tag selection is nil")
return
}
spanSelection := tdSelection.Find("span")
if spanSelection.Length() == 0 {
err = fmt.Errorf("<span> not found in <td>")
return
}
if spanSelection.First().HasClass("not-published") {
isRemovedByBookmaker = true
return
}
span := spanSelection.Get(0)
textNode := span.FirstChild
if textNode == nil {
err = fmt.Errorf("Empty total's price <span>")
return
}
if textNode.Type != xhtml.TextNode {
err = fmt.Errorf("First child of total's price <span> must be TextNode, not %s",
nodeTypes[textNode.Type])
return
}
priceStr := strings.TrimSpace(textNode.Data)
if priceStr == "-" {
isRemovedByBookmaker = true
return
}
price, err = strconv.ParseFloat(priceStr, 64)
if err != nil {
err = fmt.Errorf("Parse price: %s; priceStr=%q", err, priceStr)
return
}
return
}
func parseHandballMatchScore(str string) (homeGoals, awayGoals int, err error) {
//fmt.Printf("SCORE: %s\n", str)
list, err := ParseFlashscoreEncoding(str)
if err != nil {
return
}
// DE home с учетом ОТ
// DF away с учетом ОТ
// DG home основное время
// DH away основное время
var isHomeGoalsFound, isAwayGoalsFound bool
for _, m := range list {
if m["DG"] != "" {
homeGoals, err = strconv.Atoi(m["DG"])
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, m["DG"])
return
}
isHomeGoalsFound = true
//fmt.Printf("DH: %s, goals: %d\n", m["DH"], awayGoals)
}
if m["DH"] != "" {
awayGoals, err = strconv.Atoi(m["DH"])
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, m["DH"])
return
}
isAwayGoalsFound = true
//fmt.Printf("DG: %s, goals: %d\n", m["DG"], homeGoals)
}
}
if isHomeGoalsFound && isAwayGoalsFound {
return
}
err = fmt.Errorf("Home goals or/and Away goals not found")
return
}
func parseHandballTeamScoreByPeriods(str string) (stat teamScoreByPeriods, hasScoreByPeriods bool, err error) {
// AC÷3¬ вероятно статус
// BA÷15¬ p1 home
// BB÷10¬ p1 away
// ~BC÷15¬ p2 home
// BD÷14¬ p2 away
// ~A1÷¬~
var isP1HomeFound, isP1AwayFound, isP2HomeFound, isP2AwayFound bool
blocks := strings.Split(str, SymbolTilda)
for _, block := range blocks {
block = strings.TrimSpace(block)
if block == "" {
continue
}
pairs := strings.Split(block, SymbolNot)
for _, pair := range pairs {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
kv := strings.Split(pair, SymbolDivide)
if len(kv) != 2 {
err = fmt.Errorf("Wrong key/value: %s", pair)
return
}
key := strings.TrimSpace(kv[0])
value := strings.TrimSpace(kv[1])
switch key {
case "BA":
stat.P1HomeGoals, err = strconv.Atoi(value)
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, value)
return
}
isP1HomeFound = true
case "BB":
stat.P1AwayGoals, err = strconv.Atoi(value)
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, value)
return
}
isP1AwayFound = true
case "BC":
stat.P2HomeGoals, err = strconv.Atoi(value)
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, value)
return
}
isP2HomeFound = true
case "BD":
stat.P2AwayGoals, err = strconv.Atoi(value)
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, value)
return
}
isP2AwayFound = true
}
}
}
if isP1HomeFound && isP1AwayFound && isP2HomeFound && isP2AwayFound {
hasScoreByPeriods = true
return
}
// В этом месте нет ошибки, но hasScoreByPeriods = false
return
}
type PeriodScore struct {
PeriodNumber int
HomeScore int
AwayScore int
}
func parseTennisTeamScoreByPeriods(str string) (periods []PeriodScore, hasScoreByPeriods bool, err error) {
// Счет 2:1, по сетам: 7-6(7-2) 4-6 6-2
// Тильда ~ вероятно разделяет периоды
// AC÷3¬ вероятно статус
// BA÷7¬ s1 home
// DA÷7¬ s1 tiebreak home
// BB÷6¬ s1 away
// DB÷2¬ s1 tiebreak away
// RC÷1:05¬
// ~
// BC÷4¬ s2 home
// BD÷6¬ s2 away
// RD÷0:50¬
// ~
// BE÷6¬ s3 home
// BF÷2¬ s3 away
// RE÷0:43¬
// ~
// RB÷2:38¬
// ~
// PSPH÷WC¬
// PSPA÷11¬
// A1÷¬~
// Счет 3:2, 3-6 7-6(7-5) 7-6(7-4) 0-6 6-3
// AC÷3¬
// BA÷3¬
// BB÷6¬
// RC÷0:36¬ продолжительность периода
// ~
// BC÷7¬
// DC÷7¬
// BD÷6¬
// DD÷5¬
// RD÷1:18¬
// ~
// BE÷7¬
// DE÷7¬
// BF÷6¬
// DF÷4¬
// RE÷1:08¬
// ~
// BG÷0¬ s4 home
// BH÷6¬ s4 away
// RF÷0:29¬
// ~
// BI÷6¬ s5 home
// BJ÷3¬ s5 away
// RG÷0:38¬
// ~
// RB÷4:09¬ продолжительность матча
// ~
// PSPH÷20¬
// PSPA÷12¬
// A1÷¬~
blocks := strings.Split(str, SymbolTilda)
for _, block := range blocks {
block = strings.TrimSpace(block)
if block == "" {
continue
}
var (
period PeriodScore
periodSignature string
)
pairs := strings.Split(block, SymbolNot)
for _, pair := range pairs {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
kv := strings.Split(pair, SymbolDivide)
if len(kv) != 2 {
err = fmt.Errorf("Wrong key/value: %s", pair)
return
}
key := strings.TrimSpace(kv[0])
value := strings.TrimSpace(kv[1])
switch key {
case "BA", "BC", "BE", "BG", "BI":
period.HomeScore, err = strconv.Atoi(value)
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, value)
return
}
// записываем в начало, чтобы независимо от порядка полей в исходной
// строке получить одинаковую подпись
periodSignature = key + periodSignature
case "BB", "BD", "BF", "BH", "BJ":
period.AwayScore, err = strconv.Atoi(value)
if err != nil {
err = fmt.Errorf("Atoi: %s; string=%s", err, value)
return
}
// дописываем в конец, чтобы независимо от порядка полей в исходной
// строке получить одинаковую подпись
periodSignature += key
}
}
switch periodSignature {
case "BABB":
period.PeriodNumber = 1
case "BCBD":
period.PeriodNumber = 2
case "BEBF":
period.PeriodNumber = 3
case "BGBH":
period.PeriodNumber = 4
case "BIBJ":
period.PeriodNumber = 5
default:
if periodSignature != "" {
// Что-то нашло но не то, что нужно (например только home),
// поэтому обработку прерываем
return
}
}
if period.PeriodNumber > 0 {
periods = append(periods, period)
}
}
if len(periods) > 0 {
// Периоды найдены, обработчка не была прервана, значит все ок
hasScoreByPeriods = true
}
return
}
//type FlashscoreEncodedData
func ParseFlashscoreEncoding(str string) (_ []map[string]string, err error) {
var list []map[string]string
blocks := strings.Split(str, SymbolTilda)
for _, block := range blocks {
block = strings.TrimSpace(block)
if block == "" {
continue
}
pairs := strings.Split(block, SymbolNot)
m := make(map[string]string)
for _, pair := range pairs {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
kv := strings.Split(pair, SymbolDivide)
if len(kv) != 2 {
err = fmt.Errorf("Wrong key/value: %s", pair)
return
}
key := strings.TrimSpace(kv[0])
value := strings.TrimSpace(kv[1])
m[key] = value
}
list = append(list, m)
}
return list, nil
}
func parseFlashscoreEncodedOdds(str string) (odds Odds, err error) {
decoded, err := ParseFlashscoreEncoding(str)
if err != nil {
return
}
var (
market string
period string
total *Total
totals = make(map[string]*Total)
h1Totals = make(map[string]*Total)
)
for _, m := range decoded {
if market == "" {
// Ищем рынок
market = getMarket(m["OAI"])
} else if period == "" {
// Рынок есть - ищем период
if m["OAI"] != "" {
// Нужный период не нашли, но встретили новый рынок
market = getMarket(m["OAI"])
period = ""
} else if m["OBI"] != "" {
period = getPeriod(m["OBI"])
}
} else {
// Есть и рынок и период
switch market {
case "1x2":
// Накапливаем offers букмекеров пока не встретим OAI или OBI
if m["OAI"] != "" {
// Встретили новый рынок
market = getMarket(m["OAI"])
period = ""
} else if m["OBI"] != "" {
// Встретили новый период
period = getPeriod(m["OBI"])
} else {
// Ищем предложения букмекеров
if m["OG"] == "0" {
// Перечеркнутое предложение
continue
}
if m["XA"] != "" && m["XB"] != "" && m["XC"] != "" {
winner, perr := parseFlashscoreEncodedWinner3Way(m)
if perr != nil {
// log
fmt.Println(perr)
} else {
odds.Winner = append(odds.Winner, winner)
}
}
}
case "12":
// Накапливаем offers букмекеров пока не встретим OAI или OBI
if m["OAI"] != "" {
// Встретили новый рынок
market = getMarket(m["OAI"])
period = ""
} else if m["OBI"] != "" {
// Встретили новый период
period = getPeriod(m["OBI"])
} else {
// Ищем предложения букмекеров
if m["OG"] == "0" {
// Перечеркнутое предложение
continue
}
if m["XB"] != "" && m["XC"] != "" {
winner, perr := parseFlashscoreEncodedWinner2Way(m)
if perr != nil {
// log
fmt.Println(perr)
} else {
odds.Winner2Way = append(odds.Winner2Way, winner)
}
}
}
case "total":
switch period {
case "ft":
// Накапливаем offers букмекеров пока не встретим OAI или OBI
if m["OAI"] != "" {
// Встретили новый рынок
market = getMarket(m["OAI"])
period = ""
total = nil
//totalValue = ""
} else if m["OBI"] != "" {
period = getPeriod(m["OBI"])
total = nil
} else if m["OCT"] == "Total" || m["OCT"] == "Games" {
// Total - handball, football; Games - tennis
totalValue := m["OC"]
total = totals[totalValue]
if total == nil {
total = &Total{
Total: totalValue,
}
totals[totalValue] = total
}
} else {
if m["XB"] != "" && m["XC"] != "" {
isRemovedByBookmaker, offer, perr := parseFlashscoreEncodedTotalOffer(m)
if perr != nil {
// log
fmt.Println(perr)
} else {
if !isRemovedByBookmaker {
if total != nil {
total.Offers = append(total.Offers, offer)
}
}
}
}
}
case "1hf", "set1":
// Накапливаем offers букмекеров пока не встретим OAI или OBI
if m["OAI"] != "" {
// Встретили новый рынок
market = getMarket(m["OAI"])
period = ""
total = nil
//totalValue = ""
} else if m["OBI"] != "" {
period = getPeriod(m["OBI"])
total = nil
} else if m["OCT"] == "Total" || m["OCT"] == "Games" {
// Total - handball, football; Games - tennis
totalValue := m["OC"]
total = totals[totalValue]
if total == nil {
total = &Total{
Total: totalValue,
}
h1Totals[totalValue] = total
}
} else {
if m["XB"] != "" && m["XC"] != "" {
isRemovedByBookmaker, offer, perr := parseFlashscoreEncodedTotalOffer(m)
if perr != nil {
// log
fmt.Println(perr)
} else {
if !isRemovedByBookmaker {
if total != nil {
total.Offers = append(total.Offers, offer)
}
}
}
}
}
default:
panic(fmt.Sprintf("Bug: unknown total period %q", period))
}
}
}
}
for _, ptotal := range totals {
odds.FullTimeTotal = append(odds.FullTimeTotal, *ptotal)
}
for _, ptotal := range h1Totals {
odds.H1Total = append(odds.H1Total, *ptotal)
}
return
}
// little helper
func getMarket(code string) string {
switch code {
// football, handball
case "1x2":
return "1x2"
// football, handball, tennis
case "under-over":
return "total"
// tennis
case "moneyline":
return "12"
default:
return ""
}
}
// little helper
func getPeriod(code string) string {
switch code {
// football, handball, tennis
case "ft":
return "ft"
// football, handball
case "1hf":
return "1hf"
// tennis
case "set1":
return "set1"
default:
return ""
}
}
func parseFlashscoreEncodedWinner3Way(m map[string]string) (winner Winner3Way, err error) {
winner.Bookmaker, err = strconv.Atoi(m["OE"])
if err != nil {
err = fmt.Errorf("Parse bookmakerID: %s; string=%s", err, m["OE"])
return
}
winner.Home, err = parseFlashscoreEncodedPrice(m["XA"])
if err != nil {
return
}
winner.Away, err = parseFlashscoreEncodedPrice(m["XC"])
if err != nil {
return
}
winner.Draw, err = parseFlashscoreEncodedPrice(m["XB"])
if err != nil {
return
}
return
}
func parseFlashscoreEncodedWinner2Way(m map[string]string) (winner Winner2Way, err error) {
winner.Bookmaker, err = strconv.Atoi(m["OE"])
if err != nil {
err = fmt.Errorf("Parse bookmakerID: %s; string=%s", err, m["OE"])
return
}
winner.Home, err = parseFlashscoreEncodedPrice(m["XB"])
if err != nil {
return
}
winner.Away, err = parseFlashscoreEncodedPrice(m["XC"])
if err != nil {
return
}
return
}
func parseFlashscoreEncodedTotalOffer(m map[string]string) (isRemovedByBookmaker bool, offer TotalOffer, err error) {
offer.Bookmaker, err = strconv.Atoi(m["OE"])
if err != nil {
err = fmt.Errorf("Parse bookmakerID: %s; string=%s", err, m["OE"])
return
}
offer.Over, err = parseFlashscoreEncodedPrice(m["XB"])
if err != nil {
return
}
offer.Under, err = parseFlashscoreEncodedPrice(m["XC"])
if err != nil {
return
}
// Перечеркнутое предложение
if m["OG"] == "0" {
isRemovedByBookmaker = true
}
return
}
// Функция пасит простые поля вида "1.44" и более сложные вида "2.24[u]2.39"
func parseFlashscoreEncodedPrice(str string) (price float64, err error) {
str = strings.TrimSpace(str)
list := strings.Split(str, "]")
// Последний элемент
str = list[len(list)-1]
price, err = strconv.ParseFloat(str, 64)
if err != nil {
err = fmt.Errorf("Parse price: %s; string=%s", err, str)
return
}
return
}
///////////////
// little helper
func getH2HTab(tabName string) string {
tabName = strings.ToUpper(tabName)
if strings.Contains(tabName, "OVERALL") {
return "overall"
}
return ""
}
// little helper
func getH2HList(listName, homeTeamName, awayTeamName string) string {
// Приводим все строки к единому регистру
listName = strings.ToUpper(listName)
homeTeamName = strings.ToUpper(homeTeamName)
awayTeamName = strings.ToUpper(awayTeamName)
// 3 списка, примеры:
//
// Last matches: HSG WETZLAR
// Last matches: MT MELSUNGEN
// Head-to-head matches: HSG WETZLAR - MT MELSUNGEN
//fmt.Printf("listName: '%s'; home='%s'; away='%s'\n", listName, homeTeamName, awayTeamName)
// 'HEAD-TO-HEAD MATCHES' если h2h матчей нет
if strings.Contains(listName, homeTeamName) {
if strings.Contains(listName, awayTeamName) {
// Обе команды
return "h2h"
}
// Только home
return "home"
}
if strings.Contains(listName, awayTeamName) {
// Только away
return "away"
}
return ""
}
func parseFlashscoreEncodedH2H(str, homeTeamName, awayTeamName string) (homeTeamMatches, awayTeamMatches, h2hMatches []model.TeamMatch, err error) {
decoded, err := ParseFlashscoreEncoding(str)
if err != nil {
return
}
pretty.PPrintln("DECODED", decoded)
var (
// Tabs на H2H странице: Overall, HSG Wetzlar - Home, MT Melsungen - Away
tab string
// Списки последних матчей: хозяев, гостей, h2h
list string
)
for _, m := range decoded {
if tab == "" {
// Ищем Tab
tab = getH2HTab(m["KA"])
} else if list == "" {
// Рынок есть - ищем период
if m["KA"] != "" {
// Нужный период не нашли, но встретили новый рынок
tab = getH2HTab(m["KA"])
list = ""
} else if m["KB"] != "" {
list = getH2HList(m["KB"], homeTeamName, awayTeamName)
}
} else {
// Есть и рынок и список
if m["KA"] != "" {
// Нужный период не нашли, но встретили новый рынок
tab = getH2HTab(m["KA"])
list = ""
} else if m["KB"] != "" {
list = getH2HList(m["KB"], homeTeamName, awayTeamName)
} else {
if m["KC"] != "" && m["KP"] != "" {
//startTime, _ := strconv.ParseInt(m["KC"], 10, 64)
match := model.TeamMatch{
MatchID: m["KP"],
//ChampID: m["KJ"] + " - " + m["KK"], Чисто для дебага можно выводить
}
switch list {
case "home":
homeTeamMatches = append(homeTeamMatches, match)
case "away":
awayTeamMatches = append(awayTeamMatches, match)
case "h2h":
h2hMatches = append(h2hMatches, match)
default:
panic(fmt.Sprintf("Bug: unknown Tab's listName %q", list))
}
}
}
}
}
return
}
//ZA zone: champ name extended
//ZL "champID"
/*
AA: 2XhOe000
AD: 1595772000
ADE: 1595772000
AB: 1
CR: 1
AC: 1
CX: Southampton
ER: Round 38
RW: 0
AX: 0
BX: -1
WQ:
WM: SOU
AE: Southampton
JA: fu16YetP
WU: southampton
WN: SHU
AF: Sheffield Utd
JB: bDRnslpP
WV: sheffield-utd
AW: 1
AN: n
Расшифровка:
ZA - champ name
AA - ID матча, нужен чтобы получить статистику по матчу
AD / ADE - время начала матча в UTC
CX - home team
AF - away team
WU - home team url part (нужно ли ?)
WV - away team url part (нужно ли ?)
ER - раунд (можно не парсить, но сгруппировать)
*/
func parseFlashscoreEncodedAllGames(str string, sportID int) (matches []ScheduledMatch, err error) {
decoded, err := ParseFlashscoreEncoding(str)
if err != nil {
return
}
var (
champID string
zone model.TeamZone
fullChampName string
)
for _, m := range decoded {
zl := m["ZL"] // если поле непустое - словарь описывает чемпионат, а не матч
if zl != "" {
champID = strings.TrimSpace(zl)
champID = strings.TrimSuffix(champID, "/")
// Пример /football/vietnam/v-league-1
parts := strings.Split(champID, "/")
if len(parts) < 3 {
continue
}
zone.ZoneID = strings.Join(parts[:3], "/")
parts = strings.Split(m["ZA"], ":")
zone.Name = strings.TrimSpace(strings.Title(strings.ToLower(parts[0])))
zone.SportID = sportID
fullChampName = m["ZA"]
} else {
//if m["AA"] != "" && m["AF"] != "" && m["CX"] != "" {
if m["AA"] != "" && m["AD"] != "" {
startTime, _ := strconv.ParseInt(m["AD"], 10, 64)
match := ScheduledMatch{
MatchID: m["AA"],
StartTime: time.Unix(startTime, 0),
ChampID: champID,
Zone: zone,
SportID: sportID,
FullChampName: fullChampName,
}
matches = append(matches, match)
}
}
}
return
}
// Служебные структуры для метода parseTeamsOnMatchPage
type jsObjectTeam struct {
TeamID string `json:"id"`
Name string `json:"name"`
}
type jsObjectParticipantsData struct {
Home []jsObjectTeam `json:"home"`
Away []jsObjectTeam `json:"away"`
}
type jsVarEnvironment struct {
ParticipantsData jsObjectParticipantsData `json:"participantsData"`
}
// parseTeamsOnMatchPageInJsVar - метод извлекает имена и TeamID команд из тега <script>.
// Гандбольные матчи хранят эти данные в json в теге <script>.
func parseTeamsOnMatchPageInJsVar(doc *goquery.Document) (home, away model.Team, err error) {
var jsonVar []byte
//doc.Find("script[type='text/javascript']").EachWithBreak(func(idx int, x *goquery.Selection) bool {
doc.Find("script").EachWithBreak(func(idx int, x *goquery.Selection) bool {
txt := x.Text()
txt = strings.TrimSpace(txt)
prefixes := []string{
"window.environment = ",
"var environment = ",
}
suffix := ";"
for _, prefix := range prefixes {
if strings.HasPrefix(txt, prefix) {
txt = strings.TrimPrefix(txt, prefix)
txt = strings.TrimSuffix(txt, suffix)
jsonVar = []byte(txt)
// JS код нашли - прерываем цикл
return false
}
}
// JS код не нашли - переходим к следующему SCRIPT тегу
return true
})
//fmt.Printf("%q\n\n", jsVar[:20])
var env jsVarEnvironment
err = json.Unmarshal([]byte(jsonVar), &env)
if err != nil {
return
}
h := env.ParticipantsData.Home[0]
a := env.ParticipantsData.Away[0]
home.TeamID = h.TeamID
home.CanonicalName = h.Name
away.TeamID = a.TeamID
away.CanonicalName = a.Name
return
}
/*
//var reChampID = regexp.MustCompile(`'/football/([^/]+)/([^/]+)/'`)
var reChampID = regexp.MustCompile(`'/([^/]+)/([^/]+)/([^/]+)/'`)
func parseTeamMatchInfo(doc *goquery.Document, sportID int) (zone model.TeamZone, champ model.TeamChamp, home, away model.Team, err error) {
spanList := doc.Find(".description__country")
if spanList.Length() == 0 {
err = fmt.Errorf("Tag .description__country not found")
return
}
span := spanList.Get(0)
dirtyZone := span.FirstChild
if dirtyZone == nil {
err = fmt.Errorf("Tag .description__country have not children")
return
}
if dirtyZone.Type != xhtml.TextNode {
err = fmt.Errorf("First child of tag .description__country must be TextNode, not %s",
nodeTypes[dirtyZone.Type])
return
}
txt := strings.TrimSpace(dirtyZone.Data)
txt = strings.TrimSuffix(txt, ":")
if txt == "" {
err = fmt.Errorf("Empty zone name found in tag .description__country")
return
}
txt = strings.ToLower(txt)
txt = strings.Title(txt)
zone.Name = txt
zone.SportID = sportID
a := dirtyZone.NextSibling
if a == nil {
err = fmt.Errorf("Tag <a> with champ info not found in the tag .description__country")
return
}
var onclick string
for _, attr := range a.Attr {
if strings.ToLower(attr.Key) == "onclick" {
onclick = attr.Val
break
}
}
if onclick == "" {
err = fmt.Errorf("Attr 'onclick' not found in the tag <a> in the .description__country tag")
return
}
// window.open('/football/england/premier-league/'); return false;
groups := reChampID.FindStringSubmatch(onclick)
if len(groups) != 4 {
err = fmt.Errorf("ChampID and ZoneID not found in 'onclick' attr: %s", onclick)
return
}
zone.ZoneID = fmt.Sprintf("/%s/%s", groups[1], groups[2])
champ.ChampID = fmt.Sprintf("/%s/%s/%s", groups[1], groups[2], groups[3])
// Не забываем!
champ.ZoneID = zone.ZoneID
if a.FirstChild == nil {
err = fmt.Errorf("Empty champ name in <a> tag")
return
}
dirtyChamp := a.FirstChild
if dirtyChamp.Type != xhtml.TextNode {
err = fmt.Errorf("Tag <a> content must be TextNode, not %s",
nodeTypes[dirtyChamp.Type])
return
}
parts := strings.Split(dirtyChamp.Data, "-")
// Возможный чемпионат 'Club Friendly', где нет раунда, поэтому нужна проверка
if len(parts) > 1 {
// Последний элемент - это раунд. Выкидываем
parts = parts[:len(parts)-1]
}
champ.Name = strings.TrimSpace(strings.Join(parts, "-"))
// ВАЖНО!
// Зону для команды узнаем на странице команды!
home, err = parseTeam(doc.Find(".tname-home"))
if err != nil {
err = fmt.Errorf("Parse home team: %s", err)
return
}
away, err = parseTeam(doc.Find(".tname-away"))
if err != nil {
err = fmt.Errorf("Parse away team: %s", err)
return
}
return
}
*/
Reference in New Issue Copy Permalink