From b1161377dcb23fd88d01f587740e968f5f9efc8a Mon Sep 17 00:00:00 2001 From: Kennedy Bushnell Date: Wed, 22 Apr 2026 22:32:27 -0700 Subject: [PATCH 1/3] Avoid empty updates --- src/workers.go | 73 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/src/workers.go b/src/workers.go index 93635f0..7711c59 100644 --- a/src/workers.go +++ b/src/workers.go @@ -83,22 +83,37 @@ func (b *Builder) housesWorker(client *resty.Client) error { return 0 }() + const maxRetries = 5 + for _, town := range b.Towns { log.Printf("[info] Retrieving data about houses and guildhalls in %s.", town) ApiUrl := "https://" + TibiaDataAPIhost + "/v4/houses/" + b.Worlds[worldsIndex] + "/" + url.QueryEscape(town) - res, err := client.R().Get(ApiUrl) - if err != nil { - return fmt.Errorf("issue getting %s endpoint. Error: %s", ApiUrl, err) - } - switch res.StatusCode() { - case http.StatusOK: + var lastErr error + var success bool + + for attempt := 1; attempt <= maxRetries; attempt++ { + res, err := client.R().Get(ApiUrl) + if err != nil { + lastErr = fmt.Errorf("issue getting %s endpoint. Error: %s", ApiUrl, err) + log.Printf("[warn] Attempt %d/%d failed for %s: %s", attempt, maxRetries, town, lastErr) + time.Sleep(time.Duration(attempt) * time.Second) + continue + } + + if res.StatusCode() != http.StatusOK { + lastErr = fmt.Errorf("non-200 status retrieving houses for %s. StatusCode: %d", town, res.StatusCode()) + log.Printf("[warn] Attempt %d/%d failed for %s: %s", attempt, maxRetries, town, lastErr) + time.Sleep(time.Duration(attempt) * time.Second) + continue + } + // Get byte slice from string. bytes := []byte(res.Body()) var cont SourceHousesOverview - err := json.Unmarshal(bytes, &cont) + err = json.Unmarshal(bytes, &cont) if err != nil { return fmt.Errorf("issue when unmarshaling data. Town is %s. Err: %s", town, err) } @@ -120,8 +135,13 @@ func (b *Builder) housesWorker(client *resty.Client) error { HouseType: "guildhall", }) } - default: - log.Printf("[warn] Issue when retrieving data about houses and guildhalls in %s. StatusCode: %d", town, res.StatusCode()) + + success = true + break + } + + if !success { + return fmt.Errorf("houses refresh failed for town %s after %d attempts: %s", town, maxRetries, lastErr) } if sleepFlag { @@ -141,15 +161,24 @@ func (b *Builder) creaturesWorker(client *resty.Client) error { const raceEndpointIndexer = "&race=" var safe []string + var parseErr error creatures := doc.Find(".BoxContent .Creatures").First() creatures.Find("div").Each(func(index int, s *goquery.Selection) { + if parseErr != nil { + return + } + url, exists := s.Find("a").Attr("href") if !exists { return } raceIndex := strings.Index(url, raceEndpointIndexer) + if raceIndex == -1 { + parseErr = fmt.Errorf("unexpected HTML format from tibia.com: creature URL %q missing %q", url, raceEndpointIndexer) + return + } endpoint := strings.TrimPrefix(url[raceIndex:], raceEndpointIndexer) safe = append(safe, endpoint) pluralName := s.Find("div").First().Text() @@ -207,6 +236,10 @@ func (b *Builder) creaturesWorker(client *resty.Client) error { } }) + if parseErr != nil { + return parseErr + } + for i, s := range safe { str := SpaceMap(b.Creatures[i].Name) _, isSpecial := specialCreaturesCases[s] @@ -225,7 +258,13 @@ func (b *Builder) spellsWorker(client *resty.Client) error { return fmt.Errorf("%s, func: spellsWorker", err) } + var spellParseErr error + doc.Find(".Table3 table.TableContent tr").Each(func(index int, s *goquery.Selection) { + if spellParseErr != nil { + return + } + if index == 0 { return } @@ -233,8 +272,16 @@ func (b *Builder) spellsWorker(client *resty.Client) error { s.Find("td").EachWithBreak(func(index int, inner *goquery.Selection) bool { if index == 0 { rawText := inner.Text() - spellName := rawText[0:strings.Index(rawText, " (")] - formula := rawText[strings.Index(rawText, " (")+2 : strings.Index(rawText, ")")] + + parenOpen := strings.Index(rawText, " (") + parenClose := strings.Index(rawText, ")") + if parenOpen == -1 || parenClose == -1 { + spellParseErr = fmt.Errorf("unexpected HTML format from tibia.com: spell text %q missing expected parentheses", rawText) + return false + } + + spellName := rawText[0:parenOpen] + formula := rawText[parenOpen+2 : parenClose] var endpoint string if specialCase, isSpecial := specialSpellsCases[spellName]; isSpecial { @@ -256,5 +303,9 @@ func (b *Builder) spellsWorker(client *resty.Client) error { }) }) + if spellParseErr != nil { + return spellParseErr + } + return nil } From 88dc07f96ce227c5a54e49aaabadc05603688417 Mon Sep 17 00:00:00 2001 From: Kennedy Bushnell Date: Wed, 22 Apr 2026 22:43:15 -0700 Subject: [PATCH 2/3] simpler --- src/main.go | 4 +-- src/workers.go | 85 +++++++++++++++++++++----------------------------- 2 files changed, 38 insertions(+), 51 deletions(-) diff --git a/src/main.go b/src/main.go index 0d9facf..c7369a6 100644 --- a/src/main.go +++ b/src/main.go @@ -39,7 +39,7 @@ func main() { // Set client timeout and retry client.SetTimeout(5 * time.Second) - client.SetRetryCount(2) + client.SetRetryCount(5) // Set headers for all requests client.SetHeaders(map[string]string{ @@ -85,7 +85,7 @@ func main() { err = builder.spellsWorker(client) if err != nil { - log.Fatalf("[error] Issue with fansitesWorker. Error: %s", err) + log.Fatalf("[error] Issue with spellsWorker. Error: %s", err) } log.Println("[info] Validation of builder lists to prevent empty set of strings.") diff --git a/src/workers.go b/src/workers.go index 7711c59..2fd98e9 100644 --- a/src/workers.go +++ b/src/workers.go @@ -73,6 +73,14 @@ func (b *Builder) housesWorker(client *resty.Client) error { }) + if len(b.Worlds) == 0 { + return fmt.Errorf("no worlds found on tibia.com, possible HTML format change or maintenance") + } + + if len(b.Towns) == 0 { + return fmt.Errorf("no towns found on tibia.com, possible HTML format change or maintenance") + } + // Find the index of Antica in b.Worlds[] or fallback to first index worldsIndex := func() int { for i, world := range b.Worlds { @@ -83,65 +91,44 @@ func (b *Builder) housesWorker(client *resty.Client) error { return 0 }() - const maxRetries = 5 - for _, town := range b.Towns { log.Printf("[info] Retrieving data about houses and guildhalls in %s.", town) ApiUrl := "https://" + TibiaDataAPIhost + "/v4/houses/" + b.Worlds[worldsIndex] + "/" + url.QueryEscape(town) + res, err := client.R().Get(ApiUrl) + if err != nil { + return fmt.Errorf("issue getting %s endpoint. Error: %s", ApiUrl, err) + } - var lastErr error - var success bool - - for attempt := 1; attempt <= maxRetries; attempt++ { - res, err := client.R().Get(ApiUrl) - if err != nil { - lastErr = fmt.Errorf("issue getting %s endpoint. Error: %s", ApiUrl, err) - log.Printf("[warn] Attempt %d/%d failed for %s: %s", attempt, maxRetries, town, lastErr) - time.Sleep(time.Duration(attempt) * time.Second) - continue - } - - if res.StatusCode() != http.StatusOK { - lastErr = fmt.Errorf("non-200 status retrieving houses for %s. StatusCode: %d", town, res.StatusCode()) - log.Printf("[warn] Attempt %d/%d failed for %s: %s", attempt, maxRetries, town, lastErr) - time.Sleep(time.Duration(attempt) * time.Second) - continue - } - - // Get byte slice from string. - bytes := []byte(res.Body()) - - var cont SourceHousesOverview - err = json.Unmarshal(bytes, &cont) - if err != nil { - return fmt.Errorf("issue when unmarshaling data. Town is %s. Err: %s", town, err) - } + if res.StatusCode() != http.StatusOK { + return fmt.Errorf("non-200 status retrieving houses for %s. StatusCode: %d", town, res.StatusCode()) + } - for _, value := range cont.Houses.HouseList { - b.Houses = append(b.Houses, AssetsHouse{ - Name: value.Name, - HouseID: value.HouseID, - Town: town, - HouseType: "house", - }) - } + // Get byte slice from string. + bytes := []byte(res.Body()) - for _, value := range cont.Houses.GuildhallList { - b.Houses = append(b.Houses, AssetsHouse{ - Name: value.Name, - HouseID: value.HouseID, - Town: town, - HouseType: "guildhall", - }) - } + var cont SourceHousesOverview + err = json.Unmarshal(bytes, &cont) + if err != nil { + return fmt.Errorf("issue when unmarshaling data. Town is %s. Err: %s", town, err) + } - success = true - break + for _, value := range cont.Houses.HouseList { + b.Houses = append(b.Houses, AssetsHouse{ + Name: value.Name, + HouseID: value.HouseID, + Town: town, + HouseType: "house", + }) } - if !success { - return fmt.Errorf("houses refresh failed for town %s after %d attempts: %s", town, maxRetries, lastErr) + for _, value := range cont.Houses.GuildhallList { + b.Houses = append(b.Houses, AssetsHouse{ + Name: value.Name, + HouseID: value.HouseID, + Town: town, + HouseType: "guildhall", + }) } if sleepFlag { From 6799f956190e4a9267d7f23abd8b6e10ec9138a7 Mon Sep 17 00:00:00 2001 From: Kennedy Bushnell Date: Wed, 22 Apr 2026 22:46:45 -0700 Subject: [PATCH 3/3] exponential backoff --- src/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.go b/src/main.go index c7369a6..41e4294 100644 --- a/src/main.go +++ b/src/main.go @@ -40,6 +40,8 @@ func main() { // Set client timeout and retry client.SetTimeout(5 * time.Second) client.SetRetryCount(5) + client.SetRetryWaitTime(1 * time.Second) + client.SetRetryMaxWaitTime(30 * time.Second) // Set headers for all requests client.SetHeaders(map[string]string{