fix: character issues with umlauts

src/TibiaDataUtils.go

@@ -87,9 +87,6 @@ func TibiaDataQueryEscapeString(data string) string {
 	// switching "+" to " "
 	data = strings.ReplaceAll(data, "+", " ")
 
-	// encoding string to latin-1
-	data, _ = TibiaDataConvertEncodingtoISO88591(data)
-
 	// returning with QueryEscape function
 	return url.QueryEscape(data)
 }

src/TibiaDataUtils_test.go

@@ -150,16 +150,19 @@ func TestEscaper(t *testing.T) {
 		strOne   = "god durin"
 		strTwo   = "god+durin"
 		strThree = "gód"
+		strFour  = "Näurin"
 	)
 
 	sanitizedStrOne := TibiaDataQueryEscapeString(strOne)
 	sanitizedStrTwo := TibiaDataQueryEscapeString(strTwo)
 	sanitizedStrThree := TibiaDataQueryEscapeString(strThree)
+	sanitizedStrFour := TibiaDataQueryEscapeString(strFour)
 
 	assert := assert.New(t)
 	assert.Equal(sanitizedStrOne, "god+durin")
 	assert.Equal(sanitizedStrTwo, "god+durin")
-	assert.Equal(sanitizedStrThree, "g%F3d")
+	assert.Equal(sanitizedStrThree, "g%C3%B3d")
+	assert.Equal(sanitizedStrFour, "N%C3%A4urin")
 }
 
 func TestDateParser(t *testing.T) {

src/TibiaWorldsWorld.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"golang.org/x/net/html/charset"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,8 +54,12 @@ var (
 func TibiaWorldsWorldImpl(world string, BoxContentHTML string, url string) (WorldResponse, error) {
 	// TODO: We need to read the world name from the response rather than pass it into this func
 
+	utf8Reader, err := charset.NewReader(strings.NewReader(BoxContentHTML), "")
+	if err != nil {
+		return WorldResponse{}, fmt.Errorf("[error] TibiaWorldsWorldImpl failed at charset.NewReader, err: %s", err)
+	}
 	// Loading HTML data into ReaderHTML for goquery with NewReader
-	ReaderHTML, err := goquery.NewDocumentFromReader(strings.NewReader(BoxContentHTML))
+	ReaderHTML, err := goquery.NewDocumentFromReader(utf8Reader)
 	if err != nil {
 		return WorldResponse{}, fmt.Errorf("[error] TibiaWorldsWorldImpl failed at goquery.NewDocumentFromReader, err: %s", err)
 	}

src/TibiaWorldsWorld_test.go

@@ -1,13 +1,47 @@
 package main
 
 import (
+	"golang.org/x/text/encoding/charmap"
 	"io"
 	"testing"
 
 	"github.com/TibiaData/tibiadata-api-go/src/static"
 	"github.com/stretchr/testify/assert"
 )
 
+func TestWorldAntica_UmlautRecovery(t *testing.T) {
+	// Load the UTF‑8 HTML fixture
+	f, err := static.TestFiles.Open("testdata/worlds/world/Antica.html")
+	if err != nil {
+		t.Fatalf("opening test file: %v", err)
+	}
+	defer f.Close()
+
+	utf8Data, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatalf("reading test file: %v", err)
+	}
+
+	// Re‑encode to ISO‑8859‑1 to simulate the real‑world mis‑encoding - this is what we receive from as encoding
+	isoBytes, err := charmap.ISO8859_1.NewEncoder().Bytes(utf8Data)
+	if err != nil {
+		t.Fatalf("re‑encoding to ISO‑8859‑1 failed: %v", err)
+	}
+
+	// check function TibiaWorldsWorldImpl
+	resp, err := TibiaWorldsWorldImpl("Antica", string(isoBytes), "")
+	if err != nil {
+		t.Fatalf("parser error: %v", err)
+	}
+	got := resp.World.OnlinePlayers[0].Name
+
+	// It must *not* be the mojibake sequence
+	assert.NotEqual(t, "Näurin", got, "should not return mojibake")
+
+	// It must be the correct Umlaut
+	assert.Equal(t, "Näurin", got, "should correctly decode Umlaut from input")
+}
+
 func TestWorldEndebra(t *testing.T) {
 	file, err := static.TestFiles.Open("testdata/worlds/world/Endebra.html")
 	if err != nil {

src/webserver.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"golang.org/x/net/html/charset"
 	"log"
 	"net/http"
 	"os"
@@ -1289,14 +1290,16 @@ func TibiaDataHTMLDataCollector(TibiaDataRequest TibiaDataRequestStruct) (string
 		return string(res.Body()), nil
 	}
 
-	// Convert body to io.Reader
-	resIo := bytes.NewReader(res.Body())
-
-	// wrap reader in a converting reader from ISO 8859-1 to UTF-8
-	resIo2 := TibiaDataConvertEncodingtoUTF8(resIo)
-
-	// Load the HTML document
-	doc, err := goquery.NewDocumentFromReader(resIo2)
+	// Decode the raw response into real UTF-8 using the Content‑Type header
+	utf8Reader, err := charset.NewReader(
+		bytes.NewReader(res.Body()),
+		res.Header().Get("Content-Type"),
+	)
+	if err != nil {
+		log.Printf("[error] TibiaDataHTMLDataCollector charset.NewReader failed: %s", err)
+		return "", err
+	}
+	doc, err := goquery.NewDocumentFromReader(utf8Reader)
 	if err != nil {
 		log.Printf("[error] TibiaDataHTMLDataCollector (URL: %s) error: %s", res.Request.URL, err)
 		return "", err