go-scraper/sitemap_parser_test.go at main · JakeWritesCode/go-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
package main

import (
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"testing"
)

func TestParseSitemapForUrls_ReturnsErrorOnInvalidXML(t *testing.T) {
	t.Parallel()
	sitemap := `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>http://example.com</loc></url></urlset`
	_, err := ParseSitemapForUrls(sitemap)
	assert.Error(t, err)
}

func TestParseSitemapForUrls_ReturnsUrls(t *testing.T) {
	t.Parallel()
	sitemap := `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
	<url>
		<loc>http://example.com</loc>
	</url>
	<url>
		<loc>http://example.com/about</loc>
	</url>
	<url>
		<loc>http://example.com/contact</loc>
	</url>
	</urlset>`
	urls, err := ParseSitemapForUrls(sitemap)
	require.NoError(t, err)
	assert.Equal(t, 3, len(urls))
	assert.Contains(t, urls, "http://example.com")
	assert.Contains(t, urls, "http://example.com/about")
	assert.Contains(t, urls, "http://example.com/contact")
}

func TestParseSitemapForUrls_EmptySitemap(t *testing.T) {
	t.Parallel()
	sitemap := `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"></urlset>`
	urls, err := ParseSitemapForUrls(sitemap)
	require.NoError(t, err)
	assert.Equal(t, 0, len(urls))
}

func TestParseSitemapForUrls_NoLocTags(t *testing.T) {
	t.Parallel()
	sitemap := `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
	<url>
		<lastmod>2023-10-01</lastmod>
	</url>
	<url>
		<changefreq>daily</changefreq>
	</url>
	</urlset>`
	urls, err := ParseSitemapForUrls(sitemap)
	require.NoError(t, err)
	assert.Equal(t, 0, len(urls))
}