x/sitemap/robots.go

package sitemap

import (
	"fmt"
	"strings"
)

type RobotsBuilder struct {
	userAgents []*userAgentBuilder
}

type userAgentBuilder struct {
	name      string
	disallows []string
	allows    []string
	sitemaps  []string
}

const (
	// https://www.keycdn.com/blog/web-crawlers
	AllAgents     = "*"
	GoogleAgent   = "Googlebot"
	BingAgent     = "Bingbot"
	YahooAgent    = "Slurp"
	DuckDuckAgent = "DuckDuckBot"
	BaiduAgent    = "Baiduspider"
	YandexAgent   = "YandexBot"
	SogouAgent    = "Sogou web spider/4.0"
	ExaleadAgent  = "Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)"
	FacebookAgent = "facebot"
	AlexaAgent    = "ia_archiver"
)

func Robots() *RobotsBuilder {
	return &RobotsBuilder{}
}

func (r *RobotsBuilder) Agent(name string) *userAgentBuilder {
	agent := &userAgentBuilder{
		name: name,
	}
	r.userAgents = append(r.userAgents, agent)
	return agent
}

func (r *RobotsBuilder) ToTxt() string {
	b := strings.Builder{}
	for _, agent := range r.userAgents {
		b.WriteString(fmt.Sprintf("User-agent: %s\n", agent.name))
		for _, disallow := range agent.disallows {
			b.WriteString(fmt.Sprintf("Disallow: %s\n", disallow))
		}
		for _, allow := range agent.allows {
			b.WriteString(fmt.Sprintf("Allow: %s\n", allow))
		}
		for _, sitemap := range agent.sitemaps {
			b.WriteString(fmt.Sprintf("Sitemap: %s\n", sitemap))
		}
		b.WriteString("\n")
	}
	return b.String()
}

func (u *userAgentBuilder) AddSitemapUrl(sitemaps ...string) *userAgentBuilder {
	u.sitemaps = append(u.sitemaps, sitemaps...)
	return u
}

func (u *userAgentBuilder) Allow(allows ...string) *userAgentBuilder {
	u.allows = append(u.allows, allows...)
	return u
}

func (u *userAgentBuilder) Disallow(disallows ...string) *userAgentBuilder {
	u.disallows = append(u.disallows, disallows...)
	return u
}