go-cbox-scraper/page.go

package go_cbox_scraper

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"time"

	"github.com/PuerkitoBio/goquery"
)

var (
	digitRegex *regexp.Regexp
)

func init() {
	digitRegex = regexp.MustCompile(`\d+`)
}

type CBoxPage struct {
	Messages    map[int]*Message
	CanPaginate bool

	index         int
	previousIndex int

	canPaginatePrevious bool
	canPaginateNext     bool
	previousString      string
	nextString          string

	previousPage int
	nextPage     int

	smallestID int
	largestID  int

	section string

	ServerInfo
}

func NewCBoxPage(cbxInfo ServerInfo) *CBoxPage {
	return &CBoxPage{
		Messages:            make(map[int]*Message),
		CanPaginate:         false,
		index:               -1,
		previousIndex:       -1,
		canPaginatePrevious: false,
		canPaginateNext:     false,
		previousString:      "",
		nextString:          "",
		previousPage:        -1,
		nextPage:            -1,
		smallestID:          -1,
		largestID:           -1,
		section:             "",
		ServerInfo:          cbxInfo,
	}
}

func (p *CBoxPage) SmallestID() int {
	return p.smallestID
}

func (p *CBoxPage) LargestID() int {
	return p.largestID
}

func (p *CBoxPage) FetchMain() error {
	cboxURL := p.buildCboxURL("main")

	request, err := newHTTPRequest(cboxURL)
	if err != nil {
		return err
	}

	document, err := p.request(request)
	if err != nil {
		return err
	}

	p.parsePage(document)

	return nil
}

func (p *CBoxPage) FetchPrevious() error {
	if !p.canPaginatePrevious {
		return errors.New("can not paginate previous")
	}

	cboxURL := p.buildCboxURL("archive")
	cboxURL.RawQuery = p.previousString

	request, err := newHTTPRequest(cboxURL)
	if err != nil {
		return nil
	}

	document, err := p.request(request)
	if err != nil {
		return err
	}

	p.parsePage(document)

	return nil
}

func (p *CBoxPage) FetchNext() error {
	if !p.canPaginateNext {
		return errors.New("can not paginate next")
	}

	cboxURL := p.buildCboxURL("archive")
	cboxURL.RawQuery = p.nextString

	request, err := newHTTPRequest(cboxURL)
	if err != nil {
		return err
	}

	document, err := p.request(request)
	if err != nil {
		return err
	}

	p.parsePage(document)

	return nil
}

func (p *CBoxPage) request(request *http.Request) (*goquery.Document, error) {
	p.debugPrintln(request.URL.String())

	httpClient := &http.Client{
		Timeout: 30 * time.Second,
	}

	// Make request
	response, err := httpClient.Do(request)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	// Create a goquery document from the HTTP response
	document, err := goquery.NewDocumentFromReader(response.Body)
	if err != nil {
		return nil, err
	}

	return document, nil
}

func (p *CBoxPage) buildCboxURL(section string) *url.URL {
	// format := "https://www%d.cbox.ws/box/?boxid=%d&boxtag=%s&sec=%s"
	u := &url.URL{
		Host:   fmt.Sprintf("www%d.cbox.ws", p.WebHostID),
		Scheme: "https",
		Path:   "/box/",
	}
	q := u.Query()
	q.Set("boxid", strconv.Itoa(p.BoxID))
	q.Set("boxtag", p.BoxTag)
	q.Set("sec", section)
	u.RawQuery = q.Encode()
	return u
}

func (p *CBoxPage) parsePage(document *goquery.Document) {
	document.Find(".msg").Each(func(index int, element *goquery.Selection) {
		messageIDString, exists := element.Attr("id")
		if exists {
			messageIDInt, _ := strconv.Atoi(digitRegex.FindString(messageIDString))
			if p.smallestID == -1 || messageIDInt < p.smallestID {
				p.smallestID = messageIDInt
			}
			if p.largestID == -1 || messageIDInt > p.largestID {
				p.largestID = messageIDInt
			}
			p.Messages[messageIDInt] = p.parseMessage(messageIDInt, element)
		}
	})

	// can paginate backwards on main page
	_, p.canPaginatePrevious = document.Find("#lnkArchive").Attr("href")
	if p.canPaginatePrevious {
		cboxURL := p.buildCboxURL("archive")
		q := cboxURL.Query()
		q.Set("i", strconv.Itoa(p.smallestID))
		p.previousString = q.Encode()
	} else {
		// in an archive page
		previousString, canPrevious := document.Find("td[align='left'] a").Attr("href")
		if canPrevious {
			p.canPaginatePrevious = true
			p.previousString = previousString[3:]
		}

		nextString, canNext := document.Find("td[align='right'] a").Attr("href")
		if canNext {
			p.canPaginateNext = true
			p.nextString = nextString[3:]
		}
	}
}

func (p *CBoxPage) parseMessage(messageID int, element *goquery.Selection) *Message {
	message := Message{
		MessageID: messageID,
	}

	datetimeElement := element.Find("div").Text()
	if datetimeElement != "" {
		message.DateTime, _ = time.Parse(DatetimeFormat, datetimeElement)
	}
	element.Find("div").Remove()

	name := element.Find("b.nme").Text()
	if name != "" {
		message.Username = name
	}
	element.Find("b.nme").Remove()

	// trim the first 2 characters, colon and space
	message.Message = element.Text()[2:]

	return &message
}

func newHTTPRequest(url *url.URL) (*http.Request, error) {
	// Create and modify HTTP request before sending
	request, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, err
	}
	request.Header.Set("Accept", "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/xbm, */* ")
	request.Header.Set("Accept-Language", "en")
	request.Header.Set("Connection", "Keep-Alive")
	request.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98)")

	return request, nil
}