68 lines
1.8 KiB
Go
68 lines
1.8 KiB
Go
package go_cbox_scraper
|
|
|
|
import (
|
|
"github.com/PuerkitoBio/goquery"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
func (p *CBoxPage) parseOldPage(document *goquery.Document) {
|
|
document.Find(".msg").Each(func(index int, element *goquery.Selection) {
|
|
messageIDString, exists := element.Attr("id")
|
|
if exists {
|
|
messageIDInt, _ := strconv.Atoi(digitRegex.FindString(messageIDString))
|
|
if p.smallestID == -1 || messageIDInt < p.smallestID {
|
|
p.smallestID = messageIDInt
|
|
}
|
|
if p.largestID == -1 || messageIDInt > p.largestID {
|
|
p.largestID = messageIDInt
|
|
}
|
|
p.Messages[messageIDInt] = p.parseOldMessage(messageIDInt, element)
|
|
}
|
|
})
|
|
|
|
// can paginate backwards on main page
|
|
_, p.canPaginatePrevious = document.Find("#lnkArchive").Attr("href")
|
|
if p.canPaginatePrevious {
|
|
cboxURL := p.buildCboxURL("archive")
|
|
q := cboxURL.Query()
|
|
q.Set("i", strconv.Itoa(p.smallestID))
|
|
p.previousString = q.Encode()
|
|
} else {
|
|
// in an archive page
|
|
previousString, canPrevious := document.Find("td[align='left'] a").Attr("href")
|
|
if canPrevious {
|
|
p.canPaginatePrevious = true
|
|
p.previousString = previousString[3:]
|
|
}
|
|
|
|
nextString, canNext := document.Find("td[align='right'] a").Attr("href")
|
|
if canNext {
|
|
p.canPaginateNext = true
|
|
p.nextString = nextString[3:]
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *CBoxPage) parseOldMessage(messageID int, element *goquery.Selection) *Message {
|
|
message := Message{
|
|
MessageID: messageID,
|
|
}
|
|
|
|
datetimeElement := element.Find("div").Text()
|
|
if datetimeElement != "" {
|
|
message.DateTime, _ = time.Parse(DatetimeFormat, datetimeElement)
|
|
}
|
|
element.Find("div").Remove()
|
|
|
|
name := element.Find("b.nme").Text()
|
|
if name != "" {
|
|
message.Username = name
|
|
}
|
|
element.Find("b.nme").Remove()
|
|
|
|
// trim the first 2 characters, colon and space
|
|
message.Message = element.Text()[2:]
|
|
|
|
return &message
|
|
} |