update scraping logic to handle new style cbox pages and new unified dateformat

page_oldstyle.go

@@ -0,0 +1,68 @@
+package go_cbox_scraper
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"strconv"
+	"time"
+)
+
+func (p *CBoxPage) parseOldPage(document *goquery.Document) {
+	document.Find(".msg").Each(func(index int, element *goquery.Selection) {
+		messageIDString, exists := element.Attr("id")
+		if exists {
+			messageIDInt, _ := strconv.Atoi(digitRegex.FindString(messageIDString))
+			if p.smallestID == -1 || messageIDInt < p.smallestID {
+				p.smallestID = messageIDInt
+			}
+			if p.largestID == -1 || messageIDInt > p.largestID {
+				p.largestID = messageIDInt
+			}
+			p.Messages[messageIDInt] = p.parseOldMessage(messageIDInt, element)
+		}
+	})
+
+	// can paginate backwards on main page
+	_, p.canPaginatePrevious = document.Find("#lnkArchive").Attr("href")
+	if p.canPaginatePrevious {
+		cboxURL := p.buildCboxURL("archive")
+		q := cboxURL.Query()
+		q.Set("i", strconv.Itoa(p.smallestID))
+		p.previousString = q.Encode()
+	} else {
+		// in an archive page
+		previousString, canPrevious := document.Find("td[align='left'] a").Attr("href")
+		if canPrevious {
+			p.canPaginatePrevious = true
+			p.previousString = previousString[3:]
+		}
+
+		nextString, canNext := document.Find("td[align='right'] a").Attr("href")
+		if canNext {
+			p.canPaginateNext = true
+			p.nextString = nextString[3:]
+		}
+	}
+}
+
+func (p *CBoxPage) parseOldMessage(messageID int, element *goquery.Selection) *Message {
+	message := Message{
+		MessageID: messageID,
+	}
+
+	datetimeElement := element.Find("div").Text()
+	if datetimeElement != "" {
+		message.DateTime, _ = time.Parse(DatetimeFormat, datetimeElement)
+	}
+	element.Find("div").Remove()
+
+	name := element.Find("b.nme").Text()
+	if name != "" {
+		message.Username = name
+	}
+	element.Find("b.nme").Remove()
+
+	// trim the first 2 characters, colon and space
+	message.Message = element.Text()[2:]
+
+	return &message
+}