package go_cbox_scraper import ( "errors" "fmt" "log" "net/http" "net/url" "regexp" "strconv" "time" "github.com/PuerkitoBio/goquery" ) var ( digitRegex *regexp.Regexp chicago *time.Location ) func init() { var err error digitRegex = regexp.MustCompile(`\d+`) chicago, err = time.LoadLocation("America/Chicago") if err != nil { panic("Unable to load timezone offset for America/Chicago from local timezone database") } } type CBoxPage struct { Messages map[int]*Message CanPaginate bool index int previousIndex int canPaginatePrevious bool canPaginateNext bool previousString string nextString string previousPage int nextPage int smallestID int largestID int section string isOld bool ServerInfo } func NewCBoxPage(cbxInfo ServerInfo) *CBoxPage { return &CBoxPage{ Messages: make(map[int]*Message), CanPaginate: false, index: -1, previousIndex: -1, canPaginatePrevious: false, canPaginateNext: false, previousString: "", nextString: "", previousPage: -1, nextPage: -1, smallestID: -1, largestID: -1, section: "", isOld: true, ServerInfo: cbxInfo, } } func (p *CBoxPage) SmallestID() int { return p.smallestID } func (p *CBoxPage) LargestID() int { return p.largestID } func (p *CBoxPage) FetchMain() error { cboxURL := p.buildCboxURL("main") request, err := newHTTPRequest(cboxURL) if err != nil { return err } document, err := p.request(request) if err != nil { return err } p.parsePage(document) return nil } func (p *CBoxPage) FetchPrevious() error { if !p.canPaginatePrevious { return errors.New("can not paginate previous") } cboxURL := p.buildCboxURL("archive") cboxURL.RawQuery = p.previousString request, err := newHTTPRequest(cboxURL) if err != nil { return nil } document, err := p.request(request) if err != nil { return err } p.parsePage(document) return nil } func (p *CBoxPage) FetchNext() error { if !p.canPaginateNext { return errors.New("can not paginate next") } cboxURL := p.buildCboxURL("archive") cboxURL.RawQuery = p.nextString request, err := newHTTPRequest(cboxURL) if err != nil { return err } document, err := p.request(request) if err != nil { return err } p.parsePage(document) return nil } func (p *CBoxPage) request(request *http.Request) (*goquery.Document, error) { p.debugPrintln(request.URL.String()) httpClient := &http.Client{ Timeout: 30 * time.Second, } // Make request response, err := httpClient.Do(request) if err != nil { return nil, err } defer response.Body.Close() // Create a goquery document from the HTTP response document, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } return document, nil } func (p *CBoxPage) buildCboxURL(section string) *url.URL { // format := "https://www%d.cbox.ws/box/?boxid=%d&boxtag=%s&sec=%s" u := &url.URL{ Host: fmt.Sprintf("www%d.cbox.ws", p.WebHostID), Scheme: "https", Path: "/box/", } q := u.Query() q.Set("boxid", strconv.Itoa(p.BoxID)) q.Set("boxtag", p.BoxTag) q.Set("sec", section) u.RawQuery = q.Encode() return u } func (p *CBoxPage) parsePage(document *goquery.Document) { tableIndex := document.Find("table").First().Length() // tableIndex returns 0 when not found if tableIndex > 0{ log.Println("Parsing old style page") p.parseOldPage(document) return } p.isOld = false log.Println("Parsing new style page") p.parseNewPage(document) } func newHTTPRequest(url *url.URL) (*http.Request, error) { // Create and modify HTTP request before sending request, err := http.NewRequest("GET", url.String(), nil) if err != nil { return nil, err } request.Header.Set("Accept", "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/xbm, */* ") request.Header.Set("Accept-Language", "en") request.Header.Set("Connection", "Keep-Alive") request.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98)") return request, nil }