diff --git a/.idea/go-cbox-scraper.iml b/.idea/go-cbox-scraper.iml new file mode 100644 index 0000000..c956989 --- /dev/null +++ b/.idea/go-cbox-scraper.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..5eaeb4b --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/example/gd.gob b/example/gd.gob new file mode 100644 index 0000000..390605a Binary files /dev/null and b/example/gd.gob differ diff --git a/example/main.go b/example/main.go index 48c2083..d836a22 100644 --- a/example/main.go +++ b/example/main.go @@ -1,7 +1,7 @@ package main import ( - cboxscraper ".." + cboxscraper "github.com/Neo-Desktop/go-cbox-scraper" "log" ) @@ -12,7 +12,7 @@ func main() { WebHostID: 6, BoxID: 850801, BoxTag: "hD3VIj", - Debug: true, + Debug: false, } scraper := cboxscraper.NewScraper(info, -1, -1) diff --git a/page.go b/page.go index 1c49714..815444e 100644 --- a/page.go +++ b/page.go @@ -3,7 +3,6 @@ package go_cbox_scraper import ( "errors" "fmt" - "log" "net/http" "net/url" "regexp" @@ -136,9 +135,7 @@ func (p *CBoxPage) FetchNext() error { } func (p *CBoxPage) request(request *http.Request) (*goquery.Document, error) { - if p.Debug { - log.Println(request.URL.String()) - } + p.debugPrintln(request.URL.String()) httpClient := &http.Client{ Timeout: 30 * time.Second, diff --git a/scraper.go b/scraper.go index be6035c..952ca19 100644 --- a/scraper.go +++ b/scraper.go @@ -2,7 +2,6 @@ package go_cbox_scraper import ( "encoding/gob" - "log" "os" "time" ) @@ -26,16 +25,12 @@ func NewScraper(cboxServerInfo CBoxServerInfo, smallestID int, largestID int) *C } func (s *CBoxScraper) sleep() { - if s.Debug { - log.Println("Sleeping 10 seconds...") - } + s.debugPrintln("Sleeping 10 seconds...") time.Sleep(10 * time.Second) } func (s *CBoxScraper) Scrape(updatesOnly bool) error { - if s.Debug { - log.Println("Scraper Started...") - } + s.debugPrintln("Scraper Started...") page := NewCBoxPage(s.CBoxServerInfo) @@ -44,16 +39,12 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error { return err } - if s.Debug { - log.Printf("Main fetched, scraped %d messages\n", len(page.Messages)) - log.Printf("\tpage smallestID: %d - scraper smallestID: %d\n", page.SmallestID(), s.SmallestMessageID) - log.Printf("\tpage largestID: %d - scraper LargestID: %d\n", page.LargestID(), s.LargestMessageID) - } + s.debugPrintf("Main fetched, scraped %d messages\n", len(page.Messages)) + s.debugPrintf("\tpage smallestID: %d - scraper smallestID: %d\n", page.SmallestID(), s.SmallestMessageID) + s.debugPrintf("\tpage largestID: %d - scraper LargestID: %d\n", page.LargestID(), s.LargestMessageID) if updatesOnly && s.LargestMessageID < page.SmallestID() { - if s.Debug { - log.Println("Scraper: Case 3 - lots of new messages") - } + s.debugPrintln("Scraper: Case 3 - lots of new messages") s.sleep() for s.LargestMessageID < page.smallestID { err := page.FetchPrevious() @@ -64,11 +55,11 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error { } s.merge(page.Messages) } else if updatesOnly && s.LargestMessageID < page.LargestID() { - log.Println("Scraper: Case 2 - Some new Messages") + s.debugPrintln("Scraper: Case 2 - Some new Messages") // merge what we retrieved s.merge(page.Messages) } else if !updatesOnly { - log.Println("Scraper: Case 4 - fetch all") + s.debugPrintln("Scraper: Case 4 - fetch all") s.sleep() for { err := page.FetchPrevious() @@ -79,12 +70,10 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error { } s.merge(page.Messages) } else { - log.Println("Scraper: Case 0 - no update") + s.debugPrintln("Scraper: Case 0 - no update") } - if s.Debug { - log.Printf("Archives fetched, scraped %d messages - page smallestID: %d\n", len(page.Messages), page.SmallestID()) - } + s.debugPrintf("Archives fetched, scraped %d messages - page smallestID: %d\n", len(page.Messages), page.SmallestID()) return nil } diff --git a/serverInfo.go b/serverInfo.go index 9439e1b..08c81a9 100644 --- a/serverInfo.go +++ b/serverInfo.go @@ -1,8 +1,30 @@ package go_cbox_scraper +import ( + "log" +) + type CBoxServerInfo struct { WebHostID int BoxID int BoxTag string Debug bool } + +func (c *CBoxServerInfo) debugPrint(args ... interface{}) { + if c.Debug { + log.Print(args...) + } +} + +func (c *CBoxServerInfo) debugPrintf(format string, args ... interface{}) { + if c.Debug { + log.Printf(format, args...) + } +} + +func (c *CBoxServerInfo) debugPrintln(args ... interface{}) { + if c.Debug { + log.Println(args...) + } +} \ No newline at end of file