diff --git a/.idea/go-cbox-scraper.iml b/.idea/go-cbox-scraper.iml
new file mode 100644
index 0000000..c956989
--- /dev/null
+++ b/.idea/go-cbox-scraper.iml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..5eaeb4b
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/example/gd.gob b/example/gd.gob
new file mode 100644
index 0000000..390605a
Binary files /dev/null and b/example/gd.gob differ
diff --git a/example/main.go b/example/main.go
index 48c2083..d836a22 100644
--- a/example/main.go
+++ b/example/main.go
@@ -1,7 +1,7 @@
package main
import (
- cboxscraper ".."
+ cboxscraper "github.com/Neo-Desktop/go-cbox-scraper"
"log"
)
@@ -12,7 +12,7 @@ func main() {
WebHostID: 6,
BoxID: 850801,
BoxTag: "hD3VIj",
- Debug: true,
+ Debug: false,
}
scraper := cboxscraper.NewScraper(info, -1, -1)
diff --git a/page.go b/page.go
index 1c49714..815444e 100644
--- a/page.go
+++ b/page.go
@@ -3,7 +3,6 @@ package go_cbox_scraper
import (
"errors"
"fmt"
- "log"
"net/http"
"net/url"
"regexp"
@@ -136,9 +135,7 @@ func (p *CBoxPage) FetchNext() error {
}
func (p *CBoxPage) request(request *http.Request) (*goquery.Document, error) {
- if p.Debug {
- log.Println(request.URL.String())
- }
+ p.debugPrintln(request.URL.String())
httpClient := &http.Client{
Timeout: 30 * time.Second,
diff --git a/scraper.go b/scraper.go
index be6035c..952ca19 100644
--- a/scraper.go
+++ b/scraper.go
@@ -2,7 +2,6 @@ package go_cbox_scraper
import (
"encoding/gob"
- "log"
"os"
"time"
)
@@ -26,16 +25,12 @@ func NewScraper(cboxServerInfo CBoxServerInfo, smallestID int, largestID int) *C
}
func (s *CBoxScraper) sleep() {
- if s.Debug {
- log.Println("Sleeping 10 seconds...")
- }
+ s.debugPrintln("Sleeping 10 seconds...")
time.Sleep(10 * time.Second)
}
func (s *CBoxScraper) Scrape(updatesOnly bool) error {
- if s.Debug {
- log.Println("Scraper Started...")
- }
+ s.debugPrintln("Scraper Started...")
page := NewCBoxPage(s.CBoxServerInfo)
@@ -44,16 +39,12 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error {
return err
}
- if s.Debug {
- log.Printf("Main fetched, scraped %d messages\n", len(page.Messages))
- log.Printf("\tpage smallestID: %d - scraper smallestID: %d\n", page.SmallestID(), s.SmallestMessageID)
- log.Printf("\tpage largestID: %d - scraper LargestID: %d\n", page.LargestID(), s.LargestMessageID)
- }
+ s.debugPrintf("Main fetched, scraped %d messages\n", len(page.Messages))
+ s.debugPrintf("\tpage smallestID: %d - scraper smallestID: %d\n", page.SmallestID(), s.SmallestMessageID)
+ s.debugPrintf("\tpage largestID: %d - scraper LargestID: %d\n", page.LargestID(), s.LargestMessageID)
if updatesOnly && s.LargestMessageID < page.SmallestID() {
- if s.Debug {
- log.Println("Scraper: Case 3 - lots of new messages")
- }
+ s.debugPrintln("Scraper: Case 3 - lots of new messages")
s.sleep()
for s.LargestMessageID < page.smallestID {
err := page.FetchPrevious()
@@ -64,11 +55,11 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error {
}
s.merge(page.Messages)
} else if updatesOnly && s.LargestMessageID < page.LargestID() {
- log.Println("Scraper: Case 2 - Some new Messages")
+ s.debugPrintln("Scraper: Case 2 - Some new Messages")
// merge what we retrieved
s.merge(page.Messages)
} else if !updatesOnly {
- log.Println("Scraper: Case 4 - fetch all")
+ s.debugPrintln("Scraper: Case 4 - fetch all")
s.sleep()
for {
err := page.FetchPrevious()
@@ -79,12 +70,10 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error {
}
s.merge(page.Messages)
} else {
- log.Println("Scraper: Case 0 - no update")
+ s.debugPrintln("Scraper: Case 0 - no update")
}
- if s.Debug {
- log.Printf("Archives fetched, scraped %d messages - page smallestID: %d\n", len(page.Messages), page.SmallestID())
- }
+ s.debugPrintf("Archives fetched, scraped %d messages - page smallestID: %d\n", len(page.Messages), page.SmallestID())
return nil
}
diff --git a/serverInfo.go b/serverInfo.go
index 9439e1b..08c81a9 100644
--- a/serverInfo.go
+++ b/serverInfo.go
@@ -1,8 +1,30 @@
package go_cbox_scraper
+import (
+ "log"
+)
+
type CBoxServerInfo struct {
WebHostID int
BoxID int
BoxTag string
Debug bool
}
+
+func (c *CBoxServerInfo) debugPrint(args ... interface{}) {
+ if c.Debug {
+ log.Print(args...)
+ }
+}
+
+func (c *CBoxServerInfo) debugPrintf(format string, args ... interface{}) {
+ if c.Debug {
+ log.Printf(format, args...)
+ }
+}
+
+func (c *CBoxServerInfo) debugPrintln(args ... interface{}) {
+ if c.Debug {
+ log.Println(args...)
+ }
+}
\ No newline at end of file