finalize debug logging

.idea/go-cbox-scraper.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/go-cbox-scraper.iml" filepath="$PROJECT_DIR$/.idea/go-cbox-scraper.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

example/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	cboxscraper ".."
+	cboxscraper "github.com/Neo-Desktop/go-cbox-scraper"
 	"log"
 )
 
@@ -12,7 +12,7 @@ func main() {
 		WebHostID: 6,
 		BoxID:     850801,
 		BoxTag:    "hD3VIj",
-		Debug:     true,
+		Debug:     false,
 	}
 
 	scraper := cboxscraper.NewScraper(info, -1, -1)

page.go

@@ -3,7 +3,6 @@ package go_cbox_scraper
 import (
 	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -136,9 +135,7 @@ func (p *CBoxPage) FetchNext() error {
 }
 
 func (p *CBoxPage) request(request *http.Request) (*goquery.Document, error) {
-	if p.Debug {
+	p.debugPrintln(request.URL.String())
-		log.Println(request.URL.String())
-	}
 
 	httpClient := &http.Client{
 		Timeout: 30 * time.Second,

scraper.go

@@ -2,7 +2,6 @@ package go_cbox_scraper
 
 import (
 	"encoding/gob"
-	"log"
 	"os"
 	"time"
 )
@@ -26,16 +25,12 @@ func NewScraper(cboxServerInfo CBoxServerInfo, smallestID int, largestID int) *C
 }
 
 func (s *CBoxScraper) sleep() {
-	if s.Debug {
+	s.debugPrintln("Sleeping 10 seconds...")
-		log.Println("Sleeping 10 seconds...")
-	}
 	time.Sleep(10 * time.Second)
 }
 
 func (s *CBoxScraper) Scrape(updatesOnly bool) error {
-	if s.Debug {
+	s.debugPrintln("Scraper Started...")
-		log.Println("Scraper Started...")
-	}
 
 	page := NewCBoxPage(s.CBoxServerInfo)
 
@@ -44,16 +39,12 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error {
 		return err
 	}
 
-	if s.Debug {
+	s.debugPrintf("Main fetched, scraped %d messages\n", len(page.Messages))
-		log.Printf("Main fetched, scraped %d messages\n", len(page.Messages))
+	s.debugPrintf("\tpage smallestID: %d - scraper smallestID: %d\n", page.SmallestID(), s.SmallestMessageID)
-		log.Printf("\tpage smallestID: %d - scraper smallestID: %d\n", page.SmallestID(), s.SmallestMessageID)
+	s.debugPrintf("\tpage largestID: %d - scraper LargestID: %d\n", page.LargestID(), s.LargestMessageID)
-		log.Printf("\tpage largestID: %d - scraper LargestID: %d\n", page.LargestID(), s.LargestMessageID)
-	}
 
 	if updatesOnly && s.LargestMessageID < page.SmallestID() {
-		if s.Debug {
+		s.debugPrintln("Scraper: Case 3 - lots of new messages")
-			log.Println("Scraper: Case 3 - lots of new messages")
-		}
 		s.sleep()
 		for s.LargestMessageID < page.smallestID {
 			err := page.FetchPrevious()
@@ -64,11 +55,11 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error {
 		}
 		s.merge(page.Messages)
 	} else if updatesOnly && s.LargestMessageID < page.LargestID() {
-		log.Println("Scraper: Case 2 - Some new Messages")
+		s.debugPrintln("Scraper: Case 2 - Some new Messages")
 		// merge what we retrieved
 		s.merge(page.Messages)
 	} else if !updatesOnly {
-		log.Println("Scraper: Case 4 - fetch all")
+		s.debugPrintln("Scraper: Case 4 - fetch all")
 		s.sleep()
 		for {
 			err := page.FetchPrevious()
@@ -79,12 +70,10 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error {
 		}
 		s.merge(page.Messages)
 	} else {
-		log.Println("Scraper: Case 0 - no update")
+		s.debugPrintln("Scraper: Case 0 - no update")
 	}
 
-	if s.Debug {
+	s.debugPrintf("Archives fetched, scraped %d messages - page smallestID: %d\n", len(page.Messages), page.SmallestID())
-		log.Printf("Archives fetched, scraped %d messages - page smallestID: %d\n", len(page.Messages), page.SmallestID())
-	}
 
 	return nil
 }

serverInfo.go

@@ -1,8 +1,30 @@
 package go_cbox_scraper
 
+import (
+	"log"
+)
+
 type CBoxServerInfo struct {
 	WebHostID int
 	BoxID     int
 	BoxTag    string
 	Debug     bool
 }
+
+func (c *CBoxServerInfo) debugPrint(args ... interface{}) {
+	if c.Debug {
+		log.Print(args...)
+	}
+}
+
+func (c *CBoxServerInfo) debugPrintf(format string, args ... interface{}) {
+	if c.Debug {
+		log.Printf(format, args...)
+	}
+}
+
+func (c *CBoxServerInfo) debugPrintln(args ... interface{}) {
+	if c.Debug {
+		log.Println(args...)
+	}
+}