From 7ebc9d5acc1dbfb593f05bbb6068497912f1ab90 Mon Sep 17 00:00:00 2001 From: Amrit Panesar Date: Tue, 10 Aug 2021 04:46:12 -0700 Subject: [PATCH] remove unnecessary CBox prefix rewrite sraper/dumper examples load should not nake a new file simply the NewScraper() funciton --- example/dumper/main.go | 20 +++++++++++------ example/scraper/main.go | 40 ++++++++++++++++++++------------- message.go | 6 ++--- page.go | 16 ++++++------- scraper.go | 50 +++++++++++++++++++++++------------------ serverInfo.go | 8 +++---- 6 files changed, 80 insertions(+), 60 deletions(-) diff --git a/example/dumper/main.go b/example/dumper/main.go index 1957dcd..4b305b4 100644 --- a/example/dumper/main.go +++ b/example/dumper/main.go @@ -1,20 +1,26 @@ package main import ( - cboxscraper "github.com/Neo-Desktop/go-cbox-scraper" - "fmt" "log" + "os" "sort" + + cbox "github.com/Neo-Desktop/go-cbox-scraper" ) func main() { - info := cboxscraper.CBoxServerInfo{} - scraper := cboxscraper.NewScraper(info, -1, -1) + if len(os.Args) != 2 { + log.Fatalln("syntax:", os.Args[0], "") + } - err := scraper.Load("../test.gob") - if err != nil { - log.Println("Load failed:", err) + scraper := cbox.NewScraper() + err := scraper.Load(os.Args[1]) + + if err != nil && os.IsNotExist(err) { + log.Fatalln("error, unable to open file", os.Args[1]) + } else if err != nil { + log.Fatalln("error reading file", err) } keys := make([]int, 0, len(scraper.Messages)) diff --git a/example/scraper/main.go b/example/scraper/main.go index bf9706f..61880eb 100644 --- a/example/scraper/main.go +++ b/example/scraper/main.go @@ -2,37 +2,45 @@ package main import ( "log" + "os" - cboxscraper "github.com/Neo-Desktop/go-cbox-scraper" + cbox "github.com/Neo-Desktop/go-cbox-scraper" ) func main() { - // this example configuration is for a "new" style cbox - // this will not work with the program as-is - info := cboxscraper.CBoxServerInfo{ - WebHostID: 6, - BoxID: 850801, - BoxTag: "hD3VIj", - Debug: false, + if len(os.Args) != 2 { + log.Fatalln("syntax:", os.Args[0], "[filename]") } - scraper := cboxscraper.NewScraper(info, -1, -1) + scraper := cbox.NewScraper() + err := scraper.Load(os.Args[1]) - err := scraper.Load("../test.gob") - if err != nil { - log.Println("Load failed:", err) + if err != nil && os.IsNotExist(err) { + log.Println("Unable to open file, attempting to create...", err) + scraper.Configure(cbox.ServerInfo{ + WebHostID: 7, + BoxID: 41422, + BoxTag: "7wcls1", + Debug: false, + }) + err = scraper.Save(os.Args[1]) + if err != nil { + log.Fatalln("Unable to save file:", err) + } + } else if err != nil { + log.Fatalln("Loading file failed:", err) } err = scraper.Scrape(true) if err != nil { - log.Println("Scraper failed:", err) + log.Println("Scraping did not finish successfully: ", err) } - // change this value before saving + // all settings and messages are saved when calling Save() scraper.Debug = true - err = scraper.Save("../test.gob") + err = scraper.Save(os.Args[1]) if err != nil { - log.Println("Save failed:", err) + log.Fatalln("Unable to save file:", err) } } diff --git a/message.go b/message.go index da45851..d33cd4c 100644 --- a/message.go +++ b/message.go @@ -5,13 +5,13 @@ import ( "time" ) -type CBoxMessage struct { +type Message struct { MessageID int DateTime time.Time Username string Message string } -func (m *CBoxMessage) String() string { - return fmt.Sprintf("#%d [%s] <%s> %s", m.MessageID, m.DateTime.Format(CboxDatetimeFormat), m.Username, m.Message) +func (m *Message) String() string { + return fmt.Sprintf("#%d [%s] <%s> %s", m.MessageID, m.DateTime.Format(DatetimeFormat), m.Username, m.Message) } diff --git a/page.go b/page.go index 815444e..82db1ea 100644 --- a/page.go +++ b/page.go @@ -21,7 +21,7 @@ func init() { } type CBoxPage struct { - Messages map[int]*CBoxMessage + Messages map[int]*Message CanPaginate bool index int @@ -40,12 +40,12 @@ type CBoxPage struct { section string - CBoxServerInfo + ServerInfo } -func NewCBoxPage(cbxInfo CBoxServerInfo) *CBoxPage { +func NewCBoxPage(cbxInfo ServerInfo) *CBoxPage { return &CBoxPage{ - Messages: make(map[int]*CBoxMessage), + Messages: make(map[int]*Message), CanPaginate: false, index: -1, previousIndex: -1, @@ -58,7 +58,7 @@ func NewCBoxPage(cbxInfo CBoxServerInfo) *CBoxPage { smallestID: -1, largestID: -1, section: "", - CBoxServerInfo: cbxInfo, + ServerInfo: cbxInfo, } } @@ -210,14 +210,14 @@ func (p *CBoxPage) parsePage(document *goquery.Document) { } } -func (p *CBoxPage) parseMessage(messageID int, element *goquery.Selection) *CBoxMessage { - message := CBoxMessage{ +func (p *CBoxPage) parseMessage(messageID int, element *goquery.Selection) *Message { + message := Message{ MessageID: messageID, } datetimeElement := element.Find("div").Text() if datetimeElement != "" { - message.DateTime, _ = time.Parse(CboxDatetimeFormat, datetimeElement) + message.DateTime, _ = time.Parse(DatetimeFormat, datetimeElement) } element.Find("div").Remove() diff --git a/scraper.go b/scraper.go index 6365964..e312a82 100644 --- a/scraper.go +++ b/scraper.go @@ -6,33 +6,39 @@ import ( "time" ) -const CboxDatetimeFormat = "2006-01-02 03:04PM" +const DatetimeFormat = "2006-01-02 03:04PM" -type CBoxScraper struct { +type Scraper struct { SmallestMessageID int LargestMessageID int - Messages map[int]*CBoxMessage - CBoxServerInfo + Messages map[int]*Message + ServerInfo } -func NewScraper(cboxServerInfo CBoxServerInfo, smallestID int, largestID int) *CBoxScraper { - return &CBoxScraper{ - SmallestMessageID: smallestID, - LargestMessageID: largestID, - CBoxServerInfo: cboxServerInfo, - Messages: make(map[int]*CBoxMessage), +func NewScraper() *Scraper { + return &Scraper{ + Messages: make(map[int]*Message), } } -func (s *CBoxScraper) sleep() { +func NewScraperFromFile(filePath string) (*Scraper, error) { + output := NewScraper() + err := output.Load(filePath) + if err != nil { + return nil, err + } + return output, nil +} + +func (s *Scraper) sleep() { s.debugPrintln("Sleeping 10 seconds...") time.Sleep(10 * time.Second) } -func (s *CBoxScraper) Scrape(updatesOnly bool) error { +func (s *Scraper) Scrape(updatesOnly bool) error { s.debugPrintln("Scraper Started...") - page := NewCBoxPage(s.CBoxServerInfo) + page := NewCBoxPage(s.ServerInfo) err := page.FetchMain() if err != nil { @@ -78,14 +84,14 @@ func (s *CBoxScraper) Scrape(updatesOnly bool) error { return nil } -func (s *CBoxScraper) merge(input map[int]*CBoxMessage) { +func (s *Scraper) merge(input map[int]*Message) { for k,v := range input { s.Messages[k] = v } s.updateIndices() } -func (s *CBoxScraper) updateIndices() { +func (s *Scraper) updateIndices() { s.SmallestMessageID = -1 s.LargestMessageID = -1 for k, _ := range s.Messages { @@ -98,7 +104,11 @@ func (s *CBoxScraper) updateIndices() { } } -func (s *CBoxScraper) Save(path string) error { +func (s *Scraper) Configure(config ServerInfo) { + s.ServerInfo = config +} + +func (s *Scraper) Save(path string) error { flags := os.O_TRUNC | os.O_RDWR | os.O_EXCL file, err := os.Stat(path) if file == nil { @@ -122,12 +132,8 @@ func (s *CBoxScraper) Save(path string) error { return nil } -func (s *CBoxScraper) Load(path string) error { - file, err := os.Stat(path) - if file == nil { - err = s.Save(path) - } - if err != nil { +func (s *Scraper) Load(path string) error { + if _, err := os.Stat(os.Args[1]); os.IsNotExist(err) { return err } diff --git a/serverInfo.go b/serverInfo.go index 08c81a9..50b431d 100644 --- a/serverInfo.go +++ b/serverInfo.go @@ -4,26 +4,26 @@ import ( "log" ) -type CBoxServerInfo struct { +type ServerInfo struct { WebHostID int BoxID int BoxTag string Debug bool } -func (c *CBoxServerInfo) debugPrint(args ... interface{}) { +func (c *ServerInfo) debugPrint(args ... interface{}) { if c.Debug { log.Print(args...) } } -func (c *CBoxServerInfo) debugPrintf(format string, args ... interface{}) { +func (c *ServerInfo) debugPrintf(format string, args ... interface{}) { if c.Debug { log.Printf(format, args...) } } -func (c *CBoxServerInfo) debugPrintln(args ... interface{}) { +func (c *ServerInfo) debugPrintln(args ... interface{}) { if c.Debug { log.Println(args...) }