initial commit - working code
new file: .gitignore new file: .idea/.gitignore new file: .idea/Steamworks-Collection-Downloader.iml new file: .idea/modules.xml new file: .idea/vcs.xml new file: downloader/downloadable.go new file: downloader/downloader.go new file: go.mod new file: go.sum new file: main.go new file: manifest/entry.go new file: manifest/manifest.go new file: webapi/publishedFileDetailsResponse.go
This commit is contained in:
13 files changed
+833
No files matched your search
@@ -0,0 +1,13 @@
|
||||
package downloader
|
||||
|
||||
import "github.com/Neo-Desktop/Steamworks-Collection-Downloader/manifest"
|
||||
|
||||
type Downloadable struct {
|
||||
PreviewFilename string
|
||||
FileURL string
|
||||
PreviewURL string
|
||||
Title string
|
||||
Filesize string
|
||||
manifest.Entry
|
||||
fetched bool
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/Neo-Desktop/Steamworks-Collection-Downloader/manifest"
|
||||
"github.com/Neo-Desktop/Steamworks-Collection-Downloader/webapi"
|
||||
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
type Downloader struct {
|
||||
apikey string
|
||||
prefix string
|
||||
Fetched map[string]*Downloadable
|
||||
}
|
||||
|
||||
func (d *Downloader) Start(apiKey string, seeds []string, prefix string) error {
|
||||
d.apikey = apiKey
|
||||
d.prefix = prefix
|
||||
|
||||
err, r := d.RequestFileDetails(apiKey, seeds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, v := range seeds {
|
||||
d.AddSeed(v)
|
||||
}
|
||||
|
||||
err = d.FetchCollections(r)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *Downloader) AddSeed(id string) {
|
||||
if d.Fetched == nil {
|
||||
d.Fetched = make(map[string]*Downloadable)
|
||||
}
|
||||
d.Fetched[id] = new(Downloadable)
|
||||
}
|
||||
|
||||
func (d *Downloader) FetchCollections(r *webapi.PublishedFileDetailsResponse) error {
|
||||
var err error
|
||||
|
||||
if d.apikey == "" {
|
||||
return errors.New("API key not set")
|
||||
}
|
||||
|
||||
for {
|
||||
for _, v := range r.Response.PublishedFileDetails {
|
||||
d.Fetched[v.PublishedFileID].fetched = true
|
||||
|
||||
log.Printf("%s - has children (%t)\n", v.Title, v.NumChildren > 0)
|
||||
if v.NumChildren > 0 {
|
||||
for _, c := range v.Children {
|
||||
if _, ok := d.Fetched[c.PublishedFileID]; !ok {
|
||||
d.Fetched[c.PublishedFileID] = &Downloadable{fetched: false}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(v.Filename) > 3 && v.Filename[len(v.Filename)-3:] == "vpk" {
|
||||
d.Fetched[v.PublishedFileID] = &Downloadable{
|
||||
FileURL: v.FileURL,
|
||||
PreviewURL: v.PreviewURL,
|
||||
Title: v.Title,
|
||||
Entry: manifest.Entry{
|
||||
ID: v.PublishedFileID,
|
||||
TimeCreated: fmt.Sprintf("%d", v.TimeCreated),
|
||||
TimeUpdated: fmt.Sprintf("%d", v.TimeUpdated),
|
||||
HContentFile: v.HContentFile,
|
||||
HContentFileSize: v.FileSize,
|
||||
HContentPreview: v.HContentPreview,
|
||||
HContentPreviewSize: v.PreviewFileSize,
|
||||
},
|
||||
fetched: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
children := make([]string, 0)
|
||||
for k, v := range d.Fetched {
|
||||
if v.fetched == false {
|
||||
children = append(children, k)
|
||||
}
|
||||
}
|
||||
|
||||
if len(children) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
err, r = d.RequestFileDetails(d.apikey, children)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range d.Fetched {
|
||||
if v.FileURL == "" {
|
||||
delete(d.Fetched, k)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) RequestFileDetails(key string, fileIDs []string) (error, *webapi.PublishedFileDetailsResponse) {
|
||||
client := new(http.Client)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, "", nil)
|
||||
if err != nil {
|
||||
return err, nil
|
||||
}
|
||||
|
||||
req.URL.Scheme = "https"
|
||||
req.URL.Host = "api.steampowered.com"
|
||||
req.URL.Path = "/IPublishedFileService/GetDetails/v1"
|
||||
|
||||
query := req.URL.Query()
|
||||
query.Add("includechildren", "true")
|
||||
query.Add("key", key)
|
||||
|
||||
for i, v := range fileIDs {
|
||||
query.Add(fmt.Sprintf("publishedfileids[%d]", i), v)
|
||||
}
|
||||
|
||||
req.URL.RawQuery = query.Encode()
|
||||
|
||||
log.Printf("Requesting: %s - sleeping 5 seconds\n", req.URL)
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err, nil
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err, nil
|
||||
}
|
||||
|
||||
r := new(webapi.PublishedFileDetailsResponse)
|
||||
|
||||
err = json.Unmarshal(body, r)
|
||||
if err != nil {
|
||||
return err, nil
|
||||
}
|
||||
|
||||
return nil, r
|
||||
}
|
||||
|
||||
func (d *Downloader) DisplayFetched() {
|
||||
keys := make(map[string]string, 0)
|
||||
for k, v := range d.Fetched {
|
||||
keys[v.Title] = k
|
||||
}
|
||||
|
||||
k2 := maps.Keys(keys)
|
||||
slices.Sort(k2)
|
||||
|
||||
for _, k := range k2 {
|
||||
v := d.Fetched[keys[k]]
|
||||
fmt.Printf("%s: %s (%s) [%s]\n", v.Title, v.FileURL, v.FileNamePrefixed(d.prefix), v.PreviewFileNamePrefixed(d.prefix))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) IsContentUpdateNeeded(v *manifest.Entry, v2 *manifest.Entry) bool {
|
||||
return v.TimeUpdated != v2.TimeUpdated || v.TimeCreated != v2.TimeCreated ||
|
||||
v.HContentFile != v2.HContentFile || v.HContentFileSize != v2.HContentFileSize
|
||||
}
|
||||
|
||||
func (d *Downloader) IsPreviewUpdateNeeded(v *manifest.Entry, v2 *manifest.Entry) bool {
|
||||
return v.TimeUpdated != v2.TimeUpdated || v.TimeCreated != v2.TimeCreated ||
|
||||
v.HContentPreview != v2.HContentPreview || v.HContentPreviewSize != v2.HContentPreviewSize
|
||||
}
|
||||
|
||||
func (d *Downloader) UpdateFiles(m *manifest.Manifest) {
|
||||
for k, v := range d.Fetched {
|
||||
contentUpdate := false
|
||||
previewUpdate := false
|
||||
contentSizeOnDisk := int64(0)
|
||||
previewSizeOnDisk := int64(0)
|
||||
|
||||
if v2, ok := m.Entries[k]; ok {
|
||||
contentStat, err := os.Stat(v.FileNamePrefixed(d.prefix))
|
||||
if err == nil {
|
||||
contentSizeOnDisk = contentStat.Size()
|
||||
} else if errors.Is(err, os.ErrNotExist) {
|
||||
contentUpdate = true
|
||||
fmt.Fprintf(os.Stderr, "could not find file %s\n", v.FileNamePrefixed(d.prefix))
|
||||
} else {
|
||||
contentUpdate = true
|
||||
fmt.Fprintf(os.Stderr, "could not stat() %s: %s\n", v.FileNamePrefixed(d.prefix), err)
|
||||
}
|
||||
|
||||
previewStat, err := os.Stat(v.PreviewFileNamePrefixed(d.prefix))
|
||||
if err == nil {
|
||||
previewSizeOnDisk = previewStat.Size()
|
||||
} else if errors.Is(err, os.ErrNotExist) {
|
||||
previewUpdate = true
|
||||
fmt.Fprintf(os.Stderr, "could not find file %s\n", v.PreviewFileNamePrefixed(d.prefix))
|
||||
} else {
|
||||
previewUpdate = true
|
||||
fmt.Fprintf(os.Stderr, "could not stat() %s: %s\n", v.PreviewFileNamePrefixed(d.prefix), err)
|
||||
}
|
||||
|
||||
if !contentUpdate && !d.IsContentUpdateNeeded(&v.Entry, v2) &&
|
||||
fmt.Sprintf("%d", contentSizeOnDisk) == v.HContentFileSize {
|
||||
log.Printf("%s.vpk (%s): already up to date\n", v.ID, v.Title)
|
||||
} else {
|
||||
contentUpdate = true
|
||||
if fmt.Sprintf("%d", previewSizeOnDisk) != v.HContentPreviewSize {
|
||||
log.Printf("%s.vpk (%s): expected size %s, have %d\n", v.ID, v.Title, v.HContentPreviewSize, previewSizeOnDisk)
|
||||
}
|
||||
}
|
||||
|
||||
if !previewUpdate && !d.IsPreviewUpdateNeeded(&v.Entry, v2) &&
|
||||
fmt.Sprintf("%d", previewSizeOnDisk) == v.HContentPreviewSize {
|
||||
log.Printf("%s.jpg (%s): already up to date\n", v.ID, v.Title)
|
||||
} else {
|
||||
previewUpdate = true
|
||||
if fmt.Sprintf("%d", previewSizeOnDisk) != v.HContentPreviewSize {
|
||||
log.Printf("%s.jpg (%s): expected size %s, have %d\n", v.ID, v.Title, v.HContentPreviewSize, previewSizeOnDisk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if previewUpdate {
|
||||
err := d.DownloadFile(v.PreviewURL, v.PreviewFileNamePrefixed(d.prefix), v.HContentPreviewSize)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if contentUpdate {
|
||||
err := d.DownloadFile(v.FileURL, v.FileNamePrefixed(d.prefix), v.HContentFileSize)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
err := m.AddEntry(&v.Entry)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error adding entry for %s (id %s) to manifest\n", v.Title, v.ID)
|
||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) DownloadFile(url string, filename string, filesize string) error {
|
||||
out, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0664)
|
||||
defer out.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating file %s\n%w\n", filename, err)
|
||||
}
|
||||
|
||||
log.Printf("downloading %s to %s", url, filename)
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error downloading file %s\n%w\n", filename, err)
|
||||
}
|
||||
|
||||
n, err := io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error streaming file to %s\n%w\n", filename, err)
|
||||
}
|
||||
|
||||
if fmt.Sprintf("%d", n) != filesize {
|
||||
return fmt.Errorf("expected %s bytes, only recieved %d bytes\n", filesize, n)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user