initial commit - working code

new file:   .gitignore
	new file:   .idea/.gitignore
	new file:   .idea/Steamworks-Collection-Downloader.iml
	new file:   .idea/modules.xml
	new file:   .idea/vcs.xml
	new file:   downloader/downloadable.go
	new file:   downloader/downloader.go
	new file:   go.mod
	new file:   go.sum
	new file:   main.go
	new file:   manifest/entry.go
	new file:   manifest/manifest.go
	new file:   webapi/publishedFileDetailsResponse.go
This commit is contained in:
Neo committed 2022-10-13 17:37:52 -07:00
1 parent 2a1a8a2adc
commit ec93fc3379
13 files changed
+833

No files matched your search

+210
View File
@@ -0,0 +1,210 @@
/workshop
Steamworks-Collection-Downloader
### NotepadPP template
# Notepad++ backups #
*.bak
### VisualStudioCode template
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
*.code-workspace
# Local History for Visual Studio Code
.history/
### JetBrains template
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### Linux template
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
# .nfs files are created when an open file is removed but is still being accessed
.nfs*
### Go template
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
### Windows template
# Windows thumbnail cache files
Thumbs.db
Thumbs.db:encryptable
ehthumbs.db
ehthumbs_vista.db
# Dump file
*.stackdump
# Folder config file
[Dd]esktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
### macOS template
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### SublimeText template
# Cache files for Sublime Text
*.tmlanguage.cache
*.tmPreferences.cache
*.stTheme.cache
# Workspace files are user-specific
*.sublime-workspace
# Project files should be checked into the repository, unless a significant
# proportion of contributors will probably not be using Sublime Text
# *.sublime-project
# SFTP configuration file
sftp-config.json
sftp-config-alt*.json
# Package control specific files
Package Control.last-run
Package Control.ca-list
Package Control.ca-bundle
Package Control.system-ca-bundle
Package Control.cache/
Package Control.ca-certs/
Package Control.merged-ca-bundle
Package Control.user-ca-bundle
oscrypto-ca-bundle.crt
bh_unicode_properties.cache
# Sublime-github package stores a github token in this file
# https://packagecontrol.io/packages/sublime-github
GitHub.sublime-settings
+8
View File
@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
+9
View File
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/Steamworks-Collection-Downloader.iml" filepath="$PROJECT_DIR$/.idea/Steamworks-Collection-Downloader.iml" />
</modules>
</component>
</project>
Generated
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
+13
View File
@@ -0,0 +1,13 @@
package downloader
import "github.com/Neo-Desktop/Steamworks-Collection-Downloader/manifest"
type Downloadable struct {
PreviewFilename string
FileURL string
PreviewURL string
Title string
Filesize string
manifest.Entry
fetched bool
}
+287
View File
@@ -0,0 +1,287 @@
package downloader
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"time"
"github.com/Neo-Desktop/Steamworks-Collection-Downloader/manifest"
"github.com/Neo-Desktop/Steamworks-Collection-Downloader/webapi"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
type Downloader struct {
apikey string
prefix string
Fetched map[string]*Downloadable
}
func (d *Downloader) Start(apiKey string, seeds []string, prefix string) error {
d.apikey = apiKey
d.prefix = prefix
err, r := d.RequestFileDetails(apiKey, seeds)
if err != nil {
return err
}
for _, v := range seeds {
d.AddSeed(v)
}
err = d.FetchCollections(r)
return err
}
func (d *Downloader) AddSeed(id string) {
if d.Fetched == nil {
d.Fetched = make(map[string]*Downloadable)
}
d.Fetched[id] = new(Downloadable)
}
func (d *Downloader) FetchCollections(r *webapi.PublishedFileDetailsResponse) error {
var err error
if d.apikey == "" {
return errors.New("API key not set")
}
for {
for _, v := range r.Response.PublishedFileDetails {
d.Fetched[v.PublishedFileID].fetched = true
log.Printf("%s - has children (%t)\n", v.Title, v.NumChildren > 0)
if v.NumChildren > 0 {
for _, c := range v.Children {
if _, ok := d.Fetched[c.PublishedFileID]; !ok {
d.Fetched[c.PublishedFileID] = &Downloadable{fetched: false}
}
}
}
if len(v.Filename) > 3 && v.Filename[len(v.Filename)-3:] == "vpk" {
d.Fetched[v.PublishedFileID] = &Downloadable{
FileURL: v.FileURL,
PreviewURL: v.PreviewURL,
Title: v.Title,
Entry: manifest.Entry{
ID: v.PublishedFileID,
TimeCreated: fmt.Sprintf("%d", v.TimeCreated),
TimeUpdated: fmt.Sprintf("%d", v.TimeUpdated),
HContentFile: v.HContentFile,
HContentFileSize: v.FileSize,
HContentPreview: v.HContentPreview,
HContentPreviewSize: v.PreviewFileSize,
},
fetched: true,
}
}
}
children := make([]string, 0)
for k, v := range d.Fetched {
if v.fetched == false {
children = append(children, k)
}
}
if len(children) == 0 {
break
}
err, r = d.RequestFileDetails(d.apikey, children)
if err != nil {
return err
}
}
for k, v := range d.Fetched {
if v.FileURL == "" {
delete(d.Fetched, k)
}
}
return nil
}
func (d *Downloader) RequestFileDetails(key string, fileIDs []string) (error, *webapi.PublishedFileDetailsResponse) {
client := new(http.Client)
req, err := http.NewRequest(http.MethodGet, "", nil)
if err != nil {
return err, nil
}
req.URL.Scheme = "https"
req.URL.Host = "api.steampowered.com"
req.URL.Path = "/IPublishedFileService/GetDetails/v1"
query := req.URL.Query()
query.Add("includechildren", "true")
query.Add("key", key)
for i, v := range fileIDs {
query.Add(fmt.Sprintf("publishedfileids[%d]", i), v)
}
req.URL.RawQuery = query.Encode()
log.Printf("Requesting: %s - sleeping 5 seconds\n", req.URL)
time.Sleep(5 * time.Second)
res, err := client.Do(req)
if err != nil {
return err, nil
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return err, nil
}
r := new(webapi.PublishedFileDetailsResponse)
err = json.Unmarshal(body, r)
if err != nil {
return err, nil
}
return nil, r
}
func (d *Downloader) DisplayFetched() {
keys := make(map[string]string, 0)
for k, v := range d.Fetched {
keys[v.Title] = k
}
k2 := maps.Keys(keys)
slices.Sort(k2)
for _, k := range k2 {
v := d.Fetched[keys[k]]
fmt.Printf("%s: %s (%s) [%s]\n", v.Title, v.FileURL, v.FileNamePrefixed(d.prefix), v.PreviewFileNamePrefixed(d.prefix))
}
}
func (d *Downloader) IsContentUpdateNeeded(v *manifest.Entry, v2 *manifest.Entry) bool {
return v.TimeUpdated != v2.TimeUpdated || v.TimeCreated != v2.TimeCreated ||
v.HContentFile != v2.HContentFile || v.HContentFileSize != v2.HContentFileSize
}
func (d *Downloader) IsPreviewUpdateNeeded(v *manifest.Entry, v2 *manifest.Entry) bool {
return v.TimeUpdated != v2.TimeUpdated || v.TimeCreated != v2.TimeCreated ||
v.HContentPreview != v2.HContentPreview || v.HContentPreviewSize != v2.HContentPreviewSize
}
func (d *Downloader) UpdateFiles(m *manifest.Manifest) {
for k, v := range d.Fetched {
contentUpdate := false
previewUpdate := false
contentSizeOnDisk := int64(0)
previewSizeOnDisk := int64(0)
if v2, ok := m.Entries[k]; ok {
contentStat, err := os.Stat(v.FileNamePrefixed(d.prefix))
if err == nil {
contentSizeOnDisk = contentStat.Size()
} else if errors.Is(err, os.ErrNotExist) {
contentUpdate = true
fmt.Fprintf(os.Stderr, "could not find file %s\n", v.FileNamePrefixed(d.prefix))
} else {
contentUpdate = true
fmt.Fprintf(os.Stderr, "could not stat() %s: %s\n", v.FileNamePrefixed(d.prefix), err)
}
previewStat, err := os.Stat(v.PreviewFileNamePrefixed(d.prefix))
if err == nil {
previewSizeOnDisk = previewStat.Size()
} else if errors.Is(err, os.ErrNotExist) {
previewUpdate = true
fmt.Fprintf(os.Stderr, "could not find file %s\n", v.PreviewFileNamePrefixed(d.prefix))
} else {
previewUpdate = true
fmt.Fprintf(os.Stderr, "could not stat() %s: %s\n", v.PreviewFileNamePrefixed(d.prefix), err)
}
if !contentUpdate && !d.IsContentUpdateNeeded(&v.Entry, v2) &&
fmt.Sprintf("%d", contentSizeOnDisk) == v.HContentFileSize {
log.Printf("%s.vpk (%s): already up to date\n", v.ID, v.Title)
} else {
contentUpdate = true
if fmt.Sprintf("%d", previewSizeOnDisk) != v.HContentPreviewSize {
log.Printf("%s.vpk (%s): expected size %s, have %d\n", v.ID, v.Title, v.HContentPreviewSize, previewSizeOnDisk)
}
}
if !previewUpdate && !d.IsPreviewUpdateNeeded(&v.Entry, v2) &&
fmt.Sprintf("%d", previewSizeOnDisk) == v.HContentPreviewSize {
log.Printf("%s.jpg (%s): already up to date\n", v.ID, v.Title)
} else {
previewUpdate = true
if fmt.Sprintf("%d", previewSizeOnDisk) != v.HContentPreviewSize {
log.Printf("%s.jpg (%s): expected size %s, have %d\n", v.ID, v.Title, v.HContentPreviewSize, previewSizeOnDisk)
}
}
}
if previewUpdate {
err := d.DownloadFile(v.PreviewURL, v.PreviewFileNamePrefixed(d.prefix), v.HContentPreviewSize)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
}
if contentUpdate {
err := d.DownloadFile(v.FileURL, v.FileNamePrefixed(d.prefix), v.HContentFileSize)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
}
err := m.AddEntry(&v.Entry)
if err != nil {
fmt.Fprintf(os.Stderr, "Error adding entry for %s (id %s) to manifest\n", v.Title, v.ID)
fmt.Fprintf(os.Stderr, "%s\n", err)
}
}
}
func (d *Downloader) DownloadFile(url string, filename string, filesize string) error {
out, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0664)
defer out.Close()
if err != nil {
return fmt.Errorf("error creating file %s\n%w\n", filename, err)
}
log.Printf("downloading %s to %s", url, filename)
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("error downloading file %s\n%w\n", filename, err)
}
n, err := io.Copy(out, resp.Body)
if err != nil {
return fmt.Errorf("error streaming file to %s\n%w\n", filename, err)
}
if fmt.Sprintf("%d", n) != filesize {
return fmt.Errorf("expected %s bytes, only recieved %d bytes\n", filesize, n)
}
return nil
}
+5
View File
@@ -0,0 +1,5 @@
module github.com/Neo-Desktop/Steamworks-Collection-Downloader
go 1.18
require golang.org/x/exp v0.0.0-20221011201855-a3968a42eed6
+2
View File
@@ -0,0 +1,2 @@
golang.org/x/exp v0.0.0-20221011201855-a3968a42eed6 h1:+hSdOdB7nHAFs+EDQXTvkJj7kUMugNAcE2x+BwxlVt4=
golang.org/x/exp v0.0.0-20221011201855-a3968a42eed6/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+122
View File
@@ -0,0 +1,122 @@
package main
import (
"flag"
"fmt"
"log"
"os"
"strings"
"github.com/Neo-Desktop/Steamworks-Collection-Downloader/downloader"
"github.com/Neo-Desktop/Steamworks-Collection-Downloader/manifest"
)
var (
APIKEY string
SEEDS []string
PATH string
MANIFESTJSON string
)
const (
REPOURL = "https://github.com/Neo-Desktop/Steamworks-Collection-Downloader"
)
func init() {
flag.Usage = usage
flagAPIKEY := flag.String("apikey", "", "Required: Steamworks Web API key")
flagHELP := flag.Bool("help", false, "Displays this help message")
flagPREFIX := flag.String("prefix", "."+string(os.PathSeparator)+"workshop", "path to the <gamedir>/addons/workshop folder")
flagMANIFEST := flag.String("manifest", "workshopmanifest.json", "filename to store the current state of downloaded files (will be stored in prefix)")
envAPIKEY := os.Getenv("WSDL_APIKEY")
envPREFIX := os.Getenv("WSDL_PREFIX")
envSEEDS := os.Getenv("WSDL_SEEDS")
envMANIFEST := os.Getenv("WSDL_MANIFEST")
flag.Parse()
if flagAPIKEY != nil && *flagAPIKEY != "" {
APIKEY = *flagAPIKEY
} else if envAPIKEY != "" {
APIKEY = envAPIKEY
} else {
fmt.Fprintf(os.Stderr, "Please specify an APIKEY\n\n")
usage()
os.Exit(1)
}
if flagPREFIX != nil && *flagPREFIX != "" {
PATH = *flagPREFIX
} else if envPREFIX != "" {
PATH = envPREFIX
} else {
fmt.Fprintf(os.Stderr, "Please specify a prefix\n\n")
usage()
os.Exit(1)
}
if flagMANIFEST != nil && *flagMANIFEST != "" {
MANIFESTJSON = *flagMANIFEST
} else if envMANIFEST != "" {
MANIFESTJSON = envMANIFEST
} else {
fmt.Fprintf(os.Stderr, "Please specify a filename for the manifest\n\n")
usage()
os.Exit(1)
}
if len(flag.Args()) != 0 {
SEEDS = flag.Args()
} else if envSEEDS != "" {
SEEDS = strings.Split(envSEEDS, " ")
} else {
fmt.Fprintf(os.Stderr, "Please specify one or more items to download (as command line arguments)\n\n")
usage()
os.Exit(1)
}
if PATH[len(PATH)-1] != os.PathSeparator {
PATH = PATH + string(os.PathSeparator)
}
if *flagHELP {
usage()
os.Exit(1)
}
}
func usage() {
fmt.Fprintf(os.Stderr, "Usage: %s [options] (-apikey <key>) <id> [...]\n", os.Args[0])
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, " <id> string\n Required: one or more Steamworks collection(s) or file ID(s) to download\n")
fmt.Fprintf(os.Stderr, "\nPlease visit %s for source code and more information\n\n", REPOURL)
}
func main() {
d := new(downloader.Downloader)
m := new(manifest.Manifest)
err := d.Start(APIKEY, SEEDS, PATH)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(2)
}
d.DisplayFetched()
err = m.Open(MANIFESTJSON, PATH)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(2)
}
d.UpdateFiles(m)
m.Close()
log.Println("Completed!")
}
+19
View File
@@ -0,0 +1,19 @@
package manifest
type Entry struct {
ID string
TimeCreated string
TimeUpdated string
HContentFile string
HContentFileSize string
HContentPreview string
HContentPreviewSize string
}
func (w Entry) FileNamePrefixed(prefix string) string {
return prefix + w.ID + ".vpk"
}
func (w Entry) PreviewFileNamePrefixed(prefix string) string {
return prefix + w.ID + ".jpg"
}
+77
View File
@@ -0,0 +1,77 @@
package manifest
import (
"encoding/json"
"io/ioutil"
"os"
)
type Manifest struct {
filename string
prefix string
Entries map[string]*Entry
*os.File
}
func (m *Manifest) Open(filename string, prefix string) (err error) {
filename = prefix + filename
m.File, err = os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0664)
if err != nil {
return err
}
buffer, err := ioutil.ReadAll(m)
if err != nil {
return err
}
err = m.Parse(buffer)
if err != nil {
return err
}
return nil
}
func (m *Manifest) Parse(manifest []byte) error {
m.Entries = make(map[string]*Entry)
if len(manifest) == 0 {
return nil
}
err := json.Unmarshal(manifest, &m.Entries)
if err != nil {
return err
}
return nil
}
func (m *Manifest) AddEntry(entry *Entry) error {
m.Entries[entry.ID] = entry
_, err := m.Seek(0, 0)
if err != nil {
return err
}
err = m.Truncate(0)
if err != nil {
return err
}
b, err := json.Marshal(m.Entries)
if err != nil {
return err
}
_, err = m.Write(b)
if err != nil {
return err
}
return nil
}
+67
View File
@@ -0,0 +1,67 @@
package webapi
type PublishedFileDetailsResponse struct {
Response struct {
PublishedFileDetails []struct {
Result int `json:"result"`
PublishedFileID string `json:"publishedfileid"`
Creator string `json:"creator"`
CreatorAppid int `json:"creator_appid"`
ConsumerAppid int `json:"consumer_appid"`
ConsumerShortcutID int `json:"consumer_shortcutid"`
Filename string `json:"filename"`
FileSize string `json:"file_size"`
PreviewFileSize string `json:"preview_file_size"`
FileURL string `json:"file_url"`
PreviewURL string `json:"preview_url"`
URL string `json:"url"`
HContentFile string `json:"hcontent_file"`
HContentPreview string `json:"hcontent_preview"`
Title string `json:"title"`
FileDescription string `json:"file_description"`
TimeCreated int `json:"time_created"`
TimeUpdated int `json:"time_updated"`
Visibility int `json:"visibility"`
Flags int `json:"flags"`
WorkshopFile bool `json:"workshop_file"`
WorkshopAccepted bool `json:"workshop_accepted"`
ShowSubscribeAll bool `json:"show_subscribe_all"`
NumCommentsPublic int `json:"num_comments_public"`
Banned bool `json:"banned"`
BanReason string `json:"ban_reason"`
Banner string `json:"banner"`
CanBeDeleted bool `json:"can_be_deleted"`
AppName string `json:"app_name"`
FileType int `json:"file_type"`
CanSubscribe bool `json:"can_subscribe"`
Subscriptions int `json:"subscriptions"`
Favorited int `json:"favorited"`
Followers int `json:"followers"`
LifetimeSubscriptions int `json:"lifetime_subscriptions"`
LifetimeFavorited int `json:"lifetime_favorited"`
LifetimeFollowers int `json:"lifetime_followers"`
LifetimePlaytime string `json:"lifetime_playtime"`
LifetimePlaytimeSessions string `json:"lifetime_playtime_sessions"`
Views int `json:"views"`
ImageWidth int `json:"image_width,omitempty"`
ImageHeight int `json:"image_height,omitempty"`
NumChildren int `json:"num_children"`
NumReports int `json:"num_reports"`
Tags []struct {
Tag string `json:"tag"`
DisplayName string `json:"display_name"`
} `json:"tags"`
Children []struct {
PublishedFileID string `json:"publishedfileid"`
SortOrder int `json:"sortorder"`
FileType int `json:"file_type"`
} `json:"children,omitempty"`
Language int `json:"language"`
MaybeInappropriateSex bool `json:"maybe_inappropriate_sex"`
MaybeInappropriateViolence bool `json:"maybe_inappropriate_violence"`
RevisionChangeNumber string `json:"revision_change_number"`
Revision int `json:"revision"`
BanTextCheckResult int `json:"ban_text_check_result"`
} `json:"publishedfiledetails"`
} `json:"response"`
}