diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c01d59f..03bf207 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,10 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.13 + - name: Set up Go 1.19 uses: actions/setup-go@v1 with: - go-version: 1.13 + go-version: 1.19 id: go - name: Check out code into the Go module directory diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e8341f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/issue/ +/ZiniGo/pkg/ diff --git a/.idea/.gitignore b/.idea/.gitignore index 73f69e0..13566b8 100644 --- a/.idea/.gitignore +++ b/.idea/.gitignore @@ -1,8 +1,8 @@ # Default ignored files /shelf/ /workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ # Datasource local storage ignored files /dataSources/ /dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/Grabazine.go b/ZiniGo/main.go similarity index 87% rename from Grabazine.go rename to ZiniGo/main.go index 6f79095..00bb944 100644 --- a/Grabazine.go +++ b/ZiniGo/main.go @@ -7,11 +7,12 @@ import ( "fmt" wkhtml "github.com/SebastiaanKlippert/go-wkhtmltopdf" "github.com/icza/gox/stringsx" - "github.com/mxschmitt/playwright-go" "github.com/pdfcpu/pdfcpu/pkg/api" + "github.com/playwright-community/playwright-go" "github.com/tidwall/gjson" "io/ioutil" "log" + "math/rand" "net/http" "os" "os/exec" @@ -237,7 +238,11 @@ func GetPages(userToken LoginResponse, issue LibraryData, endpoint string) Respo req, _ := http.NewRequest("GET", "https://zinio.com/api/newsstand/newsstands/101/issues/"+strconv.Itoa(issue.Id)+"/content/pages?format=svg&application_id=9901&css_content=true&user_id="+userToken.Data.User.UserIDString, nil) req.Header.Add("Content-Type", "application/json") - req.AddCookie(&http.Cookie{Name: "zwrt", Value: userToken.Data.AccessToken}) + //req.AddCookie(&http.Cookie{Name: "zwrt", Value: userToken.Data.AccessToken}) + for _, cookie := range userToken.Data.Cookies { + req.AddCookie(cookie) + + } //req.Header.Add("Authorization", "bearer "+token) //req.Header.Add("Authorization", token) @@ -269,9 +274,13 @@ func GetInitialToken() (token string, err error) { } func GetLoginToken(username string, password string) LoginResponse { + fmt.Println("GettingLogin") + client := &http.Client{} - var jsonStr = []byte(`{"username":"` + username + `","password":"` + password + `"}`) + var jsonStr = []byte(`{"username":"` + username + `","password":"` + password + `","device":{"name":"Windows Chrome","fingerprint":"` + randSeq(32) + `","device_type":"Desktop","client_type":"Web"},"newsstand":{"currency":"ZAR","id":134,"country":"ZA","name":"South Africa","cc":"za","localeCode":"en_ZA","userLang":"en_ZA","userCountry":"ZA","userCurrency":"ZAR","requiresCookies":true,"requiresExplicitConsent":true,"requiresAdultConfirmation":true,"adWords":{"id":1,"label":""},"isDefaultNewsstand":false}}`) + //fmt.Println(string(jsonStr)) + req, _ := http.NewRequest("POST", "https://www.zinio.com/api/login?project=99&logger=null", bytes.NewBuffer(jsonStr)) req.Header.Add("Content-Type", "application/json") @@ -279,12 +288,14 @@ func GetLoginToken(username string, password string) LoginResponse { resp, _ := client.Do(req) data, _ := ioutil.ReadAll(resp.Body) + //fmt.Println(string(data)) responseType := LoginResponse{} _ = json.Unmarshal([]byte(data), &responseType) for _, cookie := range resp.Cookies() { + fmt.Println("cookie name:" + cookie.Name + "cookie value" + cookie.Value) if cookie.Name == "zwrt" { responseType.Data.AccessToken = cookie.Value } @@ -292,12 +303,15 @@ func GetLoginToken(username string, password string) LoginResponse { responseType.Data.RefreshToken = cookie.Value } } + responseType.Data.Cookies = resp.Cookies() + fmt.Println("GotLogin") return responseType } func GetLibrary(userToken LoginResponse, endpoint string) []LibraryResponse { + fmt.Println("Fetching Library") client := &http.Client{} var itemsToReturn []LibraryResponse @@ -305,10 +319,15 @@ func GetLibrary(userToken LoginResponse, endpoint string) []LibraryResponse { pageToFetch := 1 for { + fmt.Println("Fetching page:" + strconv.Itoa(pageToFetch)) + req, _ := http.NewRequest("GET", "https://zinio.com/api/newsstand/newsstands/101/users/"+userToken.Data.User.UserIDString+"/library_issues?limit="+strconv.Itoa(issuesToFetch)+"&page="+strconv.Itoa(pageToFetch), nil) req.Header.Add("Content-Type", "application/json") - req.AddCookie(&http.Cookie{Name: "zwrt", Value: userToken.Data.AccessToken}) + for _, cookie := range userToken.Data.Cookies { + req.AddCookie(cookie) + } + //req.AddCookie(&http.Cookie{Name: "zwrt", Value: userToken.Data.AccessToken}) //req.Header.Add("Authorization", "bearer "+userToken.Data.Token.AccessToken) //req.Header.Add("Authorization", initialToken) @@ -319,6 +338,7 @@ func GetLibrary(userToken LoginResponse, endpoint string) []LibraryResponse { } data, _ := ioutil.ReadAll(resp.Body) + fmt.Println(string(data)) responseType := LibraryResponse{} @@ -340,6 +360,11 @@ func fileExists(filename string) bool { if os.IsNotExist(err) { return false } + + if os.IsPermission(err) { + fmt.Println("Unable to read location - check permissions: " + filename) + return true + } return !info.IsDir() } @@ -353,6 +378,7 @@ type LoginData struct { Token Token `json:"token"` RefreshToken string `json:"refreshToken"` AccessToken string + Cookies []*http.Cookie } type User struct { @@ -378,7 +404,7 @@ type Publication struct { Name string `json:"name"` } -//https://stackoverflow.com/questions/47606761/repeat-code-if-an-error-occured +// https://stackoverflow.com/questions/47606761/repeat-code-if-an-error-occured func retry(attempts int, sleep time.Duration, f func() error) (err error) { for i := 0; ; i++ { err = f() @@ -461,3 +487,14 @@ func RemoveBadCharacters(input string) string { return temp } + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") + +func randSeq(n int) string { + + b := make([]rune, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} diff --git a/buildscripts/darwin.sh b/buildscripts/darwin.sh index 3cb2c33..1f940f4 100644 --- a/buildscripts/darwin.sh +++ b/buildscripts/darwin.sh @@ -1,15 +1,20 @@ -set GOOS = 'windows' -set GOARCH = 'amd64' +set GOOS=windows +set GOARCH=amd64 cd .. -go build -o built/Zinigo_Windows_x64.exe Grabazine.go +go build -v -o built/Zinigo_Windows_x64.exe Zinigo/main.go cd buildscripts -set GOOS = 'linux' -set GOARCH = 'amd64' +set GOOS=linux +set GOARCH=amd64 cd .. -go build -o built/Zinigo_Linux_AMD64 Grabazine.go +go build -o built/Zinigo_Linux_AMD64 Zinigo/main.go cd buildscripts set GOOS=darwin set GOARCH=amd64 cd .. -go build -o built/Zinigo_Macos_AMD64 Grabazine.go +go build -o built/Zinigo_Macos_AMD64 Zinigo/main.go +cd buildscripts +set GOOS=windows +set GOARCH=386 +cd .. +go build -o built/Zinigo_Windows.exe Zinigo/main.go cd buildscripts diff --git a/buildscripts/windows.ps1 b/buildscripts/windows.ps1 index 52d4d9f..0eccc36 100644 --- a/buildscripts/windows.ps1 +++ b/buildscripts/windows.ps1 @@ -1,10 +1,25 @@ $env:GOOS = 'windows' $env:GOARCH = 'amd64' cd .. -go build -o built/Zinigo_Windows_x64.exe Grabazine.go +go build -o built/Zinigo_Windows_x64.exe Zinigo/main.go +cd buildscripts +$env:GOOS = 'windows' +$env:GOARCH = '386' +cd .. +go build -o built/Zinigo_Windows_x86.exe Zinigo/main.go cd buildscripts $env:GOOS = 'linux' $env:GOARCH = 'amd64' cd .. -go build -o built/Zinigo_Linux_AMD64 Grabazine.go +go build -o built/Zinigo_Linux_AMD64 Zinigo/main.go +cd buildscripts +$env:GOOS = 'darwin' +$env:GOARCH = 'amd64' +cd .. +go build -o built/Zinigo_Macos_Intel Zinigo/main.go +cd buildscripts +$env:GOOS = 'darwin' +$env:GOARCH = 'arm64' +cd .. +go build -o built/Zinigo_Macos_AppleSilicon Zinigo/main.go cd buildscripts \ No newline at end of file diff --git a/built/Zinigo_Linux_AMD64 b/built/Zinigo_Linux_AMD64 index 5b72abb..8d7b056 100755 Binary files a/built/Zinigo_Linux_AMD64 and b/built/Zinigo_Linux_AMD64 differ diff --git a/built/Zinigo_Macos_AppleSilicon b/built/Zinigo_Macos_AppleSilicon new file mode 100644 index 0000000..04a3725 Binary files /dev/null and b/built/Zinigo_Macos_AppleSilicon differ diff --git a/built/Zinigo_Macos_Intel b/built/Zinigo_Macos_Intel new file mode 100644 index 0000000..7950a76 Binary files /dev/null and b/built/Zinigo_Macos_Intel differ diff --git a/built/Zinigo_Windows_x64.exe b/built/Zinigo_Windows_x64.exe index 9ee22a2..3602ee2 100755 Binary files a/built/Zinigo_Windows_x64.exe and b/built/Zinigo_Windows_x64.exe differ diff --git a/built/Zinigo_Windows_x86.exe b/built/Zinigo_Windows_x86.exe new file mode 100644 index 0000000..364ccee Binary files /dev/null and b/built/Zinigo_Windows_x86.exe differ diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2d9dde4 --- /dev/null +++ b/go.mod @@ -0,0 +1,26 @@ +module ZiniGo + +go 1.19 + +require ( + github.com/SebastiaanKlippert/go-wkhtmltopdf v1.7.2 + github.com/icza/gox v0.0.0-20220921190100-610a6663952b + github.com/pdfcpu/pdfcpu v0.3.13 + github.com/playwright-community/playwright-go v0.2000.1 + github.com/tidwall/gjson v1.14.3 +) + +require ( + github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/hhrutter/lzw v0.0.0-20190829144645-6f07a24e8650 // indirect + github.com/hhrutter/tiff v0.0.0-20190829141212-736cae8d0bc7 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb // indirect + golang.org/x/text v0.3.6 // indirect + gopkg.in/square/go-jose.v2 v2.6.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9fcb505 --- /dev/null +++ b/go.sum @@ -0,0 +1,55 @@ +github.com/SebastiaanKlippert/go-wkhtmltopdf v1.7.2 h1:LORAatv6KuKheYq8HXehiwx3f/VGuzJBNSydUDQ98EM= +github.com/SebastiaanKlippert/go-wkhtmltopdf v1.7.2/go.mod h1:TY8r0gmwEL1c5Lbd66NgQCkL4ZjGDJCMVqvbbFvUx20= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/hhrutter/lzw v0.0.0-20190827003112-58b82c5a41cc/go.mod h1:yJBvOcu1wLQ9q9XZmfiPfur+3dQJuIhYQsMGLYcItZk= +github.com/hhrutter/lzw v0.0.0-20190829144645-6f07a24e8650 h1:1yY/RQWNSBjJe2GDCIYoLmpWVidrooriUr4QS/zaATQ= +github.com/hhrutter/lzw v0.0.0-20190829144645-6f07a24e8650/go.mod h1:yJBvOcu1wLQ9q9XZmfiPfur+3dQJuIhYQsMGLYcItZk= +github.com/hhrutter/tiff v0.0.0-20190829141212-736cae8d0bc7 h1:o1wMw7uTNyA58IlEdDpxIrtFHTgnvYzA8sCQz8luv94= +github.com/hhrutter/tiff v0.0.0-20190829141212-736cae8d0bc7/go.mod h1:WkUxfS2JUu3qPo6tRld7ISb8HiC0gVSU91kooBMDVok= +github.com/icza/gox v0.0.0-20220921190100-610a6663952b h1:xd2cjCOyGBWqq79z+56XZV0BThORjuMXzFpj5rIkz0U= +github.com/icza/gox v0.0.0-20220921190100-610a6663952b/go.mod h1:VbcN86fRkkUMPX2ufM85Um8zFndLZswoIW1eYtpAcVk= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pdfcpu/pdfcpu v0.3.13 h1:VFon2Yo1PJt+sA57vPAeXWGLSZ7Ux3Jl4h02M0+s3dg= +github.com/pdfcpu/pdfcpu v0.3.13/go.mod h1:UJc5xsXg0fpmjp1zOPdyYcAQArc/Zf3V0nv5URe+9fg= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/playwright-community/playwright-go v0.2000.1 h1:2JViSHpJQ/UL/PO1Gg6gXV5IcXAAsoBJ3KG9L3wKXto= +github.com/playwright-community/playwright-go v0.2000.1/go.mod h1:1y9cM9b9dVHnuRWzED1KLM7FtbwTJC8ibDjI6MNqewU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= +github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +golang.org/x/image v0.0.0-20190823064033-3a9bac650e44/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= +gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=